diff --git a/.CodeQL.yml b/.CodeQL.yml deleted file mode 100644 index 3c93eef4979827..00000000000000 --- a/.CodeQL.yml +++ /dev/null @@ -1,10 +0,0 @@ -# This file configures CodeQL runs and TSA bug autofiling. For more information, see: -# https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/codeql/troubleshooting/bugs/generated-library-code -# (Access restricted to Microsoft employees only.) - -path_classifiers: - refs: - # The ref/ directories don't contain shipping implementations of code, so they should - # be excluded from analysis. If there is a problem at the API layer, the analysis - # engine will detect the problem in the src/ implementations anyway. - - src/libraries/**/ref/* diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 22f4226d1bfee9..2663dda99de3fe 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "9.0.0-prerelease.24161.1", + "version": "9.0.0-prerelease.24112.4", "commands": [ "xharness" ] diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d76e325e8b6cae..5a697ac088194c 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -25,5 +25,4 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libssl-dev \ libkrb5-dev \ zlib1g-dev \ - ninja-build \ - tzdata + ninja-build diff --git a/.devcontainer/scripts/onCreateCommand.sh b/.devcontainer/scripts/onCreateCommand.sh index 779ced4de99b98..066d0eccda876b 100755 --- a/.devcontainer/scripts/onCreateCommand.sh +++ b/.devcontainer/scripts/onCreateCommand.sh @@ -2,26 +2,6 @@ set -e -function wasm_common() { - # prebuild for WASM, so it is ready for wasm development - make -C src/mono/browser provision-wasm - export EMSDK_PATH=$PWD/src/mono/browser/emsdk - case "$1" in - wasm) - # Put your common commands for wasm here - ./build.sh mono+libs -os browser -c Release - ;; - wasm-multithreaded) - # Put your common commands for wasm-multithread here - ./build.sh mono+libs -os browser -c Release /p:WasmEnableThreads=true - ;; - *) - # install dotnet-serve for running wasm samples - ./dotnet.sh tool install dotnet-serve --version 1.10.172 --tool-path ./.dotnet-tools-global - ;; - esac -} - opt=$1 case "$opt" in @@ -40,11 +20,13 @@ case "$opt" in ;; wasm) - wasm_common $opt - ;; + # prebuild for WASM, so it is ready for wasm development + make -C src/mono/browser provision-wasm + export EMSDK_PATH=$PWD/src/mono/browser/emsdk + ./build.sh mono+libs -os browser -c Release - wasm-multithreaded) - wasm_common $opt + # install dotnet-serve for running wasm samples + ./dotnet.sh tool install dotnet-serve --version 1.10.172 --tool-path ./.dotnet-tools-global ;; esac diff --git a/.devcontainer/wasm-multiThreaded/Dockerfile b/.devcontainer/wasm-multiThreaded/Dockerfile deleted file mode 100644 index 75f2465b391b3c..00000000000000 --- a/.devcontainer/wasm-multiThreaded/Dockerfile +++ /dev/null @@ -1,60 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.192.0/containers/dotnet/.devcontainer/base.Dockerfile -# For details on dotnet specific container, see: https://github.com/microsoft/vscode-dev-containers/tree/main/containers/dotnet - -# [Choice] .NET version: 6.0, 7.0 -ARG VARIANT="6.0-jammy" -FROM mcr.microsoft.com/devcontainers/dotnet:0-${VARIANT} - -# Set up machine requirements to build the repo and the gh CLI -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends \ - cmake \ - llvm \ - clang \ - build-essential \ - python3 \ - curl \ - git \ - lldb \ - liblldb-dev \ - libunwind8 \ - libunwind8-dev \ - gettext \ - libicu-dev \ - liblttng-ust-dev \ - libssl-dev \ - libkrb5-dev \ - zlib1g-dev \ - ninja-build - -SHELL ["/bin/bash", "-c"] - -# Install LTS npm and node -RUN source /usr/local/share/nvm/nvm.sh && nvm install --lts - -# Install V8 Engine -RUN curl -sSL "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/linux/chromium-v8/v8-linux64-rel-10.8.168.zip" -o ./v8.zip \ - && unzip ./v8.zip -d /usr/local/v8 \ - && echo $'#!/usr/bin/env bash\n\ -"/usr/local/v8/d8" --snapshot_blob="/usr/local/v8/snapshot_blob.bin" "$@"\n' > /usr/local/bin/v8 \ - && chmod +x /usr/local/bin/v8 - -# install chromium dependencies to run debugger tests: -RUN sudo apt-get install libnss3 -y \ - && apt-get install libatk1.0-0 -y \ - && apt-get install libatk-bridge2.0-0 -y \ - && apt-get install libcups2 -y \ - && apt-get install libdrm2 -y \ - && apt-get install libxkbcommon-x11-0 -y \ - && apt-get install libxcomposite-dev -y \ - && apt-get install libxdamage1 -y \ - && apt-get install libxrandr2 -y \ - && apt-get install libgbm-dev -y \ - && apt-get install libpango-1.0-0 -y \ - && apt-get install libcairo2 -y \ - && apt-get install libasound2 -y - -# install firefox dependencies to run debugger tests: -RUN sudo apt-get install libdbus-glib-1-2 -y \ - && apt-get install libgtk-3-0 -y \ - && apt-get install libx11-xcb-dev -y diff --git a/.devcontainer/wasm-multiThreaded/devcontainer.json b/.devcontainer/wasm-multiThreaded/devcontainer.json deleted file mode 100644 index db3b2981b57149..00000000000000 --- a/.devcontainer/wasm-multiThreaded/devcontainer.json +++ /dev/null @@ -1,65 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. -{ - "name": "WASM multithreaded development (prebuilt)", - "build": { - "dockerfile": "Dockerfile", - "args": { - // Update 'VARIANT' to pick a .NET Core version: 6.0, 7.0 - "VARIANT": "6.0-jammy" - } - }, - "hostRequirements": { - "cpus": 4, - "memory": "8gb" - }, - - "features": { - "ghcr.io/devcontainers/features/github-cli:1": {} - }, - - // Configure tool-specific properties. - "customizations": { - // Configure properties specific to VS Code. - "vscode": { - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "ms-dotnettools.csharp" - ], - "settings": { - // Loading projects on demand is better for larger codebases - "omnisharp.enableMsBuildLoadProjectsOnDemand": true, - "omnisharp.enableRoslynAnalyzers": true, - "omnisharp.enableEditorConfigSupport": true, - "omnisharp.enableAsyncCompletion": true, - "omnisharp.testRunSettings": "${containerWorkspaceFolder}/artifacts/obj/vscode/.runsettings" - } - } - }, - - // Use 'onCreateCommand' to run pre-build commands inside the codespace - "onCreateCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/onCreateCommand.sh wasm-multithreaded", - - // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "${containerWorkspaceFolder}/.devcontainer/scripts/postCreateCommand.sh", - - // Add the locally installed dotnet to the path to ensure that it is activated - // This allows developers to just use 'dotnet build' on the command-line, and the local dotnet version will be used. - // Add the global tools dir to the PATH so that globally installed tools will work - "remoteEnv": { - "PATH": "${containerWorkspaceFolder}/.dotnet:${containerWorkspaceFolder}/.dotnet-tools-global:${containerEnv:PATH}", - "DOTNET_MULTILEVEL_LOOKUP": "0", - // Path to provisioned Emscripten SDK, for rebuilding the wasm runtime - "EMSDK_PATH": "${containerWorkspaceFolder}/src/mono/browser/emsdk", - }, - - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode", - - // Forward mono samples port - "forwardPorts": [8000], - "portsAttributes": { - "8000": { - "label": "mono wasm samples (8000)", - } - } -} diff --git a/.devcontainer/wasm/devcontainer.json b/.devcontainer/wasm/devcontainer.json index f4144299ff11d3..ab598dcb9a32d2 100644 --- a/.devcontainer/wasm/devcontainer.json +++ b/.devcontainer/wasm/devcontainer.json @@ -1,6 +1,6 @@ // For format details, see https://aka.ms/devcontainer.json. { - "name": "WASM singlethreaded development (prebuilt)", + "name": "WASM development (prebuilt)", "build": { "dockerfile": "Dockerfile", "args": { diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12303cdf57558d..b042cebfc6b92a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,7 +21,7 @@ /src/mono @marek-safar -/src/mono/llvm @vargaz @steveisok +/src/mono/llvm @vargaz @SamMonoRT /src/mono/mono/arch @vargaz /src/mono/mono/eglib @vargaz @lambdageek @@ -36,7 +36,7 @@ /src/mono/mono/eventpipe @lateralusX @lambdageek -/src/mono/mono/mini @vargaz @lambdageek @steveisok +/src/mono/mono/mini @vargaz @lambdageek @SamMonoRT /src/mono/mono/mini/*cfgdump* @vargaz /src/mono/mono/mini/*exceptions* @vargaz @BrzVlad /src/mono/mono/mini/*llvm* @vargaz @fanyang-mono @@ -50,7 +50,7 @@ /src/mono/mono/mini/*simd* @fanyang-mono /src/mono/mono/profiler @BrzVlad @lambdageek -/src/mono/mono/sgen @BrzVlad @lambdageek +/src/mono/mono/sgen @BrzVlad @lambdageek @SamMonoRT /src/mono/mono/utils @vargaz @lambdageek /src/mono/mono/utils/*-win* @lateralusX @lambdageek @@ -112,4 +112,4 @@ # Area ownership and repo automation /docs/area-owners.* @jeffhandley /docs/issue*.md @jeffhandley -/.github/policies/ @jeffhandley @mkArtakMSFT +/.github/fabricbot.json @jeffhandley diff --git a/.github/ISSUE_TEMPLATE/04_ci_known_issue.yml b/.github/ISSUE_TEMPLATE/04_ci_known_issue.yml new file mode 100644 index 00000000000000..17ec4e5e5ec930 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/04_ci_known_issue.yml @@ -0,0 +1,32 @@ +name: CI Known Issue Report +description: Create a known issue directly +labels: ["blocking-clean-ci","Known Build Error"] +body: + - type: markdown + attributes: + value: | + Use this template to report issues currently affecting PR stability, be it build or test failures. + - type: textarea + id: background + attributes: + label: Error Blob + description: Please identify a clear error string that can help identify future instances of this issue. For more information on how to fill this check our issue triage guidelines at [Failure Analysis](/dotnet/runtime/blob/main/docs/workflow/ci/failure-analysis.md#what-to-do-if-you-determine-the-failure-is-unrelated) + value: | + ```json + { + "ErrorMessage": "", + "BuildRetry": false, + "ErrorPattern": "", + "ExcludeConsoleLog": true + } + ``` + validations: + required: true + - type: textarea + id: repro-steps + attributes: + label: Reproduction Steps + description: | + If possible describe where you observe the issue with links and any other relevant details. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/04_blank_issue.md b/.github/ISSUE_TEMPLATE/05_blank_issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/04_blank_issue.md rename to .github/ISSUE_TEMPLATE/05_blank_issue.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index b14edd954edeef..54d8c5740bad6c 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -18,6 +18,3 @@ contact_links: - name: Issue with WPF url: https://github.com/dotnet/wpf/issues/new/choose about: Please open issues relating to WPF in dotnet/wpf. - - name: CI Known Issue Report - url: https://helix.dot.net/BuildAnalysis/CreateKnownIssues - about: Use the helper to create a Known Issue in CI if failures in your runs are unrelated to your change. See [Failure Analysis](https://github.com/dotnet/runtime/blob/main/docs/workflow/ci/failure-analysis.md#what-to-do-if-you-determine-the-failure-is-unrelated) for triage instructions. diff --git a/.github/fabricbot.json b/.github/fabricbot.json new file mode 100644 index 00000000000000..27ea96a2762f3a --- /dev/null +++ b/.github/fabricbot.json @@ -0,0 +1,2938 @@ +[ + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "Area-owners", + "labelsAndMentions": [ + { + "labels": [ + "area-AssemblyLoader-coreclr" + ], + "mentionees": [ + "vitek-karas", + "agocke", + "vsadov" + ] + }, + { + "labels": [ + "area-AssemblyLoader-mono" + ], + "mentionees": [] + }, + { + "labels": [ + "area-CodeGen-coreclr" + ], + "mentionees": [ + "JulieLeeMSFT", + "jakobbotsch" + ] + }, + { + "labels": [ + "area-Codegen-Interpreter-mono" + ], + "mentionees": [ + "brzvlad", + "kotlarmilos" + ] + }, + { + "labels": [ + "area-Codegen-JIT-Mono" + ], + "mentionees": [ + "SamMonoRT", + "vargaz" + ] + }, + { + "labels": [ + "area-CodeGen-LLVM-Mono" + ], + "mentionees": [ + "SamMonoRT", + "vargaz" + ] + }, + { + "labels": [ + "area-Codegen-Intrinsics-mono" + ], + "mentionees": [ + "SamMonoRT", + "fanyang-mono" + ] + }, + { + "labels": [ + "area-CodeGen-meta-Mono" + ], + "mentionees": [ + "SamMonoRT", + "vargaz", + "lambdageek" + ] + }, + { + "labels": [ + "area-System.DateTime" + ], + "mentionees": [ + "dotnet/area-system-datetime" + ] + }, + { + "labels": [ + "area-Debugger-mono" + ], + "mentionees": [ + "thaystg" + ] + }, + { + "labels": [ + "area-DependencyModel" + ], + "mentionees": [ + "dotnet/area-dependencymodel" + ] + }, + { + "labels": [ + "area-Diagnostics-coreclr" + ], + "mentionees": [ + "tommcdon" + ] + }, + { + "labels": [ + "area-Extensions-Caching" + ], + "mentionees": [ + "dotnet/area-extensions-caching" + ] + }, + { + "labels": [ + "area-Extensions-Configuration" + ], + "mentionees": [ + "dotnet/area-extensions-configuration" + ] + }, + { + "labels": [ + "area-Extensions-DependencyInjection" + ], + "mentionees": [ + "dotnet/area-extensions-dependencyinjection" + ] + }, + { + "labels": [ + "area-Extensions-FileSystem" + ], + "mentionees": [ + "dotnet/area-extensions-filesystem" + ] + }, + { + "labels": [ + "area-Extensions-Hosting" + ], + "mentionees": [ + "dotnet/area-extensions-hosting" + ] + }, + { + "labels": [ + "area-Extensions-HttpClientFactory" + ], + "mentionees": [ + "dotnet/ncl" + ] + }, + { + "labels": [ + "area-Extensions-Logging" + ], + "mentionees": [ + "dotnet/area-extensions-logging" + ] + }, + { + "labels": [ + "area-Extensions-Options" + ], + "mentionees": [ + "dotnet/area-extensions-options" + ] + }, + { + "labels": [ + "area-Extensions-Primitives" + ], + "mentionees": [ + "dotnet/area-extensions-primitives" + ] + }, + { + "labels": [ + "area-GC-coreclr" + ], + "mentionees": [ + "dotnet/gc" + ] + }, + { + "labels": [ + "area-GC-mono" + ], + "mentionees": [ + "brzvlad" + ] + }, + { + "labels": [ + "area-Host" + ], + "mentionees": [ + "vitek-karas", + "agocke", + "vsadov" + ] + }, + { + "labels": [ + "area-HostModel" + ], + "mentionees": [ + "vitek-karas", + "agocke" + ] + }, + { + "labels": [ + "area-ILTools-coreclr" + ], + "mentionees": [ + "JulieLeeMSFT" + ] + }, + { + "labels": [ + "area-Tools-ILVerification" + ], + "mentionees": [ + "JulieLeeMSFT" + ] + }, + { + "labels": [ + "area-Infrastructure" + ], + "mentionees": [ + "dotnet/runtime-infrastructure" + ] + }, + { + "labels": [ + "area-Infrastructure-coreclr" + ], + "mentionees": [ + "hoyosjs" + ] + }, + { + "labels": [ + "area-Infrastructure-libraries" + ], + "mentionees": [ + "dotnet/area-infrastructure-libraries" + ] + }, + { + "labels": [ + "area-Infrastructure-mono" + ], + "mentionees": [ + "directhex" + ] + }, + { + "labels": [ + "area-Meta" + ], + "mentionees": [ + "dotnet/area-meta" + ] + }, + { + "labels": [ + "area-Microsoft.CSharp" + ], + "mentionees": [ + "cston" + ] + }, + { + "labels": [ + "area-Microsoft.Extensions" + ], + "mentionees": [ + "dotnet/area-microsoft-extensions" + ] + }, + { + "labels": [ + "area-Microsoft.VisualBasic" + ], + "mentionees": [ + "cston" + ] + }, + { + "labels": [ + "area-Microsoft.Win32" + ], + "mentionees": [ + "dotnet/area-microsoft-win32" + ] + }, + { + "labels": [ + "area-NativeAOT-coreclr" + ], + "mentionees": [ + "agocke", + "MichalStrehovsky", + "jkotas" + ] + }, + { + "labels": [ + "area-Single-File" + ], + "mentionees": [ + "agocke", + "vitek-karas", + "vsadov" + ] + }, + { + "labels": [ + "area-System.Buffers" + ], + "mentionees": [ + "dotnet/area-system-buffers" + ] + }, + { + "labels": [ + "area-System.CodeDom" + ], + "mentionees": [ + "dotnet/area-system-codedom" + ] + }, + { + "labels": [ + "area-System.Collections" + ], + "mentionees": [ + "dotnet/area-system-collections" + ] + }, + { + "labels": [ + "area-System.ComponentModel" + ], + "mentionees": [ + "dotnet/area-system-componentmodel" + ] + }, + { + "labels": [ + "area-System.ComponentModel.Composition" + ], + "mentionees": [ + "dotnet/area-system-componentmodel-composition" + ] + }, + { + "labels": [ + "area-System.ComponentModel.DataAnnotations" + ], + "mentionees": [ + "dotnet/area-system-componentmodel-dataannotations" + ] + }, + { + "labels": [ + "area-System.Composition" + ], + "mentionees": [ + "dotnet/area-system-composition" + ] + }, + { + "labels": [ + "area-System.Configuration" + ], + "mentionees": [ + "dotnet/area-system-configuration" + ] + }, + { + "labels": [ + "area-System.Console" + ], + "mentionees": [ + "dotnet/area-system-console" + ] + }, + { + "labels": [ + "area-System.Data" + ], + "mentionees": [ + "roji", + "ajcvickers" + ] + }, + { + "labels": [ + "area-System.Data.Odbc" + ], + "mentionees": [ + "roji", + "ajcvickers" + ] + }, + { + "labels": [ + "area-System.Data.OleDB" + ], + "mentionees": [ + "roji", + "ajcvickers" + ] + }, + { + "labels": [ + "area-System.Data.SqlClient" + ], + "mentionees": [ + "davoudeshtehari", + "david-engel", + "jrahnama" + ] + }, + { + "labels": [ + "area-System.Diagnostics" + ], + "mentionees": [ + "tommcdon" + ] + }, + { + "labels": [ + "area-System.Diagnostics.Activity" + ], + "mentionees": [ + "dotnet/area-system-diagnostics-activity" + ] + }, + { + "labels": [ + "area-System.Diagnostics.EventLog" + ], + "mentionees": [ + "dotnet/area-system-diagnostics-eventlog" + ] + }, + { + "labels": [ + "area-System.Diagnostics.PerformanceCounter" + ], + "mentionees": [ + "dotnet/area-system-diagnostics-performancecounter" + ] + }, + { + "labels": [ + "area-System.Diagnostics.Process" + ], + "mentionees": [ + "dotnet/area-system-diagnostics-process" + ] + }, + { + "labels": [ + "area-System.Diagnostics.TraceSource" + ], + "mentionees": [ + "dotnet/area-system-diagnostics-tracesource" + ] + }, + { + "labels": [ + "area-System.Diagnostics.Tracing" + ], + "mentionees": [ + "tarekgh", + "tommcdon", + "pjanotti" + ] + }, + { + "labels": [ + "area-System.DirectoryServices" + ], + "mentionees": [ + "dotnet/area-system-directoryservices", + "jay98014" + ] + }, + { + "labels": [ + "area-System.Drawing" + ], + "mentionees": [ + "dotnet/area-system-drawing" + ] + }, + { + "labels": [ + "area-System.Dynamic.Runtime" + ], + "mentionees": [ + "cston" + ] + }, + { + "labels": [ + "area-System.Formats.Asn1" + ], + "mentionees": [ + "dotnet/area-system-formats-asn1", + "bartonjs", + "vcsjones" + ] + }, + { + "labels": [ + "area-System.Formats.Cbor" + ], + "mentionees": [ + "dotnet/area-system-formats-cbor", + "bartonjs", + "vcsjones" + ] + }, + { + "labels": [ + "area-System.Formats.Tar" + ], + "mentionees": [ + "dotnet/area-system-formats-tar" + ] + }, + { + "labels": [ + "area-System.Globalization" + ], + "mentionees": [ + "dotnet/area-system-globalization" + ] + }, + { + "labels": [ + "area-System.IO" + ], + "mentionees": [ + "dotnet/area-system-io" + ] + }, + { + "labels": [ + "area-System.IO.Compression" + ], + "mentionees": [ + "dotnet/area-system-io-compression" + ] + }, + { + "labels": [ + "area-System.IO.Hashing" + ], + "mentionees": [ + "dotnet/area-system-io-hashing", + "bartonjs", + "vcsjones" + ] + }, + { + "labels": [ + "area-System.IO.Ports" + ], + "mentionees": [ + "dotnet/area-system-io-ports" + ] + }, + { + "labels": [ + "area-System.Linq" + ], + "mentionees": [ + "dotnet/area-system-linq" + ] + }, + { + "labels": [ + "area-System.Linq.Expressions" + ], + "mentionees": [ + "cston" + ] + }, + { + "labels": [ + "area-System.Linq.Parallel" + ], + "mentionees": [ + "dotnet/area-system-linq-parallel" + ] + }, + { + "labels": [ + "area-System.Management" + ], + "mentionees": [ + "dotnet/area-system-management" + ] + }, + { + "labels": [ + "area-System.Memory" + ], + "mentionees": [ + "dotnet/area-system-memory" + ] + }, + { + "labels": [ + "area-System.Net" + ], + "mentionees": [ + "dotnet/ncl" + ] + }, + { + "labels": [ + "area-System.Net.Http" + ], + "mentionees": [ + "dotnet/ncl" + ] + }, + { + "labels": [ + "area-System.Net.Quic" + ], + "mentionees": [ + "dotnet/ncl" + ] + }, + { + "labels": [ + "area-System.Net.Security" + ], + "mentionees": [ + "dotnet/ncl", + "bartonjs", + "vcsjones" + ] + }, + { + "labels": [ + "area-System.Net.Sockets" + ], + "mentionees": [ + "dotnet/ncl" + ] + }, + { + "labels": [ + "area-System.Numerics" + ], + "mentionees": [ + "dotnet/area-system-numerics" + ] + }, + { + "labels": [ + "area-System.Numerics.Tensors" + ], + "mentionees": [ + "dotnet/area-system-numerics-tensors" + ] + }, + { + "labels": [ + "area-System.Reflection" + ], + "mentionees": [ + "dotnet/area-system-reflection" + ] + }, + { + "labels": [ + "area-System.Reflection.Emit" + ], + "mentionees": [ + "dotnet/area-system-reflection-emit" + ] + }, + { + "labels": [ + "area-System.Reflection.Metadata" + ], + "mentionees": [ + "dotnet/area-system-reflection-metadata" + ] + }, + { + "labels": [ + "area-System.Resources" + ], + "mentionees": [ + "dotnet/area-system-resources" + ] + }, + { + "labels": [ + "area-System.Runtime" + ], + "mentionees": [ + "dotnet/area-system-runtime" + ] + }, + { + "labels": [ + "area-System.Runtime.CompilerServices" + ], + "mentionees": [ + "dotnet/area-system-runtime-compilerservices" + ] + }, + { + "labels": [ + "area-System.Runtime.InteropServices" + ], + "mentionees": [ + "dotnet/interop-contrib" + ] + }, + { + "labels": [ + "area-System.Runtime.Intrinsics" + ], + "mentionees": [ + "dotnet/area-system-runtime-intrinsics" + ] + }, + { + "labels": [ + "area-System.Security" + ], + "mentionees": [ + "dotnet/area-system-security", + "bartonjs", + "vcsjones" + ] + }, + { + "labels": [ + "area-System.ServiceProcess" + ], + "mentionees": [ + "dotnet/area-system-serviceprocess" + ] + }, + { + "labels": [ + "area-System.Speech" + ], + "mentionees": [ + "dotnet/area-system-speech" + ] + }, + { + "labels": [ + "area-System.Text.Encoding" + ], + "mentionees": [ + "dotnet/area-system-text-encoding" + ] + }, + { + "labels": [ + "area-System.Text.Encodings.Web" + ], + "mentionees": [ + "dotnet/area-system-text-encodings-web" + ] + }, + { + "labels": [ + "area-System.Text.Json" + ], + "mentionees": [ + "dotnet/area-system-text-json", + "gregsdennis" + ] + }, + { + "labels": [ + "area-System.Text.RegularExpressions" + ], + "mentionees": [ + "dotnet/area-system-text-regularexpressions" + ] + }, + { + "labels": [ + "area-System.Threading" + ], + "mentionees": [ + "mangod9" + ] + }, + { + "labels": [ + "area-System.Threading.Channels" + ], + "mentionees": [ + "dotnet/area-system-threading-channels" + ] + }, + { + "labels": [ + "area-System.Threading.Tasks" + ], + "mentionees": [ + "dotnet/area-system-threading-tasks" + ] + }, + { + "labels": [ + "area-System.Transactions" + ], + "mentionees": [ + "roji", + "ajcvickers" + ] + }, + { + "labels": [ + "area-System.Xml" + ], + "mentionees": [ + "dotnet/area-system-xml" + ] + }, + { + "labels": [ + "area-Tools-ILLink" + ], + "mentionees": [ + "agocke", + "sbomer", + "vitek-karas" + ] + }, + { + "labels": [ + "area-vm-coreclr" + ], + "mentionees": [ + "mangod9" + ] + } + ], + "replyTemplate": "Tagging subscribers to this area: ${mentionees}\nSee info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed.", + "enableForPullRequests": true + }, + "disabled": false + }, + { + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssuesOnlyResponder", + "version": "1.0", + "config": { + "conditions": { + "operator": "and", + "operands": [ + { + "name": "labelAdded", + "parameters": { + "label": "breaking-change" + } + } + ] + }, + "eventType": "issue", + "eventNames": [ + "issues" + ], + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "needs-breaking-change-doc-created" + } + }, + { + "name": "addReply", + "parameters": { + "comment": "Added `needs-breaking-change-doc-created` label because this issue has the `breaking-change` label. \n\n1. [ ] Create and link to this issue a matching issue in the dotnet/docs repo using the [breaking change documentation template](https://aka.ms/dotnet/docs/new-breaking-change-issue), then remove this `needs-breaking-change-doc-created` label.\n\nTagging @dotnet/compat for awareness of the breaking change." + } + } + ], + "taskName": "Add breaking change doc label to issue" + }, + "disabled": false + }, + { + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "conditions": { + "operator": "and", + "operands": [ + { + "name": "labelAdded", + "parameters": { + "label": "breaking-change" + } + }, + { + "name": "isPr", + "parameters": {} + } + ] + }, + "eventType": "pull_request", + "eventNames": [ + "pull_request", + "issues" + ], + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "needs-breaking-change-doc-created" + } + }, + { + "name": "addReply", + "parameters": { + "comment": "Added `needs-breaking-change-doc-created` label because this PR has the `breaking-change` label. \n\nWhen you commit this breaking change:\n\n1. [ ] Create and link to this PR and the issue a matching issue in the dotnet/docs repo using the [breaking change documentation template](https://aka.ms/dotnet/docs/new-breaking-change-issue), then remove this `needs-breaking-change-doc-created` label.\n2. [ ] Ask a committer to mail the `.NET Breaking Change Notification` DL.\n\nTagging @dotnet/compat for awareness of the breaking change." + } + } + ], + "taskName": "Add breaking change doc label to PR" + }, + "disabled": false + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "@Mention for linkable-framework", + "labelsAndMentions": [ + { + "labels": [ + "linkable-framework" + ], + "mentionees": [ + "eerhardt", + "vitek-karas", + "LakshanF", + "sbomer", + "joperezr", + "marek-safar" + ] + } + ], + "replyTemplate": "Tagging subscribers to 'linkable-framework': ${mentionees}\nSee info in area-owners.md if you want to be subscribed.", + "enableForPullRequests": true + } + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "@Mention for size-reduction", + "replyTemplate": "Tagging subscribers to 'size-reduction': ${mentionees}\nSee info in area-owners.md if you want to be subscribed.", + "labelsAndMentions": [ + { + "labels": [ + "size-reduction" + ], + "mentionees": [ + "eerhardt", + "SamMonoRT", + "marek-safar" + ] + } + ], + "enableForPullRequests": true + } + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "@Mention for wasm", + "labelsAndMentions": [ + { + "labels": [ + "arch-wasm" + ], + "mentionees": [ + "lewing" + ] + } + ], + "replyTemplate": "Tagging subscribers to 'arch-wasm': ${mentionees}\nSee info in area-owners.md if you want to be subscribed.", + "enableForPullRequests": true + } + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "@Mention for ios", + "labelsAndMentions": [ + { + "labels": [ + "os-ios" + ], + "mentionees": [ + "steveisok", + "akoeplinger", + "kotlarmilos" + ] + } + ], + "enableForPullRequests": true, + "replyTemplate": "Tagging subscribers to 'os-ios': ${mentionees}\nSee info in area-owners.md if you want to be subscribed." + } + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "@Mention for android", + "labelsAndMentions": [ + { + "labels": [ + "os-android" + ], + "mentionees": [ + "steveisok", + "akoeplinger" + ] + } + ], + "enableForPullRequests": true, + "replyTemplate": "Tagging subscribers to 'arch-android': ${mentionees}\nSee info in area-owners.md if you want to be subscribed." + } + }, + { + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "conditions": { + "operator": "and", + "operands": [ + { + "operator": "or", + "operands": [ + { + "name": "prMatchesPattern", + "parameters": { + "matchRegex": ".*ILLink.*" + } + }, + { + "name": "prMatchesPattern", + "parameters": { + "matchRegex": ".*illink.*" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "linkable-framework" + } + } + ] + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + }, + "eventType": "pull_request", + "eventNames": [ + "pull_request", + "issues" + ], + "taskName": "[Linkable-framework workgroup] Add linkable-framework label to new Prs that touch files with *ILLink* that not have it already", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "linkable-framework" + } + } + ] + } + }, + { + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "conditions": { + "operator": "and", + "operands": [ + { + "operator": "or", + "operands": [ + { + "name": "prMatchesPattern", + "parameters": { + "matchRegex": ".*ILLink.*" + } + }, + { + "name": "prMatchesPattern", + "parameters": { + "matchRegex": ".*illink.*" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "linkable-framework" + } + } + ] + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "isAction", + "parameters": { + "action": "synchronize" + } + } + ] + }, + "eventType": "pull_request", + "eventNames": [ + "pull_request", + "issues" + ], + "taskName": "[Linkable-framework workgroup] Add linkable-framework label to Prs that get changes pushed where they touch *ILLInk* files", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "linkable-framework" + } + } + ] + } + }, + { + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssuesOnlyResponder", + "version": "1.0", + "config": { + "conditions": { + "operator": "and", + "operands": [ + { + "name": "labelAdded", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ] + }, + "eventType": "issue", + "eventNames": [ + "issues" + ], + "taskName": "Manual Issue Cleanup", + "actions": [ + { + "name": "addReply", + "parameters": { + "comment": "Due to lack of recent activity, this issue has been marked as a candidate for backlog cleanup. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will undo this process.\n\nThis process is part of our [issue cleanup automation](https://github.com/dotnet/runtime/blob/main/docs/issue-cleanup.md)." + } + }, + { + "name": "addLabel", + "parameters": { + "label": "no-recent-activity" + } + } + ] + } + }, + { + "taskType": "scheduled", + "capabilityId": "ScheduledSearch", + "subCapability": "ScheduledSearch", + "version": "1.1", + "config": { + "frequency": [ + { + "weekDay": 0, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 1, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 2, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 3, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 4, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 5, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 6, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + } + ], + "searchTerms": [ + { + "name": "noActivitySince", + "parameters": { + "days": 1644 + } + }, + { + "name": "isIssue", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "noLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ], + "taskName": "Automated Issue cleanup", + "actions": [ + { + "name": "addReply", + "parameters": { + "comment": "Due to lack of recent activity, this issue has been marked as a candidate for backlog cleanup. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will undo this process.\n\nThis process is part of our [issue cleanup automation](https://github.com/dotnet/runtime/blob/main/docs/issue-cleanup.md)." + } + }, + { + "name": "addLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + }, + { + "name": "addLabel", + "parameters": { + "label": "no-recent-activity" + } + } + ] + } + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "taskName": "@Mention for tvos", + "labelsAndMentions": [ + { + "labels": [ + "os-tvos" + ], + "mentionees": [ + "steveisok", + "akoeplinger" + ] + } + ], + "enableForPullRequests": true, + "replyTemplate": "Tagging subscribers to 'os-tvos': ${mentionees}\nSee info in area-owners.md if you want to be subscribed." + } + }, + { + "taskType": "scheduledAndTrigger", + "capabilityId": "IssueRouting", + "subCapability": "@Mention", + "version": "1.0", + "config": { + "labelsAndMentions": [ + { + "labels": [ + "os-maccatalyst" + ], + "mentionees": [ + "steveisok", + "akoeplinger" + ] + } + ], + "replyTemplate": "Tagging subscribers to 'os-maccatalyst': ${mentionees}\nSee info in area-owners.md if you want to be subscribed.", + "enableForPullRequests": true, + "taskName": "@Mention for maccatalyst" + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssuesOnlyResponder", + "version": "1.0", + "config": { + "taskName": "Add untriaged label to new/reopened issues without a milestone", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "untriaged" + } + } + ], + "eventType": "issue", + "eventNames": [ + "issues" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "operator": "or", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "opened" + } + }, + { + "name": "isAction", + "parameters": { + "action": "reopened" + } + } + ] + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "operator": "not", + "operands": [ + { + "name": "isInMilestone", + "parameters": {} + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "untriaged" + } + } + ] + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssuesOnlyResponder", + "version": "1.0", + "config": { + "taskName": "Remove untriaged label from issues when closed or added to a milestone", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "untriaged" + } + } + ], + "eventType": "issue", + "eventNames": [ + "issues" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "operator": "or", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "closed" + } + }, + { + "name": "isInMilestone", + "parameters": {} + } + ] + }, + { + "name": "hasLabel", + "parameters": { + "label": "untriaged" + } + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "InPrLabel", + "subCapability": "InPrLabel", + "version": "1.0", + "config": { + "taskName": "Add `in-pr` label on issue when an open pull request is targeting it", + "inPrLabelText": "There is an active PR which will close this issue when it is merged", + "fixedLabelEnabled": false, + "label_inPr": "in-pr" + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "taskName": "Assign Team PRs to author", + "actions": [ + { + "name": "assignToUser", + "parameters": { + "user": { + "type": "prAuthor" + } + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "opened" + } + }, + { + "operator": "not", + "operands": [ + { + "name": "activitySenderHasPermissions", + "parameters": { + "permissions": "read" + } + } + ] + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "taskName": "Label community PRs", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "community-contribution" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "opened" + } + }, + { + "name": "isPr", + "parameters": {} + }, + { + "operator": "and", + "operands": [ + { + "operator": "not", + "operands": [ + { + "name": "activitySenderHasPermissions", + "parameters": { + "permissions": "admin" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "activitySenderHasPermissions", + "parameters": { + "permissions": "write" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": "github-actions[bot]" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": "dotnet-maestro[bot]" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": "dotnet-maestro-bot[bot]" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": "dotnet-maestro-bot" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": "dotnet-maestro" + } + } + ] + }, + { + "operator": "not", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": "github-actions" + } + } + ] + } + ] + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssuesOnlyResponder", + "version": "1.0", + "config": { + "taskName": "Needs-author-action notification", + "actions": [ + { + "name": "addReply", + "parameters": { + "comment": "This issue has been marked `needs-author-action` and may be missing some important information." + } + } + ], + "eventType": "issue", + "eventNames": [ + "issues" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "labelAdded", + "parameters": { + "label": "needs-author-action" + } + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestReviewResponder", + "version": "1.0", + "config": { + "taskName": "PR reviews with \"changes requested\" applies the needs-author-action label", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "needs-author-action" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request_review" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "operator": "not", + "operands": [ + { + "name": "activitySenderHasPermissions", + "parameters": { + "state": "changes_requested", + "permissions": "read" + } + } + ] + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isAction", + "parameters": { + "action": "submitted" + } + }, + { + "name": "isReviewState", + "parameters": { + "state": "changes_requested" + } + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssueCommentResponder", + "version": "1.0", + "config": { + "taskName": "Replace `needs-author-action` label with `needs-further-triage` label when the author comments on an issue that is not still untriaged", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "needs-further-triage" + } + }, + { + "name": "removeLabel", + "parameters": { + "label": "needs-author-action" + } + } + ], + "eventType": "issue", + "eventNames": [ + "issue_comment" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "created" + } + }, + { + "name": "isActivitySender", + "parameters": { + "user": { + "type": "author" + } + } + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + }, + { + "operator": "not", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "untriaged" + } + } + ] + }, + { + "name": "isIssue", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssueCommentResponder", + "version": "1.0", + "config": { + "taskName": "Remove `needs-author-action` label when the author comments on an `untriaged` issue", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "needs-author-action" + } + } + ], + "eventType": "issue", + "eventNames": [ + "issue_comment" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "created" + } + }, + { + "name": "isActivitySender", + "parameters": { + "user": { + "type": "author" + } + } + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + }, + { + "name": "hasLabel", + "parameters": { + "label": "untriaged" + } + }, + { + "name": "isIssue", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "taskName": "Pushing changes to PR branch removes the needs-author-action label", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "needs-author-action" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isAction", + "parameters": { + "action": "synchronize" + } + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestCommentResponder", + "version": "1.0", + "config": { + "taskName": "Author commenting in PR removes the needs-author-action label", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "needs-author-action" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "issue_comment" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": { + "type": "author" + } + } + }, + { + "name": "isAction", + "parameters": { + "action": "created" + } + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestReviewResponder", + "version": "1.0", + "config": { + "taskName": "Author responding to a pull request review comment removes the needs-author-action label", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "needs-author-action" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request_review" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isActivitySender", + "parameters": { + "user": { + "type": "author" + } + } + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + }, + { + "name": "isAction", + "parameters": { + "action": "submitted" + } + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "scheduled", + "capabilityId": "ScheduledSearch", + "subCapability": "ScheduledSearch", + "version": "1.1", + "config": { + "taskName": "Add no-recent-activity label to issues", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "addReply", + "parameters": { + "comment": "This issue has been automatically marked `no-recent-activity` because it has not had any activity for 14 days. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will remove `no-recent-activity`." + } + } + ], + "frequency": [ + { + "weekDay": 0, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 1, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 2, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 3, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 4, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 5, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 6, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + } + ], + "searchTerms": [ + { + "name": "isIssue", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + }, + { + "name": "noActivitySince", + "parameters": { + "days": 14 + } + }, + { + "name": "noLabel", + "parameters": { + "label": "no-recent-activity" + } + } + ] + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "scheduled", + "capabilityId": "ScheduledSearch", + "subCapability": "ScheduledSearch", + "version": "1.1", + "config": { + "taskName": "Add no-recent-activity label to PRs", + "actions": [ + { + "name": "addLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "addReply", + "parameters": { + "comment": "This pull request has been automatically marked `no-recent-activity` because it has not had any activity for 14 days. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will remove `no-recent-activity`." + } + } + ], + "frequency": [ + { + "weekDay": 0, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 1, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 2, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 3, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 4, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 5, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + }, + { + "weekDay": 6, + "hours": [ + 4, + 10, + 16, + 22 + ], + "timezoneOffset": 1 + } + ], + "searchTerms": [ + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "hasLabel", + "parameters": { + "label": "needs-author-action" + } + }, + { + "name": "noActivitySince", + "parameters": { + "days": 14 + } + }, + { + "name": "noLabel", + "parameters": { + "label": "no-recent-activity" + } + } + ] + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssuesOnlyResponder", + "version": "1.0", + "config": { + "taskName": "Remove `no-recent-activity` label from issues when issue is modified", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "removeLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ], + "eventType": "issue", + "eventNames": [ + "issues" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "operator": "not", + "operands": [ + { + "name": "isAction", + "parameters": { + "action": "closed" + } + } + ] + }, + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "operator": "not", + "operands": [ + { + "name": "labelAdded", + "parameters": { + "label": "no-recent-activity" + } + } + ] + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "IssueCommentResponder", + "version": "1.0", + "config": { + "taskName": "Remove `no-recent-activity` label when an issue is commented on", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "removeLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ], + "eventType": "issue", + "eventNames": [ + "issue_comment" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "isIssue", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestResponder", + "version": "1.0", + "config": { + "taskName": "Remove `no-recent-activity` label from PRs when modified", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "removeLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "operator": "not", + "operands": [ + { + "name": "labelAdded", + "parameters": { + "label": "no-recent-activity" + } + } + ] + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestCommentResponder", + "version": "1.0", + "config": { + "taskName": "Remove `no-recent-activity` label from PRs when commented on", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "removeLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "issue_comment" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "trigger", + "capabilityId": "IssueResponder", + "subCapability": "PullRequestReviewResponder", + "version": "1.0", + "config": { + "taskName": "Remove `no-recent-activity` label from PRs when new review is added", + "actions": [ + { + "name": "removeLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "removeLabel", + "parameters": { + "label": "backlog-cleanup-candidate" + } + } + ], + "eventType": "pull_request", + "eventNames": [ + "pull_request_review" + ], + "conditions": { + "operator": "and", + "operands": [ + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + } + ] + } + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "scheduled", + "capabilityId": "ScheduledSearch", + "subCapability": "ScheduledSearch", + "version": "1.1", + "config": { + "taskName": "Close issues with no recent activity", + "actions": [ + { + "name": "addReply", + "parameters": { + "comment": "This issue will now be closed since it had been marked `no-recent-activity` but received no further activity in the past 14 days. It is still possible to reopen or comment on the issue, but please note that the issue will be locked if it remains inactive for another 30 days." + } + }, + { + "name": "closeIssue", + "parameters": {} + } + ], + "frequency": [ + { + "weekDay": 0, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 1, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 2, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 3, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 4, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 5, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 6, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + } + ], + "searchTerms": [ + { + "name": "isIssue", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "noActivitySince", + "parameters": { + "days": 14 + } + } + ] + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "scheduled", + "capabilityId": "ScheduledSearch", + "subCapability": "ScheduledSearch", + "version": "1.1", + "config": { + "taskName": "Close PRs with no-recent-activity", + "actions": [ + { + "name": "addReply", + "parameters": { + "comment": "This pull request will now be closed since it had been marked `no-recent-activity` but received no further activity in the past 14 days. It is still possible to reopen or comment on the pull request, but please note that it will be locked if it remains inactive for another 30 days." + } + }, + { + "name": "closeIssue", + "parameters": {} + } + ], + "frequency": [ + { + "weekDay": 0, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 1, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 2, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 3, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 4, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 5, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 6, + "hours": [ + 0, + 6, + 12, + 18 + ], + "timezoneOffset": 0 + } + ], + "searchTerms": [ + { + "name": "isPr", + "parameters": {} + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "hasLabel", + "parameters": { + "label": "no-recent-activity" + } + }, + { + "name": "noActivitySince", + "parameters": { + "days": 14 + } + } + ] + } + }, + { + "taskSource": "fabricbot-config", + "taskType": "scheduled", + "capabilityId": "ScheduledSearch", + "subCapability": "ScheduledSearch", + "version": "1.1", + "config": { + "taskName": "Close inactive Draft PRs", + "actions": [ + { + "name": "closeIssue", + "parameters": {} + }, + { + "name": "addReply", + "parameters": { + "comment": "Draft Pull Request was automatically closed for 30 days of inactivity. Please [let us know](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you'd like to reopen it." + } + } + ], + "frequency": [ + { + "weekDay": 0, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 1, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 2, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 3, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 4, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 5, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + }, + { + "weekDay": 6, + "hours": [ + 5, + 11, + 17, + 23 + ], + "timezoneOffset": 0 + } + ], + "searchTerms": [ + { + "name": "isDraftPr", + "parameters": { + "value": "true" + } + }, + { + "name": "isOpen", + "parameters": {} + }, + { + "name": "noActivitySince", + "parameters": { + "days": 30 + } + } + ] + } + } +] diff --git a/.github/policies/resourceManagement.yml b/.github/policies/resourceManagement.yml deleted file mode 100644 index 94a9b162a4569a..00000000000000 --- a/.github/policies/resourceManagement.yml +++ /dev/null @@ -1,1882 +0,0 @@ -id: -name: GitOps.PullRequestIssueManagement -description: GitOps.PullRequestIssueManagement primitive -owner: -resource: repository -disabled: false -where: -configuration: - resourceManagementConfiguration: - scheduledSearches: - - description: Automated Issue cleanup - frequencies: - - hourly: - hour: 6 - filters: - - noActivitySince: - days: 1644 - - isIssue - - isOpen - - isNotLabeledWith: - label: backlog-cleanup-candidate - actions: - - addReply: - reply: >- - Due to lack of recent activity, this issue has been marked as a candidate for backlog cleanup. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will undo this process. - - - This process is part of our [issue cleanup automation](https://github.com/dotnet/runtime/blob/main/docs/issue-cleanup.md). - - addLabel: - label: backlog-cleanup-candidate - - addLabel: - label: no-recent-activity - - description: Add no-recent-activity label to issues - frequencies: - - hourly: - hour: 6 - filters: - - isIssue - - isOpen - - hasLabel: - label: needs-author-action - - noActivitySince: - days: 14 - - isNotLabeledWith: - label: no-recent-activity - actions: - - addLabel: - label: no-recent-activity - - addReply: - reply: This issue has been automatically marked `no-recent-activity` because it has not had any activity for 14 days. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will remove `no-recent-activity`. - - description: Add no-recent-activity label to PRs - frequencies: - - hourly: - hour: 6 - filters: - - isPullRequest - - isOpen - - hasLabel: - label: needs-author-action - - noActivitySince: - days: 14 - - isNotLabeledWith: - label: no-recent-activity - actions: - - addLabel: - label: no-recent-activity - - addReply: - reply: This pull request has been automatically marked `no-recent-activity` because it has not had any activity for 14 days. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will remove `no-recent-activity`. - - description: Close issues with no recent activity - frequencies: - - hourly: - hour: 6 - filters: - - isIssue - - isOpen - - hasLabel: - label: no-recent-activity - - noActivitySince: - days: 14 - actions: - - addReply: - reply: This issue will now be closed since it had been marked `no-recent-activity` but received no further activity in the past 14 days. It is still possible to reopen or comment on the issue, but please note that the issue will be locked if it remains inactive for another 30 days. - - closeIssue - - description: Close PRs with no-recent-activity - frequencies: - - hourly: - hour: 6 - filters: - - isPullRequest - - isOpen - - hasLabel: - label: no-recent-activity - - noActivitySince: - days: 14 - actions: - - addReply: - reply: This pull request will now be closed since it had been marked `no-recent-activity` but received no further activity in the past 14 days. It is still possible to reopen or comment on the pull request, but please note that it will be locked if it remains inactive for another 30 days. - - closeIssue - - description: Close inactive Draft PRs - frequencies: - - hourly: - hour: 6 - filters: - - isDraftPullRequest - - isOpen - - noActivitySince: - days: 30 - actions: - - closeIssue - - addReply: - reply: Draft Pull Request was automatically closed for 30 days of inactivity. Please [let us know](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you'd like to reopen it. - eventResponderTasks: - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: area-AssemblyLoader-coreclr - then: - - mentionUsers: - mentionees: - - vitek-karas - - agocke - - vsadov - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-AssemblyLoader-mono - then: - - mentionUsers: - mentionees: [] - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-CodeGen-coreclr - then: - - mentionUsers: - mentionees: - - JulieLeeMSFT - - jakobbotsch - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Codegen-Interpreter-mono - then: - - mentionUsers: - mentionees: - - brzvlad - - kotlarmilos - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Codegen-JIT-Mono - then: - - mentionUsers: - mentionees: - - SamMonoRT - - vargaz - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-CodeGen-LLVM-Mono - then: - - mentionUsers: - mentionees: - - SamMonoRT - - vargaz - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Codegen-Intrinsics-mono - then: - - mentionUsers: - mentionees: - - SamMonoRT - - fanyang-mono - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-CodeGen-meta-Mono - then: - - mentionUsers: - mentionees: - - SamMonoRT - - vargaz - - lambdageek - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.DateTime - then: - - mentionUsers: - mentionees: - - dotnet/area-system-datetime - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Debugger-mono - then: - - mentionUsers: - mentionees: - - thaystg - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-DependencyModel - then: - - mentionUsers: - mentionees: - - dotnet/area-dependencymodel - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Diagnostics-coreclr - then: - - mentionUsers: - mentionees: - - tommcdon - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-Caching - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-caching - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-Configuration - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-configuration - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-DependencyInjection - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-dependencyinjection - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-FileSystem - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-filesystem - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-Hosting - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-hosting - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-HttpClientFactory - then: - - mentionUsers: - mentionees: - - dotnet/ncl - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-Logging - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-logging - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-Options - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-options - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Extensions-Primitives - then: - - mentionUsers: - mentionees: - - dotnet/area-extensions-primitives - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-GC-coreclr - then: - - mentionUsers: - mentionees: - - dotnet/gc - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-GC-mono - then: - - mentionUsers: - mentionees: - - brzvlad - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Host - then: - - mentionUsers: - mentionees: - - vitek-karas - - agocke - - vsadov - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-HostModel - then: - - mentionUsers: - mentionees: - - vitek-karas - - agocke - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-ILTools-coreclr - then: - - mentionUsers: - mentionees: - - JulieLeeMSFT - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Tools-ILVerification - then: - - mentionUsers: - mentionees: - - JulieLeeMSFT - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Infrastructure - then: - - mentionUsers: - mentionees: - - dotnet/runtime-infrastructure - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Infrastructure-coreclr - then: - - mentionUsers: - mentionees: - - hoyosjs - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Infrastructure-libraries - then: - - mentionUsers: - mentionees: - - dotnet/area-infrastructure-libraries - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Infrastructure-mono - then: - - mentionUsers: - mentionees: - - directhex - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Meta - then: - - mentionUsers: - mentionees: - - dotnet/area-meta - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Microsoft.CSharp - then: - - mentionUsers: - mentionees: - - cston - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Microsoft.Extensions - then: - - mentionUsers: - mentionees: - - dotnet/area-microsoft-extensions - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Microsoft.VisualBasic - then: - - mentionUsers: - mentionees: - - cston - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Microsoft.Win32 - then: - - mentionUsers: - mentionees: - - dotnet/area-microsoft-win32 - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-NativeAOT-coreclr - then: - - mentionUsers: - mentionees: - - agocke - - MichalStrehovsky - - jkotas - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Single-File - then: - - mentionUsers: - mentionees: - - agocke - - vitek-karas - - vsadov - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Buffers - then: - - mentionUsers: - mentionees: - - dotnet/area-system-buffers - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.CodeDom - then: - - mentionUsers: - mentionees: - - dotnet/area-system-codedom - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Collections - then: - - mentionUsers: - mentionees: - - dotnet/area-system-collections - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.ComponentModel - then: - - mentionUsers: - mentionees: - - dotnet/area-system-componentmodel - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.ComponentModel.Composition - then: - - mentionUsers: - mentionees: - - dotnet/area-system-componentmodel-composition - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.ComponentModel.DataAnnotations - then: - - mentionUsers: - mentionees: - - dotnet/area-system-componentmodel-dataannotations - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Composition - then: - - mentionUsers: - mentionees: - - dotnet/area-system-composition - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Configuration - then: - - mentionUsers: - mentionees: - - dotnet/area-system-configuration - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Console - then: - - mentionUsers: - mentionees: - - dotnet/area-system-console - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Data - then: - - mentionUsers: - mentionees: - - roji - - ajcvickers - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Data.Odbc - then: - - mentionUsers: - mentionees: - - roji - - ajcvickers - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Data.OleDB - then: - - mentionUsers: - mentionees: - - roji - - ajcvickers - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Data.SqlClient - then: - - mentionUsers: - mentionees: - - davoudeshtehari - - david-engel - - jrahnama - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics - then: - - mentionUsers: - mentionees: - - tommcdon - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics.Activity - then: - - mentionUsers: - mentionees: - - dotnet/area-system-diagnostics-activity - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics.EventLog - then: - - mentionUsers: - mentionees: - - dotnet/area-system-diagnostics-eventlog - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics.PerformanceCounter - then: - - mentionUsers: - mentionees: - - dotnet/area-system-diagnostics-performancecounter - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics.Process - then: - - mentionUsers: - mentionees: - - dotnet/area-system-diagnostics-process - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics.TraceSource - then: - - mentionUsers: - mentionees: - - dotnet/area-system-diagnostics-tracesource - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Diagnostics.Tracing - then: - - mentionUsers: - mentionees: - - tarekgh - - tommcdon - - pjanotti - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.DirectoryServices - then: - - mentionUsers: - mentionees: - - dotnet/area-system-directoryservices - - jay98014 - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Drawing - then: - - mentionUsers: - mentionees: - - dotnet/area-system-drawing - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Dynamic.Runtime - then: - - mentionUsers: - mentionees: - - cston - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Formats.Asn1 - then: - - mentionUsers: - mentionees: - - dotnet/area-system-formats-asn1 - - bartonjs - - vcsjones - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Formats.Cbor - then: - - mentionUsers: - mentionees: - - dotnet/area-system-formats-cbor - - bartonjs - - vcsjones - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Formats.Tar - then: - - mentionUsers: - mentionees: - - dotnet/area-system-formats-tar - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Globalization - then: - - mentionUsers: - mentionees: - - dotnet/area-system-globalization - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.IO - then: - - mentionUsers: - mentionees: - - dotnet/area-system-io - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.IO.Compression - then: - - mentionUsers: - mentionees: - - dotnet/area-system-io-compression - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.IO.Hashing - then: - - mentionUsers: - mentionees: - - dotnet/area-system-io-hashing - - bartonjs - - vcsjones - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.IO.Ports - then: - - mentionUsers: - mentionees: - - dotnet/area-system-io-ports - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Linq - then: - - mentionUsers: - mentionees: - - dotnet/area-system-linq - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Linq.Expressions - then: - - mentionUsers: - mentionees: - - cston - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Linq.Parallel - then: - - mentionUsers: - mentionees: - - dotnet/area-system-linq-parallel - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Management - then: - - mentionUsers: - mentionees: - - dotnet/area-system-management - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Memory - then: - - mentionUsers: - mentionees: - - dotnet/area-system-memory - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Net - then: - - mentionUsers: - mentionees: - - dotnet/ncl - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Net.Http - then: - - mentionUsers: - mentionees: - - dotnet/ncl - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Net.Quic - then: - - mentionUsers: - mentionees: - - dotnet/ncl - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Net.Security - then: - - mentionUsers: - mentionees: - - dotnet/ncl - - bartonjs - - vcsjones - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Net.Sockets - then: - - mentionUsers: - mentionees: - - dotnet/ncl - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Numerics - then: - - mentionUsers: - mentionees: - - dotnet/area-system-numerics - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Numerics.Tensors - then: - - mentionUsers: - mentionees: - - dotnet/area-system-numerics-tensors - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Reflection - then: - - mentionUsers: - mentionees: - - dotnet/area-system-reflection - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Reflection.Emit - then: - - mentionUsers: - mentionees: - - dotnet/area-system-reflection-emit - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Reflection.Metadata - then: - - mentionUsers: - mentionees: - - dotnet/area-system-reflection-metadata - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Resources - then: - - mentionUsers: - mentionees: - - dotnet/area-system-resources - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Runtime - then: - - mentionUsers: - mentionees: - - dotnet/area-system-runtime - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Runtime.CompilerServices - then: - - mentionUsers: - mentionees: - - dotnet/area-system-runtime-compilerservices - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Runtime.InteropServices - then: - - mentionUsers: - mentionees: - - dotnet/interop-contrib - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Runtime.Intrinsics - then: - - mentionUsers: - mentionees: - - dotnet/area-system-runtime-intrinsics - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Security - then: - - mentionUsers: - mentionees: - - dotnet/area-system-security - - bartonjs - - vcsjones - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.ServiceProcess - then: - - mentionUsers: - mentionees: - - dotnet/area-system-serviceprocess - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Speech - then: - - mentionUsers: - mentionees: - - dotnet/area-system-speech - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Text.Encoding - then: - - mentionUsers: - mentionees: - - dotnet/area-system-text-encoding - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Text.Encodings.Web - then: - - mentionUsers: - mentionees: - - dotnet/area-system-text-encodings-web - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Text.Json - then: - - mentionUsers: - mentionees: - - dotnet/area-system-text-json - - gregsdennis - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Text.RegularExpressions - then: - - mentionUsers: - mentionees: - - dotnet/area-system-text-regularexpressions - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Threading - then: - - mentionUsers: - mentionees: - - mangod9 - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Threading.Channels - then: - - mentionUsers: - mentionees: - - dotnet/area-system-threading-channels - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Threading.Tasks - then: - - mentionUsers: - mentionees: - - dotnet/area-system-threading-tasks - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Transactions - then: - - mentionUsers: - mentionees: - - roji - - ajcvickers - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-System.Xml - then: - - mentionUsers: - mentionees: - - dotnet/area-system-xml - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-Tools-ILLink - then: - - mentionUsers: - mentionees: - - agocke - - sbomer - - vitek-karas - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - - if: - - hasLabel: - label: area-vm-coreclr - then: - - mentionUsers: - mentionees: - - mangod9 - replyTemplate: >- - Tagging subscribers to this area: ${mentionees} - - See info in [area-owners.md](https://github.com/dotnet/runtime/blob/main/docs/area-owners.md) if you want to be subscribed. - assignMentionees: False - description: Area-owners - - if: - - payloadType: Issues - - labelAdded: - label: breaking-change - then: - - addLabel: - label: needs-breaking-change-doc-created - - addReply: - reply: >- - Added `needs-breaking-change-doc-created` label because this issue has the `breaking-change` label. - - - 1. [ ] Create and link to this issue a matching issue in the dotnet/docs repo using the [breaking change documentation template](https://aka.ms/dotnet/docs/new-breaking-change-issue), then remove this `needs-breaking-change-doc-created` label. - - - Tagging @dotnet/compat for awareness of the breaking change. - description: Add breaking change doc label to issue - - if: - - payloadType: Pull_Request - - labelAdded: - label: breaking-change - - isPullRequest - then: - - addLabel: - label: needs-breaking-change-doc-created - - addReply: - reply: >- - Added `needs-breaking-change-doc-created` label because this PR has the `breaking-change` label. - - - When you commit this breaking change: - - - 1. [ ] Create and link to this PR and the issue a matching issue in the dotnet/docs repo using the [breaking change documentation template](https://aka.ms/dotnet/docs/new-breaking-change-issue), then remove this `needs-breaking-change-doc-created` label. - - 2. [ ] Ask a committer to mail the `.NET Breaking Change Notification` DL. - - - Tagging @dotnet/compat for awareness of the breaking change. - description: Add breaking change doc label to PR - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: linkable-framework - then: - - mentionUsers: - mentionees: - - eerhardt - - vitek-karas - - LakshanF - - sbomer - - joperezr - - marek-safar - replyTemplate: >- - Tagging subscribers to 'linkable-framework': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for linkable-framework' - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: size-reduction - then: - - mentionUsers: - mentionees: - - eerhardt - - SamMonoRT - - marek-safar - replyTemplate: >- - Tagging subscribers to 'size-reduction': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for size-reduction' - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: arch-wasm - then: - - mentionUsers: - mentionees: - - lewing - replyTemplate: >- - Tagging subscribers to 'arch-wasm': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for wasm' - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: os-ios - then: - - mentionUsers: - mentionees: - - steveisok - - akoeplinger - - kotlarmilos - replyTemplate: >- - Tagging subscribers to 'os-ios': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for ios' - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: os-android - then: - - mentionUsers: - mentionees: - - steveisok - - akoeplinger - replyTemplate: >- - Tagging subscribers to 'arch-android': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for android' - - if: - - payloadType: Pull_Request - - or: - - filesMatchPattern: - pattern: .*ILLink.* - - filesMatchPattern: - pattern: .*illink.* - - not: - hasLabel: - label: linkable-framework - - isPullRequest - - isOpen - then: - - addLabel: - label: linkable-framework - description: '[Linkable-framework workgroup] Add linkable-framework label to new Prs that touch files with *ILLink* that not have it already' - - if: - - payloadType: Pull_Request - - or: - - filesMatchPattern: - pattern: .*ILLink.* - - filesMatchPattern: - pattern: .*illink.* - - not: - hasLabel: - label: linkable-framework - - isPullRequest - - isOpen - - isAction: - action: Synchronize - then: - - addLabel: - label: linkable-framework - description: '[Linkable-framework workgroup] Add linkable-framework label to Prs that get changes pushed where they touch *ILLInk* files' - - if: - - payloadType: Issues - - labelAdded: - label: backlog-cleanup-candidate - then: - - addReply: - reply: >- - Due to lack of recent activity, this issue has been marked as a candidate for backlog cleanup. It will be closed if no further activity occurs within 14 more days. Any new comment (by anyone, not necessarily the author) will undo this process. - - - This process is part of our [issue cleanup automation](https://github.com/dotnet/runtime/blob/main/docs/issue-cleanup.md). - - addLabel: - label: no-recent-activity - description: Manual Issue Cleanup - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: os-tvos - then: - - mentionUsers: - mentionees: - - steveisok - - akoeplinger - replyTemplate: >- - Tagging subscribers to 'os-tvos': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for tvos' - - if: - - or: - - payloadType: Issues - - payloadType: Pull_Request - - isAction: - action: Opened - then: - - if: - - hasLabel: - label: os-maccatalyst - then: - - mentionUsers: - mentionees: - - steveisok - - akoeplinger - replyTemplate: >- - Tagging subscribers to 'os-maccatalyst': ${mentionees} - - See info in area-owners.md if you want to be subscribed. - assignMentionees: False - description: '@Mention for maccatalyst' - - if: - - payloadType: Issues - - or: - - isAction: - action: Opened - - isAction: - action: Reopened - - isOpen - - not: isPartOfAnyMilestone - - not: - hasLabel: - label: untriaged - then: - - addLabel: - label: untriaged - description: Add untriaged label to new/reopened issues without a milestone - - if: - - payloadType: Issues - - or: - - isAction: - action: Closed - - isPartOfAnyMilestone - - hasLabel: - label: untriaged - then: - - removeLabel: - label: untriaged - description: Remove untriaged label from issues when closed or added to a milestone - - if: - - payloadType: Pull_Request - then: - - inPrLabel: - label: in-pr - description: Add `in-pr` label on issue when an open pull request is targeting it - - if: - - payloadType: Pull_Request - - isAction: - action: Opened - - not: - activitySenderHasPermission: - permission: Read - then: - - assignTo: - author: True - description: Assign Team PRs to author - - if: - - payloadType: Pull_Request - - isAction: - action: Opened - - isPullRequest - - and: - - not: - activitySenderHasPermission: - permission: Admin - - not: - activitySenderHasPermission: - permission: Write - - not: - isActivitySender: - user: github-actions[bot] - issueAuthor: False - - not: - isActivitySender: - user: dotnet-maestro[bot] - issueAuthor: False - - not: - isActivitySender: - user: dotnet-maestro-bot[bot] - issueAuthor: False - - not: - isActivitySender: - user: dotnet-maestro-bot - issueAuthor: False - - not: - isActivitySender: - user: dotnet-maestro - issueAuthor: False - - not: - isActivitySender: - user: github-actions - issueAuthor: False - then: - - addLabel: - label: community-contribution - description: Label community PRs - - if: - - payloadType: Issues - - labelAdded: - label: needs-author-action - then: - - addReply: - reply: This issue has been marked `needs-author-action` and may be missing some important information. - description: Needs-author-action notification - - if: - - payloadType: Pull_Request_Review - - not: - activitySenderHasPermission: - permission: Read - - isPullRequest - - isAction: - action: Submitted - - isReviewState: - reviewState: Changes_requested - then: - - addLabel: - label: needs-author-action - description: PR reviews with "changes requested" applies the needs-author-action label - - if: - - payloadType: Issue_Comment - - isAction: - action: Created - - isActivitySender: - issueAuthor: True - - hasLabel: - label: needs-author-action - - not: - hasLabel: - label: untriaged - - isIssue - - isOpen - then: - - addLabel: - label: needs-further-triage - - removeLabel: - label: needs-author-action - description: Replace `needs-author-action` label with `needs-further-triage` label when the author comments on an issue that is not still untriaged - - if: - - payloadType: Issue_Comment - - isAction: - action: Created - - isActivitySender: - issueAuthor: True - - hasLabel: - label: needs-author-action - - hasLabel: - label: untriaged - - isIssue - - isOpen - then: - - removeLabel: - label: needs-author-action - description: Remove `needs-author-action` label when the author comments on an `untriaged` issue - - if: - - payloadType: Pull_Request - - isPullRequest - - isAction: - action: Synchronize - - hasLabel: - label: needs-author-action - then: - - removeLabel: - label: needs-author-action - description: Pushing changes to PR branch removes the needs-author-action label - - if: - - payloadType: Issue_Comment - - isActivitySender: - issueAuthor: True - - isAction: - action: Created - - hasLabel: - label: needs-author-action - - isPullRequest - - isOpen - then: - - removeLabel: - label: needs-author-action - description: Author commenting in PR removes the needs-author-action label - - if: - - payloadType: Pull_Request_Review - - isActivitySender: - issueAuthor: True - - hasLabel: - label: needs-author-action - - isAction: - action: Submitted - - isPullRequest - - isOpen - then: - - removeLabel: - label: needs-author-action - description: Author responding to a pull request review comment removes the needs-author-action label - - if: - - payloadType: Issues - - not: - isAction: - action: Closed - - hasLabel: - label: no-recent-activity - - not: - labelAdded: - label: no-recent-activity - then: - - removeLabel: - label: no-recent-activity - - removeLabel: - label: backlog-cleanup-candidate - description: Remove `no-recent-activity` label from issues when issue is modified - - if: - - payloadType: Issue_Comment - - hasLabel: - label: no-recent-activity - - isIssue - then: - - removeLabel: - label: no-recent-activity - - removeLabel: - label: backlog-cleanup-candidate - description: Remove `no-recent-activity` label when an issue is commented on - - if: - - payloadType: Pull_Request - - isPullRequest - - isOpen - - hasLabel: - label: no-recent-activity - - not: - labelAdded: - label: no-recent-activity - then: - - removeLabel: - label: no-recent-activity - - removeLabel: - label: backlog-cleanup-candidate - description: Remove `no-recent-activity` label from PRs when modified - - if: - - payloadType: Issue_Comment - - hasLabel: - label: no-recent-activity - - isPullRequest - - isOpen - then: - - removeLabel: - label: no-recent-activity - - removeLabel: - label: backlog-cleanup-candidate - description: Remove `no-recent-activity` label from PRs when commented on - - if: - - payloadType: Pull_Request_Review - - hasLabel: - label: no-recent-activity - - isPullRequest - - isOpen - then: - - removeLabel: - label: no-recent-activity - - removeLabel: - label: backlog-cleanup-candidate - description: Remove `no-recent-activity` label from PRs when new review is added -onFailure: -onSuccess: diff --git a/Directory.Build.props b/Directory.Build.props index b1ac2559f9ed1e..26e112fab56e16 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -11,9 +11,6 @@ all cases, rather than ordinarily where we build them during mobile or wasm build legs. This makes the manifests available on source-only builds. --> true - - false @@ -316,8 +313,6 @@ '$(OfficialBuildId)' == ''">true true - - ClrFullNativeBuild;ClrRuntimeSubset;ClrJitSubset;ClrPalTestsSubset;ClrAllJitsSubset;ClrILToolsSubset;ClrNativeAotSubset;ClrSpmiSubset;ClrCrossComponentsSubset;ClrDebugSubset;HostArchitecture;PgoInstrument;NativeOptimizationDataSupported;CMakeArgs diff --git a/THIRD-PARTY-NOTICES.TXT b/THIRD-PARTY-NOTICES.TXT index 065abb6033e966..ff5aaacd21b76a 100644 --- a/THIRD-PARTY-NOTICES.TXT +++ b/THIRD-PARTY-NOTICES.TXT @@ -73,7 +73,7 @@ https://github.com/madler/zlib https://zlib.net/zlib_license.html /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.3.1, January 22nd, 2024 + version 1.2.13, October 13th, 2022 Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler diff --git a/docs/area-owners.md b/docs/area-owners.md index aaa81100379d7a..52cb16d8d8d720 100644 --- a/docs/area-owners.md +++ b/docs/area-owners.md @@ -73,9 +73,9 @@ Note: Editing this file doesn't update the mapping used by `@msftbot` for area-s | area-System.Composition | @ericstj | @dotnet/area-system-composition | | | area-System.Configuration | @ericstj | @dotnet/area-system-configuration | | | area-System.Console | @jeffhandley | @dotnet/area-system-console | | -| area-System.Data | @sammonort | @ajcvickers @davoudeshtehari @david-engel @roji | | -| area-System.Data.Odbc | @sammonort | @ajcvickers @roji | | -| area-System.Data.OleDB | @sammonort | @ajcvickers @roji | | +| area-System.Data | @ajcvickers | @ajcvickers @davoudeshtehari @david-engel @roji | | +| area-System.Data.Odbc | @ajcvickers | @ajcvickers @roji | | +| area-System.Data.OleDB | @ajcvickers | @ajcvickers @roji | | | area-System.Data.SqlClient | @David-Engel | @davoudeshtehari @david-engel @jrahnama | Archived component - limited churn/contributions (see https://devblogs.microsoft.com/dotnet/introducing-the-new-microsoftdatasqlclient/) | | area-System.DateTime | @ericstj | @dotnet/area-system-datetime | System namespace APIs related to dates and times, including DateOnly, DateTime, DateTimeKind, DateTimeOffset, DayOfWeek, TimeOnly, TimeSpan, TimeZone, and TimeZoneInfo | | area-System.Diagnostics | @tommcdon | @dotnet/area-system-diagnostics | | @@ -135,7 +135,7 @@ Note: Editing this file doesn't update the mapping used by `@msftbot` for area-s | area-System.Threading.Channels | @ericstj | @dotnet/area-system-threading-channels | Consultants: @stephentoub | | area-System.Threading.RateLimiting | @rafikiassumani-msft | @BrennanConroy @halter73 | | | area-System.Threading.Tasks | @ericstj | @dotnet/area-system-threading-tasks | Consultants: @stephentoub | -| area-System.Transactions | @sammonort | @roji | | +| area-System.Transactions | @ajcvickers | @roji | | | area-System.Xml | @jeffhandley | @dotnet/area-system-xml | | | area-TieredCompilation-coreclr | @mangod9 | @kouvel | | | area-Tools-ILLink | @agocke | @dotnet/illink | | @@ -162,16 +162,16 @@ Note: Editing this file doesn't update the mapping used by `@msftbot` for area-s > dedicated OS lead/owner, rather ownership falls back to the `area-*` label. However, > Windows is a supported operating system of course. -| Operating System | Lead | Owners (area experts to tag in PRs and issues) | Description | -|------------------|---------------|----------------------------------------------------|-----------------| -| os-android | @vitek-karas | @akoeplinger | | -| os-freebsd | | @wfurt @Thefrank @sec | | -| os-maccatalyst | @vitek-karas | @kotlarmilos | | -| os-ios | @vitek-karas | @vargaz, @kotlarmilos | | -| os-tizen | @gbalykov | @dotnet/samsung | | -| os-tvos | @vitek-karas | @vargaz, @kotlarmilos | | -| os-wasi | @lewing | @pavelsavara | | -| os-browser | @lewing | @pavelsavara | | +| Operating System | Lead | Owners (area experts to tag in PRs and issues) | Description | +|------------------|---------------|---------------------------------------------------------------------|-----------------| +| os-android | @vitek-karas | @akoeplinger | | +| os-freebsd | | @wfurt @Thefrank @sec | | +| os-maccatalyst | @vitek-karas | @kotlarmilos | | +| os-ios | @vitek-karas | @vargaz, @kotlarmilos | | +| os-tizen | @gbalykov | @hjleee, @wscho77, @clamp03, @JongHeonChoi, @t-mustafin, @viewizard | | +| os-tvos | @vitek-karas | @vargaz, @kotlarmilos | | +| os-wasi | @lewing | @pavelsavara | | +| os-browser | @lewing | @pavelsavara | | ## Architectures @@ -180,12 +180,12 @@ Note: Editing this file doesn't update the mapping used by `@msftbot` for area-s > [!NOTE] > Ownership isn't the same as supported. See [operating systems](#operating-systems) for details. -| Architecture | Lead | Owners (area experts to tag in PRs and issues) | Description | -|------------------|---------------|----------------------------------------------------|-----------------| -| arch-loongarch64 | @shushanhf | @LuckyXu-HF | | -| arch-riscv | @gbalykov | @dotnet/samsung | | -| arch-s390x | @uweigand | @uweigand | | -| arch-wasm | @lewing | @lewing, @pavelsavara | | +| Architecture | Lead | Owners (area experts to tag in PRs and issues) | Description | +|------------------|---------------|---------------------------------------------------------------------|--------------| +| arch-loongarch64 | @shushanhf | @LuckyXu-HF | | +| arch-riscv | @gbalykov | @hjleee, @wscho77, @clamp03, @JongHeonChoi, @t-mustafin, @viewizard | | +| arch-s390x | @uweigand | @uweigand | | +| arch-wasm | @lewing | @lewing, @pavelsavara | | ## Community Triagers diff --git a/docs/design/coreclr/botr/guide-for-porting.md b/docs/design/coreclr/botr/guide-for-porting.md index f7ca105bf165db..5d2c01aa52d066 100644 --- a/docs/design/coreclr/botr/guide-for-porting.md +++ b/docs/design/coreclr/botr/guide-for-porting.md @@ -413,6 +413,12 @@ Here is an annotated list of the stubs implemented for Unix on Arm64. Today use of this feature on Unix requires hand-written IL. On Windows this feature is commonly used by C++/CLI +3. EH Correctness. Some helpers are written in assembly to provide well known + locations for NullReferenceExceptions to be generated out of a SIGSEGV + signal. + + 1. `JIT_MemSet`, and `JIT_MemCpy` have this requirement + #### cgencpu.h This header is included by various code in the VM directory. It provides a large diff --git a/docs/design/coreclr/jit/first-class-structs.md b/docs/design/coreclr/jit/first-class-structs.md index 4211f75ff745f8..dc017aee75f2e6 100644 --- a/docs/design/coreclr/jit/first-class-structs.md +++ b/docs/design/coreclr/jit/first-class-structs.md @@ -94,6 +94,10 @@ encountered by most phases of the JIT: [#21705](https://github.com/dotnet/coreclr/pull/21705) they are no longer large nodes. * `GT_STORE_OBJ` and `GT_STORE_BLK` have the same structure as `GT_OBJ` and `GT_BLK`, respectively * `Data()` is op2 + * `GT_STORE_DYN_BLK` (GenTreeStoreDynBlk extends GenTreeBlk) + * Additional child `gtDynamicSize` + * Note that these aren't really struct stores; they represent dynamically sized blocks + of arbitrary data. * For `GT_LCL_FLD` nodes, we store a pointer to `ClassLayout` in the node. * For `GT_LCL_VAR` nodes, the `ClassLayout` is obtained from the `LclVarDsc`. diff --git a/docs/design/coreclr/jit/ryujit-overview.md b/docs/design/coreclr/jit/ryujit-overview.md index 5e63d38e98f664..cdb17002ee1974 100644 --- a/docs/design/coreclr/jit/ryujit-overview.md +++ b/docs/design/coreclr/jit/ryujit-overview.md @@ -222,7 +222,6 @@ The top-level function of interest is `Compiler::compCompile`. It invokes the fo | [Common Subexpression Elimination (CSE)](#cse) | Elimination of redundant subexressions based on value numbers. | | [Assertion Propagation](#assertion-propagation) | Utilizes value numbers to propagate and transform based on properties such as non-nullness. | | [Range analysis](#range-analysis) | Eliminate array index range checks based on value numbers and assertions | -| [Induction variable optimization](#iv-opts) | Optimize induction variables used inside natural loops based on scalar evolution analysis | | [VN-based dead store elimination](#vn-based-dead-store-elimination) | Eliminate stores that do not change the value of a local. | | [If conversion](#if-conversion) | Transform conditional definitions into `GT_SELECT` operators. | | [Rationalization](#rationalization) | Flowgraph order changes from `FGOrderTree` to `FGOrderLinear`. All `GT_COMMA` nodes are transformed. | @@ -348,11 +347,6 @@ reused. Utilizes value numbers to propagate and transform based on properties such as non-nullness. -### Induction variable optimization - -Performs scalar evolution analysis and utilized it to optimize induction variables inside loops. -Currently this entails IV widening which is done on x64 only. - ### Range analysis Optimize array index range checks based on value numbers and assertions. diff --git a/docs/design/coreclr/jit/ryujit-tutorial.md b/docs/design/coreclr/jit/ryujit-tutorial.md index ec900ccc8cd937..34466e45afbcdc 100644 --- a/docs/design/coreclr/jit/ryujit-tutorial.md +++ b/docs/design/coreclr/jit/ryujit-tutorial.md @@ -447,10 +447,6 @@ This is the same diagram as before, but with additional links to indicate execut - Determine initial value for dependent phis - Eliminate checks where the range of the index is within the check range -### Induction Variable Optimization -- Perform scalar evolution analysis to describe values of IR nodes inside loops -- Perform IV widening on x64 to avoid unnecessary zero extensions for array/span indexing - ## RyuJIT Back-End ### Rationalization diff --git a/docs/design/features/byreflike-generics.md b/docs/design/features/byreflike-generics.md index 99aa905ebf0f58..d644a25e7f3f22 100644 --- a/docs/design/features/byreflike-generics.md +++ b/docs/design/features/byreflike-generics.md @@ -127,129 +127,3 @@ The following are IL sequences involving the `box` instruction. They are used fo `box` ; `isinst` ; `unbox.any` – The box, `isint`, and unbox target types are all equal. `box` ; `isinst` ; `br_true/false` – The box target type is equal to the unboxed target type or the box target type is `Nullable` and target type equalities can be computed. - -## Examples - -Below are valid and invalid examples of ByRefLike as Generic parameters. All examples use the **not official** syntax, `allows ref struct`, for indicating the Generic permits ByRefLike types. - -**1) Valid** -```csharp -class A where T1: allows ref struct -{ - public void M(); -} - -// The derived class is okay to lack the 'allows' -// because the base permits non-ByRefLike (default) -// _and_ ByRefLike types. -class B : A -{ - public void N() - => M(); // Any T2 satisfies the constraints from A<> -} -``` - -**2) Invalid** -```csharp -class A -{ - public void M(); -} - -// The derived class cannot push up the allows -// constraint for ByRefLike types. -class B : A where T2: allows ref struct -{ - public void N() - => M(); // A<> may not permit a T2 -} -``` - -**3) Valid** -```csharp -interface IA -{ - void M(); -} - -ref struct A : IA -{ - public void M() { } -} - -class B -{ - // This call is permitted because no boxing is needed - // to dispatch to the method - it is implemented on A. - public static void C(T t) where T: IA, allows ref struct - => t.M(); -} -``` - -**4) Invalid** -```csharp -interface IA -{ - public void M() { } -} - -ref struct A : IA -{ - // Relies on IA::M() implementation. -} - -class B -{ - // Reliance on a DIM forces the generic parameter - // to be boxed, which is invalid for ByRefLike types. - public static void C(T t) where T: IA, allows ref struct - => t.M(); -} -``` - -**5) Valid** -```csharp -class A where T1: allows ref struct -{ -} - -class B -{ - // The type parameter is okay to lack the 'allows' - // because the field permits non-ByRefLike (default) - // _and_ ByRefLike types. - A Field; -} -``` - -**6) Invalid** -```csharp -class A -{ -} - -class B where T2: allows ref struct -{ - // The type parameter can be passed to - // the field type, but will fail if - // T2 is a ByRefLike type. - A Field; -} -``` - -**7) Invalid** -```csharp -class A -{ - virtual void M() where T1: allows ref struct; -} - -class B : A -{ - // Override methods need to match be at least - // as restrictive with respect to constraints. - // If a user has an instance of A, they are - // not aware they could be calling B. - override void M(); -} -``` \ No newline at end of file diff --git a/docs/design/features/globalization-icu-wasm.md b/docs/design/features/globalization-icu-wasm.md index ed5c03e88aa20d..956807b30c5cd5 100644 --- a/docs/design/features/globalization-icu-wasm.md +++ b/docs/design/features/globalization-icu-wasm.md @@ -28,7 +28,7 @@ Removing specific feature data might result in an exception that starts with `[C * For prerequisites run `.devcontainer/postCreateCommand.sh` (it is run automatically on creation if using Codespaces) * Building: ``` - ./build.sh /p:TargetOS=Browser /p:TargetArchitecture=wasm + ./build.sh /p:TargetOS=Browser /p:TargetArchitecture=wasm /p:IcuTracing=true ``` Output is located in `artifacts/bin/icu-browser-wasm`. @@ -45,7 +45,7 @@ Removing specific feature data might result in an exception that starts with `[C ``` * Building: ```bash - ./build.sh /p:TargetOS=Android /p:TargetArchitecture=x64 + ./build.sh /p:TargetOS=Android /p:TargetArchitecture=x64 /p:IcuTracing=true ``` Output from both builds will be located in subdirectories of `artifacts/bin`. Copy the generated `.dat` files to your project location and provide the path to it in the `.csproj`, e.g.: diff --git a/docs/infra/automation.md b/docs/infra/automation.md index a4ed601cf33e23..6b15e2a9171662 100644 --- a/docs/infra/automation.md +++ b/docs/infra/automation.md @@ -1,9 +1,13 @@ ## Automation -### Policy Service Bot +### Fabric Bot -This repository uses the Policy Service bot to automate issue and pull request management. All automation rules are defined in the [`.github/policies`](../../.github/policies) folder. +This repository uses Fabric Bot to automate issue and pull request management. All automation rules are defined in the [`.github/fabricbot.json`](../../.github/fabricbot.json) file. #### Notifications -You are welcome to enable notifications for yourself for one or more areas. You will be tagged whenever there are new issues and PR's in the area. You do not need to have commit access for this. To add or remove notifications for yourself, please offer a PR that edits the "mentionees" value for that area in the policy YAML file. +You are welcome to enable notifications for yourself for one or more areas. You will be tagged whenever there are new issues and PR's in the area. You do not need to have commit access for this. To add or remove notifications for yourself, please offer a PR that edits the "mentionees" value for that area. [Here is an example](https://github.com/dotnet/runtime/commit/c28b13f0cf4e2127a74285b65188413ca7e677d4). + +#### Other changes + +For any other changes, you will need access to the [`Fabric Bot portal`](https://portal.fabricbot.ms/bot/) which is only available to Microsoft employees at present. Ensure you are signed out from the portal, choose "Import Configuration" option and make changes using the editor. It's necessary to use the portal because there is at present no published JSON schema for the configuration format. diff --git a/docs/project/list-of-diagnostics.md b/docs/project/list-of-diagnostics.md index 9ad0f02f5f88bd..4cae3a85a87d8d 100644 --- a/docs/project/list-of-diagnostics.md +++ b/docs/project/list-of-diagnostics.md @@ -108,7 +108,6 @@ The PR that reveals the implementation of the ``/``/`` The `corflags.exe` tool that ships with the .NET Framework SDK can show whether a binary is delay-signed or strong-named. For a delay-signed assembly it may show: diff --git a/docs/workflow/building/coreclr/nativeaot.md b/docs/workflow/building/coreclr/nativeaot.md index a0f782aad4774b..31783578b9fde0 100644 --- a/docs/workflow/building/coreclr/nativeaot.md +++ b/docs/workflow/building/coreclr/nativeaot.md @@ -22,6 +22,8 @@ The Native AOT toolchain can be currently built for Linux (x64/arm64), macOS (x6 The paths to major components can be overridden using `IlcToolsPath`, `IlcSdkPath`, `IlcFrameworkPath`, `IlcFrameworkNativePath` and `IlcMibcPath` properties for `dotnet publish`. For example, `/p:IlcToolsPath=\artifacts\bin\coreclr\windows.x64.Debug\ilc` can be used to override the compiler with a local debug build for troubleshooting or quick iterations. +The component that writes out object files (objwriter.dll/libobjwriter.so/libobjwriter.dylib) is based on LLVM and doesn't build in the runtime repo. It gets published as a NuGet package out of the [dotnet/llvm-project](https://github.com/dotnet/llvm-project) repo (branch [objwriter/12.x](https://github.com/dotnet/llvm-project/tree/objwriter/12.x)). If you're working on ObjWriter or bringing up a new platform that doesn't have ObjWriter packages yet, as additional pre-requisites you need to build objwriter out of that repo and replace the file in the output. + ### Building packages Run `build[.cmd|.sh] -c Release` from the repo root to build the NativeAOT toolchain packages. The build will place the toolchain packages at `artifacts\packages\Release\Shipping`. To publish your project using these packages: diff --git a/docs/workflow/ci/failure-analysis.md b/docs/workflow/ci/failure-analysis.md index 58a11c06bdfa4f..57917c841316a8 100644 --- a/docs/workflow/ci/failure-analysis.md +++ b/docs/workflow/ci/failure-analysis.md @@ -12,19 +12,6 @@ ## Triaging errors seen in CI -## Summary - -**Passing Build Analysis is required to merge into the runtime repo**. - -To resolve failures, do the following, in order: - -1. Fix the problem if your PR is the cause. -2. For all failures not in the "Known test errors" section, [try to file a Known Build Error issue](#what-to-do-if-you-determine-the-failure-is-unrelated). -3. If all else fails, perform a [manual bypass](#bypassing-build-analysis). - - -## Details - In case of failure, any PR on the runtime will have a failed GitHub check - PR Build Analysis - which has a summary of all failures, including a list of matching known issues as well as any regressions introduced to the build or the tests. This tab should be your first stop for analyzing the PR failures. ![Build analysis check](analysis-check.png) @@ -91,7 +78,6 @@ If you have considered all the diagnostic artifacts and determined the failure i ```` It already contains most of the essential information, but *it is very important that you fill out the json blob*. - - You can now use the [Build Analysis Known Issue Helper](https://helix.dot.net/BuildAnalysis/CreateKnownIssues) to create an issue. It assists in adding the right set of labels, fill the necessary paths in the json blob, and it will validate that it matches the text presented for the issue found in the logs. - You can add into the `ErrorMessage` field the string that you found uniquely identifies the issue. In case you need to use a regex, use the `ErrorPattern` field instead. This is a limited to a single-line, non-backtracking regex as described [here](https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssues.md#regex-matching). This regex also needs to be appropriately escaped. Check the [arcade known issues](https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssues.md#filling-out-known-issues-json-blob) documentation for a good guide on proper regex and JSON escaping. - The field `ExcludeConsoleLog` describes if the execution logs should be considered on top of the individual test results. **For most cases, this should be set to `true` as the failure will happen within a single test**. Setting it to `false` will mean all failures within an xUnit set of tests will also get attributed to this particular error, since there's one log describing all the problems. Due to limitations in Known Issues around rate limiting and xUnit resiliency, setting `ExcludeConsoleLog=false` is necessary in two scenarios: + Nested tests as reported to Azure DevOps. Essentially this means theory failures, which look like this when reported in Azure DevOps: ![xUnit theory seen in azure devops](theory-azdo.png). @@ -109,16 +95,6 @@ After you do this, if the failure is occurring frequently as per the data captur There are plenty of intermittent failures that won't manifest again on a retry. Therefore these steps should be followed for every iteration of the PR build, e.g. before retrying/rebuilding. -### Bypassing build analysis - -To unconditionally bypass the build analysis check (turn it green), you can add a comment to your PR with the following text: - -``` -/ba-g -``` - -For more information, see https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/EscapeMechanismforBuildAnalysis.md - ### Examples of Build Analysis #### Good usage examples diff --git a/docs/workflow/ci/triaging-failures.md b/docs/workflow/ci/triaging-failures.md index bf5e80f7522e6b..1baa5605277475 100644 --- a/docs/workflow/ci/triaging-failures.md +++ b/docs/workflow/ci/triaging-failures.md @@ -8,7 +8,7 @@ stress mode test configuration failures, such as failures in a JIT stress test r One goal of failure investigation is to quickly route failures to the correct area owner. The ownership of various product areas is detailed [here](../../area-owners.md). The GitHub auto-tagging bot uses the ownership information -in the file [Policy Service configuration](../../../.github/policies). +in the file [fabricbot.json](../../../.github/fabricbot.json). ## Platform configuration diff --git a/docs/workflow/debugging/coreclr/debugging-aot-compilers.md b/docs/workflow/debugging/coreclr/debugging-aot-compilers.md index 7896e1b8bb5021..341e5489548e1d 100644 --- a/docs/workflow/debugging/coreclr/debugging-aot-compilers.md +++ b/docs/workflow/debugging/coreclr/debugging-aot-compilers.md @@ -85,7 +85,7 @@ The object files generated by the ILC compiler contain debug information for met The ILC compiler typically compiles the whole program - it loosely corresponds to the composite mode of crossgen2. There is a multifile mode, where each managed assembly corresponds to a single object file, but this mode is not shipping. -The supported object files generated by the ILC compiler are PE/ELF/Mach-O formats. +The object files generated by the ILC compiler are written out using an LLVM-based object writer (consumed as a NuGet package built out of the dotnet/llvm-project repo, branch objwriter/12.x). The object writer uses the LLVM assembler APIs (APIs meant to be used by tools that convert textual assembly into machine code) to emit object files in PE/ELF/Mach-O formats. ## Example of debugging a test application in Crossgen2 diff --git a/docs/workflow/debugging/coreclr/debugging-runtime.md b/docs/workflow/debugging/coreclr/debugging-runtime.md index 6edce3c7646e0a..dd92fe93cfaf2e 100644 --- a/docs/workflow/debugging/coreclr/debugging-runtime.md +++ b/docs/workflow/debugging/coreclr/debugging-runtime.md @@ -150,7 +150,7 @@ It might also be the case that you would need the latest changes in SOS, or you' **NOTE**: Only `lldb` is supported to use with SOS. You can also use `gdb`, `cgdb`, or other debuggers, but you might not have access to SOS. 1. Perform a build of the _clr_ subset of the runtime repo. -2. Start lldb passing `corerun`, the app to run (e.g. `HelloWorld.dll`), and any arguments this app might need: `lldb -- /path/to/corerun /path/to/app.dll ` +2. Start lldb passing `corerun`, the app to run (e.g. `HelloWorld.dll`), and any arguments this app might need: `lldb /path/to/corerun /path/to/app.dll ` 3. If you're using the installed version of SOS, you can skip this step. If you built SOS manually, you have to load it before starting the debugging session: `plugin load /path/to/built/sos/libsosplugin.so`. Note that `.so` is for Linux, and `.dylib` is for macOS. You can find more information in the diagnostics repo [private sos build doc](https://github.com/dotnet/diagnostics/blob/main/documentation/using-sos-private-build.md). 4. Launch program: `process launch -s` 5. To stop breaks on _SIGUSR1_ signals used by the runtime run the following command: `process handle -s false SIGUSR1` diff --git a/docs/workflow/debugging/mono/android-debugging.md b/docs/workflow/debugging/mono/android-debugging.md index 7e86eb7753242b..918ac1503efa5e 100644 --- a/docs/workflow/debugging/mono/android-debugging.md +++ b/docs/workflow/debugging/mono/android-debugging.md @@ -57,27 +57,25 @@ Since you're debugging an optimized release build, it is likely the debugger wil ## Native debugging using a local debug build of Mono -Ensure the prerequisites are met for [Testing Android](../../testing/libraries/testing-android.md#prerequisites). +Build the runtime for your android architecture: `ANDROID_NDK_ROOT= ./build.sh --os android --arch x86 -c Debug`. See the instructions for [Testing Android](../../testing/libraries/testing-android.md) for details. -Build the runtime for your android architecture `` and keep debug symbols in the binary: -`./build.sh -s mono+libs -os android -arch -c Debug /p:KeepNativeSymbols=true` - -In the source code for the C# project, add the following to the .csproj (replacing `` by the appropriate location and `` with the built android architecture): +In the source code for the C# project, add the following to the .csproj (replacing `` by the appropriate location): ``` - ``` -Then rebuild and reinstall the project, open the apk in Android Studio (File > Profile or Debug APK), and debug. - -Note: If debugging in Android Studio stops at signals `SIGPWR` and `SIGXCPU` during startup, configure LLDB to not stop the process for those signals via `process handle -p true -s false -n true SIGPWR` and `process handle -p true -s false -n true SIGXCPU` in Android Studio's LLDB tab. +Then rebuild and reinstall the project, open the apk in Android Studio, and debug. The +runtime native libraries will be stripped, so to make use of debug symbols, you +will need to follow the steps above (rename `*.so.dbg` in the artifacts to +`*.so.so` and add them to the APK project in Android Studio) ## Native and managed debugging or debugging the managed debugger diff --git a/docs/workflow/testing/host/testing.md b/docs/workflow/testing/host/testing.md index bb45307ecf46f3..35c7359c411a9a 100644 --- a/docs/workflow/testing/host/testing.md +++ b/docs/workflow/testing/host/testing.md @@ -77,11 +77,6 @@ The `category!=failing` is to respect the [filtering traits](../libraries/filter ### Visual Studio The [Microsoft.DotNet.CoreSetup.sln](/src/installer/Microsoft.DotNet.CoreSetup.sln) can be used to run and debug host tests through Visual Studio. When using the solution, the product should have already been [built](#building-tests) and the [test context](#test-context) set up. -If you built the runtime or libraries with a different configuration from the host, you have to specify this when starting visual studio: - -```console -build.cmd -vs Microsoft.DotNet.CoreSetup -rc Release -lc Release -``` ### Preserving test artifacts diff --git a/eng/CodeAnalysis.src.globalconfig b/eng/CodeAnalysis.src.globalconfig index 307e910d1628e9..2677ac469e6671 100644 --- a/eng/CodeAnalysis.src.globalconfig +++ b/eng/CodeAnalysis.src.globalconfig @@ -274,12 +274,6 @@ dotnet_diagnostic.CA1512.severity = warning # CA1513: Use ObjectDisposedException throw helper dotnet_diagnostic.CA1513.severity = warning -# CA1514: Avoid redundant length argument -dotnet_diagnostic.CA1514.severity = warning - -# CA1515: Consider making public types internal -dotnet_diagnostic.CA1515.severity = none - # CA1700: Do not name enum values 'Reserved' dotnet_diagnostic.CA1700.severity = none @@ -498,9 +492,6 @@ dotnet_diagnostic.CA1869.severity = warning # CA1870: Use a cached 'SearchValues' instance dotnet_diagnostic.CA1870.severity = warning -# CA1871: Do not pass a nullable struct to 'ArgumentNullException.ThrowIfNull' -dotnet_diagnostic.CA1871.severity = warning - # CA2000: Dispose objects before losing scope dotnet_diagnostic.CA2000.severity = none @@ -688,18 +679,6 @@ dotnet_diagnostic.CA2260.severity = warning # CA2261: Do not use ConfigureAwaitOptions.SuppressThrowing with Task dotnet_diagnostic.CA2261.severity = warning -# CA2262: Set 'MaxResponseHeadersLength' properly -dotnet_diagnostic.CA2262.severity = warning - -# CA2263: Prefer generic overload when type is known -dotnet_diagnostic.CA2263.severity = info - -# CA2264: Do not pass a non-nullable value to 'ArgumentNullException.ThrowIfNull' -dotnet_diagnostic.CA2264.severity = warning - -# CA2265: Do not compare Span to 'null' or 'default' -dotnet_diagnostic.CA2265.severity = warning - # CA2300: Do not use insecure deserializer BinaryFormatter dotnet_diagnostic.CA2300.severity = none diff --git a/eng/CodeAnalysis.test.globalconfig b/eng/CodeAnalysis.test.globalconfig index 3f58c3aab64ea9..79e35931782f56 100644 --- a/eng/CodeAnalysis.test.globalconfig +++ b/eng/CodeAnalysis.test.globalconfig @@ -273,12 +273,6 @@ dotnet_diagnostic.CA1512.severity = none # CA1513: Use ObjectDisposedException throw helper dotnet_diagnostic.CA1513.severity = none -# CA1514: Avoid redundant length argument -dotnet_diagnostic.CA1514.severity = none - -# CA1515: Consider making public types internal -dotnet_diagnostic.CA1515.severity = none - # CA1700: Do not name enum values 'Reserved' dotnet_diagnostic.CA1700.severity = none @@ -495,9 +489,6 @@ dotnet_diagnostic.CA1869.severity = none # CA1870: Use a cached 'SearchValues' instance dotnet_diagnostic.CA1870.severity = none -# CA1871: Do not pass a nullable struct to 'ArgumentNullException.ThrowIfNull' -dotnet_diagnostic.CA1871.severity = none - # CA2000: Dispose objects before losing scope dotnet_diagnostic.CA2000.severity = none @@ -684,18 +675,6 @@ dotnet_diagnostic.CA2260.severity = none # CA2261: Do not use ConfigureAwaitOptions.SuppressThrowing with Task dotnet_diagnostic.CA2261.severity = none -# CA2262: Set 'MaxResponseHeadersLength' properly -dotnet_diagnostic.CA2262.severity = none - -# CA2263: Prefer generic overload when type is known -dotnet_diagnostic.CA2263.severity = none - -# CA2264: Do not pass a non-nullable value to 'ArgumentNullException.ThrowIfNull' -dotnet_diagnostic.CA2264.severity = none - -# CA2265: Do not compare Span to 'null' or 'default' -dotnet_diagnostic.CA2265.severity = none - # CA2300: Do not use insecure deserializer BinaryFormatter dotnet_diagnostic.CA2300.severity = none diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index c5da9ec68e4517..3c68bf354bae17 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -21,10 +21,6 @@ <_hostArch>$(_hostRid.Substring($(_hostRidPlatformIndex)).TrimStart('-')) minimal - - - true @@ -47,7 +43,7 @@ Properties that control flags from the VMR build, and the expected output for the VMR build should be added to this file. --> $(InnerBuildArgs) $(FlagParameterPrefix)arch $(TargetArch) $(InnerBuildArgs) $(FlagParameterPrefix)os $(TargetOS) - $(InnerBuildArgs) $(FlagParameterPrefix)cross + $(InnerBuildArgs) $(FlagParameterPrefix)cross $(InnerBuildArgs) $(FlagParameterPrefix)configuration $(Configuration) $(InnerBuildArgs) $(FlagParameterPrefix)allconfigurations $(InnerBuildArgs) $(FlagParameterPrefix)verbosity $(LogVerbosity) @@ -69,7 +65,6 @@ $(InnerBuildArgs) /p:DotNetBuildRepo=true - $(InnerBuildArgs) /p:DotNetBuildOrchestrator=true $(InnerBuildArgs) /p:OfficialBuildId=$(OfficialBuildId) $(InnerBuildArgs) /p:ContinuousIntegrationBuild=$(ContinuousIntegrationBuild) $(InnerBuildArgs) /p:PortableBuild=$(PortableBuild) diff --git a/eng/Publishing.props b/eng/Publishing.props index 8b796225f8274b..920e79cbbd2f7c 100644 --- a/eng/Publishing.props +++ b/eng/Publishing.props @@ -1,7 +1,6 @@ - + - true + 3 - - + \ No newline at end of file diff --git a/eng/Subsets.props b/eng/Subsets.props index cbe9cfc7d73c99..dd284ea6d99776 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -119,9 +119,7 @@ - <_NativeAotSupportedOS Condition="'$(TargetOS)' == 'windows' or '$(TargetOS)' == 'linux' or '$(TargetOS)' == 'osx' or '$(TargetOS)' == 'maccatalyst' or '$(TargetOS)' == 'iossimulator' or '$(TargetOS)' == 'ios' or '$(TargetOS)' == 'tvossimulator' or '$(TargetOS)' == 'tvos' or '$(TargetOS)' == 'freebsd'">true - <_NativeAotSupportedArch Condition="'$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'arm64' or '$(TargetArchitecture)' == 'arm' or ('$(TargetOS)' == 'windows' and '$(TargetArchitecture)' == 'x86')">true - true + true true @@ -257,7 +255,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 26ce2a8139000a..61fe9ade1dbb81 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,52 +1,84 @@ - + https://github.com/dotnet/icu - 9712d1680642c09dfd46827f5e850b9027ba6086 + 1b9c03a1103cf622ee2f7850d16aa1095a719e56 - + https://github.com/dotnet/msquic - 6281631a8328ffdbb1b63b231af1aaa803915b23 + 3fb2583170384341dbbc444cd5bb3d2319433fb6 https://github.com/dotnet/wcf 7f504aabb1988e9a093c1e74d8040bd52feb2f01 - + https://github.com/dotnet/emsdk - 5dd0620274178dd73cac5049e5187c00e07ecf0c + 8afd92448d03a80001c9cac5f2acb53b336263a4 - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d + + + https://github.com/dotnet/llvm-project + 9885e5aecc176ca701fc3527877d608bf7ccfb7d https://github.com/dotnet/command-line-api @@ -58,307 +90,307 @@ a045dd54a4c44723c215d992288160eb1401bb7f - + https://github.com/dotnet/cecil - 896cafe8d0683f74a5ff3eff1f92b6b2ed3500f3 + ca7e93445acbd94bfa696c16fa039f2a6130f2cb - + https://github.com/dotnet/cecil - 896cafe8d0683f74a5ff3eff1f92b6b2ed3500f3 + ca7e93445acbd94bfa696c16fa039f2a6130f2cb - + https://github.com/dotnet/emsdk - 5dd0620274178dd73cac5049e5187c00e07ecf0c + 8afd92448d03a80001c9cac5f2acb53b336263a4 - + https://github.com/dotnet/emsdk - 5dd0620274178dd73cac5049e5187c00e07ecf0c + 8afd92448d03a80001c9cac5f2acb53b336263a4 - + https://github.com/dotnet/source-build-reference-packages - 768378e775fc5ddc99d41f2c4d1c78182f326ea7 + 8ee50f75f960fbfb20fce0fefc5a3b05d15b1d21 - + https://github.com/dotnet/source-build-externals - 472629e451a5a87410ea3670606f7235a4dd5a02 + ddfb60463c966af55fd0e222c2266170e83d1324 - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/llvm-project - 3ac1a8b8d575b4bfe28f3cfb11a32589b6a05eca + 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/runtime - 2909fe8a13b361bd83727cef1af9360a0949ad2b + 963626276e11bf5587aaed69826b62682b05d9c4 - + https://github.com/dotnet/xharness - 134035492ed8154fc9c5a930a4ca52c422b21afb + 11ae3663fe3de366ea3566d7ae9b4731adee2ca3 - + https://github.com/dotnet/xharness - 134035492ed8154fc9c5a930a4ca52c422b21afb + 11ae3663fe3de366ea3566d7ae9b4731adee2ca3 - + https://github.com/dotnet/xharness - 134035492ed8154fc9c5a930a4ca52c422b21afb + 11ae3663fe3de366ea3566d7ae9b4731adee2ca3 - + https://github.com/dotnet/arcade - 39839f3007d9f3bbabf7a4b6a96ef5dd6be9e5ac + c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 78a5b978e1965c1335edb4b9a22bc4d6ff5a77a6 + db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 78a5b978e1965c1335edb4b9a22bc4d6ff5a77a6 + db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 78a5b978e1965c1335edb4b9a22bc4d6ff5a77a6 + db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 78a5b978e1965c1335edb4b9a22bc4d6ff5a77a6 + db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 - + https://github.com/dotnet/hotreload-utils - afafaea82e66771c6c4e874dc9560c06adad4a13 + 465874b5842702bf69bbb6bacd94a52d8ea2a073 - + https://github.com/dotnet/runtime-assets - fe7f6de587d1867b71a8e846ffee3b2035afbfe5 + b5ac2d9031d4b2dc40683b31de86b05a20b670af https://github.com/dotnet/roslyn @@ -367,18 +399,19 @@ https://github.com/dotnet/roslyn 77372c66fd54927312b5b0a2e399e192f74445c9 + https://github.com/dotnet/roslyn 77372c66fd54927312b5b0a2e399e192f74445c9 - + https://github.com/dotnet/roslyn-analyzers - 94749ce487be31b74bae5629b5af5d2392377f6d + 68c643b4667c6808bd21910ef32f7e2f7bd776c5 - + https://github.com/dotnet/roslyn-analyzers - 94749ce487be31b74bae5629b5af5d2392377f6d + 68c643b4667c6808bd21910ef32f7e2f7bd776c5 @@ -386,23 +419,23 @@ 77372c66fd54927312b5b0a2e399e192f74445c9 - + https://github.com/dotnet/sdk - 1aed5c88d3f91dd1610cd60b03782a82a6a07e81 + 0962c1f89f5daf924a9fe876c80e80b0bde63b0d - + https://github.com/dotnet/sdk - 1aed5c88d3f91dd1610cd60b03782a82a6a07e81 + 0962c1f89f5daf924a9fe876c80e80b0bde63b0d - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 78a5b978e1965c1335edb4b9a22bc4d6ff5a77a6 + db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 - + https://dev.azure.com/dnceng/internal/_git/dotnet-optimization - 78a5b978e1965c1335edb4b9a22bc4d6ff5a77a6 + db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 @@ -410,9 +443,9 @@ https://github.com/NuGet/NuGet.Client 8fef55f5a55a3b4f2c96cd1a9b5ddc51d4b927f8 - + https://github.com/dotnet/installer - e911f5c82cc02aea96e227596e16c830d54cf03a + ab44b49a395ac6a524628cdd8c5d99614248c584 diff --git a/eng/Versions.props b/eng/Versions.props index e32b450af54776..bb1645abaf2b4c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,11 +7,11 @@ 0 0 9.0.100 - 8.0.2 + 8.0.0 7.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet8)').Build),14)) 6.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet7)').Build),11)) preview - 3 + 2 false release @@ -34,8 +34,8 @@ - 3.11.0-beta1.24158.2 - 9.0.0-preview.24158.2 + 3.11.0-beta1.24109.1 + 9.0.0-preview.24109.1 - 9.0.100-preview.3.24161.5 + 9.0.100-preview.2.24112.1 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 2.6.7-beta.24161.5 - 9.0.0-beta.24161.5 - 2.6.7-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 - 9.0.0-beta.24161.5 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 2.6.7-beta.24112.1 + 9.0.0-beta.24112.1 + 2.6.7-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 + 9.0.0-beta.24112.1 1.4.0 6.0.0-preview.1.102 - 9.0.0-preview.3.24161.1 + 9.0.0-preview.2.24115.1 6.0.0 - 9.0.0-preview.3.24161.1 + 9.0.0-preview.2.24115.1 + + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 6.0.0 1.1.1 @@ -119,45 +128,45 @@ 8.0.0 5.0.0 4.5.5 - 9.0.0-preview.3.24161.1 - 9.0.0-preview.3.24161.1 + 9.0.0-preview.2.24115.1 + 9.0.0-preview.2.24115.1 6.0.0 5.0.0 5.0.0 5.0.0 7.0.0 - 9.0.0-preview.3.24161.1 + 9.0.0-preview.2.24115.1 6.0.0 7.0.0 4.5.4 4.5.0 - 9.0.0-preview.3.24161.1 + 9.0.0-preview.2.24115.1 8.0.0 8.0.0 8.0.0 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 - 9.0.0-beta.24161.1 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 + 9.0.0-beta.24108.2 - 1.0.0-prerelease.24106.4 - 1.0.0-prerelease.24106.4 - 1.0.0-prerelease.24106.4 - 1.0.0-prerelease.24106.4 - 1.0.0-prerelease.24106.4 - 1.0.0-prerelease.24106.4 + 1.0.0-prerelease.24104.2 + 1.0.0-prerelease.24104.2 + 1.0.0-prerelease.24104.2 + 1.0.0-prerelease.24104.2 + 1.0.0-prerelease.24104.2 + 1.0.0-prerelease.24104.2 2.0.0 17.8.0-beta1.23475.2 @@ -178,10 +187,10 @@ 1.4.0 17.4.0-preview-20220707-01 - 9.0.0-prerelease.24161.1 - 9.0.0-prerelease.24161.1 - 9.0.0-prerelease.24161.1 - 9.0.0-alpha.0.24161.1 + 9.0.0-prerelease.24112.4 + 9.0.0-prerelease.24112.4 + 9.0.0-prerelease.24112.4 + 9.0.0-alpha.0.24112.1 3.12.0 4.5.0 6.0.0 @@ -207,53 +216,53 @@ 8.0.0-preview-20230918.1 - 0.11.4-alpha.24161.3 + 0.11.4-alpha.24112.1 - 9.0.0-preview.3.24161.1 + 9.0.0-preview.2.24115.1 - 9.0.0-preview.3.24155.1 + 9.0.0-preview.2.24112.1 - 2.3.5 - 9.0.0-alpha.1.24162.1 + 2.2.3 + 9.0.0-alpha.1.24067.1 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 - 9.0.0-preview.3.24160.1 + 9.0.0-preview.2.24114.5 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100TransportVersion) - 9.0.0-preview.3.24160.1 + 9.0.0-preview.2.24114.5 1.1.87-gba258badda 1.0.0-v3.14.0.5722 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 - 16.0.5-alpha.1.24154.3 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 + 16.0.5-alpha.1.24112.1 3.1.7 1.0.406601 - 9.0.100-preview.3.24161.2 + 9.0.100-preview.2.24112.1 $(MicrosoftDotnetSdkInternalVersion) diff --git a/eng/build-analysis-configuration.json b/eng/build-analysis-configuration.json index d647594ab402ae..7f532220c1001f 100644 --- a/eng/build-analysis-configuration.json +++ b/eng/build-analysis-configuration.json @@ -3,10 +3,6 @@ { "PipelineId": 129, "PipelineName": "runtime" - }, - { - "PipelineId": 133, - "PipelineName": "runtime-dev-innerloop" } ] } diff --git a/eng/build.ps1 b/eng/build.ps1 index 474e55e40f2e58..db18267f33e1c5 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -261,12 +261,6 @@ if ($vs) { $env:RUNTIMECONFIGURATION=$runtimeConfiguration } - if ($librariesConfiguration) - { - # Respect the LibrariesConfiguration variable for building inside VS with different libraries configurations - $env:LIBRARIESCONFIGURATION=$librariesConfiguration - } - # Respect the RuntimeFlavor variable for building inside VS with a different CoreLib and runtime if ($runtimeFlavor) { @@ -331,9 +325,6 @@ if ($env:TreatWarningsAsErrors -eq 'false') { $arguments += " -warnAsError 0" } -# disable terminal logger for now: https://github.com/dotnet/runtime/issues/97211 -$arguments += " /tl:false" - # Disable targeting pack caching as we reference a partially constructed targeting pack and update it later. # The later changes are ignored when using the cache. $env:DOTNETSDK_ALLOW_TARGETING_PACK_CACHING=0 diff --git a/eng/build.sh b/eng/build.sh index 75fe2cdc39c5d0..67f3cfeea47278 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -553,9 +553,6 @@ if [[ "${TreatWarningsAsErrors:-}" == "false" ]]; then arguments="$arguments -warnAsError 0" fi -# disable terminal logger for now: https://github.com/dotnet/runtime/issues/97211 -arguments="$arguments -tl:false" - initDistroRid "$os" "$arch" "$crossBuild" # Disable targeting pack caching as we reference a partially constructed targeting pack and update it later. diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index efa2fd72bfaa22..6c65e81925f2a3 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,11 +48,12 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern else { Write-Host "Package source $SourceName already present." } - AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd + + AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $pwd) { +function AddCredential($creds, $source, $username, $password) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -81,18 +82,17 @@ function AddCredential($creds, $source, $username, $pwd) { $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - - $passwordElement.SetAttribute("value", $pwd) + $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } } -$doc.Save($filename) \ No newline at end of file +$doc.Save($filename) diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 83e6d82e027a82..510458eb35b84b 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -125,6 +125,7 @@ function Build { /p:Test=$test ` /p:Pack=$pack ` /p:DotNetBuildRepo=$($productBuild -or $verticalBuild) ` + /p:ArcadeBuildVertical=$verticalBuild ` /p:IntegrationTest=$integrationTest ` /p:PerformanceTest=$performanceTest ` /p:Sign=$sign ` diff --git a/eng/common/build.sh b/eng/common/build.sh index d82ebf7428080f..bec7d02594f668 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -241,6 +241,7 @@ function Build { /p:DotNetBuildRepo=$product_build \ /p:ArcadeBuildFromSource=$source_build \ /p:DotNetBuildSourceOnly=$source_build \ + /p:ArcadeBuildVertical=$vertical_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 5dcbfd700f0362..de1687b2ccbe79 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # getNonPortableDistroRid # @@ -11,20 +11,21 @@ # non-portable rid getNonPortableDistroRid() { - targetOs="$1" - targetArch="$2" - rootfsDir="$3" - nonPortableRid="" + local targetOs="$1" + local targetArch="$2" + local rootfsDir="$3" + local nonPortableRid="" if [ "$targetOs" = "linux" ]; then - # shellcheck disable=SC1091 if [ -e "${rootfsDir}/etc/os-release" ]; then - . "${rootfsDir}/etc/os-release" - if [ "${ID}" = "rhel" ] || [ "${ID}" = "rocky" ] || [ "${ID}" = "alpine" ]; then - VERSION_ID="${VERSION_ID%.*}" # Remove the last version digit for these distros + source "${rootfsDir}/etc/os-release" + + if [[ "${ID}" == "rhel" || "${ID}" == "rocky" || "${ID}" == "alpine" ]]; then + # remove the last version digit + VERSION_ID="${VERSION_ID%.*}" fi - if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then + if [[ "${VERSION_ID:-}" =~ ^([[:digit:]]|\.)+$ ]]; then nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" else # Rolling release distros either do not set VERSION_ID, set it as blank or @@ -32,45 +33,45 @@ getNonPortableDistroRid() # so omit it here to be consistent with everything else. nonPortableRid="${ID}-${targetArch}" fi + elif [ -e "${rootfsDir}/android_platform" ]; then - # shellcheck disable=SC1091 - . "${rootfsDir}/android_platform" + source "$rootfsDir"/android_platform nonPortableRid="$RID" fi fi if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is a shell script and should always work. - __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) + # $rootfsDir can be empty. freebsd-version is shell script and it should always work. + __freebsd_major_version=$($rootfsDir/bin/freebsd-version | { read v; echo "${v%%.*}"; }) nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then + elif command -v getprop && getprop ro.product.system.model 2>&1 | grep -qi android; then __android_sdk_version=$(getprop ro.build.version.sdk) nonPortableRid="android.$__android_sdk_version-${targetArch}" elif [ "$targetOs" = "illumos" ]; then __uname_version=$(uname -v) case "$__uname_version" in omnios-*) - __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="omnios.$__omnios_major_version-${targetArch}" - ;; + __omnios_major_version=$(echo "${__uname_version:8:2}") + nonPortableRid=omnios."$__omnios_major_version"-"$targetArch" + ;; joyent_*) - __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="smartos.$__smartos_major_version-${targetArch}" - ;; - *) - nonPortableRid="illumos-${targetArch}" - ;; + __smartos_major_version=$(echo "${__uname_version:7:4}") + nonPortableRid=smartos."$__smartos_major_version"-"$targetArch" + ;; + illumos_*) + nonPortableRid=openindiana-"$targetArch" + ;; esac elif [ "$targetOs" = "solaris" ]; then __uname_version=$(uname -v) - __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) - nonPortableRid="solaris.$__solaris_major_version-${targetArch}" + __solaris_major_version=$(echo "${__uname_version%.*}") + nonPortableRid=solaris."$__solaris_major_version"-"$targetArch" elif [ "$targetOs" = "haiku" ]; then - __uname_release="$(uname -r)" + __uname_release=$(uname -r) nonPortableRid=haiku.r"$__uname_release"-"$targetArch" fi - echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' + echo "$(echo $nonPortableRid | tr '[:upper:]' '[:lower:]')" } # initDistroRidGlobal @@ -84,23 +85,26 @@ getNonPortableDistroRid() # None # # Notes: -# It is important to note that the function does not return anything, but it -# exports the following variables on success: -# __DistroRid : Non-portable rid of the target platform. -# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. +# +# It is important to note that the function does not return anything, but it +# exports the following variables on success: +# +# __DistroRid : Non-portable rid of the target platform. +# __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. +# initDistroRidGlobal() { - targetOs="$1" - targetArch="$2" - rootfsDir="" - if [ $# -ge 3 ]; then + local targetOs="$1" + local targetArch="$2" + local rootfsDir="" + if [ "$#" -ge 3 ]; then rootfsDir="$3" fi if [ -n "${rootfsDir}" ]; then # We may have a cross build. Check for the existence of the rootfsDir if [ ! -e "${rootfsDir}" ]; then - echo "Error: rootfsDir has been passed, but the location is not valid." + echo "Error rootfsDir has been passed, but the location is not valid." exit 1 fi fi @@ -115,7 +119,7 @@ initDistroRidGlobal() STRINGS="$(command -v llvm-strings || true)" fi - # Check for musl-based distros (e.g. Alpine Linux, Void Linux). + # Check for musl-based distros (e.g Alpine Linux, Void Linux). if "${rootfsDir}/usr/bin/ldd" --version 2>&1 | grep -q musl || ( [ -n "$STRINGS" ] && "$STRINGS" "${rootfsDir}/usr/bin/ldd" 2>&1 | grep -q musl ); then __PortableTargetOS="linux-musl" diff --git a/eng/common/native/init-os-and-arch.sh b/eng/common/native/init-os-and-arch.sh index 38921d4338f744..caa448ff030070 100644 --- a/eng/common/native/init-os-and-arch.sh +++ b/eng/common/native/init-os-and-arch.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash # Use uname to determine what the OS is. OSName=$(uname -s | tr '[:upper:]' '[:lower:]') @@ -54,7 +54,6 @@ case "$CPUName" in ;; armv7l|armv8l) - # shellcheck disable=SC1091 if (NAME=""; . /etc/os-release; test "$NAME" = "Tizen"); then arch=armel else diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml deleted file mode 100644 index dfc3c0cf07ceae..00000000000000 --- a/eng/common/templates-official/job/job.yml +++ /dev/null @@ -1,263 +0,0 @@ -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -parameters: -# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - cancelTimeoutInMinutes: '' - condition: '' - container: '' - continueOnError: false - dependsOn: '' - displayName: '' - pool: '' - steps: [] - strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' - templateContext: '' - -# Job base template specific parameters - # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md - artifacts: '' - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishBuildAssets: false - enablePublishTestResults: false - enablePublishUsingPipelines: false - enableBuildRetry: false - disableComponentGovernance: '' - componentGovernanceIgnoreDirectories: '' - mergeTestResults: false - testRunTitle: '' - testResultsFormat: '' - name: '' - preSteps: [] - runAsPublic: false -# Sbom related params - enableSbom: true - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - -jobs: -- job: ${{ parameters.name }} - - ${{ if ne(parameters.cancelTimeoutInMinutes, '') }}: - cancelTimeoutInMinutes: ${{ parameters.cancelTimeoutInMinutes }} - - ${{ if ne(parameters.condition, '') }}: - condition: ${{ parameters.condition }} - - ${{ if ne(parameters.container, '') }}: - container: ${{ parameters.container }} - - ${{ if ne(parameters.continueOnError, '') }}: - continueOnError: ${{ parameters.continueOnError }} - - ${{ if ne(parameters.dependsOn, '') }}: - dependsOn: ${{ parameters.dependsOn }} - - ${{ if ne(parameters.displayName, '') }}: - displayName: ${{ parameters.displayName }} - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - - ${{ if ne(parameters.strategy, '') }}: - strategy: ${{ parameters.strategy }} - - ${{ if ne(parameters.timeoutInMinutes, '') }}: - timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - - variables: - - ${{ if ne(parameters.enableTelemetry, 'false') }}: - - name: DOTNET_CLI_TELEMETRY_PROFILE - value: '$(Build.Repository.Uri)' - - ${{ if eq(parameters.enableRichCodeNavigation, 'true') }}: - - name: EnableRichCodeNavigation - value: 'true' - # Retry signature validation up to three times, waiting 2 seconds between attempts. - # See https://learn.microsoft.com/en-us/nuget/reference/errors-and-warnings/nu3028#retry-untrusted-root-failures - - name: NUGET_EXPERIMENTAL_CHAIN_BUILD_RETRY_POLICY - value: 3,2000 - - ${{ each variable in parameters.variables }}: - # handle name-value variable syntax - # example: - # - name: [key] - # value: [value] - - ${{ if ne(variable.name, '') }}: - - name: ${{ variable.name }} - value: ${{ variable.value }} - - # handle variable groups - - ${{ if ne(variable.group, '') }}: - - group: ${{ variable.group }} - - # handle template variable syntax - # example: - # - template: path/to/template.yml - # parameters: - # [key]: [value] - - ${{ if ne(variable.template, '') }}: - - template: ${{ variable.template }} - ${{ if ne(variable.parameters, '') }}: - parameters: ${{ variable.parameters }} - - # handle key-value variable syntax. - # example: - # - [key]: [value] - - ${{ if and(eq(variable.name, ''), eq(variable.group, ''), eq(variable.template, '')) }}: - - ${{ each pair in variable }}: - - name: ${{ pair.key }} - value: ${{ pair.value }} - - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds - - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: DotNet-HelixApi-Access - - ${{ if ne(parameters.workspace, '') }}: - workspace: ${{ parameters.workspace }} - - steps: - - ${{ if ne(parameters.preSteps, '') }}: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - env: - TeamName: $(_TeamName) - continueOnError: ${{ parameters.continueOnError }} - condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - - - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - - task: NuGetAuthenticate@1 - - - ${{ if and(ne(parameters.artifacts.download, 'false'), ne(parameters.artifacts.download, '')) }}: - - task: DownloadPipelineArtifact@2 - inputs: - buildType: current - artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} - targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} - itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} - - - ${{ each step in parameters.steps }}: - - ${{ step }} - - - ${{ if eq(parameters.enableRichCodeNavigation, true) }}: - - task: RichCodeNavIndexer@0 - displayName: RichCodeNav Upload - inputs: - languages: ${{ coalesce(parameters.richCodeNavigationLanguage, 'csharp') }} - environment: ${{ coalesce(parameters.richCodeNavigationEnvironment, 'internal') }} - richNavLogOutputDirectory: $(Build.SourcesDirectory)/artifacts/bin - uploadRichNavArtifacts: ${{ coalesce(parameters.richCodeNavigationUploadArtifacts, false) }} - continueOnError: true - - - template: /eng/common/templates-official/steps/component-governance.yml - parameters: - ${{ if eq(parameters.disableComponentGovernance, '') }}: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.runAsPublic, 'false'), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/dotnet/'), startsWith(variables['Build.SourceBranch'], 'refs/heads/microsoft/'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}: - disableComponentGovernance: false - ${{ else }}: - disableComponentGovernance: true - ${{ else }}: - disableComponentGovernance: ${{ parameters.disableComponentGovernance }} - componentGovernanceIgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: MicroBuildCleanup@1 - displayName: Execute Microbuild cleanup tasks - condition: and(always(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: - TeamName: $(_TeamName) - - - ${{ if ne(parameters.artifacts.publish, '') }}: - - ${{ if and(ne(parameters.artifacts.publish.artifacts, 'false'), ne(parameters.artifacts.publish.artifacts, '')) }}: - - task: CopyFiles@2 - displayName: Gather binaries for publish to artifacts - inputs: - SourceFolder: 'artifacts/bin' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' - - task: CopyFiles@2 - displayName: Gather packages for publish to artifacts - inputs: - SourceFolder: 'artifacts/packages' - Contents: '**' - TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish pipeline artifacts - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} - continueOnError: true - condition: always() - - ${{ if and(ne(parameters.artifacts.publish.logs, 'false'), ne(parameters.artifacts.publish.logs, '')) }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: 'artifacts/log' - artifactName: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} - displayName: 'Publish logs' - continueOnError: true - condition: always() - - - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' - PublishLocation: Container - ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} - continueOnError: true - condition: always() - - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'xunit')) }}: - - task: PublishTestResults@2 - displayName: Publish XUnit Test Results - inputs: - testResultsFormat: 'xUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - ${{ if or(and(eq(parameters.enablePublishTestResults, 'true'), eq(parameters.testResultsFormat, '')), eq(parameters.testResultsFormat, 'vstest')) }}: - - task: PublishTestResults@2 - displayName: Publish TRX Test Results - inputs: - testResultsFormat: 'VSTest' - testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx - mergeTestResults: ${{ parameters.mergeTestResults }} - continueOnError: true - condition: always() - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.enableSbom, 'true')) }}: - - template: /eng/common/templates-official/steps/generate-sbom.yml - parameters: - PackageVersion: ${{ parameters.packageVersion}} - BuildDropPath: ${{ parameters.buildDropPath }} - IgnoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} - - - ${{ if eq(parameters.enableBuildRetry, 'true') }}: - - task: 1ES.PublishPipelineArtifact@1 - inputs: - targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' - artifactName: 'BuildConfiguration' - displayName: 'Publish build retry configuration' - continueOnError: true \ No newline at end of file diff --git a/eng/common/templates-official/job/onelocbuild.yml b/eng/common/templates-official/job/onelocbuild.yml deleted file mode 100644 index ba9ba49303292a..00000000000000 --- a/eng/common/templates-official/job/onelocbuild.yml +++ /dev/null @@ -1,112 +0,0 @@ -parameters: - # Optional: dependencies of the job - dependsOn: '' - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: '' - - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - SourcesDirectory: $(Build.SourcesDirectory) - CreatePr: true - AutoCompletePr: false - ReusePr: true - UseLfLineEndings: true - UseCheckedInLocProjectJson: false - SkipLocProjectJsonGeneration: false - LanguageSet: VS_Main_Languages - LclSource: lclFilesInRepo - LclPackageId: '' - RepoType: gitHub - GitHubOrg: dotnet - MirrorRepo: '' - MirrorBranch: main - condition: '' - JobNameSuffix: '' - -jobs: -- job: OneLocBuild${{ parameters.JobNameSuffix }} - - dependsOn: ${{ parameters.dependsOn }} - - displayName: OneLocBuild${{ parameters.JobNameSuffix }} - - variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - - name: _GenerateLocProjectArguments - value: -SourcesDirectory ${{ parameters.SourcesDirectory }} - -LanguageSet "${{ parameters.LanguageSet }}" - -CreateNeutralXlfs - - ${{ if eq(parameters.UseCheckedInLocProjectJson, 'true') }}: - - name: _GenerateLocProjectArguments - value: ${{ variables._GenerateLocProjectArguments }} -UseCheckedInLocProjectJson - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt - os: windows - - steps: - - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - - task: Powershell@2 - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 - arguments: $(_GenerateLocProjectArguments) - displayName: Generate LocProject.json - condition: ${{ parameters.condition }} - - - task: OneLocBuild@2 - displayName: OneLocBuild - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - inputs: - locProj: eng/Localize/LocProject.json - outDir: $(Build.ArtifactStagingDirectory) - lclSource: ${{ parameters.LclSource }} - lclPackageId: ${{ parameters.LclPackageId }} - isCreatePrSelected: ${{ parameters.CreatePr }} - isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }} - ${{ if eq(parameters.CreatePr, true) }}: - isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - isShouldReusePrSelected: ${{ parameters.ReusePr }} - packageSourceAuth: patAuth - patVariable: ${{ parameters.CeapexPat }} - ${{ if eq(parameters.RepoType, 'gitHub') }}: - repoType: ${{ parameters.RepoType }} - gitHubPatVariable: "${{ parameters.GithubPat }}" - ${{ if ne(parameters.MirrorRepo, '') }}: - isMirrorRepoSelected: true - gitHubOrganization: ${{ parameters.GitHubOrg }} - mirrorRepo: ${{ parameters.MirrorRepo }} - mirrorBranch: ${{ parameters.MirrorBranch }} - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Localization Files - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/loc' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish LocProject.json - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/Localize/' - PublishLocation: Container - ArtifactName: Loc - condition: ${{ parameters.condition }} \ No newline at end of file diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml deleted file mode 100644 index 5f54135569b290..00000000000000 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ /dev/null @@ -1,157 +0,0 @@ -parameters: - configuration: 'Debug' - - # Optional: condition for the job to run - condition: '' - - # Optional: 'true' if future jobs should run even if this job fails - continueOnError: false - - # Optional: dependencies of the job - dependsOn: '' - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool - pool: {} - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishUsingPipelines: false - - # Optional: whether the build's artifacts will be published using release pipelines or direct feed publishing - publishAssetsImmediately: false - - artifactsPublishingAdditionalParameters: '' - - signingValidationAdditionalParameters: '' - -jobs: -- job: Asset_Registry_Publish - - dependsOn: ${{ parameters.dependsOn }} - timeoutInMinutes: 150 - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - displayName: Publish Assets - ${{ else }}: - displayName: Publish to Build Asset Registry - - variables: - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: Publish-Build-Assets - - group: AzureDevOps-Artifact-Feeds-Pats - - name: runCodesignValidationInjection - value: false - # unconditional - needed for logs publishing (redactor tool version) - - template: /eng/common/templates-official/post-build/common-variables.yml - - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ if ne(variables['System.TeamProject'], 'DevDiv') }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt - os: windows - steps: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: self - fetchDepth: 3 - clean: true - - - task: DownloadBuildArtifacts@0 - displayName: Download artifact - inputs: - artifactName: AssetManifests - downloadPath: '$(Build.StagingDirectory)/Download' - checkDownloadedFiles: true - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Build Assets - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet - /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro.dot.net - /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} - /p:OfficialBuildId=$(Build.BuildNumber) - condition: ${{ parameters.condition }} - continueOnError: ${{ parameters.continueOnError }} - - - task: powershell@2 - displayName: Create ReleaseConfigs Artifact - inputs: - targetType: inline - script: | - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" - Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish ReleaseConfigs Artifact - inputs: - PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - task: powershell@2 - displayName: Check if SymbolPublishingExclusionsFile.txt exists - inputs: - targetType: inline - script: | - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" - if(Test-Path -Path $symbolExclusionfile) - { - Write-Host "SymbolExclusionFile exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]true" - } - else{ - Write-Host "Symbols Exclusion file does not exists" - Write-Host "##vso[task.setvariable variable=SymbolExclusionFile]false" - } - - - task: 1ES.PublishBuildArtifacts@1 - displayName: Publish SymbolPublishingExclusionsFile Artifact - condition: eq(variables['SymbolExclusionFile'], 'true') - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' - PublishLocation: Container - ArtifactName: ReleaseConfigs - - - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}: - - template: /eng/common/templates-official/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' - - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - - template: /eng/common/templates-official/steps/publish-logs.yml - parameters: - JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml deleted file mode 100644 index 50f04e642a3543..00000000000000 --- a/eng/common/templates-official/job/source-build.yml +++ /dev/null @@ -1,67 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. The template produces a server job with a - # default ID 'Source_Build_Complete' to put in a dependency list if necessary. - - # Specifies the prefix for source-build jobs added to pipeline. Use this if disambiguation needed. - jobNamePrefix: 'Source_Build' - - # Defines the platform on which to run the job. By default, a linux-x64 machine, suitable for - # managed-only repositories. This is an object with these properties: - # - # name: '' - # The name of the job. This is included in the job ID. - # targetRID: '' - # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false - # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than - # linux-x64), and compiling against distro-provided packages rather than portable ones. - # skipPublishValidation: false - # Disables publishing validation. By default, a check is performed to ensure no packages are - # published by source-build. - # container: '' - # A container to use. Runs in docker. - # pool: {} - # A pool to use. Runs directly on an agent. - # buildScript: '' - # Specifies the build script to invoke to perform the build in the repo. The default - # './build.sh' should work for typical Arcade repositories, but this is customizable for - # difficult situations. - # jobProperties: {} - # A list of job properties to inject at the top level, for potential extensibility beyond - # container and pool. - platform: {} - -jobs: -- job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} - displayName: Source-Build (${{ parameters.platform.name }}) - - ${{ each property in parameters.platform.jobProperties }}: - ${{ property.key }}: ${{ property.value }} - - ${{ if ne(parameters.platform.container, '') }}: - container: ${{ parameters.platform.container }} - - ${{ if eq(parameters.platform.pool, '') }}: - # The default VM host AzDO pool. This should be capable of running Docker containers: almost all - # source-build builds run in Docker, including the default managed platform. - # /eng/common/templates-official/variables/pool-providers.yml can't be used here (some customers declare variables already), so duplicate its logic - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.1804.amd64 - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2-pt - os: linux - - ${{ if ne(parameters.platform.pool, '') }}: - pool: ${{ parameters.platform.pool }} - - workspace: - clean: all - - steps: - - template: /eng/common/templates-official/steps/source-build.yml - parameters: - platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml deleted file mode 100644 index 53a9ef51fd82d2..00000000000000 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ /dev/null @@ -1,67 +0,0 @@ -parameters: - runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20240129.2 - sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json - sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" - preSteps: [] - binlogPath: artifacts/log/Debug/Build.binlog - condition: '' - dependsOn: '' - pool: '' - -jobs: -- job: SourceIndexStage1 - dependsOn: ${{ parameters.dependsOn }} - condition: ${{ parameters.condition }} - variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} - - name: SourceIndexPackageSource - value: ${{ parameters.sourceIndexPackageSource }} - - name: BinlogPath - value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - - template: /eng/common/templates-official/variables/pool-providers.yml - - ${{ if ne(parameters.pool, '') }}: - pool: ${{ parameters.pool }} - ${{ if eq(parameters.pool, '') }}: - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - image: windows.vs2022.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - name: $(DncEngInternalBuildPool) - image: windows.vs2022.amd64 - - steps: - - ${{ each preStep in parameters.preSteps }}: - - ${{ preStep }} - - - task: UseDotNet@2 - displayName: Use .NET 8 SDK - inputs: - packageType: sdk - version: 8.0.x - installationPath: $(Agent.TempDirectory)/dotnet - workingDirectory: $(Agent.TempDirectory) - - - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - displayName: Download Tools - # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. - workingDirectory: $(Agent.TempDirectory) - - - script: ${{ parameters.sourceIndexBuildCommand }} - displayName: Build Repository - - - script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i $(BinlogPath) -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output - displayName: Process Binlog into indexable sln - - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) - displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/codeql-build.yml b/eng/common/templates-official/jobs/codeql-build.yml deleted file mode 100644 index b68d3c2f31990f..00000000000000 --- a/eng/common/templates-official/jobs/codeql-build.yml +++ /dev/null @@ -1,31 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - # Optional: if specified, restore and use this version of Guardian instead of the default. - overrideGuardianVersion: '' - -jobs: -- template: /eng/common/templates-official/jobs/jobs.yml - parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: false - enableTelemetry: true - - variables: - - group: Publish-Build-Assets - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.109.0 - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - - name: GuardianVersion - value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - jobs: ${{ parameters.jobs }} - diff --git a/eng/common/templates-official/jobs/jobs.yml b/eng/common/templates-official/jobs/jobs.yml deleted file mode 100644 index 857a0f8ba43e84..00000000000000 --- a/eng/common/templates-official/jobs/jobs.yml +++ /dev/null @@ -1,97 +0,0 @@ -parameters: - # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md - continueOnError: false - - # Optional: Include PublishBuildArtifacts task - enablePublishBuildArtifacts: false - - # Optional: Enable publishing using release pipelines - enablePublishUsingPipelines: false - - # Optional: Enable running the source-build jobs to build repo from source - enableSourceBuild: false - - # Optional: Parameters for source-build template. - # See /eng/common/templates-official/jobs/source-build.yml for options - sourceBuildParameters: [] - - graphFileGeneration: - # Optional: Enable generating the graph files at the end of the build - enabled: false - # Optional: Include toolset dependencies in the generated graph files - includeToolset: false - - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job - jobs: [] - - # Optional: Override automatically derived dependsOn value for "publish build assets" job - publishBuildAssetsDependsOn: '' - - # Optional: Publish the assets as soon as the publish to BAR stage is complete, rather doing so in a separate stage. - publishAssetsImmediately: false - - # Optional: If using publishAssetsImmediately and additional parameters are needed, can be used to send along additional parameters (normally sent to post-build.yml) - artifactsPublishingAdditionalParameters: '' - signingValidationAdditionalParameters: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. - runAsPublic: false - - enableSourceIndex: false - sourceIndexParams: {} - -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - -jobs: -- ${{ each job in parameters.jobs }}: - - template: ../job/job.yml - parameters: - # pass along parameters - ${{ each parameter in parameters }}: - ${{ if ne(parameter.key, 'jobs') }}: - ${{ parameter.key }}: ${{ parameter.value }} - - # pass along job properties - ${{ each property in job }}: - ${{ if ne(property.key, 'job') }}: - ${{ property.key }}: ${{ property.value }} - - name: ${{ job.job }} - -- ${{ if eq(parameters.enableSourceBuild, true) }}: - - template: /eng/common/templates-official/jobs/source-build.yml - parameters: - allCompletedJobId: Source_Build_Complete - ${{ each parameter in parameters.sourceBuildParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if eq(parameters.enableSourceIndex, 'true') }}: - - template: ../job/source-index-stage1.yml - parameters: - runAsPublic: ${{ parameters.runAsPublic }} - ${{ each parameter in parameters.sourceIndexParams }}: - ${{ parameter.key }}: ${{ parameter.value }} - -- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - - ${{ if eq(parameters.enableSourceBuild, true) }}: - - Source_Build_Complete - - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - publishAssetsImmediately: ${{ parameters.publishAssetsImmediately }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} - signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml deleted file mode 100644 index 08e5db9bb11616..00000000000000 --- a/eng/common/templates-official/jobs/source-build.yml +++ /dev/null @@ -1,46 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. A job is created for each platform, as - # well as an optional server job that completes when all platform jobs complete. - - # The name of the "join" job for all source-build platforms. If set to empty string, the job is - # not included. Existing repo pipelines can use this job depend on all source-build jobs - # completing without maintaining a separate list of every single job ID: just depend on this one - # server job. By default, not included. Recommended name if used: 'Source_Build_Complete'. - allCompletedJobId: '' - - # See /eng/common/templates-official/job/source-build.yml - jobNamePrefix: 'Source_Build' - - # This is the default platform provided by Arcade, intended for use by a managed-only repo. - defaultManagedPlatform: - name: 'Managed' - container: 'mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8' - - # Defines the platforms on which to run build jobs. One job is created for each platform, and the - # object in this array is sent to the job template as 'platform'. If no platforms are specified, - # one job runs on 'defaultManagedPlatform'. - platforms: [] - -jobs: - -- ${{ if ne(parameters.allCompletedJobId, '') }}: - - job: ${{ parameters.allCompletedJobId }} - displayName: Source-Build Complete - pool: server - dependsOn: - - ${{ each platform in parameters.platforms }}: - - ${{ parameters.jobNamePrefix }}_${{ platform.name }} - - ${{ if eq(length(parameters.platforms), 0) }}: - - ${{ parameters.jobNamePrefix }}_${{ parameters.defaultManagedPlatform.name }} - -- ${{ each platform in parameters.platforms }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ platform }} - -- ${{ if eq(length(parameters.platforms), 0) }}: - - template: /eng/common/templates-official/job/source-build.yml - parameters: - jobNamePrefix: ${{ parameters.jobNamePrefix }} - platform: ${{ parameters.defaultManagedPlatform }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml deleted file mode 100644 index b9ede10bf099ae..00000000000000 --- a/eng/common/templates-official/post-build/common-variables.yml +++ /dev/null @@ -1,24 +0,0 @@ -variables: - - group: Publish-Build-Assets - - # Whether the build is internal or not - - name: IsInternalBuild - value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - - # Default Maestro++ API Endpoint and API Version - - name: MaestroApiEndPoint - value: "https://maestro.dot.net" - - name: MaestroApiAccessToken - value: $(MaestroAccessToken) - - name: MaestroApiVersion - value: "2020-02-20" - - - name: SourceLinkCLIVersion - value: 3.0.0 - - name: SymbolToolVersion - value: 1.0.1 - - name: BinlogToolVersion - value: 1.0.11 - - - name: runCodesignValidationInjection - value: false diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml deleted file mode 100644 index 5c98fe1c0f3a96..00000000000000 --- a/eng/common/templates-official/post-build/post-build.yml +++ /dev/null @@ -1,285 +0,0 @@ -parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - -stages: -- ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - - stage: Validate - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Validate Build Assets - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: NuGet Validation - condition: and(succeededOrFailed(), eq( ${{ parameters.enableNugetValidation }}, 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt - os: windows - - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - -ToolDestinationPath $(Agent.BuildDirectory)/Extract/ - - - job: - displayName: Signing Validation - condition: and( eq( ${{ parameters.enableSigningValidation }}, 'true'), ne( variables['PostBuildSign'], 'true')) - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - itemPattern: | - ** - !**/Microsoft.SourceBuild.Intermediate.*.nupkg - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: ../steps/publish-logs.yml - parameters: - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) - - - job: - displayName: SourceLink Validation - condition: eq( ${{ parameters.enableSourceLinkValidation }}, 'true') - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true - -- ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - - stage: publish_using_darc - ${{ if or(eq(parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: - dependsOn: ${{ parameters.publishDependsOn }} - ${{ else }}: - dependsOn: ${{ parameters.validateDependsOn }} - displayName: Publish using Darc - variables: - - template: common-variables.yml - - template: /eng/common/templates-official/variables/pool-providers.yml - jobs: - - job: - displayName: Publish Using Darc - timeoutInMinutes: 120 - pool: - # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) - ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: - name: AzurePipelines-EO - image: 1ESPT-Windows2022 - demands: Cmd - os: windows - # If it's not devdiv, it's dnceng - ${{ else }}: - name: $(DncEngInternalBuildPool) - image: 1es-windows-2022-pt - os: windows - steps: - - template: setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - - task: NuGetAuthenticate@1 - - - task: PowerShell@2 - displayName: Publish Using Darc - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) - -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' - -WaitPublishingFinish true - -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' - -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/post-build/setup-maestro-vars.yml b/eng/common/templates-official/post-build/setup-maestro-vars.yml deleted file mode 100644 index 0c87f149a4ad77..00000000000000 --- a/eng/common/templates-official/post-build/setup-maestro-vars.yml +++ /dev/null @@ -1,70 +0,0 @@ -parameters: - BARBuildId: '' - PromoteToChannelIds: '' - -steps: - - ${{ if eq(coalesce(parameters.PromoteToChannelIds, 0), 0) }}: - - task: DownloadBuildArtifacts@0 - displayName: Download Release Configs - inputs: - buildType: current - artifactName: ReleaseConfigs - checkDownloadedFiles: true - - - task: PowerShell@2 - name: setReleaseVars - displayName: Set Release Configs Vars - inputs: - targetType: inline - pwsh: true - script: | - try { - if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { - $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt - - $BarId = $Content | Select -Index 0 - $Channels = $Content | Select -Index 1 - $IsStableBuild = $Content | Select -Index 2 - - $AzureDevOpsProject = $Env:System_TeamProject - $AzureDevOpsBuildDefinitionId = $Env:System_DefinitionId - $AzureDevOpsBuildId = $Env:Build_BuildId - } - else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" - - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId - $Channels = $Env:PromoteToMaestroChannels -split "," - $Channels = $Channels -join "][" - $Channels = "[$Channels]" - - $IsStableBuild = $buildInfo.stable - $AzureDevOpsProject = $buildInfo.azureDevOpsProject - $AzureDevOpsBuildDefinitionId = $buildInfo.azureDevOpsBuildDefinitionId - $AzureDevOpsBuildId = $buildInfo.azureDevOpsBuildId - } - - Write-Host "##vso[task.setvariable variable=BARBuildId]$BarId" - Write-Host "##vso[task.setvariable variable=TargetChannels]$Channels" - Write-Host "##vso[task.setvariable variable=IsStableBuild]$IsStableBuild" - - Write-Host "##vso[task.setvariable variable=AzDOProjectName]$AzureDevOpsProject" - Write-Host "##vso[task.setvariable variable=AzDOPipelineId]$AzureDevOpsBuildDefinitionId" - Write-Host "##vso[task.setvariable variable=AzDOBuildId]$AzureDevOpsBuildId" - } - catch { - Write-Host $_ - Write-Host $_.Exception - Write-Host $_.ScriptStackTrace - exit 1 - } - env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates-official/post-build/trigger-subscription.yml b/eng/common/templates-official/post-build/trigger-subscription.yml deleted file mode 100644 index da669030daf6e9..00000000000000 --- a/eng/common/templates-official/post-build/trigger-subscription.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Triggering subscriptions - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/trigger-subscriptions.ps1 - arguments: -SourceRepo $(Build.Repository.Uri) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/add-build-to-channel.yml b/eng/common/templates-official/steps/add-build-to-channel.yml deleted file mode 100644 index f67a210d62f3e5..00000000000000 --- a/eng/common/templates-official/steps/add-build-to-channel.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - ChannelId: 0 - -steps: -- task: PowerShell@2 - displayName: Add Build to Channel - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/add-build-to-channel.ps1 - arguments: -BuildId $(BARBuildId) - -ChannelId ${{ parameters.ChannelId }} - -MaestroApiAccessToken $(MaestroApiAccessToken) - -MaestroApiEndPoint $(MaestroApiEndPoint) - -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates-official/steps/component-governance.yml b/eng/common/templates-official/steps/component-governance.yml deleted file mode 100644 index 0ecec47b0c9177..00000000000000 --- a/eng/common/templates-official/steps/component-governance.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - disableComponentGovernance: false - componentGovernanceIgnoreDirectories: '' - -steps: -- ${{ if eq(parameters.disableComponentGovernance, 'true') }}: - - script: "echo ##vso[task.setvariable variable=skipComponentGovernanceDetection]true" - displayName: Set skipComponentGovernanceDetection variable -- ${{ if ne(parameters.disableComponentGovernance, 'true') }}: - - task: ComponentGovernanceComponentDetection@0 - continueOnError: true - inputs: - ignoreDirectories: ${{ parameters.componentGovernanceIgnoreDirectories }} \ No newline at end of file diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml deleted file mode 100644 index 488b560e8ba4eb..00000000000000 --- a/eng/common/templates-official/steps/generate-sbom.yml +++ /dev/null @@ -1,48 +0,0 @@ -# BuildDropPath - The root folder of the drop directory for which the manifest file will be generated. -# PackageName - The name of the package this SBOM represents. -# PackageVersion - The version of the package this SBOM represents. -# ManifestDirPath - The path of the directory where the generated manifest files will be placed -# IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. - -parameters: - PackageVersion: 7.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' - PackageName: '.NET' - ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom - IgnoreDirectories: '' - sbomContinueOnError: true - -steps: -- task: PowerShell@2 - displayName: Prep for SBOM generation in (Non-linux) - condition: or(eq(variables['Agent.Os'], 'Windows_NT'), eq(variables['Agent.Os'], 'Darwin')) - inputs: - filePath: ./eng/common/generate-sbom-prep.ps1 - arguments: ${{parameters.manifestDirPath}} - -# Chmodding is a workaround for https://github.com/dotnet/arcade/issues/8461 -- script: | - chmod +x ./eng/common/generate-sbom-prep.sh - ./eng/common/generate-sbom-prep.sh ${{parameters.manifestDirPath}} - displayName: Prep for SBOM generation in (Linux) - condition: eq(variables['Agent.Os'], 'Linux') - continueOnError: ${{ parameters.sbomContinueOnError }} - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'Generate SBOM manifest' - continueOnError: ${{ parameters.sbomContinueOnError }} - inputs: - PackageName: ${{ parameters.packageName }} - BuildDropPath: ${{ parameters.buildDropPath }} - PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} - ${{ if ne(parameters.IgnoreDirectories, '') }}: - AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish SBOM manifest - continueOnError: ${{parameters.sbomContinueOnError}} - inputs: - targetPath: '${{parameters.manifestDirPath}}' - artifactName: $(ARTIFACT_NAME) - diff --git a/eng/common/templates-official/steps/publish-logs.yml b/eng/common/templates-official/steps/publish-logs.yml deleted file mode 100644 index 84b2f559c56e40..00000000000000 --- a/eng/common/templates-official/steps/publish-logs.yml +++ /dev/null @@ -1,49 +0,0 @@ -parameters: - StageLabel: '' - JobLabel: '' - CustomSensitiveDataList: '' - # A default - in case value from eng/common/templates-official/post-build/common-variables.yml is not passed - BinlogToolVersion: '1.0.11' - -steps: -- task: Powershell@2 - displayName: Prepare Binlogs to Upload - inputs: - targetType: inline - script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - continueOnError: true - condition: always() - -- task: PowerShell@2 - displayName: Redact Logs - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 - # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - # If the file exists - sensitive data for redaction will be sourced from it - # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' - '$(publishing-dnceng-devdiv-code-r-build-re)' - '$(MaestroAccessToken)' - '$(dn-bot-all-orgs-artifact-feeds-rw)' - '$(akams-client-id)' - '$(akams-client-secret)' - '$(microsoft-symbol-server-pat)' - '$(symweb-symbol-server-pat)' - '$(dn-bot-all-orgs-build-rw-code-rw)' - ${{parameters.CustomSensitiveDataList}} - continueOnError: true - condition: always() - -- task: 1ES.PublishBuildArtifacts@1 - displayName: Publish Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' - PublishLocation: Container - ArtifactName: PostBuildLogs - continueOnError: true - condition: always() diff --git a/eng/common/templates-official/steps/retain-build.yml b/eng/common/templates-official/steps/retain-build.yml deleted file mode 100644 index 83d97a26a01ff9..00000000000000 --- a/eng/common/templates-official/steps/retain-build.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: - # Optional azure devops PAT with build execute permissions for the build's organization, - # only needed if the build that should be retained ran on a different organization than - # the pipeline where this template is executing from - Token: '' - # Optional BuildId to retain, defaults to the current running build - BuildId: '' - # Azure devops Organization URI for the build in the https://dev.azure.com/ format. - # Defaults to the organization the current pipeline is running on - AzdoOrgUri: '$(System.CollectionUri)' - # Azure devops project for the build. Defaults to the project the current pipeline is running on - AzdoProject: '$(System.TeamProject)' - -steps: - - task: powershell@2 - inputs: - targetType: 'filePath' - filePath: eng/common/retain-build.ps1 - pwsh: true - arguments: > - -AzdoOrgUri: ${{parameters.AzdoOrgUri}} - -AzdoProject ${{parameters.AzdoProject}} - -Token ${{coalesce(parameters.Token, '$env:SYSTEM_ACCESSTOKEN') }} - -BuildId ${{coalesce(parameters.BuildId, '$env:BUILD_ID')}} - displayName: Enable permanent build retention - env: - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - BUILD_ID: $(Build.BuildId) \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml deleted file mode 100644 index 68fa739c4ab215..00000000000000 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ /dev/null @@ -1,93 +0,0 @@ -# Please remember to update the documentation if you make changes to these parameters! -parameters: - HelixSource: 'pr/default' # required -- sources must start with pr/, official/, prodcon/, or agent/ - HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' - HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number - HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group - HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY - HelixProjectArguments: '' # optional -- arguments passed to the build command - HelixConfiguration: '' # optional -- additional property attached to a job - HelixPreCommands: '' # optional -- commands to run before Helix work item execution - HelixPostCommands: '' # optional -- commands to run after Helix work item execution - WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects - WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects - WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects - CorrelationPayloadDirectory: '' # optional -- a directory to zip up and send to Helix as a correlation payload - XUnitProjects: '' # optional -- semicolon-delimited list of XUnitProjects to parse and send to Helix; requires XUnitRuntimeTargetFramework, XUnitPublishTargetFramework, XUnitRunnerVersion, and IncludeDotNetCli=true - XUnitWorkItemTimeout: '' # optional -- the workitem timeout in seconds for all workitems created from the xUnit projects specified by XUnitProjects - XUnitPublishTargetFramework: '' # optional -- framework to use to publish your xUnit projects - XUnitRuntimeTargetFramework: '' # optional -- framework to use for the xUnit console runner - XUnitRunnerVersion: '' # optional -- version of the xUnit nuget package you wish to use on Helix; required for XUnitProjects - IncludeDotNetCli: false # optional -- true will download a version of the .NET CLI onto the Helix machine as a correlation payload; requires DotNetCliPackageType and DotNetCliVersion - DotNetCliPackageType: '' # optional -- either 'sdk', 'runtime' or 'aspnetcore-runtime'; determines whether the sdk or runtime will be sent to Helix; see https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - DotNetCliVersion: '' # optional -- version of the CLI to send to Helix; based on this: https://raw.githubusercontent.com/dotnet/core/main/release-notes/releases-index.json - WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." - IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set - HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) - Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - env: - BuildConfig: $(_BuildConfig) - HelixSource: ${{ parameters.HelixSource }} - HelixType: ${{ parameters.HelixType }} - HelixBuild: ${{ parameters.HelixBuild }} - HelixConfiguration: ${{ parameters.HelixConfiguration }} - HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} - HelixPreCommands: ${{ parameters.HelixPreCommands }} - HelixPostCommands: ${{ parameters.HelixPostCommands }} - WorkItemDirectory: ${{ parameters.WorkItemDirectory }} - WorkItemCommand: ${{ parameters.WorkItemCommand }} - WorkItemTimeout: ${{ parameters.WorkItemTimeout }} - CorrelationPayloadDirectory: ${{ parameters.CorrelationPayloadDirectory }} - XUnitProjects: ${{ parameters.XUnitProjects }} - XUnitWorkItemTimeout: ${{ parameters.XUnitWorkItemTimeout }} - XUnitPublishTargetFramework: ${{ parameters.XUnitPublishTargetFramework }} - XUnitRuntimeTargetFramework: ${{ parameters.XUnitRuntimeTargetFramework }} - XUnitRunnerVersion: ${{ parameters.XUnitRunnerVersion }} - IncludeDotNetCli: ${{ parameters.IncludeDotNetCli }} - DotNetCliPackageType: ${{ parameters.DotNetCliPackageType }} - DotNetCliVersion: ${{ parameters.DotNetCliVersion }} - WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - HelixBaseUri: ${{ parameters.HelixBaseUri }} - Creator: ${{ parameters.Creator }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml deleted file mode 100644 index 53ed57b6d48abc..00000000000000 --- a/eng/common/templates-official/steps/source-build.yml +++ /dev/null @@ -1,131 +0,0 @@ -parameters: - # This template adds arcade-powered source-build to CI. - - # This is a 'steps' template, and is intended for advanced scenarios where the existing build - # infra has a careful build methodology that must be followed. For example, a repo - # (dotnet/runtime) might choose to clone the GitHub repo only once and store it as a pipeline - # artifact for all subsequent jobs to use, to reduce dependence on a strong network connection to - # GitHub. Using this steps template leaves room for that infra to be included. - - # Defines the platform on which to run the steps. See 'eng/common/templates-official/job/source-build.yml' - # for details. The entire object is described in the 'job' template for simplicity, even though - # the usage of the properties on this object is split between the 'job' and 'steps' templates. - platform: {} - -steps: -# Build. Keep it self-contained for simple reusability. (No source-build-specific job variables.) -- script: | - set -x - df -h - - # If building on the internal project, the artifact feeds variable may be available (usually only if needed) - # In that case, call the feed setup script to add internal feeds corresponding to public ones. - # In addition, add an msbuild argument to copy the WIP from the repo to the target build location. - # This is because SetupNuGetSources.sh will alter the current NuGet.config file, and we need to preserve those - # changes. - internalRestoreArgs= - if [ '$(dn-bot-dnceng-artifact-feeds-rw)' != '$''(dn-bot-dnceng-artifact-feeds-rw)' ]; then - # Temporarily work around https://github.com/dotnet/arcade/issues/7709 - chmod +x $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh $(Build.SourcesDirectory)/NuGet.config $(dn-bot-dnceng-artifact-feeds-rw) - internalRestoreArgs='/p:CopyWipIntoInnerSourceBuildRepo=true' - - # The 'Copy WIP' feature of source build uses git stash to apply changes from the original repo. - # This only works if there is a username/email configured, which won't be the case in most CI runs. - git config --get user.email - if [ $? -ne 0 ]; then - git config user.email dn-bot@microsoft.com - git config user.name dn-bot - fi - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - buildConfig=Release - # Check if AzDO substitutes in a build config from a variable, and use it if so. - if [ '$(_BuildConfig)' != '$''(_BuildConfig)' ]; then - buildConfig='$(_BuildConfig)' - fi - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:DotNetPublishUsingPipelines=true /p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - targetRidArgs= - if [ '${{ parameters.platform.targetRID }}' != '' ]; then - targetRidArgs='/p:TargetRid=${{ parameters.platform.targetRID }}' - fi - - runtimeOsArgs= - if [ '${{ parameters.platform.runtimeOS }}' != '' ]; then - runtimeOsArgs='/p:RuntimeOS=${{ parameters.platform.runtimeOS }}' - fi - - baseOsArgs= - if [ '${{ parameters.platform.baseOS }}' != '' ]; then - baseOsArgs='/p:BaseOS=${{ parameters.platform.baseOS }}' - fi - - publishArgs= - if [ '${{ parameters.platform.skipPublishValidation }}' != 'true' ]; then - publishArgs='--publish' - fi - - assetManifestFileName=SourceBuild_RidSpecific.xml - if [ '${{ parameters.platform.name }}' != '' ]; then - assetManifestFileName=SourceBuild_${{ parameters.platform.name }}.xml - fi - - ${{ coalesce(parameters.platform.buildScript, './build.sh') }} --ci \ - --configuration $buildConfig \ - --restore --build --pack $publishArgs -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - $internalRestoreArgs \ - $targetRidArgs \ - $runtimeOsArgs \ - $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ - /p:ArcadeBuildFromSource=true \ - /p:DotNetBuildSourceOnly=true \ - /p:DotNetBuildRepo=true \ - /p:AssetManifestFileName=$assetManifestFileName - displayName: Build - -# Upload build logs for diagnosis. -- task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - artifacts/sb/prebuilt-report/** - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - -- task: 1ES.PublishPipelineArtifact@1 - displayName: Publish BuildLogs - inputs: - targetPath: '$(Build.StagingDirectory)/BuildLogs' - artifactName: BuildLogs_SourceBuild_${{ parameters.platform.name }}_Attempt$(System.JobAttempt) - continueOnError: true - condition: succeededOrFailed() - -# Manually inject component detection so that we can ignore the source build upstream cache, which contains -# a nupkg cache of input packages (a local feed). -# This path must match the upstream cache path in property 'CurrentRepoSourceBuiltNupkgCacheDir' -# in src\Microsoft.DotNet.Arcade.Sdk\tools\SourceBuild\SourceBuildArcade.targets -- task: ComponentGovernanceComponentDetection@0 - displayName: Component Detection (Exclude upstream cache) - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/artifacts/sb/src/artifacts/obj/source-built-upstream-cache' diff --git a/eng/common/templates-official/variables/pool-providers.yml b/eng/common/templates-official/variables/pool-providers.yml deleted file mode 100644 index beab7d1bfba062..00000000000000 --- a/eng/common/templates-official/variables/pool-providers.yml +++ /dev/null @@ -1,45 +0,0 @@ -# Select a pool provider based off branch name. Anything with branch name containing 'release' must go into an -Svc pool, -# otherwise it should go into the "normal" pools. This separates out the queueing and billing of released branches. - -# Motivation: -# Once a given branch of a repository's output has been officially "shipped" once, it is then considered to be COGS -# (Cost of goods sold) and should be moved to a servicing pool provider. This allows both separation of queueing -# (allowing release builds and main PR builds to not intefere with each other) and billing (required for COGS. -# Additionally, the pool provider name itself may be subject to change when the .NET Core Engineering Services -# team needs to move resources around and create new and potentially differently-named pools. Using this template -# file from an Arcade-ified repo helps guard against both having to update one's release/* branches and renaming. - -# How to use: -# This yaml assumes your shipped product branches use the naming convention "release/..." (which many do). -# If we find alternate naming conventions in broad usage it can be added to the condition below. -# -# First, import the template in an arcade-ified repo to pick up the variables, e.g.: -# -# variables: -# - template: /eng/common/templates-official/variables/pool-providers.yml -# -# ... then anywhere specifying the pool provider use the runtime variables, -# $(DncEngInternalBuildPool) -# -# pool: -# name: $(DncEngInternalBuildPool) -# image: 1es-windows-2022-pt - -variables: - # Coalesce the target and source branches so we know when a PR targets a release branch - # If these variables are somehow missing, fall back to main (tends to have more capacity) - - # Any new -Svc alternative pools should have variables added here to allow for splitting work - - - name: DncEngInternalBuildPool - value: $[ - replace( - replace( - eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), - True, - 'NetCore1ESPool-Svc-Internal' - ), - False, - 'NetCore1ESPool-Internal' - ) - ] \ No newline at end of file diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml deleted file mode 100644 index dbdd66d4a4b3a0..00000000000000 --- a/eng/common/templates-official/variables/sdl-variables.yml +++ /dev/null @@ -1,7 +0,0 @@ -variables: -# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in -# sync with the packages.config file. -- name: DefaultGuardianVersion - value: 0.109.0 -- name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index a3277bf15c51ff..01c0dd995e4b61 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -15,7 +15,6 @@ parameters: timeoutInMinutes: '' variables: [] workspace: '' - templateContext: '' # Job base template specific parameters # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md @@ -69,9 +68,6 @@ jobs: ${{ if ne(parameters.timeoutInMinutes, '') }}: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - ${{ if ne(parameters.templateContext, '') }}: - templateContext: ${{ parameters.templateContext }} - variables: - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index a58aef2847e1c8..7d8dc89b919bc8 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,11 +65,6 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } -# True if the build is a product build -[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } - -[String[]]$properties = if (Test-Path variable:properties) { $properties } else { @() } - function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -855,8 +850,7 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and -not($properties -like "*DotNetBuildRepo=true*")) { + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index db64e298ff6314..ece4b730795360 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,9 +68,6 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} -# True if the build is a product build -product_build=${product_build:-false} - # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -144,7 +141,7 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="${repo_root}.dotnet" + dotnet_root="$repo_root/.dotnet" export DOTNET_INSTALL_DIR="$dotnet_root" @@ -506,8 +503,7 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - # Skip this when the build is a child of the VMR orchestrator build. - if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && "$properties" != *"DotNetBuildRepo=true"* ]]; then + if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/native/configurecompiler.cmake b/eng/native/configurecompiler.cmake index 0ab4a178b055fa..166fd52ab96562 100644 --- a/eng/native/configurecompiler.cmake +++ b/eng/native/configurecompiler.cmake @@ -832,32 +832,10 @@ if (MSVC) add_compile_options($<$:/we4013>) # 'function' undefined - assuming extern returning int. add_compile_options($<$:/we4102>) # "'%$S' : unreferenced label". add_compile_options($<$:/we4551>) # Function call missing argument list. + add_compile_options($<$:/we4700>) # Local used w/o being initialized. add_compile_options($<$:/we4640>) # 'instance' : construction of local static object is not thread-safe add_compile_options($<$:/we4806>) # Unsafe operation involving type 'bool'. - # SDL requires the below warnings to be treated as errors: - # More info: https://liquid.microsoft.com/Web/Object/Read/ms.security/Requirements/Microsoft.Security.SystemsADM.10086 - # (Access to that URL restricted to Microsoft employees.) - add_compile_options($<$:/we4055>) # 'conversion' : from data pointer 'type1' to function pointer 'type2' - add_compile_options($<$:/we4146>) # unary minus operator applied to unsigned type, result still unsigned - add_compile_options($<$:/we4242>) # 'identifier' : conversion from 'type1' to 'type2', possible loss of data - add_compile_options($<$:/we4244>) # 'conversion' conversion from 'type1' to 'type2', possible loss of data - add_compile_options($<$:/we4267>) # 'var' : conversion from 'size_t' to 'type', possible loss of data - add_compile_options($<$:/we4302>) # 'conversion' : truncation from 'type 1' to 'type 2' - add_compile_options($<$:/we4308>) # negative integral constant converted to unsigned type - add_compile_options($<$:/we4509>) # nonstandard extension used: 'function' uses SEH and 'object' has destructor - add_compile_options($<$:/we4510>) # 'class' : default constructor could not be generated - add_compile_options($<$:/we4532>) # 'continue' : jump out of __finally/finally block has undefined behavior during termination handling - add_compile_options($<$:/we4533>) # initialization of 'variable' is skipped by 'instruction' - add_compile_options($<$:/we4610>) # object 'class' can never be instantiated - user-defined constructor required - add_compile_options($<$:/we4611>) # interaction between 'function' and C++ object destruction is non-portable - add_compile_options($<$:/we4700>) # uninitialized local variable 'name' used - add_compile_options($<$:/we4701>) # Potentially uninitialized local variable 'name' used - add_compile_options($<$:/we4703>) # Potentially uninitialized local pointer variable 'name' used - add_compile_options($<$:/we4789>) # destination of memory copy is too small - add_compile_options($<$:/we4995>) # 'function': name was marked as #pragma deprecated - add_compile_options($<$:/we4996>) # 'function': was declared deprecated - # Set Warning Level 3: add_compile_options($<$:/w34092>) # Sizeof returns 'unsigned long'. add_compile_options($<$:/w34121>) # Structure is sensitive to alignment. diff --git a/eng/native/configureplatform.cmake b/eng/native/configureplatform.cmake index c7c378ab0e41b3..f5b5753e129eb5 100644 --- a/eng/native/configureplatform.cmake +++ b/eng/native/configureplatform.cmake @@ -27,8 +27,6 @@ if(CLR_CMAKE_HOST_OS STREQUAL linux) endif() elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL i686) set(CLR_CMAKE_HOST_UNIX_X86 1) - elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL aarch64) - set(CLR_CMAKE_HOST_UNIX_ARM64 1) else() clr_unknown_arch() endif() diff --git a/eng/native/functions.cmake b/eng/native/functions.cmake index e10e008d775e44..543722a9c0a59e 100644 --- a/eng/native/functions.cmake +++ b/eng/native/functions.cmake @@ -1,8 +1,8 @@ function(clr_unknown_arch) if (WIN32) - message(FATAL_ERROR "Only AMD64, ARM64, ARM and I386 hosts are supported. Found: ${CMAKE_SYSTEM_PROCESSOR}") + message(FATAL_ERROR "Only AMD64, ARM64, ARM and I386 are supported. Found: ${CMAKE_SYSTEM_PROCESSOR}") elseif(CLR_CROSS_COMPONENTS_BUILD) - message(FATAL_ERROR "Only AMD64, ARM64 and I386 hosts are supported for linux cross-architecture component. Found: ${CMAKE_SYSTEM_PROCESSOR}") + message(FATAL_ERROR "Only AMD64, I386 host are supported for linux cross-architecture component. Found: ${CMAKE_SYSTEM_PROCESSOR}") else() message(FATAL_ERROR "'${CMAKE_SYSTEM_PROCESSOR}' is an unsupported architecture.") endif() @@ -228,12 +228,9 @@ function(preprocess_file inputFilename outputFilename) COMMENT "Preprocessing ${inputFilename}. Outputting to ${outputFilename}" ) else() - if (CMAKE_CXX_COMPILER_TARGET AND CMAKE_CXX_COMPILER_ID MATCHES "Clang") - set(_LOCAL_CROSS_TARGET "--target=${CMAKE_CXX_COMPILER_TARGET}") - endif() add_custom_command( OUTPUT ${outputFilename} - COMMAND ${CMAKE_CXX_COMPILER} ${_LOCAL_CROSS_TARGET} -E -P ${PREPROCESS_DEFINITIONS} ${PREPROCESS_INCLUDE_DIRECTORIES} -o ${outputFilename} -x c ${inputFilename} + COMMAND ${CMAKE_CXX_COMPILER} -E -P ${PREPROCESS_DEFINITIONS} ${PREPROCESS_INCLUDE_DIRECTORIES} -o ${outputFilename} -x c ${inputFilename} DEPENDS ${inputFilename} COMMENT "Preprocessing ${inputFilename}. Outputting to ${outputFilename}" ) diff --git a/eng/pipelines/common/evaluate-default-paths.yml b/eng/pipelines/common/evaluate-default-paths.yml index edbc1c618f6066..a5b40862c30c6a 100644 --- a/eng/pipelines/common/evaluate-default-paths.yml +++ b/eng/pipelines/common/evaluate-default-paths.yml @@ -28,7 +28,7 @@ parameters: src/mono/nuget/Microsoft.NET.Runtime.wasm.Sample.Mono/* src/mono/nuget/Microsoft.NET.Sdk.WebAssembly.Pack/* src/mono/nuget/Microsoft.NETCore.BrowserDebugHost.Transport/* - src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/**/* + src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/* src/mono/nuget/Microsoft.NET.Workload* src/mono/sample/wasm/* src/mono/browser/* @@ -213,7 +213,7 @@ jobs: - eng/testing/scenarios/BuildWasmAppsJobsList.txt - eng/testing/tests.browser.targets - eng/testing/tests.was*.targets - - src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/Sdk/WorkloadTesting.Core.targets + - src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/WorkloadTesting.Core.targets - eng/testing/workloads-browser.targets - eng/testing/workloads-testing.targets - eng/testing/workloads-wasi.targets @@ -303,7 +303,7 @@ jobs: exclude: - eng/testing/scenarios/BuildWasiAppsJobsList.txt - eng/testing/scenarios/BuildWasmAppsJobsList.txt - src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/Sdk/WorkloadTesting.Core.targets + src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/WorkloadTesting.Core.targets - eng/testing/workloads-browser.targets - eng/testing/workloads-testing.targets - eng/testing/workloads-wasi.targets diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index a846faca46e3c4..776cdae314c429 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -158,30 +158,6 @@ jobs: crossBuild: true ${{ insert }}: ${{ parameters.jobParameters }} -# Linux Bionic arm - -- ${{ if containsValue(parameters.platforms, 'linux_bionic_arm') }}: - - template: xplat-setup.yml - parameters: - jobTemplate: ${{ parameters.jobTemplate }} - helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }} - variables: ${{ parameters.variables }} - osGroup: linux - osSubgroup: _bionic - archType: arm - targetRid: linux-bionic-arm - platform: linux_bionic_arm - shouldContinueOnError: ${{ parameters.shouldContinueOnError }} - container: linux_bionic - jobParameters: - runtimeFlavor: mono - # We build on Linux, but the test queue runs Windows, so - # we need to override the test script generation - runScriptWindowsCmd: true - buildConfig: ${{ parameters.buildConfig }} - helixQueueGroup: ${{ parameters.helixQueueGroup }} - ${{ insert }}: ${{ parameters.jobParameters }} - # Linux Bionic arm64 - ${{ if containsValue(parameters.platforms, 'linux_bionic_arm64') }}: diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index a063c2127ecc71..a2e13dca489deb 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -389,7 +389,7 @@ jobs: - jitstress1_tiered - jitstress2 - jitstress2_tiered - - disabler2r + - zapdisable - tailcallstress ${{ if in(parameters.testGroup, 'jitstress-random') }}: scenarios: @@ -507,9 +507,9 @@ jobs: ${{ if in(parameters.testGroup, 'gcstress-extra') }}: scenarios: - heapverify1 - - gcstress0xc_disabler2r - - gcstress0xc_disabler2r_jitstress2 - - gcstress0xc_disabler2r_heapverify1 + - gcstress0xc_zapdisable + - gcstress0xc_zapdisable_jitstress2 + - gcstress0xc_zapdisable_heapverify1 - gcstress0xc_jitstress1 - gcstress0xc_jitstress2 - gcstress0xc_tailcallstress @@ -585,7 +585,7 @@ jobs: - jitobjectstackallocation - jitphysicalpromotion_only - jitphysicalpromotion_full - - jitrlcse + - jitcrossblocklocalassertionprop ${{ if in(parameters.testGroup, 'jit-cfg') }}: scenarios: - jitcfg diff --git a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml index c72da916d6f094..79dce1867bf695 100644 --- a/eng/pipelines/coreclr/perf-non-wasm-jobs.yml +++ b/eng/pipelines/coreclr/perf-non-wasm-jobs.yml @@ -281,24 +281,6 @@ jobs: logicalmachine: 'perfowl' experimentName: 'gdv3' - # run coreclr perfowl microbenchmarks perf rlcse jobs - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - linux_x64 - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perfowl' - experimentName: 'rlcse' - # run coreclr crossgen perf job - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -334,29 +316,30 @@ jobs: parameters: name: MonoRuntimePacks - # build PerfBDN app - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - ios_arm64 - jobParameters: - dependsOn: - - Build_android_arm64_release_Mono_Packs - buildArgs: -s mono -c $(_BuildConfig) - nameSuffix: PerfBDNApp - isOfficialBuild: false - pool: - vmImage: 'macos-12' - postBuildSteps: - - template: /eng/pipelines/coreclr/templates/build-perf-bdn-app.yml - parameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: Android BDN App Artifacts - artifactName: PerfBDNAppArm - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + # Disabled due to: https://github.com/dotnet/performance/issues/3655 + # # build PerfBDN app + # - template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - ios_arm64 + # jobParameters: + # dependsOn: + # - Build_android_arm64_release_Mono_Packs + # buildArgs: -s mono -c $(_BuildConfig) + # nameSuffix: PerfBDNApp + # isOfficialBuild: false + # pool: + # vmImage: 'macos-12' + # postBuildSteps: + # - template: /eng/pipelines/coreclr/templates/build-perf-bdn-app.yml + # parameters: + # rootFolder: '$(Build.SourcesDirectory)/artifacts/' + # includeRootFolder: true + # displayName: Android BDN App Artifacts + # artifactName: PerfBDNAppArm + # archiveExtension: '.tar.gz' + # archiveType: tar + # tarCompression: gz diff --git a/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml b/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml index de23519f9c62ed..164485e7d00770 100644 --- a/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml +++ b/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml @@ -60,17 +60,14 @@ extends: - osx_x64 - osx_arm64 - linux_x64 - - linux_arm - linux_arm64 - linux_musl_x64 - - linux_musl_arm64 jobParameters: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Libs - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false timeoutInMinutes: 300 # doesn't normally take this long, but I've seen Helix queues backed up for 160 minutes - includeAllPlatforms: true # extra steps, run tests postBuildSteps: - template: /eng/pipelines/libraries/helix.yml @@ -94,7 +91,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Checked_Libs - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false timeoutInMinutes: 360 # extra steps, run tests postBuildSteps: @@ -119,7 +116,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Checked_Libs_SizeOpt - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Size /p:IlcUseServerGc=false /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Size /p:IlcUseServerGc=false timeoutInMinutes: 240 # extra steps, run tests postBuildSteps: @@ -144,7 +141,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Checked_Libs_SpeedOpt - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Speed /p:IlcUseServerGc=false /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Speed /p:IlcUseServerGc=false timeoutInMinutes: 240 # extra steps, run tests postBuildSteps: @@ -165,7 +162,6 @@ extends: platforms: - windows_x64 - linux_x64 - - linux_arm variables: - name: timeoutPerTestInMinutes value: 60 @@ -174,7 +170,7 @@ extends: jobParameters: timeoutInMinutes: 240 nameSuffix: NativeAOT_Pri0 - buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: diff --git a/eng/pipelines/coreclr/templates/build-perf-bdn-app.yml b/eng/pipelines/coreclr/templates/build-perf-bdn-app.yml index 02c96372885772..ddcea7b914715c 100644 --- a/eng/pipelines/coreclr/templates/build-perf-bdn-app.yml +++ b/eng/pipelines/coreclr/templates/build-perf-bdn-app.yml @@ -16,7 +16,7 @@ parameters: archiveExtension: '' archiveType: '' tarCompression: '' - framework: 'net9.0' # Framework version to get versions for and build for + framework: 'net8.0' # Framework version to get versions for and build for perfRepo: 'main' # Perf repo to pull for the PerfLabExporter @@ -61,13 +61,13 @@ steps: echo '{ }' > ./global.json curl -o NuGet.config 'https://raw.githubusercontent.com/dotnet/maui/${{parameters.framework}}/NuGet.config' curl -o dotnet-install.sh 'https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh' - curl -Lo maui-supported-sdk-version.json 'https://maui.blob.core.windows.net/metadata/sdks/${{parameters.framework}}.json' + curl -Lo maui-supported-sdk-version.json 'https://aka.ms/dotnet/sdk/maui/${{parameters.framework}}.json' version=$(sed -nr 's/\s*"version": "(.*)"/\1/p' ./maui-supported-sdk-version.json) chmod -R a+rx . ./dotnet-install.sh --version $version --install-dir . ./dotnet --info - ./dotnet workload install maui --from-rollback-file https://maui.blob.core.windows.net/metadata/rollbacks/${{parameters.framework}}.json --configfile NuGet.config - ./dotnet workload install android --from-rollback-file https://maui.blob.core.windows.net/metadata/rollbacks/${{parameters.framework}}.json --configfile NuGet.config + ./dotnet workload install maui --from-rollback-file https://aka.ms/dotnet/maui/${{parameters.framework}}.json --configfile NuGet.config + ./dotnet workload install android --from-rollback-file https://aka.ms/dotnet/maui/${{parameters.framework}}.json --configfile NuGet.config displayName: Install MAUI workload workingDirectory: $(Build.SourcesDirectory) @@ -147,7 +147,7 @@ steps: # Remove the embed assemblies from source - script: | - ../dotnet build ./src/Core/tests/Benchmarks.Droid/Benchmarks.Droid.csproj --configuration Release -bl:BenchmarksDroid.binlog /p:TF_Build=False /p:ForceNet8Current=true + ../dotnet build ./src/Core/tests/Benchmarks.Droid/Benchmarks.Droid.csproj --configuration Release -bl:BenchmarksDroid.binlog /p:TF_Build=False mv ./src/Core/tests/Benchmarks.Droid/bin/Release/${{parameters.framework}}-android/android-arm64/com.microsoft.maui.benchmarks-Signed.apk ./MonoBenchmarksDroid.apk displayName: Build BDN Android App workingDirectory: $(Build.SourcesDirectory)/maui diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 93391941114482..7b4ce6c6c7f431 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -86,9 +86,9 @@ jobs: # Linux musl arm32 - ${{ if eq(parameters.platform, 'linux_musl_arm') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Alpine.316.Arm32.Open)Ubuntu.2004.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm32v7 + - (Alpine.316.Arm32.Open)Ubuntu.2004.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.16-helix-arm32v7 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.316.Arm32)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm32v7 + - (Alpine.316.Arm32)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.16-helix-arm32v7 # Linux musl arm64 - ${{ if eq(parameters.platform, 'linux_musl_arm64') }}: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 450cd2799fa0ce..7adba086e45c11 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -90,7 +90,7 @@ jobs: - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }} - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: - ${{ 'build_android_arm64_release_AndroidMono' }} - - ${{ 'Build_ios_arm64_release_PerfBDNApp' }} + # - ${{ 'Build_ios_arm64_release_PerfBDNApp' }} Disabled per: https://github.com/dotnet/performance/issues/3655 - ${{ if eq(parameters.runtimeType, 'iOSMono')}}: - ${{ 'build_ios_arm64_release_iOSMono' }} - ${{ if eq(parameters.runtimeType, 'iOSNativeAOT')}}: @@ -228,13 +228,14 @@ jobs: artifactFileName: 'AndroidMonoarm64.tar.gz' artifactName: 'AndroidMonoarm64' displayName: 'Mono Android HelloWorld' - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory) - cleanUnpackFolder: false - artifactFileName: 'AndroidBDNApk.tar.gz' - artifactName: 'AndroidBDNApk' - displayName: 'Mono Android BDN Apk' + # Disabled per: https://github.com/dotnet/performance/issues/3655 + # - template: /eng/pipelines/common/download-artifact-step.yml + # parameters: + # unpackFolder: $(Build.SourcesDirectory) + # cleanUnpackFolder: false + # artifactFileName: 'AndroidBDNApk.tar.gz' + # artifactName: 'AndroidBDNApk' + # displayName: 'Mono Android BDN Apk' # Download iOSMono and Native AOT tests - ${{ if or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: diff --git a/eng/pipelines/extra-platforms/runtime-extra-platforms-wasm.yml b/eng/pipelines/extra-platforms/runtime-extra-platforms-wasm.yml index fc8d757233cd45..31d15946c50da6 100644 --- a/eng/pipelines/extra-platforms/runtime-extra-platforms-wasm.yml +++ b/eng/pipelines/extra-platforms/runtime-extra-platforms-wasm.yml @@ -290,18 +290,17 @@ jobs: # ff tests are unstable currently shouldContinueOnError: true - # Active Issue https://github.com/dotnet/runtime/issues/98771 - # - template: /eng/pipelines/common/templates/wasm-debugger-tests.yml - # parameters: - # platforms: - # - Browser_wasm - # - Browser_wasm_win - # extraBuildArgs: /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) - # nameSuffix: DebuggerTests_MultiThreaded - # alwaysRun: ${{ parameters.isWasmOnlyBuild }} - # isExtraPlatformsBuild: ${{ parameters.isExtraPlatformsBuild }} - # isWasmOnlyBuild: ${{ parameters.isWasmOnlyBuild }} - # runOnlyOnWasmOnlyPipelines: true + - template: /eng/pipelines/common/templates/wasm-debugger-tests.yml + parameters: + platforms: + - Browser_wasm + - Browser_wasm_win + extraBuildArgs: /p:WasmEnableThreads=true /p:AotHostArchitecture=x64 /p:AotHostOS=$(_hostedOS) + nameSuffix: DebuggerTests_MultiThreaded + alwaysRun: ${{ parameters.isWasmOnlyBuild }} + isExtraPlatformsBuild: ${{ parameters.isExtraPlatformsBuild }} + isWasmOnlyBuild: ${{ parameters.isWasmOnlyBuild }} + runOnlyOnWasmOnlyPipelines: true # Disable for now #- template: /eng/pipelines/coreclr/perf-wasm-jobs.yml diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 6baf34455ee792..d6f52839e600ac 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -96,7 +96,7 @@ jobs: # Android - ${{ if in(parameters.platform, 'android_x86', 'android_x64', 'linux_bionic_x64') }}: - Ubuntu.2204.Amd64.Android.29.Open - - ${{ if in(parameters.platform, 'android_arm', 'android_arm64', 'linux_bionic_arm', 'linux_bionic_arm64') }}: + - ${{ if in(parameters.platform, 'android_arm', 'android_arm64', 'linux_bionic_arm64') }}: - Windows.11.Amd64.Android.Open # iOS Simulator/Mac Catalyst arm64 diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index f6f452199d627b..5c68b4377ee143 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -183,7 +183,7 @@ jobs: - jitstress1_tiered - jitstress2 - jitstress2_tiered - - disabler2r + - zapdisable - tailcallstress ${{ if in(parameters.coreclrTestGroup, 'jitstress-random') }}: scenarios: @@ -220,9 +220,9 @@ jobs: ${{ if in(parameters.coreclrTestGroup, 'gcstress-extra') }}: scenarios: - heapverify1 - - gcstress0xc_disabler2r - - gcstress0xc_disabler2r_jitstress2 - - gcstress0xc_disabler2r_heapverify1 + - gcstress0xc_zapdisable + - gcstress0xc_zapdisable_jitstress2 + - gcstress0xc_zapdisable_heapverify1 - gcstress0xc_jitstress1 - gcstress0xc_jitstress2 - gcstress0xc_jitminopts_heapverify1 @@ -242,7 +242,7 @@ jobs: - jitosr_stress_random - syntheticpgo - syntheticpgo_blend - - jitrlcse + - jitcrossblocklocalassertionprop - ${{ if eq(parameters.SuperPmiCollect, true) }}: - template: /eng/pipelines/libraries/superpmi-postprocess-step.yml diff --git a/eng/pipelines/runtime-official.yml b/eng/pipelines/runtime-official.yml index 0cc897455e2e06..ba4964eca0eb2c 100644 --- a/eng/pipelines/runtime-official.yml +++ b/eng/pipelines/runtime-official.yml @@ -316,7 +316,6 @@ extends: - linux_musl_arm - linux_musl_arm64 - linux_bionic_x64 - - linux_bionic_arm - linux_bionic_arm64 - windows_x64 - windows_arm64 diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index d7a1f1847eb11a..be2870611e617b 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -254,7 +254,7 @@ extends: jobParameters: timeoutInMinutes: 120 nameSuffix: NativeAOT - buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: @@ -293,7 +293,7 @@ extends: jobParameters: timeoutInMinutes: 180 nameSuffix: NativeAOT - buildArgs: -s clr.aot+host.native+libs.native+libs.sfx -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs.native+libs.sfx -rc $(_BuildConfig) -lc Release -hc Release postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: @@ -338,7 +338,7 @@ extends: testGroup: innerloop timeoutInMinutes: 120 nameSuffix: NativeAOT - buildArgs: -s clr.aot+host.native+libs+tools.illink -c $(_BuildConfig) -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs+tools.illink -c $(_BuildConfig) -rc $(_BuildConfig) -lc Release -hc Release postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: @@ -375,7 +375,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Libraries - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true /p:ArchiveTests=true /p:RunAnalyzers=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true /p:ArchiveTests=true timeoutInMinutes: 240 # Doesn't actually take long, but we've seen the ARM64 Helix queue often get backlogged for 2+ hours # extra steps, run tests postBuildSteps: diff --git a/eng/targetingpacks.targets b/eng/targetingpacks.targets index 8a6a8b96a2f56c..befd249f231d2a 100644 --- a/eng/targetingpacks.targets +++ b/eng/targetingpacks.targets @@ -43,7 +43,7 @@ LatestRuntimeFrameworkVersion="$(ProductVersion)" RuntimeFrameworkName="$(LocalFrameworkOverrideName)" RuntimePackNamePatterns="$(LocalFrameworkOverrideName).Runtime.**RID**" - RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;tizen.4.0.0-armel;tizen.5.0.0-armel;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;maccatalyst-x64;maccatalyst-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86;freebsd-x64;freebsd-arm64;linux-ppc64le;linux-riscv64;linux-musl-riscv64" + RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;tizen.4.0.0-armel;tizen.5.0.0-armel;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;maccatalyst-x64;maccatalyst-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86;freebsd-x64;freebsd-arm64;linux-ppc64le" TargetFramework="$(NetCoreAppCurrent)" TargetingPackName="$(LocalFrameworkOverrideName).Ref" TargetingPackVersion="$(ProductVersion)" @@ -53,7 +53,7 @@ RuntimeFrameworkName="$(LocalFrameworkOverrideName)" LatestRuntimeFrameworkVersion="$(ProductVersion)" RuntimePackNamePatterns="$(LocalFrameworkOverrideName).Runtime.Mono.**RID**" - RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;linux-riscv64;linux-musl-riscv64;rhel.6-x64;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86;browser-wasm;ios-arm64;ios-arm;iossimulator-arm64;iossimulator-x64;iossimulator-x86;tvos-arm64;tvossimulator-arm64;tvossimulator-x64;maccatalyst-x64;maccatalyst-arm64;android-arm64;android-arm;android-x64;android-x86" + RuntimePackRuntimeIdentifiers="linux-arm;linux-arm64;linux-musl-arm64;linux-musl-x64;linux-x64;osx-x64;rhel.6-x64;win-arm64;win-x64;win-x86;linux-musl-arm;osx-arm64;linux-s390x;linux-bionic-arm;linux-bionic-arm64;linux-bionic-x64;linux-bionic-x86;browser-wasm;ios-arm64;ios-arm;iossimulator-arm64;iossimulator-x64;iossimulator-x86;tvos-arm64;tvossimulator-arm64;tvossimulator-x64;maccatalyst-x64;maccatalyst-arm64;android-arm64;android-arm;android-x64;android-x86" RuntimePackLabels="Mono" Condition="'$(UseLocalTargetingRuntimePack)' == 'true' and ('@(KnownRuntimePack)' == '' or @(KnownRuntimePack->WithMetadataValue('Identity', 'Microsoft.NETCore.App')->WithMetadataValue('RuntimePackLabels', 'Mono')->WithMetadataValue('TargetFramework', '$(NetCoreAppCurrent)')) == '')" /> @@ -78,7 +78,7 @@ TargetFramework="$(NetCoreAppCurrent)" Crossgen2PackNamePattern="$(LocalFrameworkOverrideName).Crossgen2.**RID**" Crossgen2PackVersion="$(ProductVersion)" - Crossgen2RuntimeIdentifiers="linux-musl-x64;linux-x64;win-x64;linux-arm;linux-arm64;linux-musl-arm;linux-musl-arm64;osx-arm64;osx-x64;win-arm64;win-x86;linux-riscv64;linux-musl-riscv64" + Crossgen2RuntimeIdentifiers="linux-musl-x64;linux-x64;win-x64;linux-arm;linux-arm64;linux-musl-arm;linux-musl-arm64;osx-arm64;osx-x64;win-arm64;win-x86" Condition="'$(UseLocalCrossgen2Pack)' == 'true' and '@(KnownCrossgen2Pack->AnyHaveMetadataValue('TargetFramework', '$(NetCoreAppCurrent)'))' != 'true'" /> - 122.0.6261.69 - 1250580 - https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/1250580 - 12.2.281 - 122.0.6261.69 - 1250580 - https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/1250586 - 12.2.281 + 121.0.6167.184 + 1233107 + https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/1233114 + 12.1.285 + 121.0.6167.185 + 1233107 + https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/1233136 + 12.1.285 \ No newline at end of file diff --git a/eng/testing/WasmRunnerTemplate.cmd b/eng/testing/WasmRunnerTemplate.cmd index f92cee17cc9df7..83aeb53cad03a6 100644 --- a/eng/testing/WasmRunnerTemplate.cmd +++ b/eng/testing/WasmRunnerTemplate.cmd @@ -59,9 +59,6 @@ if /I [%XHARNESS_COMMAND%] == [test] ( if [%BROWSER_PATH%] == [] if not [%HELIX_CORRELATION_PAYLOAD%] == [] ( set "BROWSER_PATH=--browser-path^=%HELIX_CORRELATION_PAYLOAD%\chrome-win\chrome.exe" ) - if [%JS_ENGINE_ARGS%] == [] ( - set "JS_ENGINE_ARGS=--browser-arg^=--js-flags^=--stack-trace-limit^=1000" - ) ) if [%XHARNESS_ARGS%] == [] ( diff --git a/eng/testing/WasmRunnerTemplate.sh b/eng/testing/WasmRunnerTemplate.sh index 4f5856546fc56b..71347666cde802 100644 --- a/eng/testing/WasmRunnerTemplate.sh +++ b/eng/testing/WasmRunnerTemplate.sh @@ -58,10 +58,6 @@ if [[ "$XHARNESS_COMMAND" == "test" ]]; then fi fi fi -else - if [[ -z "$JS_ENGINE_ARGS" ]]; then - JS_ENGINE_ARGS="--browser-arg=--js-flags=--stack-trace-limit=1000" - fi fi if [[ -z "$XHARNESS_ARGS" ]]; then diff --git a/eng/testing/linker/project.csproj.template b/eng/testing/linker/project.csproj.template index d31c8df9c9271e..41466e8d4492af 100644 --- a/eng/testing/linker/project.csproj.template +++ b/eng/testing/linker/project.csproj.template @@ -75,12 +75,6 @@ {AdditionalProjectReferences} - - - <_BoolPropertiesThatTriggerRelinking Remove="InvariantGlobalization" /> - - - diff --git a/eng/testing/outerBuild.targets b/eng/testing/outerBuild.targets index 6465a272bca8b0..c071944c21d93d 100644 --- a/eng/testing/outerBuild.targets +++ b/eng/testing/outerBuild.targets @@ -1,19 +1,12 @@ - + - - + - - - - false - - \ No newline at end of file diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 4d0aad300cd99d..c2f3e7b1955e26 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -35,24 +35,24 @@ $(Python) test.py sod --scenario-name "%(Identity)" $(Python) post.py - + + diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 08225ae75004d2..f7d321930627a0 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -140,7 +140,7 @@ if ($NoR2R) { } if ($ExperimentName) { - $SetupArguments = "$SetupArguments --experiment-name $ExperimentName" + $SetupArguments = "$SetupArguments --experiment-name '$ExperimentName'" } if ($UseLocalCommitTime) { diff --git a/eng/testing/performance/performance-setup.sh b/eng/testing/performance/performance-setup.sh index 0a735a796a6e0a..6eeb7223ffb430 100755 --- a/eng/testing/performance/performance-setup.sh +++ b/eng/testing/performance/performance-setup.sh @@ -492,7 +492,7 @@ if [[ "$nor2r" == "true" ]]; then fi if [[ ! -z "$experimentname" ]]; then - setup_arguments="$setup_arguments --experiment-name $experimentname" + setup_arguments="$setup_arguments --experiment-name '$experimentname'" fi if [[ "$monoaot" == "true" ]]; then diff --git a/eng/testing/scenarios/BuildWasmAppsJobsList.txt b/eng/testing/scenarios/BuildWasmAppsJobsList.txt index 5ccb34b25e18ac..5519c03e6ef7db 100644 --- a/eng/testing/scenarios/BuildWasmAppsJobsList.txt +++ b/eng/testing/scenarios/BuildWasmAppsJobsList.txt @@ -36,13 +36,12 @@ Wasm.Build.Tests.TestAppScenarios.AppSettingsTests Wasm.Build.Tests.TestAppScenarios.LazyLoadingTests Wasm.Build.Tests.TestAppScenarios.LibraryInitializerTests Wasm.Build.Tests.TestAppScenarios.SatelliteLoadingTests -Wasm.Build.Tests.TestAppScenarios.DownloadResourceProgressTests -Wasm.Build.Tests.TestAppScenarios.SignalRClientTests Wasm.Build.Tests.WasmBuildAppTest Wasm.Build.Tests.WasmNativeDefaultsTests Wasm.Build.Tests.WasmRunOutOfAppBundleTests Wasm.Build.Tests.WasmSIMDTests Wasm.Build.Tests.WasmTemplateTests Wasm.Build.Tests.WorkloadTests +Wasm.Build.Tests.TestAppScenarios.DownloadResourceProgressTests Wasm.Build.Tests.MT.Blazor.SimpleMultiThreadedTests Wasm.Build.Tests.TestAppScenarios.DebugLevelTests diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index bce044984e9379..d27fa412d490b1 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -87,8 +87,8 @@ <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(WasmMainAssemblyFileName)' != ''">--run $(WasmMainAssemblyFileName) <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll + <_XUnitBackgroundExec Condition="'$(_XUnitBackgroundExec)' == '' and '$(WasmEnableThreads)' == 'true'">true $(WasmTestAppArgs) -backgroundExec - $(WasmXHarnessMonoArgs) --setenv=IsWasmBackgroundExec=true <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true diff --git a/global.json b/global.json index cd39f3cc389a3c..dc5aff95f8db27 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,18 @@ { "sdk": { - "version": "9.0.100-preview.1.24101.2", + "version": "9.0.100-alpha.1.23615.4", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "9.0.100-preview.1.24101.2" + "dotnet": "9.0.100-alpha.1.23615.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24161.5", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24161.5", - "Microsoft.DotNet.SharedFramework.Sdk": "9.0.0-beta.24161.5", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24112.1", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24112.1", + "Microsoft.DotNet.SharedFramework.Sdk": "9.0.0-beta.24112.1", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", - "Microsoft.NET.Sdk.IL": "9.0.0-preview.3.24161.1" + "Microsoft.NET.Sdk.IL": "9.0.0-preview.2.24115.1" } } diff --git a/src/coreclr/CMakeLists.txt b/src/coreclr/CMakeLists.txt index 4aa45914ff54b0..1c314d9bf624e0 100644 --- a/src/coreclr/CMakeLists.txt +++ b/src/coreclr/CMakeLists.txt @@ -147,7 +147,7 @@ add_subdirectory(tools/aot/jitinterface) if(NOT CLR_CROSS_COMPONENTS_BUILD) # NativeAOT only buildable for a subset of CoreCLR-supported configurations - if(CLR_CMAKE_HOST_ARCH_ARM64 OR CLR_CMAKE_HOST_ARCH_AMD64 OR CLR_CMAKE_HOST_ARCH_ARM OR (CLR_CMAKE_HOST_ARCH_I386 AND CLR_CMAKE_HOST_WIN32)) + if(CLR_CMAKE_HOST_ARCH_ARM64 OR CLR_CMAKE_HOST_ARCH_AMD64 OR CLR_CMAKE_HOST_ARCH_ARM) add_subdirectory(nativeaot) endif() endif(NOT CLR_CROSS_COMPONENTS_BUILD) diff --git a/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj index 9ef1024c449da8..6b3ddff0cc868c 100644 --- a/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -1,4 +1,4 @@ - + false @@ -46,7 +46,8 @@ $(ProductVersion) $(ProductVersion) - $(NoWarn),0419,0649 + + $(NoWarn),0419,0649;AD0001 enable @@ -136,6 +137,8 @@ + + @@ -200,17 +203,18 @@ + - - - - - - + + + + + + @@ -222,10 +226,12 @@ + + diff --git a/src/coreclr/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml b/src/coreclr/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml index d7fd368adac7c2..fb07d67f37c68e 100644 --- a/src/coreclr/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml +++ b/src/coreclr/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml @@ -2,6 +2,11 @@ + + + + + diff --git a/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs b/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs index 77f64abd1b42c0..db8d4ead4659b9 100644 --- a/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs +++ b/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs @@ -203,6 +203,7 @@ private static void ClassRegistrationScenarioForType(ComActivationContext cxt, b // Finally validate signature ReadOnlySpan methParams = method.GetParametersAsSpan(); if (method.ReturnType != typeof(void) + || methParams == null || methParams.Length != 1 || (methParams[0].ParameterType != typeof(string) && methParams[0].ParameterType != typeof(Type))) { diff --git a/src/coreclr/System.Private.CoreLib/src/System/ArgIterator.cs b/src/coreclr/System.Private.CoreLib/src/System/ArgIterator.cs index d49d1dcb270b56..e7c2a99eeefa28 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/ArgIterator.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/ArgIterator.cs @@ -24,7 +24,7 @@ private struct SigPointer private int _remainingArgs; // # of remaining args. #if TARGET_WINDOWS // Native Varargs are not supported on Unix - // ArgIterator is a ref struct. It does not require pinning, therefore Unsafe.AsPointer is safe. + // ArgIterator is a ref struct. It does not require pinning. // This method null checks the this pointer as a side-effect. private ArgIterator* ThisPtr => (ArgIterator*)Unsafe.AsPointer(ref _argCookie); diff --git a/src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs index de7b3021c458fe..16d9067567ee58 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Array.CoreCLR.cs @@ -74,7 +74,7 @@ private static unsafe void CopyImpl(Array sourceArray, int sourceIndex, Array de if (pMT->ContainsGCPointers) Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount); else - SpanHelpers.Memmove(ref dst, ref src, byteCount); + Buffer.Memmove(ref dst, ref src, byteCount); // GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray return; @@ -184,7 +184,7 @@ private static unsafe void CopyImplUnBoxEachElement(Array sourceArray, int sourc } else { - SpanHelpers.Memmove(ref dest, ref obj.GetRawData(), destSize); + Buffer.Memmove(ref dest, ref obj.GetRawData(), destSize); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.Internal.cs b/src/coreclr/System.Private.CoreLib/src/System/Diagnostics/Eventing/EventPipe.CoreCLR.cs similarity index 98% rename from src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.Internal.cs rename to src/coreclr/System.Private.CoreLib/src/System/Diagnostics/Eventing/EventPipe.CoreCLR.cs index 6039bcfaa4f7a0..db76c99413e6bb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.Internal.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Diagnostics/Eventing/EventPipe.CoreCLR.cs @@ -5,6 +5,8 @@ using System.Runtime.InteropServices; using System.Threading; +#if FEATURE_PERFTRACING + namespace System.Diagnostics.Tracing { internal static partial class EventPipeInternal @@ -66,3 +68,5 @@ internal static unsafe partial IntPtr CreateProvider(string providerName, internal static unsafe partial bool WaitForSessionSignal(ulong sessionID, int timeoutMs); } } + +#endif // FEATURE_PERFTRACING diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.Internal.cs b/src/coreclr/System.Private.CoreLib/src/System/Diagnostics/Eventing/NativeRuntimeEventSource.Threading.NativeSinks.CoreCLR.cs similarity index 57% rename from src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.Internal.cs rename to src/coreclr/System.Private.CoreLib/src/System/Diagnostics/Eventing/NativeRuntimeEventSource.Threading.NativeSinks.CoreCLR.cs index 7e9368dd3e929d..95942b6291c36a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.Internal.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Diagnostics/Eventing/NativeRuntimeEventSource.Threading.NativeSinks.CoreCLR.cs @@ -10,23 +10,12 @@ namespace System.Diagnostics.Tracing // It contains the runtime specific interop to native event sinks. internal sealed partial class NativeRuntimeEventSource : EventSource { -#if NATIVEAOT - // We don't have these keywords defined from the genRuntimeEventSources.py, so we need to manually define them here. - public static partial class Keywords - { - public const EventKeywords ContentionKeyword = (EventKeywords)0x4000; - public const EventKeywords ThreadingKeyword = (EventKeywords)0x10000; - public const EventKeywords ThreadTransferKeyword = (EventKeywords)0x80000000; - public const EventKeywords WaitHandleKeyword = (EventKeywords)0x40000000000; - } -#endif - - [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogContentionLockCreated")] + [NonEvent] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogContentionLockCreated(nint LockID, nint AssociatedObjectID, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogContentionStart")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogContentionStart( ContentionFlagsMap ContentionFlags, ushort ClrInstanceID, @@ -35,38 +24,38 @@ private static partial void LogContentionStart( ulong LockOwnerThreadID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogContentionStop")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogContentionStop( ContentionFlagsMap ContentionFlags, ushort ClrInstanceID, double DurationNs); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkerThreadStart(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkerThreadStop(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkerThreadWait(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolMinMaxThreads")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolMinMaxThreads(ushort MinWorkerThreads, ushort MaxWorkerThreads, ushort MinIOCompletionThreads, ushort MaxIOCompletionThreads, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint NewWorkerThreadCount, ThreadAdjustmentReasonMap Reason, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkerThreadAdjustmentStats( double Duration, double Throughput, @@ -81,7 +70,7 @@ private static partial void LogThreadPoolWorkerThreadAdjustmentStats( ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolIOEnqueue")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolIOEnqueue( IntPtr NativeOverlapped, IntPtr Overlapped, @@ -89,34 +78,37 @@ private static partial void LogThreadPoolIOEnqueue( ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolIODequeue")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolIODequeue( IntPtr NativeOverlapped, IntPtr Overlapped, ushort ClrInstanceID); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolWorkingThreadCount( uint Count, - ushort ClrInstanceID); + ushort ClrInstanceID + ); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogThreadPoolIOPack")] + [LibraryImport(RuntimeHelpers.QCall)] private static partial void LogThreadPoolIOPack( IntPtr NativeOverlapped, IntPtr Overlapped, ushort ClrInstanceID); +#pragma warning disable IDE0060 // Remove unused parameter [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogWaitHandleWaitStart")] - private static partial void LogWaitHandleWaitStart( + private static void LogWaitHandleWaitStart( WaitHandleWaitSourceMap WaitSource, IntPtr AssociatedObjectID, - ushort ClrInstanceID); + ushort ClrInstanceID) => + Debug.Fail("This event is currently not expected to be raised by managed code in CoreCLR."); [NonEvent] - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "NativeRuntimeEventSource_LogWaitHandleWaitStop")] - private static partial void LogWaitHandleWaitStop(ushort ClrInstanceID); + private static void LogWaitHandleWaitStop(ushort ClrInstanceID) => + Debug.Fail("This event is currently not expected to be raised by managed code in CoreCLR."); +#pragma warning restore IDE0060 } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Environment.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Environment.CoreCLR.cs index e0a24a42ef3223..8bbd6e98ddaabb 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Environment.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Environment.CoreCLR.cs @@ -5,7 +5,6 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Security; using System.Threading; namespace System @@ -34,15 +33,12 @@ public static extern int ExitCode set; } + // Note: The CLR's Watson bucketization code looks at the caller of the FCALL method + // to assign blame for crashes. Don't mess with this, such as by making it call + // another managed helper method, unless you consult with some CLR Watson experts. [DoesNotReturn] - [DynamicSecurityMethod] // Methods containing StackCrawlMark local var has to be marked DynamicSecurityMethod - public static void FailFast(string? message) - { - // Note: The CLR's Watson bucketization code looks at the our caller - // to assign blame for crashes. - StackCrawlMark mark = StackCrawlMark.LookForMyCaller; - FailFast(ref mark, message, exception: null, errorMessage: null); - } + [MethodImpl(MethodImplOptions.InternalCall)] + public static extern void FailFast(string? message); // This overload of FailFast will allow you to specify the exception object // whose bucket details *could* be used when undergoing the failfast process. @@ -58,34 +54,12 @@ public static void FailFast(string? message) // IP for bucketing. If the exception object is not preallocated, it will use the bucket // details contained in the object (if any). [DoesNotReturn] - [DynamicSecurityMethod] // Methods containing StackCrawlMark local var has to be marked DynamicSecurityMethod - public static void FailFast(string? message, Exception? exception) - { - // Note: The CLR's Watson bucketization code looks at the our caller - // to assign blame for crashes. - StackCrawlMark mark = StackCrawlMark.LookForMyCaller; - FailFast(ref mark, message, exception, errorMessage: null); - } - - [DoesNotReturn] - [DynamicSecurityMethod] // Methods containing StackCrawlMark local var has to be marked DynamicSecurityMethod - internal static void FailFast(string? message, Exception? exception, string? errorMessage) - { - // Note: The CLR's Watson bucketization code looks at the our caller - // to assign blame for crashes. - StackCrawlMark mark = StackCrawlMark.LookForMyCaller; - FailFast(ref mark, message, exception, errorMessage); - } - - [DoesNotReturn] - private static void FailFast(ref StackCrawlMark mark, string? message, Exception? exception, string? errorMessage) - { - FailFast(new StackCrawlMarkHandle(ref mark), message, ObjectHandleOnStack.Create(ref exception), errorMessage); - } + [MethodImpl(MethodImplOptions.InternalCall)] + public static extern void FailFast(string? message, Exception? exception); - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "Environment_FailFast", StringMarshalling = StringMarshalling.Utf16)] [DoesNotReturn] - private static partial void FailFast(StackCrawlMarkHandle mark, string? message, ObjectHandleOnStack exception, string? errorMessage); + [MethodImpl(MethodImplOptions.InternalCall)] + internal static extern void FailFast(string? message, Exception? exception, string? errorMessage); private static unsafe string[] InitializeCommandLineArgs(char* exePath, int argc, char** argv) // invoked from VM { diff --git a/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs index 697788316ba031..590fd5b18cee09 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs @@ -865,9 +865,7 @@ public static unsafe IReadOnlyDictionary GetConfigurationVariabl Configurations = new Dictionary() }; -#pragma warning disable CS8500 // takes address of managed type - _EnumerateConfigurationValues(&context, &ConfigCallback); -#pragma warning restore CS8500 + _EnumerateConfigurationValues(Unsafe.AsPointer(ref context), &ConfigCallback); return context.Configurations!; } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs index 940d1622bad188..70cff629fc28e6 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics; using System.Runtime.CompilerServices; namespace System @@ -20,9 +19,7 @@ public partial class Object [Intrinsic] protected internal unsafe object MemberwiseClone() { - object clone = this; - RuntimeHelpers.AllocateUninitializedClone(ObjectHandleOnStack.Create(ref clone)); - Debug.Assert(clone != this); + object clone = RuntimeHelpers.AllocateUninitializedClone(this); // copy contents of "this" to the clone @@ -33,7 +30,7 @@ protected internal unsafe object MemberwiseClone() if (RuntimeHelpers.GetMethodTable(clone)->ContainsGCPointers) Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount); else - SpanHelpers.Memmove(ref dst, ref src, byteCount); + Buffer.Memmove(ref dst, ref src, byteCount); return clone; } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs b/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs index 327113c63f9a3c..2b695f1baf5b0f 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs @@ -417,7 +417,7 @@ private int GetMemberRefToken(MethodInfo methodInfo, Type[]? optionalParameterTy throw new ArgumentException(SR.Argument_MustBeRuntimeMethodInfo, nameof(methodInfo)); ReadOnlySpan paramInfo = methodInfo.GetParametersAsSpan(); - if (paramInfo.Length != 0) + if (paramInfo != null && paramInfo.Length != 0) { parameterTypes = new Type[paramInfo.Length]; requiredCustomModifiers = new Type[parameterTypes.Length][]; diff --git a/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs b/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs index 53f2690948df45..b5cff2f1e42ecd 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs @@ -645,29 +645,27 @@ public override Assembly GetSatelliteAssembly(CultureInfo culture, Version? vers { ArgumentNullException.ThrowIfNull(culture); - return InternalGetSatelliteAssembly(this, culture, version, throwOnFileNotFound: true)!; + return InternalGetSatelliteAssembly(culture, version, throwOnFileNotFound: true)!; } [DynamicSecurityMethod] // Methods containing StackCrawlMark local var has to be marked DynamicSecurityMethod - internal static Assembly? InternalGetSatelliteAssembly(Assembly assembly, - CultureInfo culture, + internal Assembly? InternalGetSatelliteAssembly(CultureInfo culture, Version? version, bool throwOnFileNotFound) { var an = new AssemblyName(); - RuntimeAssembly runtimeAssembly = (RuntimeAssembly)assembly; - an.SetPublicKey(runtimeAssembly.GetPublicKey()); - an.Flags = runtimeAssembly.GetFlags() | AssemblyNameFlags.PublicKey; - an.Version = version ?? runtimeAssembly.GetVersion(); + an.SetPublicKey(GetPublicKey()); + an.Flags = GetFlags() | AssemblyNameFlags.PublicKey; + an.Version = version ?? GetVersion(); an.CultureInfo = culture; - an.Name = runtimeAssembly.GetSimpleName() + ".resources"; + an.Name = GetSimpleName() + ".resources"; // This stack crawl mark is never used because the requesting assembly is explicitly specified, // so the value could be anything. StackCrawlMark unused = default; - RuntimeAssembly? retAssembly = InternalLoad(an, ref unused, requestingAssembly: runtimeAssembly, throwOnFileNotFound: throwOnFileNotFound); + RuntimeAssembly? retAssembly = InternalLoad(an, ref unused, requestingAssembly: this, throwOnFileNotFound: throwOnFileNotFound); - if (retAssembly == runtimeAssembly) + if (retAssembly == this) { retAssembly = null; } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.CoreCLR.cs new file mode 100644 index 00000000000000..05805072cd7cf3 --- /dev/null +++ b/src/coreclr/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.CoreCLR.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Reflection; + +namespace System.Resources +{ + internal sealed partial class ManifestBasedResourceGroveler + { + // Internal version of GetSatelliteAssembly that avoids throwing FileNotFoundException + private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, + CultureInfo culture, + Version? version) + { + return ((RuntimeAssembly)mainAssembly).InternalGetSatelliteAssembly(culture, version, throwOnFileNotFound: false); + } + } +} diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 733e3a664bcc52..4e75d7db895ce4 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -139,32 +139,8 @@ public static unsafe void PrepareMethod(RuntimeMethodHandle method, RuntimeTypeH [MethodImpl(MethodImplOptions.InternalCall)] internal static extern int TryGetHashCode(object o); - public static new unsafe bool Equals(object? o1, object? o2) - { - // Compare by ref for normal classes, by value for value types. - - if (ReferenceEquals(o1, o2)) - return true; - - if (o1 is null || o2 is null) - return false; - - MethodTable* pMT = GetMethodTable(o1); - - // If it's not a value class, don't compare by value - if (!pMT->IsValueType) - return false; - - // Make sure they are the same type. - if (pMT != GetMethodTable(o2)) - return false; - - // Compare the contents - return ContentEquals(o1, o2); - } - [MethodImpl(MethodImplOptions.InternalCall)] - private static extern unsafe bool ContentEquals(object o1, object o2); + public static extern new bool Equals(object? o1, object? o2); [Obsolete("OffsetToStringData has been deprecated. Use string.GetPinnableReference() instead.")] public static int OffsetToStringData @@ -218,8 +194,8 @@ public static object GetUninitializedObject( return rt.GetUninitializedObject(); } - [LibraryImport(QCall, EntryPoint = "ObjectNative_AllocateUninitializedClone")] - internal static partial void AllocateUninitializedClone(ObjectHandleOnStack objHandle); + [MethodImpl(MethodImplOptions.InternalCall)] + internal static extern object AllocateUninitializedClone(object obj); /// true if given type is reference type or value type that contains references [Intrinsic] @@ -486,13 +462,7 @@ internal unsafe struct MethodTable // Additional conditional fields (see methodtable.h). // m_pModule - - /// - /// A pointer to auxiliary data that is cold for method table. - /// - [FieldOffset(AuxiliaryDataOffset)] - public MethodTableAuxiliaryData* AuxiliaryData; - + // m_pAuxiliaryData // union { // m_pEEClass (pointer to the EE class) // m_pCanonMT (pointer to the canonical method table) @@ -553,12 +523,6 @@ internal unsafe struct MethodTable private const int ParentMethodTableOffset = 0x10 + DebugClassNamePtr; -#if TARGET_64BIT - private const int AuxiliaryDataOffset = 0x20 + DebugClassNamePtr; -#else - private const int AuxiliaryDataOffset = 0x18 + DebugClassNamePtr; -#endif - #if TARGET_64BIT private const int ElementTypeOffset = 0x30 + DebugClassNamePtr; #else @@ -646,28 +610,6 @@ public TypeHandle GetArrayElementTypeHandle() public extern uint GetNumInstanceFieldBytes(); } - // Subset of src\vm\methodtable.h - [StructLayout(LayoutKind.Explicit)] - internal unsafe struct MethodTableAuxiliaryData - { - [FieldOffset(0)] - private uint Flags; - - private const uint enum_flag_CanCompareBitsOrUseFastGetHashCode = 0x0001; // Is any field type or sub field type overrode Equals or GetHashCode - private const uint enum_flag_HasCheckedCanCompareBitsOrUseFastGetHashCode = 0x0002; // Whether we have checked the overridden Equals or GetHashCode - - public bool HasCheckedCanCompareBitsOrUseFastGetHashCode => (Flags & enum_flag_HasCheckedCanCompareBitsOrUseFastGetHashCode) != 0; - - public bool CanCompareBitsOrUseFastGetHashCode - { - get - { - Debug.Assert(HasCheckedCanCompareBitsOrUseFastGetHashCode); - return (Flags & enum_flag_CanCompareBitsOrUseFastGetHashCode) != 0; - } - } - } - /// /// A type handle, which can wrap either a pointer to a TypeDesc or to a . /// diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/AsmOffsets.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/AsmOffsets.cs index 7db188808e26a4..fb70ddcc703f13 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/AsmOffsets.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/AsmOffsets.cs @@ -48,14 +48,14 @@ class AsmOffsets #if TARGET_64BIT public const int OFFSETOF__REGDISPLAY__m_pCurrentContext = 0x8; - public const int SIZEOF__StackFrameIterator = 0x358; - public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x33A; - public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x350; + public const int SIZEOF__StackFrameIterator = 0x370; + public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x352; + public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x368; #else // TARGET_64BIT public const int OFFSETOF__REGDISPLAY__m_pCurrentContext = 0x4; - public const int SIZEOF__StackFrameIterator = 0x2c8; - public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x2b6; - public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x2c4; + public const int SIZEOF__StackFrameIterator = 0x2d8; + public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x2c2; + public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x2d0; #endif // TARGET_64BIT #else // DEBUG @@ -94,14 +94,14 @@ class AsmOffsets #if TARGET_64BIT public const int OFFSETOF__REGDISPLAY__m_pCurrentContext = 0x8; - public const int SIZEOF__StackFrameIterator = 0x350; - public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x332; - public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x348; + public const int SIZEOF__StackFrameIterator = 0x370; + public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x34a; + public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x360; #else // TARGET_64BIT public const int OFFSETOF__REGDISPLAY__m_pCurrentContext = 0x4; - public const int SIZEOF__StackFrameIterator = 0x2c0; - public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x2ae; - public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x2bc; + public const int SIZEOF__StackFrameIterator = 0x2d0; + public const int OFFSETOF__StackFrameIterator__m_isRuntimeWrappedExceptions = 0x2ba; + public const int OFFSETOF__StackFrameIterator__m_AdjustedControlPC = 0x2c8; #endif // TARGET_64BIT #endif // DEBUG @@ -155,7 +155,7 @@ class AsmOffsets public const int OFFSETOF__ExInfo__m_kind = 0xd0; public const int OFFSETOF__ExInfo__m_passNumber = 0xd1; public const int OFFSETOF__ExInfo__m_idxCurClause = 0xd4; - public const int OFFSETOF__ExInfo__m_frameIter = 0xd8; + public const int OFFSETOF__ExInfo__m_frameIter = 0xe0; public const int OFFSETOF__ExInfo__m_notifyDebuggerSP = OFFSETOF__ExInfo__m_frameIter + SIZEOF__StackFrameIterator; #else // TARGET_64BIT public const int SIZEOF__EHEnum = 0x10; diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs index 228f58c0ea4d06..4ae608fc17d23d 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs @@ -42,7 +42,7 @@ internal static unsafe partial bool RhpCallFilterFunclet( [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "EHEnumInitFromStackFrameIterator")] [return: MarshalAs(UnmanagedType.Bool)] - internal static unsafe partial bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, out EH.MethodRegionInfo pMethodRegionInfo, void* pEHEnum); + internal static unsafe partial bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, byte** pMethodStartAddress, void* pEHEnum); [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "EHEnumNext")] [return: MarshalAs(UnmanagedType.Bool)] diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.CoreCLR.cs index bbdccc6cd2eed4..c04665aa6c22f4 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.CoreCLR.cs @@ -266,7 +266,7 @@ public static unsafe void StructureToPtr(object structure, IntPtr ptr, bool fDel } else { - SpanHelpers.Memmove(ref *(byte*)ptr, ref structure.GetRawData(), size); + Buffer.Memmove(ref *(byte*)ptr, ref structure.GetRawData(), size); } } @@ -291,7 +291,7 @@ private static unsafe void PtrToStructureHelper(IntPtr ptr, object structure, bo } else { - SpanHelpers.Memmove(ref structure.GetRawData(), ref *(byte*)ptr, size); + Buffer.Memmove(ref structure.GetRawData(), ref *(byte*)ptr, size); } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/Versioning/CompatibilitySwitch.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/Versioning/CompatibilitySwitch.cs new file mode 100644 index 00000000000000..d90f81d48e986d --- /dev/null +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/Versioning/CompatibilitySwitch.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Runtime.Versioning +{ + internal static class CompatibilitySwitch + { + [MethodImpl(MethodImplOptions.InternalCall)] + internal static extern string? GetValueInternal(string compatibilitySwitchName); + } +} diff --git a/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs index 00a8d78685d4d9..c74d76388b91a9 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs @@ -2757,12 +2757,7 @@ public override InterfaceMapping GetInterfaceMap([DynamicallyAccessedMembers(Dyn MethodBase? rtTypeMethodBase = GetMethodBase(reflectedType, classRtMethodHandle); // a class may not implement all the methods of an interface (abstract class) so null is a valid value Debug.Assert(rtTypeMethodBase is null || rtTypeMethodBase is RuntimeMethodInfo); - RuntimeMethodInfo? targetMethod = (RuntimeMethodInfo?)rtTypeMethodBase; - // the TargetMethod provided to us by runtime internals may be a generic method instance, - // potentially with invalid arguments. TargetMethods in the InterfaceMap should never be - // instances, only definitions. - im.TargetMethods[i] = (targetMethod is { IsGenericMethod: true, IsGenericMethodDefinition: false }) - ? targetMethod.GetGenericMethodDefinition() : targetMethod!; + im.TargetMethods[i] = (MethodInfo)rtTypeMethodBase!; } return im; diff --git a/src/libraries/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs b/src/coreclr/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs similarity index 100% rename from src/libraries/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs rename to src/coreclr/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs diff --git a/src/coreclr/System.Private.CoreLib/src/System/String.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/String.CoreCLR.cs index d19cb01034a74e..f15ad03d82182b 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/String.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/String.CoreCLR.cs @@ -23,12 +23,13 @@ public static string Intern(string str) } [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "String_IsInterned")] + [return: MarshalAs(UnmanagedType.Bool)] private static partial void IsInterned(StringHandleOnStack src); public static string? IsInterned(string str) { ArgumentNullException.ThrowIfNull(str); - IsInterned(new StringHandleOnStack(ref str!)); + Intern(new StringHandleOnStack(ref str!)); return str; } @@ -38,7 +39,7 @@ internal static unsafe void InternalCopy(string src, IntPtr dest, int len) { if (len != 0) { - SpanHelpers.Memmove(ref *(byte*)dest, ref Unsafe.As(ref src.GetRawStringData()), (nuint)len); + Buffer.Memmove(ref *(byte*)dest, ref Unsafe.As(ref src.GetRawStringData()), (nuint)len); } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs index 81c0dd8e1afecd..9874eef6dc2292 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/StubHelpers.cs @@ -103,7 +103,7 @@ internal static unsafe IntPtr ConvertToNative(int flags, string strManaged, IntP // + 1 for the null character from the user. + 1 for the null character we put in. pbNativeBuffer = (byte*)Marshal.AllocCoTaskMem(nb + 2); - SpanHelpers.Memmove(ref *pbNativeBuffer, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nb); + Buffer.Memmove(ref *pbNativeBuffer, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nb); } } @@ -360,7 +360,7 @@ internal static unsafe IntPtr ConvertToNative(string strManaged, bool fBestFit, Debug.Assert(nbytesused >= 0 && nbytesused < nbytes, "Insufficient buffer allocated in VBByValStrMarshaler.ConvertToNative"); - SpanHelpers.Memmove(ref *pNative, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nbytesused); + Buffer.Memmove(ref *pNative, ref MemoryMarshal.GetArrayDataReference(bytes), (nuint)nbytesused); pNative[nbytesused] = 0; *pLength = nbytesused; @@ -409,7 +409,7 @@ internal static unsafe IntPtr ConvertToNative(int flags, string strManaged) IntPtr bstr = Marshal.AllocBSTRByteLen(length); if (bytes != null) { - SpanHelpers.Memmove(ref *(byte*)bstr, ref MemoryMarshal.GetArrayDataReference(bytes), length); + Buffer.Memmove(ref *(byte*)bstr, ref MemoryMarshal.GetArrayDataReference(bytes), length); } return bstr; @@ -1484,7 +1484,7 @@ internal static unsafe void FmtClassUpdateNativeInternal(object obj, byte* pNati } else { - SpanHelpers.Memmove(ref *pNative, ref obj.GetRawData(), size); + Buffer.Memmove(ref *pNative, ref obj.GetRawData(), size); } } @@ -1503,7 +1503,7 @@ internal static unsafe void FmtClassUpdateCLRInternal(object obj, byte* pNative) } else { - SpanHelpers.Memmove(ref obj.GetRawData(), ref *pNative, size); + Buffer.Memmove(ref obj.GetRawData(), ref *pNative, size); } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs b/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs index f4c3acb31adf88..cc13e37e083f01 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs @@ -10,17 +10,15 @@ ** ===========================================================*/ -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Reflection; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace System { [Serializable] [TypeForwardedFrom("mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089")] - public abstract partial class ValueType + public abstract class ValueType { [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2075:UnrecognizedReflectionPattern", Justification = "Trimmed fields don't make a difference for equality")] @@ -38,7 +36,7 @@ public override unsafe bool Equals([NotNullWhen(true)] object? obj) // if there are no GC references in this object we can avoid reflection // and do a fast memcmp - if (CanCompareBitsOrUseFastGetHashCode(RuntimeHelpers.GetMethodTable(obj))) // MethodTable kept alive by access to object below + if (CanCompareBits(this)) { return SpanHelpers.SequenceEqual( ref RuntimeHelpers.GetRawData(this), @@ -68,23 +66,8 @@ ref RuntimeHelpers.GetRawData(obj), return true; } - // Return true if the valuetype does not contain pointer, is tightly packed, - // does not have floating point number field and does not override Equals method. - private static unsafe bool CanCompareBitsOrUseFastGetHashCode(MethodTable* pMT) - { - MethodTableAuxiliaryData* pAuxData = pMT->AuxiliaryData; - - if (pAuxData->HasCheckedCanCompareBitsOrUseFastGetHashCode) - { - return pAuxData->CanCompareBitsOrUseFastGetHashCode; - } - - return CanCompareBitsOrUseFastGetHashCodeHelper(pMT); - } - - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "MethodTable_CanCompareBitsOrUseFastGetHashCode")] - [return: MarshalAs(UnmanagedType.Bool)] - private static unsafe partial bool CanCompareBitsOrUseFastGetHashCodeHelper(MethodTable* pMT); + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern bool CanCompareBits(object obj); /*=================================GetHashCode================================== **Action: Our algorithm for returning the hashcode is a little bit complex. We look @@ -96,74 +79,8 @@ private static unsafe bool CanCompareBitsOrUseFastGetHashCode(MethodTable* pMT) **Arguments: None. **Exceptions: None. ==============================================================================*/ - public override unsafe int GetHashCode() - { - // The default implementation of GetHashCode() for all value types. - // Note that this implementation reveals the value of the fields. - // So if the value type contains any sensitive information it should - // implement its own GetHashCode(). - - MethodTable* pMT = RuntimeHelpers.GetMethodTable(this); - ref byte rawData = ref RuntimeHelpers.GetRawData(this); - HashCode hashCode = default; - - // To get less colliding and more evenly distributed hash codes, - // we munge the class index into the hashcode - hashCode.Add((IntPtr)pMT); - - if (CanCompareBitsOrUseFastGetHashCode(pMT)) - { - // this is a struct with no refs and no "strange" offsets - uint size = pMT->GetNumInstanceFieldBytes(); - hashCode.AddBytes(MemoryMarshal.CreateReadOnlySpan(ref rawData, (int)size)); - } - else - { - object thisRef = this; - switch (GetHashCodeStrategy(pMT, ObjectHandleOnStack.Create(ref thisRef), out uint fieldOffset, out uint fieldSize, out MethodTable* fieldMT)) - { - case ValueTypeHashCodeStrategy.ReferenceField: - hashCode.Add(Unsafe.As(ref Unsafe.AddByteOffset(ref rawData, fieldOffset)).GetHashCode()); - break; - - case ValueTypeHashCodeStrategy.DoubleField: - hashCode.Add(Unsafe.As(ref Unsafe.AddByteOffset(ref rawData, fieldOffset)).GetHashCode()); - break; - - case ValueTypeHashCodeStrategy.SingleField: - hashCode.Add(Unsafe.As(ref Unsafe.AddByteOffset(ref rawData, fieldOffset)).GetHashCode()); - break; - - case ValueTypeHashCodeStrategy.FastGetHashCode: - Debug.Assert(fieldSize != 0); - hashCode.AddBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AddByteOffset(ref rawData, fieldOffset), (int)fieldSize)); - break; - - case ValueTypeHashCodeStrategy.ValueTypeOverride: - Debug.Assert(fieldMT != null); - // Box the field to handle complicated cases like mutable method and shared generic - hashCode.Add(RuntimeHelpers.Box(fieldMT, ref Unsafe.AddByteOffset(ref rawData, fieldOffset))?.GetHashCode() ?? 0); - break; - } - } - - return hashCode.ToHashCode(); - } - - // Must match the definition in src\vm\comutilnative.cpp - private enum ValueTypeHashCodeStrategy - { - None, - ReferenceField, - DoubleField, - SingleField, - FastGetHashCode, - ValueTypeOverride, - } - - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ValueType_GetHashCodeStrategy")] - private static unsafe partial ValueTypeHashCodeStrategy GetHashCodeStrategy( - MethodTable* pMT, ObjectHandleOnStack objHandle, out uint fieldOffset, out uint fieldSize, out MethodTable* fieldMT); + [MethodImpl(MethodImplOptions.InternalCall)] + public extern override int GetHashCode(); public override string? ToString() { diff --git a/src/coreclr/classlibnative/bcltype/objectnative.cpp b/src/coreclr/classlibnative/bcltype/objectnative.cpp index afbda5fad99167..4622955b44adf6 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.cpp +++ b/src/coreclr/classlibnative/bcltype/objectnative.cpp @@ -123,22 +123,48 @@ FCIMPL1(INT32, ObjectNative::TryGetHashCode, Object* obj) { } FCIMPLEND -FCIMPL2(FC_BOOL_RET, ObjectNative::ContentEquals, Object *pThisRef, Object *pCompareRef) +// +// Compare by ref for normal classes, by value for value types. +// +// @todo: it would be nice to customize this method based on the +// defining class rather than doing a runtime check whether it is +// a value type. +// + +FCIMPL2(FC_BOOL_RET, ObjectNative::Equals, Object *pThisRef, Object *pCompareRef) { - FCALL_CONTRACT; + CONTRACTL + { + FCALL_CHECK; + INJECT_FAULT(FCThrow(kOutOfMemoryException);); + } + CONTRACTL_END; + + if (pThisRef == pCompareRef) + FC_RETURN_BOOL(TRUE); - // Should be ensured by caller - _ASSERTE(pThisRef != NULL); - _ASSERTE(pCompareRef != NULL); - _ASSERTE(pThisRef->GetMethodTable() == pCompareRef->GetMethodTable()); + // Since we are in FCALL, we must handle NULL specially. + if (pThisRef == NULL || pCompareRef == NULL) + FC_RETURN_BOOL(FALSE); MethodTable *pThisMT = pThisRef->GetMethodTable(); - // Compare the contents + // If it's not a value class, don't compare by value + if (!pThisMT->IsValueType()) + FC_RETURN_BOOL(FALSE); + + // Make sure they are the same type. + if (pThisMT != pCompareRef->GetMethodTable()) + FC_RETURN_BOOL(FALSE); + + // Compare the contents (size - vtable - sync block index). + DWORD dwBaseSize = pThisMT->GetBaseSize(); + if(pThisMT == g_pStringClass) + dwBaseSize -= sizeof(WCHAR); BOOL ret = memcmp( - pThisRef->GetData(), - pCompareRef->GetData(), - pThisMT->GetNumInstanceFieldBytes()) == 0; + (void *) (pThisRef+1), + (void *) (pCompareRef+1), + dwBaseSize - sizeof(Object) - sizeof(int)) == 0; FC_GC_POLL_RET(); @@ -189,34 +215,36 @@ FCIMPL1(Object*, ObjectNative::GetClass, Object* pThis) } FCIMPLEND -extern "C" void QCALLTYPE ObjectNative_AllocateUninitializedClone(QCall::ObjectHandleOnStack objHandle) +FCIMPL1(Object*, ObjectNative::AllocateUninitializedClone, Object* pObjUNSAFE) { - QCALL_CONTRACT; + FCALL_CONTRACT; - BEGIN_QCALL; + // Delegate error handling to managed side (it will throw NullReferenceException) + if (pObjUNSAFE == NULL) + return NULL; - GCX_COOP(); + OBJECTREF refClone = ObjectToOBJECTREF(pObjUNSAFE); + + HELPER_METHOD_FRAME_BEGIN_RET_1(refClone); - OBJECTREF refClone = objHandle.Get(); - _ASSERTE(refClone != NULL); // Should be handled at managed side MethodTable* pMT = refClone->GetMethodTable(); - + // assert that String has overloaded the Clone() method _ASSERTE(pMT != g_pStringClass); - - if (pMT->IsArray()) - { - objHandle.Set(DupArrayForCloning((BASEARRAYREF)refClone)); - } - else - { + + if (pMT->IsArray()) { + refClone = DupArrayForCloning((BASEARRAYREF)refClone); + } else { // We don't need to call the because we know // that it has been called....(It was called before this was created) - objHandle.Set(AllocateObject(pMT)); + refClone = AllocateObject(pMT); } - END_QCALL; + HELPER_METHOD_FRAME_END(); + + return OBJECTREFToObject(refClone); } +FCIMPLEND extern "C" BOOL QCALLTYPE Monitor_Wait(QCall::ObjectHandleOnStack pThis, INT32 Timeout) { diff --git a/src/coreclr/classlibnative/bcltype/objectnative.h b/src/coreclr/classlibnative/bcltype/objectnative.h index 418fd2561d7cc8..d8948922dd0b74 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.h +++ b/src/coreclr/classlibnative/bcltype/objectnative.h @@ -27,12 +27,12 @@ class ObjectNative static FCDECL1(INT32, GetHashCode, Object* vThisRef); static FCDECL1(INT32, TryGetHashCode, Object* vThisRef); - static FCDECL2(FC_BOOL_RET, ContentEquals, Object *pThisRef, Object *pCompareRef); + static FCDECL2(FC_BOOL_RET, Equals, Object *pThisRef, Object *pCompareRef); + static FCDECL1(Object*, AllocateUninitializedClone, Object* pObjUNSAFE); static FCDECL1(Object*, GetClass, Object* pThis); static FCDECL1(FC_BOOL_RET, IsLockHeld, Object* pThisUNSAFE); }; -extern "C" void QCALLTYPE ObjectNative_AllocateUninitializedClone(QCall::ObjectHandleOnStack objHandle); extern "C" BOOL QCALLTYPE Monitor_Wait(QCall::ObjectHandleOnStack pThis, INT32 Timeout); extern "C" void QCALLTYPE Monitor_Pulse(QCall::ObjectHandleOnStack pThis); extern "C" void QCALLTYPE Monitor_PulseAll(QCall::ObjectHandleOnStack pThis); diff --git a/src/coreclr/classlibnative/bcltype/system.cpp b/src/coreclr/classlibnative/bcltype/system.cpp index 5d2f00cd849db5..ef02743b36696f 100644 --- a/src/coreclr/classlibnative/bcltype/system.cpp +++ b/src/coreclr/classlibnative/bcltype/system.cpp @@ -133,67 +133,124 @@ extern "C" INT32 QCALLTYPE Environment_GetProcessorCount() return processorCount; } -struct FindFailFastCallerStruct { - StackCrawlMark* pStackMark; - UINT_PTR retAddress; -}; - -// This method is called by the GetMethod function and will crawl backward -// up the stack for integer methods. -static StackWalkAction FindFailFastCallerCallback(CrawlFrame* frame, VOID* data) { +// FailFast is supported in BCL.small as internal to support failing fast in places where EEE used to be thrown. +// +// Static message buffer used by SystemNative::FailFast to avoid reliance on a +// managed string object buffer. This buffer is not always used, see comments in +// the method below. +WCHAR g_szFailFastBuffer[256]; +WCHAR *g_pFailFastBuffer = g_szFailFastBuffer; + +#define FAIL_FAST_STATIC_BUFFER_LENGTH (sizeof(g_szFailFastBuffer) / sizeof(WCHAR)) + +// This is the common code for FailFast processing that is wrapped by the two +// FailFast FCalls below. +void SystemNative::GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExceptionForWatsonBucketing, UINT_PTR retAddress, UINT exitCode, STRINGREF refErrorSourceString) +{ CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - MODE_ANY; - } - CONTRACTL_END; - - FindFailFastCallerStruct* pFindCaller = (FindFailFastCallerStruct*) data; - - // The check here is between the address of a local variable - // (the stack mark) and a pointer to the EIP for a frame - // (which is actually the pointer to the return address to the - // function from the previous frame). So we'll actually notice - // which frame the stack mark was in one frame later. This is - // fine since we only implement LookForMyCaller. - _ASSERTE(*pFindCaller->pStackMark == LookForMyCaller); - if (!frame->IsInCalleesFrames(pFindCaller->pStackMark)) - return SWA_CONTINUE; - - pFindCaller->retAddress = GetControlPC(frame->GetRegisterSet()); - return SWA_ABORT; -} + THROWS; + GC_TRIGGERS; + MODE_COOPERATIVE; + }CONTRACTL_END; -extern "C" void QCALLTYPE Environment_FailFast(QCall::StackCrawlMarkHandle mark, PCWSTR message, QCall::ObjectHandleOnStack exception, PCWSTR errorSource) -{ - QCALL_CONTRACT; + struct + { + STRINGREF refMesgString; + EXCEPTIONREF refExceptionForWatsonBucketing; + STRINGREF refErrorSourceString; + } gc; + gc.refMesgString = refMesgString; + gc.refExceptionForWatsonBucketing = refExceptionForWatsonBucketing; + gc.refErrorSourceString = refErrorSourceString; + + GCPROTECT_BEGIN(gc); + + // Managed code injected FailFast maps onto the unmanaged version + // (EEPolicy::HandleFatalError) in the following manner: the exit code is + // always set to COR_E_FAILFAST and the address passed (usually a failing + // EIP) is in fact the address of a unicode message buffer (explaining the + // reason for the fault). + // The message string comes from a managed string object so we can't rely on + // the buffer remaining in place below our feet. But equally we don't want + // to inject failure points (by, for example, allocating a heap buffer or a + // pinning handle) when we have a much higher chance than usual of actually + // tripping those failure points and eradicating useful debugging info. + // We employ various strategies to deal with this: + // o If the message is small enough we copy it into a static buffer + // (g_szFailFastBuffer). + // o Otherwise we try to allocate a buffer of the required size on the + // heap. This buffer will be leaked. + // o If the allocation above fails we return to the static buffer and + // truncate the message. + // + // Another option would seem to be to implement a new frame type that + // protects object references as pinned, but that seems like overkill for + // just this problem. + WCHAR *pszMessageBuffer = NULL; + DWORD cchMessage = (gc.refMesgString == NULL) ? 0 : gc.refMesgString->GetStringLength(); + + WCHAR * errorSourceString = NULL; + + if (gc.refErrorSourceString != NULL) + { + DWORD cchErrorSource = gc.refErrorSourceString->GetStringLength(); + errorSourceString = new (nothrow) WCHAR[cchErrorSource + 1]; - BEGIN_QCALL; + if (errorSourceString != NULL) + { + memcpyNoGCRefs(errorSourceString, gc.refErrorSourceString->GetBuffer(), cchErrorSource * sizeof(WCHAR)); + errorSourceString[cchErrorSource] = W('\0'); + } + } - GCX_COOP(); + if (cchMessage < FAIL_FAST_STATIC_BUFFER_LENGTH) + { + // The static buffer can be used only once to avoid race condition with other threads + pszMessageBuffer = InterlockedExchangeT(&g_pFailFastBuffer, NULL); + } - FindFailFastCallerStruct findCallerData; - findCallerData.pStackMark = mark; - findCallerData.retAddress = 0; - GetThread()->StackWalkFrames(FindFailFastCallerCallback, &findCallerData, FUNCTIONSONLY | QUICKUNWIND); + if (pszMessageBuffer == NULL) + { + // We can fail here, but we can handle the fault. + CONTRACT_VIOLATION(FaultViolation); + pszMessageBuffer = new (nothrow) WCHAR[cchMessage + 1]; + if (pszMessageBuffer == NULL) + { + // Truncate the message to what will fit in the static buffer. + cchMessage = FAIL_FAST_STATIC_BUFFER_LENGTH - 1; + pszMessageBuffer = InterlockedExchangeT(&g_pFailFastBuffer, NULL); + } + } - if (message == NULL || message[0] == W('\0')) + const WCHAR *pszMessage; + if (pszMessageBuffer != NULL) { - WszOutputDebugString(W("CLR: Managed code called FailFast without specifying a reason.\r\n")); + if (cchMessage > 0) + memcpyNoGCRefs(pszMessageBuffer, gc.refMesgString->GetBuffer(), cchMessage * sizeof(WCHAR)); + pszMessageBuffer[cchMessage] = W('\0'); + pszMessage = pszMessageBuffer; } else { + pszMessage = W("There is not enough memory to print the supplied FailFast message."); + cchMessage = (DWORD)u16_strlen(pszMessage); + } + + if (cchMessage == 0) { + WszOutputDebugString(W("CLR: Managed code called FailFast without specifying a reason.\r\n")); + } + else { WszOutputDebugString(W("CLR: Managed code called FailFast.\r\n")); - WszOutputDebugString(message); + WszOutputDebugString(pszMessage); WszOutputDebugString(W("\r\n")); } LPCWSTR argExceptionString = NULL; StackSString msg; - if (exception.Get() != NULL) + if (gc.refExceptionForWatsonBucketing != NULL) { - GetExceptionMessage(exception.Get(), msg); + GetExceptionMessage(gc.refExceptionForWatsonBucketing, msg); argExceptionString = msg.GetUnicode(); } @@ -206,11 +263,11 @@ extern "C" void QCALLTYPE Environment_FailFast(QCall::StackCrawlMarkHandle mark, // skip this, if required. if (IsWatsonEnabled()) { - if ((exception.Get() == NULL) || !SetupWatsonBucketsForFailFast((EXCEPTIONREF)exception.Get())) + if ((gc.refExceptionForWatsonBucketing == NULL) || !SetupWatsonBucketsForFailFast(gc.refExceptionForWatsonBucketing)) { PTR_EHWatsonBucketTracker pUEWatsonBucketTracker = pThread->GetExceptionState()->GetUEWatsonBucketTracker(); _ASSERTE(pUEWatsonBucketTracker != NULL); - pUEWatsonBucketTracker->SaveIpForWatsonBucket(findCallerData.retAddress); + pUEWatsonBucketTracker->SaveIpForWatsonBucket(retAddress); pUEWatsonBucketTracker->CaptureUnhandledInfoForWatson(TypeOfReportedError::FatalError, pThread, NULL); if (pUEWatsonBucketTracker->RetrieveWatsonBuckets() == NULL) { @@ -222,13 +279,90 @@ extern "C" void QCALLTYPE Environment_FailFast(QCall::StackCrawlMarkHandle mark, // stash the user-provided exception object. this will be used as // the inner exception object to the FatalExecutionEngineException. - if (exception.Get() != NULL) - pThread->SetLastThrownObject(exception.Get()); + if (gc.refExceptionForWatsonBucketing != NULL) + pThread->SetLastThrownObject(gc.refExceptionForWatsonBucketing); - EEPolicy::HandleFatalError(COR_E_FAILFAST, findCallerData.retAddress, message, NULL, errorSource, argExceptionString); + EEPolicy::HandleFatalError(exitCode, retAddress, pszMessage, NULL, errorSourceString, argExceptionString); - END_QCALL; + GCPROTECT_END(); +} + +// Note: Do not merge this FCALL method with any other FailFast overloads. +// Watson uses the managed FailFast method with one String for crash dump bucketization. +FCIMPL1(VOID, SystemNative::FailFast, StringObject* refMessageUNSAFE) +{ + FCALL_CONTRACT; + + STRINGREF refMessage = (STRINGREF)refMessageUNSAFE; + + HELPER_METHOD_FRAME_BEGIN_1(refMessage); + + // The HelperMethodFrame knows how to get the return address. + UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); + + // Call the actual worker to perform failfast + GenericFailFast(refMessage, NULL, retaddr, COR_E_FAILFAST, NULL); + + HELPER_METHOD_FRAME_END(); +} +FCIMPLEND + +FCIMPL2(VOID, SystemNative::FailFastWithExitCode, StringObject* refMessageUNSAFE, UINT exitCode) +{ + FCALL_CONTRACT; + + STRINGREF refMessage = (STRINGREF)refMessageUNSAFE; + + HELPER_METHOD_FRAME_BEGIN_1(refMessage); + + // The HelperMethodFrame knows how to get the return address. + UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); + + // Call the actual worker to perform failfast + GenericFailFast(refMessage, NULL, retaddr, exitCode, NULL); + + HELPER_METHOD_FRAME_END(); } +FCIMPLEND + +FCIMPL2(VOID, SystemNative::FailFastWithException, StringObject* refMessageUNSAFE, ExceptionObject* refExceptionUNSAFE) +{ + FCALL_CONTRACT; + + STRINGREF refMessage = (STRINGREF)refMessageUNSAFE; + EXCEPTIONREF refException = (EXCEPTIONREF)refExceptionUNSAFE; + + HELPER_METHOD_FRAME_BEGIN_2(refMessage, refException); + + // The HelperMethodFrame knows how to get the return address. + UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); + + // Call the actual worker to perform failfast + GenericFailFast(refMessage, refException, retaddr, COR_E_FAILFAST, NULL); + + HELPER_METHOD_FRAME_END(); +} +FCIMPLEND + +FCIMPL3(VOID, SystemNative::FailFastWithExceptionAndSource, StringObject* refMessageUNSAFE, ExceptionObject* refExceptionUNSAFE, StringObject* errorSourceUNSAFE) +{ + FCALL_CONTRACT; + + STRINGREF refMessage = (STRINGREF)refMessageUNSAFE; + EXCEPTIONREF refException = (EXCEPTIONREF)refExceptionUNSAFE; + STRINGREF errorSource = (STRINGREF)errorSourceUNSAFE; + + HELPER_METHOD_FRAME_BEGIN_3(refMessage, refException, errorSource); + + // The HelperMethodFrame knows how to get the return address. + UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); + + // Call the actual worker to perform failfast + GenericFailFast(refMessage, refException, retaddr, COR_E_FAILFAST, errorSource); + + HELPER_METHOD_FRAME_END(); +} +FCIMPLEND FCIMPL0(FC_BOOL_RET, SystemNative::IsServerGC) { diff --git a/src/coreclr/classlibnative/bcltype/system.h b/src/coreclr/classlibnative/bcltype/system.h index 9c5ab7ada84a4b..b4a773a847c398 100644 --- a/src/coreclr/classlibnative/bcltype/system.h +++ b/src/coreclr/classlibnative/bcltype/system.h @@ -43,16 +43,23 @@ class SystemNative static FCDECL1(VOID,SetExitCode,INT32 exitcode); static FCDECL0(INT32, GetExitCode); + static FCDECL1(VOID, FailFast, StringObject* refMessageUNSAFE); + static FCDECL2(VOID, FailFastWithExitCode, StringObject* refMessageUNSAFE, UINT exitCode); + static FCDECL2(VOID, FailFastWithException, StringObject* refMessageUNSAFE, ExceptionObject* refExceptionUNSAFE); + static FCDECL3(VOID, FailFastWithExceptionAndSource, StringObject* refMessageUNSAFE, ExceptionObject* refExceptionUNSAFE, StringObject* errorSourceUNSAFE); + static FCDECL0(FC_BOOL_RET, IsServerGC); // Return a method info for the method were the exception was thrown static FCDECL1(ReflectMethodObject*, GetMethodFromStackTrace, ArrayBase* pStackTraceUNSAFE); + +private: + // Common processing code for FailFast + static void GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExceptionForWatsonBucketing, UINT_PTR retAddress, UINT exitCode, STRINGREF errorSource); }; extern "C" void QCALLTYPE Environment_Exit(INT32 exitcode); -extern "C" void QCALLTYPE Environment_FailFast(QCall::StackCrawlMarkHandle mark, PCWSTR message, QCall::ObjectHandleOnStack exception, PCWSTR errorSource); - // Returns the number of logical processors that can be used by managed code extern "C" INT32 QCALLTYPE Environment_GetProcessorCount(); diff --git a/src/coreclr/clr.featuredefines.props b/src/coreclr/clr.featuredefines.props index 7905f8a573d738..dccd5d0f150c14 100644 --- a/src/coreclr/clr.featuredefines.props +++ b/src/coreclr/clr.featuredefines.props @@ -1,26 +1,37 @@ true + true true true true true + true true true + true true + true + true + true true true true + true true true true + true + true + true + true @@ -34,16 +45,23 @@ $(DefineConstants);FEATURE_ARRAYSTUB_AS_IL $(DefineConstants);FEATURE_MULTICASTSTUB_AS_IL + $(DefineConstants);FEATURE_INSTANTIATINGSTUB_AS_IL + $(DefineConstants);FEATURE_STUBS_AS_IL + $(DefineConstants);FEATURE_COLLECTIBLE_ALC $(DefineConstants);FEATURE_COMWRAPPERS $(DefineConstants);FEATURE_COMINTEROP $(DefineConstants);FEATURE_COMINTEROP_APARTMENT_SUPPORT $(DefineConstants);FEATURE_OBJCMARSHAL $(DefineConstants);FEATURE_PERFTRACING $(DefineConstants);FEATURE_EVENTSOURCE_XPLAT + $(DefineConstants);FEATURE_WIN32_REGISTRY $(DefineConstants);FEATURE_TYPEEQUIVALENCE + $(DefineConstants);FEATURE_BASICFREEZE + $(DefineConstants);FEATURE_PORTABLE_SHUFFLE_THUNKS $(DefineConstants);FEATURE_ICASTABLE $(DefineConstants);FEATURE_EH_FUNCLETS $(DefineConstants);PROFILING_SUPPORTED + $(DefineConstants);FEATURE_PROFAPI_ATTACH_DETACH diff --git a/src/coreclr/clrdefinitions.cmake b/src/coreclr/clrdefinitions.cmake index 2ffcfb00c0c05e..fb8d095b5606d7 100644 --- a/src/coreclr/clrdefinitions.cmake +++ b/src/coreclr/clrdefinitions.cmake @@ -203,6 +203,9 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGE add_definitions(-DFEATURE_MANUALLY_MANAGED_CARD_BUNDLES) endif(CLR_CMAKE_TARGET_ARCH_AMD64 OR CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGET_ARCH_LOONGARCH64 OR CLR_CMAKE_TARGET_ARCH_RISCV64) +if(NOT CLR_CMAKE_TARGET_UNIX) + add_definitions(-DFEATURE_WIN32_REGISTRY) +endif(NOT CLR_CMAKE_TARGET_UNIX) add_definitions(-D_SECURE_SCL=0) add_definitions(-DUNICODE) add_definitions(-D_UNICODE) @@ -255,7 +258,7 @@ function(set_target_definitions_to_custom_os_and_arch) if (TARGETDETAILS_OS STREQUAL "unix_anyos") target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_UNIX_ANYOS) endif() - elseif (TARGETDETAILS_OS MATCHES "^win") + elseif (TARGETDETAILS_OS STREQUAL "win") target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_WINDOWS) endif((TARGETDETAILS_OS MATCHES "^unix")) @@ -284,7 +287,7 @@ function(set_target_definitions_to_custom_os_and_arch) target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE ARM_SOFTFP) endif() - if (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix") OR (TARGETDETAILS_OS MATCHES "win_aot")) + if (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix")) target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE FEATURE_EH_FUNCLETS) - endif (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix") OR (TARGETDETAILS_OS MATCHES "win_aot")) + endif (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix")) endfunction() diff --git a/src/coreclr/crosscomponents.cmake b/src/coreclr/crosscomponents.cmake index b06b7060704892..11e923805a6ea0 100644 --- a/src/coreclr/crosscomponents.cmake +++ b/src/coreclr/crosscomponents.cmake @@ -25,13 +25,6 @@ if (CLR_CMAKE_HOST_OS STREQUAL CLR_CMAKE_TARGET_OS OR CLR_CMAKE_TARGET_IOS OR CL DESTINATIONS . COMPONENT crosscomponents ) - if (CLR_CMAKE_TARGET_ARCH_I386) - install_clr (TARGETS - clrjit_win_aot_${ARCH_TARGET_NAME}_${ARCH_HOST_NAME} - DESTINATIONS . - COMPONENT crosscomponents - ) - endif() endif() endif() endif() diff --git a/src/coreclr/crossgen-corelib.proj b/src/coreclr/crossgen-corelib.proj index 1d0a6e2262ef21..ae2741ea9c9cac 100644 --- a/src/coreclr/crossgen-corelib.proj +++ b/src/coreclr/crossgen-corelib.proj @@ -23,6 +23,7 @@ true false + false false true diff --git a/src/coreclr/debug/daccess/dacdbiimpl.cpp b/src/coreclr/debug/daccess/dacdbiimpl.cpp index b316a0769d3f67..bc1e6ad8475459 100644 --- a/src/coreclr/debug/daccess/dacdbiimpl.cpp +++ b/src/coreclr/debug/daccess/dacdbiimpl.cpp @@ -7455,13 +7455,13 @@ HRESULT DacDbiInterfaceImpl::GetILCodeVersionNodeData(VMPTR_ILCodeVersionNode vm #ifdef FEATURE_REJIT ILCodeVersion ilCode(vmILCodeVersionNode.GetDacPtr()); pData->m_state = ilCode.GetRejitState(); - pData->m_pbIL = PTR_TO_CORDB_ADDRESS(dac_cast(ilCode.GetIL())); + pData->m_pbIL = PTR_TO_CORDB_ADDRESS(dac_cast(ilCode.GetIL())); pData->m_dwCodegenFlags = ilCode.GetJitFlags(); const InstrumentedILOffsetMapping* pMapping = ilCode.GetInstrumentedILMap(); if (pMapping) { pData->m_cInstrumentedMapEntries = (ULONG)pMapping->GetCount(); - pData->m_rgInstrumentedMapEntries = PTR_TO_CORDB_ADDRESS(dac_cast(pMapping->GetOffsets())); + pData->m_rgInstrumentedMapEntries = PTR_TO_CORDB_ADDRESS(dac_cast(pMapping->GetOffsets())); } else { diff --git a/src/coreclr/debug/daccess/request.cpp b/src/coreclr/debug/daccess/request.cpp index 3993d8a6f52ee4..1e0912ea05cdde 100644 --- a/src/coreclr/debug/daccess/request.cpp +++ b/src/coreclr/debug/daccess/request.cpp @@ -2577,7 +2577,7 @@ ClrDataAccess::GetAssemblyData(CLRDATA_ADDRESS cdBaseDomainPtr, CLRDATA_ADDRESS } assemblyData->AssemblyPtr = HOST_CDADDR(pAssembly); - assemblyData->ClassLoader = 0; + assemblyData->ClassLoader = HOST_CDADDR(pAssembly->GetLoader()); assemblyData->ParentDomain = HOST_CDADDR(AppDomain::GetCurrentDomain()); assemblyData->isDynamic = pAssembly->IsDynamic(); assemblyData->ModuleCount = 0; @@ -3810,13 +3810,9 @@ ClrDataAccess::GetJumpThunkTarget(T_CONTEXT *ctx, CLRDATA_ADDRESS *targetIP, CLR #ifdef TARGET_AMD64 SOSDacEnter(); - TADDR tempTargetIP, tempTargetMD; - if (!GetAnyThunkTarget(ctx, &tempTargetIP, &tempTargetMD)) + if (!GetAnyThunkTarget(ctx, targetIP, targetMD)) hr = E_FAIL; - *targetIP = TO_CDADDR(tempTargetIP); - *targetMD = TO_CDADDR(tempTargetMD); - SOSDacLeave(); return hr; #else diff --git a/src/coreclr/debug/di/rspriv.h b/src/coreclr/debug/di/rspriv.h index 68080a65cb8a6f..ceadc7eedafe77 100644 --- a/src/coreclr/debug/di/rspriv.h +++ b/src/coreclr/debug/di/rspriv.h @@ -7325,8 +7325,7 @@ class CordbJITILFrame : public CordbBase, public ICorDebugILFrame, public ICorDe GENERICS_TYPE_TOKEN exactGenericArgsToken, DWORD dwExactGenericArgsTokenIndex, bool fVarArgFnx, - CordbReJitILCode * pReJitCode, - bool fAdjustedIP); + CordbReJitILCode * pReJitCode); HRESULT Init(); virtual ~CordbJITILFrame(); virtual void Neuter(); @@ -7437,7 +7436,6 @@ class CordbJITILFrame : public CordbBase, public ICorDebugILFrame, public ICorDe CordbILCode* GetOriginalILCode(); CordbReJitILCode* GetReJitILCode(); - void AdjustIPAfterException(); private: void RefreshCachedVarArgSigParserIfNeeded(); @@ -7505,7 +7503,6 @@ class CordbJITILFrame : public CordbBase, public ICorDebugILFrame, public ICorDe // if this frame is instrumented with rejit, this will point to the instrumented IL code RSSmartPtr m_pReJitCode; - BOOL m_adjustedIP; }; /* ------------------------------------------------------------------------- * diff --git a/src/coreclr/debug/di/rsstackwalk.cpp b/src/coreclr/debug/di/rsstackwalk.cpp index f2bf3777bb6bb3..751d18dcc17972 100644 --- a/src/coreclr/debug/di/rsstackwalk.cpp +++ b/src/coreclr/debug/di/rsstackwalk.cpp @@ -776,8 +776,7 @@ HRESULT CordbStackWalk::GetFrameWorker(ICorDebugFrame ** ppFrame) frameData.v.exactGenericArgsToken, frameData.v.dwExactGenericArgsTokenIndex, !!frameData.v.fVarArgs, - pReJitCode, - pJITFuncData->justAfterILThrow)); + pReJitCode)); // Initialize the frame. This is a nop if the method is not a vararg method. hr = pJITILFrame->Init(); diff --git a/src/coreclr/debug/di/rsthread.cpp b/src/coreclr/debug/di/rsthread.cpp index 3c5024fc80fab4..7b969ee65d3d55 100644 --- a/src/coreclr/debug/di/rsthread.cpp +++ b/src/coreclr/debug/di/rsthread.cpp @@ -7396,8 +7396,7 @@ CordbJITILFrame::CordbJITILFrame(CordbNativeFrame * pNativeFrame, GENERICS_TYPE_TOKEN exactGenericArgsToken, DWORD dwExactGenericArgsTokenIndex, bool fVarArgFnx, - CordbReJitILCode * pRejitCode, - bool fAdjustedIP) + CordbReJitILCode * pRejitCode) : CordbBase(pNativeFrame->GetProcess(), 0, enumCordbJITILFrame), m_nativeFrame(pNativeFrame), m_ilCode(pCode), @@ -7412,8 +7411,7 @@ CordbJITILFrame::CordbJITILFrame(CordbNativeFrame * pNativeFrame, m_genericArgsLoaded(false), m_frameParamsToken(exactGenericArgsToken), m_dwFrameParamsTokenIndex(dwExactGenericArgsTokenIndex), - m_pReJitCode(pRejitCode), - m_adjustedIP(fAdjustedIP) + m_pReJitCode(pRejitCode) { // We'll initialize the SigParser in CordbJITILFrame::Init(). m_sigParserCached = SigParser(NULL, 0); @@ -9029,21 +9027,6 @@ CordbReJitILCode* CordbJITILFrame::GetReJitILCode() return m_pReJitCode; } -void CordbJITILFrame::AdjustIPAfterException() -{ - CordbNativeFrame* nativeFrameToAdjustIP = m_nativeFrame; - if (!m_adjustedIP) - { - DWORD nativeOffsetToMap = (DWORD)nativeFrameToAdjustIP->m_ip - STACKWALK_CONTROLPC_ADJUST_OFFSET; - CorDebugMappingResult mappingType; - ULONG uILOffset = nativeFrameToAdjustIP->m_nativeCode->GetSequencePoints()->MapNativeOffsetToIL( - nativeOffsetToMap, - &mappingType); - m_ip= uILOffset; - m_adjustedIP = true; - } -} - /* ------------------------------------------------------------------------- * * Eval class * ------------------------------------------------------------------------- */ diff --git a/src/coreclr/debug/di/shimpriv.h b/src/coreclr/debug/di/shimpriv.h index ff0f16436a1f2c..1ce2f6857d4808 100644 --- a/src/coreclr/debug/di/shimpriv.h +++ b/src/coreclr/debug/di/shimpriv.h @@ -780,8 +780,6 @@ class ShimStackWalk // Indicate whether we are processing a converted frame. bool m_fHasConvertedFrame; - - bool m_fHasException; }; // A ShimStackWalk is deleted when a process is continued, or when the stack is changed in any way diff --git a/src/coreclr/debug/di/shimstackwalk.cpp b/src/coreclr/debug/di/shimstackwalk.cpp index 46213d4ca36491..c47620c7bc0904 100644 --- a/src/coreclr/debug/di/shimstackwalk.cpp +++ b/src/coreclr/debug/di/shimstackwalk.cpp @@ -312,7 +312,6 @@ void ShimStackWalk::Populate() // because of the leaf STUBFRAME_EXCEPTION. chainInfo.CancelUMChain(); swInfo.m_fSkipChain = true; - swInfo.m_fHasException = true; } } @@ -989,20 +988,6 @@ CorDebugInternalFrameType ShimStackWalk::GetInternalFrameType(ICorDebugInternalF void ShimStackWalk::AppendFrame(ICorDebugFrame * pFrame, StackWalkInfo * pStackWalkInfo) { - // We've detected we're in a stackwalk where we have an exception and no further managed frames - // are on top of this frame. To ensure our IP points to the user line that threw the exception, - // we ask the frame to adjust the IP to the call instruction as currently it points to the instruction after it. - if (pStackWalkInfo->m_fHasException && pStackWalkInfo->m_cFrame == 0) - { - RSExtSmartPtr pNFrame3; - HRESULT hr = pFrame->QueryInterface(IID_ICorDebugILFrame, reinterpret_cast(&pNFrame3)); - if (pNFrame3 != NULL) - { - CordbJITILFrame* JITILFrameToAdjustIP = (static_cast(pNFrame3.GetValue())); - JITILFrameToAdjustIP->AdjustIPAfterException(); - pStackWalkInfo->m_fHasException = false; - } - } // grow the ICorDebugFrame ** ppFrame = m_stackFrames.AppendThrowing(); @@ -1484,8 +1469,7 @@ ShimStackWalk::StackWalkInfo::StackWalkInfo() m_fProcessingInternalFrame(false), m_fSkipChain(false), m_fLeafFrame(true), - m_fHasConvertedFrame(false), - m_fHasException(false) + m_fHasConvertedFrame(false) { m_pChildFrame.Assign(NULL); m_pConvertedInternalFrame2.Assign(NULL); diff --git a/src/coreclr/debug/ee/controller.h b/src/coreclr/debug/ee/controller.h index b838e11c0f85a3..a2d8dc2e2602f3 100644 --- a/src/coreclr/debug/ee/controller.h +++ b/src/coreclr/debug/ee/controller.h @@ -827,7 +827,7 @@ class DebuggerPatchTable : private CHashTableAndData DebuggerControllerPatch * GetPatch(PTR_CORDB_ADDRESS_TYPE address) { SUPPORTS_DAC; - ARM_ONLY(_ASSERTE(dac_cast(address) & THUMB_CODE)); + ARM_ONLY(_ASSERTE(dac_cast(address) & THUMB_CODE)); DebuggerControllerPatch * pPatch = dac_cast(Find(HashAddress(address), (SIZE_T)(dac_cast(address)))); diff --git a/src/coreclr/debug/ee/debugger.h b/src/coreclr/debug/ee/debugger.h index 2b8573e31b3656..ac2a3218f73569 100644 --- a/src/coreclr/debug/ee/debugger.h +++ b/src/coreclr/debug/ee/debugger.h @@ -3892,6 +3892,8 @@ HANDLE OpenWin32EventOrThrow( // Returns true if the specified IL offset has a special meaning (eg. prolog, etc.) bool DbgIsSpecialILOffset(DWORD offset); +#if !defined(TARGET_X86) void FixupDispatcherContext(T_DISPATCHER_CONTEXT* pDispatcherContext, T_CONTEXT* pContext, PEXCEPTION_ROUTINE pUnwindPersonalityRoutine = NULL); +#endif #endif /* DEBUGGER_H_ */ diff --git a/src/coreclr/debug/inc/arm64/primitives.h b/src/coreclr/debug/inc/arm64/primitives.h index 5f8b5262d993e4..05c03c7b3094f8 100644 --- a/src/coreclr/debug/inc/arm64/primitives.h +++ b/src/coreclr/debug/inc/arm64/primitives.h @@ -153,9 +153,9 @@ inline void CORDbgSetInstruction(CORDB_ADDRESS_TYPE* address, #if !defined(DBI_COMPILE) && !defined(DACCESS_COMPILE) && defined(HOST_OSX) ExecutableWriterHolder instructionWriterHolder((LPVOID)address, sizeof(PRD_TYPE)); - TADDR ptraddr = dac_cast(instructionWriterHolder.GetRW()); + ULONGLONG ptraddr = dac_cast(instructionWriterHolder.GetRW()); #else // !DBI_COMPILE && !DACCESS_COMPILE && HOST_OSX - TADDR ptraddr = dac_cast(address); + ULONGLONG ptraddr = dac_cast(address); #endif // !DBI_COMPILE && !DACCESS_COMPILE && HOST_OSX *(PRD_TYPE *)ptraddr = instruction; FlushInstructionCache(GetCurrentProcess(), @@ -167,7 +167,7 @@ inline PRD_TYPE CORDbgGetInstruction(UNALIGNED CORDB_ADDRESS_TYPE* address) { LIMITED_METHOD_CONTRACT; - TADDR ptraddr = dac_cast(address); + ULONGLONG ptraddr = dac_cast(address); return *(PRD_TYPE *)ptraddr; } diff --git a/src/coreclr/debug/inc/loongarch64/primitives.h b/src/coreclr/debug/inc/loongarch64/primitives.h index b30e7dcdd2ea91..97e4fb9541a2ab 100644 --- a/src/coreclr/debug/inc/loongarch64/primitives.h +++ b/src/coreclr/debug/inc/loongarch64/primitives.h @@ -135,7 +135,7 @@ inline void CORDbgSetInstruction(CORDB_ADDRESS_TYPE* address, // In a DAC build, this function assumes the input is an host address. LIMITED_METHOD_DAC_CONTRACT; - TADDR ptraddr = dac_cast(address); + ULONGLONG ptraddr = dac_cast(address); *(PRD_TYPE *)ptraddr = instruction; FlushInstructionCache(GetCurrentProcess(), address, @@ -146,7 +146,7 @@ inline PRD_TYPE CORDbgGetInstruction(UNALIGNED CORDB_ADDRESS_TYPE* address) { LIMITED_METHOD_CONTRACT; - TADDR ptraddr = dac_cast(address); + ULONGLONG ptraddr = dac_cast(address); return *(PRD_TYPE *)ptraddr; } diff --git a/src/coreclr/debug/inc/riscv64/primitives.h b/src/coreclr/debug/inc/riscv64/primitives.h index 17ace22981c77d..066397fcda7146 100644 --- a/src/coreclr/debug/inc/riscv64/primitives.h +++ b/src/coreclr/debug/inc/riscv64/primitives.h @@ -137,7 +137,7 @@ inline void CORDbgSetInstruction(CORDB_ADDRESS_TYPE* address, // In a DAC build, this function assumes the input is an host address. LIMITED_METHOD_DAC_CONTRACT; - TADDR ptraddr = dac_cast(address); + ULONGLONG ptraddr = dac_cast(address); *(PRD_TYPE *)ptraddr = instruction; FlushInstructionCache(GetCurrentProcess(), address, @@ -148,7 +148,7 @@ inline PRD_TYPE CORDbgGetInstruction(UNALIGNED CORDB_ADDRESS_TYPE* address) { LIMITED_METHOD_CONTRACT; - TADDR ptraddr = dac_cast(address); + ULONGLONG ptraddr = dac_cast(address); return *(PRD_TYPE *)ptraddr; } diff --git a/src/coreclr/gc/env/common.h b/src/coreclr/gc/env/common.h index a3f6539aa3a491..78562ef0438b40 100644 --- a/src/coreclr/gc/env/common.h +++ b/src/coreclr/gc/env/common.h @@ -22,7 +22,6 @@ #include #include #include -#include #include diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 4d281b16251a1e..0471326c0af5f7 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2367,7 +2367,6 @@ int gc_heap::conserve_mem_setting = 0; bool gc_heap::spin_count_unit_config_p = false; uint64_t gc_heap::suspended_start_time = 0; -uint64_t gc_heap::change_heap_count_time = 0; uint64_t gc_heap::end_gc_time = 0; uint64_t gc_heap::total_suspended_time = 0; uint64_t gc_heap::process_start_time = 0; @@ -22016,7 +22015,7 @@ void gc_heap::update_end_gc_time_per_heap() if (heap_number == 0) { - dprintf (3, ("prev gen%d GC end time: prev start %I64d + prev gc elapsed %Id = %I64d", + dprintf (6666, ("prev gen%d GC end time: prev start %I64d + prev gc elapsed %Id = %I64d", gen_number, dd_previous_time_clock (dd), dd_gc_elapsed_time (dd), (dd_previous_time_clock (dd) + dd_gc_elapsed_time (dd)))); } @@ -22024,53 +22023,45 @@ void gc_heap::update_end_gc_time_per_heap() if (heap_number == 0) { - dprintf (3, ("updated NGC%d %Id elapsed time to %I64d - %I64d = %I64d", gen_number, dd_gc_clock (dd), end_gc_time, dd_time_clock (dd), dd_gc_elapsed_time (dd))); + dprintf (6666, ("updated NGC%d %Id elapsed time to %I64d - %I64d = %I64d", gen_number, dd_gc_clock (dd), end_gc_time, dd_time_clock (dd), dd_gc_elapsed_time (dd))); } } #ifdef DYNAMIC_HEAP_COUNT if ((heap_number == 0) && (dynamic_adaptation_mode == dynamic_adaptation_to_application_sizes)) { - if (settings.gc_index > 1) - { - dynamic_heap_count_data_t::sample& sample = dynamic_heap_count_data.samples[dynamic_heap_count_data.sample_index]; - sample.elapsed_between_gcs = end_gc_time - last_suspended_end_time; - sample.gc_pause_time = dd_gc_elapsed_time (dynamic_data_of (0)); - sample.msl_wait_time = get_msl_wait_time (); - // could cache this - we will get it again soon in do_post_gc - sample.gc_survived_size = get_total_promoted (); + dynamic_heap_count_data_t::sample& sample = dynamic_heap_count_data.samples[dynamic_heap_count_data.sample_index]; + sample.elapsed_between_gcs = end_gc_time - last_suspended_end_time; + sample.gc_pause_time = dd_gc_elapsed_time (dynamic_data_of (0)); + sample.msl_wait_time = get_msl_wait_time(); - dprintf (6666, ("sample#%d: this GC end %I64d - last sus end %I64d = %I64d, this GC pause %I64d, msl wait %I64d", - dynamic_heap_count_data.sample_index, end_gc_time, last_suspended_end_time, sample.elapsed_between_gcs, sample.gc_pause_time, sample.msl_wait_time)); + dprintf (6666, ("sample#%d: this GC end %I64d - last sus end %I64d = %I64d, this GC pause %I64d, msl wait %I64d", + dynamic_heap_count_data.sample_index, end_gc_time, last_suspended_end_time, sample.elapsed_between_gcs, sample.gc_pause_time, sample.msl_wait_time)); - GCEventFireHeapCountSample_V1 ( - (uint64_t)VolatileLoadWithoutBarrier (&settings.gc_index), - sample.elapsed_between_gcs, - sample.gc_pause_time, - sample.msl_wait_time); + last_suspended_end_time = end_gc_time; - dynamic_heap_count_data.sample_index = (dynamic_heap_count_data.sample_index + 1) % dynamic_heap_count_data_t::sample_size; - (dynamic_heap_count_data.current_samples_count)++; + GCEventFireHeapCountSample_V1 ( + (uint64_t)VolatileLoadWithoutBarrier (&settings.gc_index), + sample.elapsed_between_gcs, + sample.gc_pause_time, + sample.msl_wait_time); - if (settings.condemned_generation == max_generation) - { - gc_index_full_gc_end = dd_gc_clock (dynamic_data_of (0)); - size_t elapsed_between_gen2_gcs = end_gc_time - prev_gen2_end_time; - size_t gen2_elapsed_time = sample.gc_pause_time; - dynamic_heap_count_data_t::gen2_sample& g2_sample = dynamic_heap_count_data.gen2_samples[dynamic_heap_count_data.gen2_sample_index]; - g2_sample.gc_index = VolatileLoadWithoutBarrier (&(settings.gc_index)); - g2_sample.gc_percent = (float)gen2_elapsed_time * 100.0f / elapsed_between_gen2_gcs; - (dynamic_heap_count_data.current_gen2_samples_count)++; + dynamic_heap_count_data.sample_index = (dynamic_heap_count_data.sample_index + 1) % dynamic_heap_count_data_t::sample_size; - dprintf (6666, ("gen2 sample#%d: this GC end %I64d - last gen2 end %I64d = %I64d, GC elapsed %I64d, percent %.3f", - dynamic_heap_count_data.gen2_sample_index, end_gc_time, prev_gen2_end_time, elapsed_between_gen2_gcs, gen2_elapsed_time, g2_sample.gc_percent)); - dynamic_heap_count_data.gen2_sample_index = (dynamic_heap_count_data.gen2_sample_index + 1) % dynamic_heap_count_data_t::sample_size; - } + if (settings.condemned_generation == max_generation) + { + gc_index_full_gc_end = dd_gc_clock (dynamic_data_of (0)); + size_t elapsed_between_gen2_gcs = end_gc_time - prev_gen2_end_time; + size_t gen2_elapsed_time = sample.gc_pause_time; + dynamic_heap_count_data.gen2_gc_percents[dynamic_heap_count_data.gen2_sample_index] = (float)gen2_elapsed_time * 100.0f / elapsed_between_gen2_gcs; - calculate_new_heap_count (); + dprintf (6666, ("gen2 sample#%d: this GC end %I64d - last gen2 end %I64d = %I64d, GC elapsed %I64d, percent %.3f", + dynamic_heap_count_data.gen2_sample_index, end_gc_time, prev_gen2_end_time, elapsed_between_gen2_gcs, + gen2_elapsed_time, dynamic_heap_count_data.gen2_gc_percents[dynamic_heap_count_data.gen2_sample_index])); + dynamic_heap_count_data.gen2_sample_index = (dynamic_heap_count_data.gen2_sample_index + 1) % dynamic_heap_count_data_t::sample_size; } - last_suspended_end_time = end_gc_time; + calculate_new_heap_count (); } #endif //DYNAMIC_HEAP_COUNT } @@ -22237,16 +22228,11 @@ void gc_heap::gc1() dprintf (6666, ("updating BGC %Id elapsed time to %I64d - %I64d = %I64d", dd_gc_clock (dd), end_gc_time, dd_time_clock (dd), dd_gc_elapsed_time (dd))); float bgc_percent = (float)dd_gc_elapsed_time (dd) * 100.0f / (float)time_since_last_gen2; - dynamic_heap_count_data_t::gen2_sample& g2_sample = dynamic_heap_count_data.gen2_samples[dynamic_heap_count_data.gen2_sample_index]; - g2_sample.gc_index = VolatileLoadWithoutBarrier (&(settings.gc_index)); - g2_sample.gc_percent = bgc_percent; + dynamic_heap_count_data.gen2_gc_percents[dynamic_heap_count_data.gen2_sample_index] = bgc_percent; dprintf (6666, ("gen2 sample %d elapsed %Id * 100 / time inbetween gen2 %Id = %.3f", dynamic_heap_count_data.gen2_sample_index, dd_gc_elapsed_time (dd), time_since_last_gen2, bgc_percent)); dynamic_heap_count_data.gen2_sample_index = (dynamic_heap_count_data.gen2_sample_index + 1) % dynamic_heap_count_data_t::sample_size; - (dynamic_heap_count_data.current_gen2_samples_count)++; gc_index_full_gc_end = dd_gc_clock (dynamic_data_of (0)); - - calculate_new_heap_count (); } #endif //DYNAMIC_HEAP_COUNT @@ -25089,6 +25075,7 @@ void gc_heap::recommission_heap() // copy some fields from heap0 + // this is copied to dd_previous_time_clock at the start of GC dd_time_clock (dd) = dd_time_clock (heap0_dd); @@ -25165,90 +25152,37 @@ float median_of_3 (float a, float b, float c) return b; } -float log_with_base (float x, float base) +size_t gc_heap::get_num_completed_gcs () { - assert (x > base); - - return (float)(log(x) / log(base)); -} - -float mean (float* arr, int size) -{ - float sum = 0.0; - - for (int i = 0; i < size; i++) - { - sum += arr[i]; - } - return (sum / size); -} - -// Change it to a desired number if you want to print. -int max_times_to_print_tcp = 0; - -// Return the slope, and the average values in the avg arg. -float slope (float* y, int n, float* avg) -{ - assert (n > 0); - - if (n == 1) - { - dprintf (6666, ("only 1 tcp: %.3f, no slope", y[0])); - *avg = y[0]; - return 0.0; - } - - int sum_x = 0; - - for (int i = 0; i < n; i++) - { - sum_x += i; - - if (max_times_to_print_tcp >= 0) - { - dprintf (6666, ("%.3f, ", y[i])); - } - } - - float avg_x = (float)sum_x / n; - float avg_y = mean (y, n); - *avg = avg_y; - - float numerator = 0.0; - float denominator = 0.0; - - for (int i = 0; i < n; ++i) + size_t num_completed_gcs = settings.gc_index; +#ifdef BACKGROUND_GC + if (g_heaps[0]->is_bgc_in_progress ()) { - numerator += ((float)i - avg_x) * (y[i] - avg_y); - denominator += ((float)i - avg_x) * (i - avg_x); + num_completed_gcs--; + dprintf (6666, ("BGC in prog, completed GCs -> %Id", num_completed_gcs)); } +#endif //BACKGROUND_GC - max_times_to_print_tcp--; - - return (numerator / denominator); + return num_completed_gcs; } int gc_heap::calculate_new_heap_count () { assert (dynamic_adaptation_mode == dynamic_adaptation_to_application_sizes); - dprintf (6666, ("current num of samples %Id (g2: %Id) prev processed %Id (g2: %Id), last full GC happened at index %Id", - dynamic_heap_count_data.current_samples_count, dynamic_heap_count_data.current_gen2_samples_count, - dynamic_heap_count_data.processed_samples_count, dynamic_heap_count_data.processed_gen2_samples_count, gc_index_full_gc_end)); + size_t num_completed_gcs = get_num_completed_gcs (); + + dprintf (6666, ("current GC %Id(completed: %Id), prev completed GCs %Id, last full GC happened at index %Id", + VolatileLoadWithoutBarrier (&settings.gc_index), num_completed_gcs, dynamic_heap_count_data.prev_num_completed_gcs, gc_index_full_gc_end)); - if ((dynamic_heap_count_data.current_samples_count < (dynamic_heap_count_data.processed_samples_count + dynamic_heap_count_data_t::sample_size)) && - (dynamic_heap_count_data.current_gen2_samples_count < (dynamic_heap_count_data.processed_gen2_samples_count + dynamic_heap_count_data_t::sample_size))) + if (num_completed_gcs < (dynamic_heap_count_data.prev_num_completed_gcs + dynamic_heap_count_data_t::sample_size)) { dprintf (6666, ("not enough GCs, skipping")); return n_heaps; } - bool process_eph_samples_p = (dynamic_heap_count_data.current_samples_count >= (dynamic_heap_count_data.processed_samples_count + dynamic_heap_count_data_t::sample_size)); - bool process_gen2_samples_p = (dynamic_heap_count_data.current_gen2_samples_count >= (dynamic_heap_count_data.processed_gen2_samples_count + dynamic_heap_count_data_t::sample_size)); - - size_t current_gc_index = VolatileLoadWithoutBarrier (&settings.gc_index); float median_gen2_tcp_percent = 0.0f; - if (dynamic_heap_count_data.current_gen2_samples_count >= (dynamic_heap_count_data.processed_gen2_samples_count + dynamic_heap_count_data_t::sample_size)) + if (gc_index_full_gc_end >= (settings.gc_index - dynamic_heap_count_data_t::sample_size)) { median_gen2_tcp_percent = dynamic_heap_count_data.get_median_gen2_gc_percent (); } @@ -25268,43 +25202,6 @@ int gc_heap::calculate_new_heap_count () } float median_throughput_cost_percent = median_of_3 (throughput_cost_percents[0], throughput_cost_percents[1], throughput_cost_percents[2]); - float avg_throughput_cost_percent = (float)((throughput_cost_percents[0] + throughput_cost_percents[1] + throughput_cost_percents[2]) / 3.0); - - // One of the reasons for outliers is something temporarily affected GC work. We pick the min tcp if the survival is very stable to avoid counting these outliers. - float min_tcp = throughput_cost_percents[0]; - size_t min_survived = dynamic_heap_count_data.samples[0].gc_survived_size; - uint64_t min_pause = dynamic_heap_count_data.samples[0].gc_pause_time; - for (int i = 1; i < dynamic_heap_count_data_t::sample_size; i++) - { - min_tcp = min (throughput_cost_percents[i], min_tcp); - min_survived = min (dynamic_heap_count_data.samples[i].gc_survived_size, min_survived); - min_pause = min (dynamic_heap_count_data.samples[i].gc_pause_time, min_pause); - } - - dprintf (6666, ("checking if samples are stable %Id %Id %Id, min tcp %.3f, min pause %I64d", - dynamic_heap_count_data.samples[0].gc_survived_size, dynamic_heap_count_data.samples[1].gc_survived_size, dynamic_heap_count_data.samples[2].gc_survived_size, - min_tcp, min_pause)); - - bool survived_stable_p = true; - if (min_survived > 0) - { - for (int i = 0; i < dynamic_heap_count_data_t::sample_size; i++) - { - dynamic_heap_count_data_t::sample& sample = dynamic_heap_count_data.samples[i]; - float diff = (float)(sample.gc_survived_size - min_survived) / (float)min_survived; - dprintf (6666, ("sample %d diff from min is %Id -> %.3f", i, (sample.gc_survived_size - min_survived), diff)); - if (diff >= 0.15) - { - survived_stable_p = false; - } - } - } - - if (survived_stable_p) - { - dprintf (6666, ("survived is stable, so we pick min tcp %.3f", min_tcp)); - median_throughput_cost_percent = min_tcp; - } // apply exponential smoothing and use 1/3 for the smoothing factor const float smoothing = 3; @@ -25319,13 +25216,10 @@ int gc_heap::calculate_new_heap_count () smoothed_median_throughput_cost_percent = median_throughput_cost_percent; } - dprintf (6666, ("median tcp: %.3f, smoothed tcp: %.3f, avg tcp: %.3f, gen2 tcp %.3f(%.3f, %.3f, %.3f)", - median_throughput_cost_percent, smoothed_median_throughput_cost_percent, avg_throughput_cost_percent, median_gen2_tcp_percent, - dynamic_heap_count_data.gen2_samples[0].gc_percent, dynamic_heap_count_data.gen2_samples[1].gc_percent, dynamic_heap_count_data.gen2_samples[2].gc_percent)); + dprintf (6666, ("median tcp: %.3f, smoothed tcp: %.3f, gen2 tcp %.3f(%.3f, %.3f, %.3f)", + median_throughput_cost_percent, smoothed_median_throughput_cost_percent, median_gen2_tcp_percent, + dynamic_heap_count_data.gen2_gc_percents[0], dynamic_heap_count_data.gen2_gc_percents[1], dynamic_heap_count_data.gen2_gc_percents[2])); - // - // I'm keeping the old logic for now just to handle gen2. - // size_t heap_size = 0; for (int i = 0; i < n_heaps; i++) { @@ -25353,9 +25247,7 @@ int gc_heap::calculate_new_heap_count () // we don't go all the way to the number of CPUs, but stay 1 or 2 short int step_up = (n_heaps + 1) / 2; int extra_heaps = 1 + (n_max_heaps >= 32); - int actual_n_max_heaps = n_max_heaps - extra_heaps; - int max_growth = max ((n_max_heaps / 4), 2); - step_up = min (step_up, (actual_n_max_heaps - n_heaps)); + step_up = min (step_up, n_max_heaps - extra_heaps - n_heaps); // on the way down, we essentially divide the heap count by 1.5 int step_down = (n_heaps + 1) / 3; @@ -25393,310 +25285,49 @@ int gc_heap::calculate_new_heap_count () dprintf (6666, ("stress %d -> %d", n_heaps, new_n_heaps)); #else //STRESS_DYNAMIC_HEAP_COUNT int new_n_heaps = n_heaps; - - // target_tcp should be configurable. - float target_tcp = 5.0; - float target_gen2_tcp = 10.0; - float log_base = (float)1.1; - - dynamic_heap_count_data.add_to_recorded_tcp (median_throughput_cost_percent); - - // This is the average of whatever is in the recorded tcp buffer. - float avg_recorded_tcp = 0.0; - - if (process_eph_samples_p) + if (median_throughput_cost_percent > 10.0f) { - dynamic_heap_count_data.last_processed_stcp = smoothed_median_throughput_cost_percent; - - if ((median_throughput_cost_percent > 10.0f) || (smoothed_median_throughput_cost_percent > target_tcp)) + // ramp up more agressively - use as many heaps as it would take to bring + // the tcp down to 5% + new_n_heaps = (int)(n_heaps * (median_throughput_cost_percent / 5.0)); + dprintf (6666, ("[CHP0] tcp %.3f -> %d * %.3f = %d", median_throughput_cost_percent, n_heaps, (median_throughput_cost_percent / 5.0), new_n_heaps)); + new_n_heaps = min (new_n_heaps, n_max_heaps - extra_heaps); + } + // if the median tcp is 10% or less, react slower + else if ((smoothed_median_throughput_cost_percent > 5.0f) || (median_gen2_tcp_percent > 10.0f)) + { + if (smoothed_median_throughput_cost_percent > 5.0f) { - // If median is high but stcp is lower than target, and if this situation continues, stcp will quickly be above target anyway; otherwise - // we treat it as an outlier. - if (smoothed_median_throughput_cost_percent > target_tcp) - { - float step_up_percent = log_with_base ((smoothed_median_throughput_cost_percent - target_tcp + log_base), log_base); - float step_up_float = (float)(step_up_percent / 100.0 * actual_n_max_heaps); - int step_up_int = (int)step_up_float; - - dprintf (6666, ("[CHP0] inc %d(%.3f), last inc %d, %Id GCs elapsed, last stcp %.3f", - step_up_int, step_up_float, (int)dynamic_heap_count_data.last_changed_count, - (current_gc_index - dynamic_heap_count_data.last_changed_gc_index), dynamic_heap_count_data.last_changed_stcp)); - - // Don't adjust if we just adjusted last time we checked, unless we are in an extreme situation. - if ((smoothed_median_throughput_cost_percent < 20.0f) && - (avg_throughput_cost_percent < 20.0f) && - ((current_gc_index - dynamic_heap_count_data.last_changed_gc_index) < (2 * dynamic_heap_count_data_t::sample_size))) - { - dprintf (6666, ("[CHP0] we just adjusted %Id GCs ago, skipping", (current_gc_index - dynamic_heap_count_data.last_changed_gc_index))); - } - else - { - if (step_up_int) - { - if (dynamic_heap_count_data.dec_failure_count) - { - dprintf (6666, ("[CHP0] intending to grow, reset dec failure count (was %d)", dynamic_heap_count_data.dec_failure_count)); - dynamic_heap_count_data.dec_failure_count = 0; - } - - if (((int)dynamic_heap_count_data.last_changed_count > 0) && (dynamic_heap_count_data.last_changed_gc_index > 0.0) && - ((current_gc_index - dynamic_heap_count_data.last_changed_gc_index) <= (3 * dynamic_heap_count_data_t::sample_size))) - { - dprintf (6666, ("[CHP0-0] just grew %d GCs ago, no change", (current_gc_index - dynamic_heap_count_data.last_changed_gc_index))); - step_up_int = 0; - } - else - { - // If the calculation tells us to grow, we should check to see if the slope has been coming down rapidly, if so there's no reason to grow. - int above_target_tcp_count = dynamic_heap_count_data.rearrange_recorded_tcp (); - float above_target_tcp_slope = slope (dynamic_heap_count_data.recorded_tcp_rearranged, above_target_tcp_count, &avg_recorded_tcp); - float diff_pct = (target_tcp - avg_recorded_tcp) / target_tcp; - float adjusted_target_tcp = dynamic_heap_count_data.get_range_upper (target_tcp); - - dprintf (6666, ("[CHP0] slope of last %d samples is %.3f. avg %.3f (%.3f%%), current tcp %.3f, adjusted target is %.3f, failure count is %d", - above_target_tcp_count, above_target_tcp_slope, avg_recorded_tcp, (diff_pct * 100.0), - median_throughput_cost_percent, adjusted_target_tcp, dynamic_heap_count_data.inc_failure_count)); - - if (dynamic_heap_count_data.is_tcp_in_range (diff_pct, above_target_tcp_slope)) - { - step_up_int = 0; - dprintf (6666, ("[CHP0-1] slope %.3f and already close to target %.3f (%.3f%%), no change", above_target_tcp_slope, avg_recorded_tcp, (diff_pct * 100.0))); - } - else - { - if (above_target_tcp_slope < 0.0) - { - // If we are already trending down and the tcp is small enough, just wait. - if ((median_throughput_cost_percent < adjusted_target_tcp) || (avg_recorded_tcp < adjusted_target_tcp)) - { - step_up_int = 0; - dprintf (6666, ("[CHP0-2] trending down, slope is %.3f, tcp is %.3f, avg is %.3f, already below adjusted target %.3f, no change", - above_target_tcp_slope, median_throughput_cost_percent, avg_recorded_tcp, adjusted_target_tcp)); - } - } - else - { - // We are trending up, but we have too few samples and the avg is already small enough. - if ((above_target_tcp_count <= dynamic_heap_count_data.inc_recheck_threshold) && (avg_recorded_tcp < adjusted_target_tcp)) - { - step_up_int = 0; - dprintf (6666, ("[CHP0-3] trending up, only %d samples, slope is %.3f, avg is %.3f already below adjusted target %.3f, no change", - above_target_tcp_count, above_target_tcp_slope, avg_recorded_tcp, adjusted_target_tcp)); - } - } - } - } - - // If we still decided to grow, check if we need to grow aggressively. - if (step_up_int) - { - if (((int)dynamic_heap_count_data.last_changed_count > 0) && (dynamic_heap_count_data.last_changed_gc_index > 0.0)) - { - (dynamic_heap_count_data.inc_failure_count)++; - dprintf (6666, ("[CHP0-4] just grew %d GCs ago, grow more aggressively from %d -> %d more heaps", - (current_gc_index - dynamic_heap_count_data.last_changed_gc_index), step_up_int, (step_up_int * (dynamic_heap_count_data.inc_failure_count + 1)))); - step_up_int *= dynamic_heap_count_data.inc_failure_count + 1; - } - } - } - - step_up_int = min (step_up_int, max_growth); - - new_n_heaps = n_heaps + step_up_int; - new_n_heaps = min (new_n_heaps, actual_n_max_heaps); - - // If we are going to grow to be very close to max heap, it's better to just grow to it. - if ((new_n_heaps < actual_n_max_heaps) && dynamic_heap_count_data.is_close_to_max (new_n_heaps, actual_n_max_heaps)) - { - dprintf (6666, ("[CHP0-5] %d is close to max heaps %d, grow to max", new_n_heaps, actual_n_max_heaps)); - new_n_heaps = actual_n_max_heaps; - } - - if (new_n_heaps > n_heaps) - { - dynamic_heap_count_data.last_changed_gc_index = current_gc_index; - dynamic_heap_count_data.last_changed_count = step_up_float; - dynamic_heap_count_data.last_changed_stcp = smoothed_median_throughput_cost_percent; - } - - dprintf (6666, ("[CHP0] tcp %.3f, stcp %.3f -> (%d * %.3f%% = %.3f) -> %d + %d = %d -> %d", - median_throughput_cost_percent, smoothed_median_throughput_cost_percent, - actual_n_max_heaps, step_up_percent, step_up_float, step_up_int, n_heaps, (n_heaps + step_up_int), new_n_heaps)); - } - } + dprintf (6666, ("[CHP1] stcp %.3f > 5, %d + %d = %d", smoothed_median_throughput_cost_percent, n_heaps, step_up, (n_heaps + step_up))); } else { - // When we are below target, we accumulate the distance to target and only adjust when we've accumulated enough in this state. Note that - // this can include tcp's that are slightly above target, as long as it's not high enough for us to adjust the heap count. If we are just - // oscillating around target, this makes those tcp's cancel each other out. - if (dynamic_heap_count_data.below_target_accumulation == 0) - { - dynamic_heap_count_data.first_below_target_gc_index = current_gc_index; - dynamic_heap_count_data.init_recorded_tcp (); - dynamic_heap_count_data.add_to_recorded_tcp (median_throughput_cost_percent); - } - dprintf (6666, ("[CHP1] last time adjusted %s by %d at GC#%Id (%Id GCs since), stcp was %.3f, now stcp is %.3f", - ((dynamic_heap_count_data.last_changed_count > 0.0) ? "up" : "down"), (int)dynamic_heap_count_data.last_changed_count, - dynamic_heap_count_data.last_changed_gc_index, (current_gc_index - dynamic_heap_count_data.last_changed_gc_index), - dynamic_heap_count_data.last_changed_stcp, smoothed_median_throughput_cost_percent)); - - float below_target_diff = target_tcp - median_throughput_cost_percent; - dynamic_heap_count_data.below_target_accumulation += below_target_diff; - - dprintf (6666, ("[CHP1] below target for the past %Id GCs, accumulated %.3f, min (10%% of max is %.2f, 20%% of hc is %.2f)", - (current_gc_index - dynamic_heap_count_data.first_below_target_gc_index), dynamic_heap_count_data.below_target_accumulation, - (actual_n_max_heaps * 0.1), (n_heaps * 0.2))); - - if (dynamic_heap_count_data.below_target_accumulation >= dynamic_heap_count_data.below_target_threshold) - { - int below_target_tcp_count = dynamic_heap_count_data.rearrange_recorded_tcp (); - float below_target_tcp_slope = slope (dynamic_heap_count_data.recorded_tcp, below_target_tcp_count, &avg_recorded_tcp); - float diff_pct = (target_tcp - smoothed_median_throughput_cost_percent) / target_tcp; - int step_down_int = (int)(diff_pct / 2.0 * n_heaps); - dprintf (6666, ("[CHP1] observed %d tcp's <= or ~ target, avg %.3f, slope %.3f, stcp %.3f below target, shrink by %.3f * %d = %d heaps", - below_target_tcp_count, avg_recorded_tcp, below_target_tcp_slope, (diff_pct * 100.0), (diff_pct * 50.0), n_heaps, step_down_int)); - - bool shrink_p = false; - if (dynamic_heap_count_data.is_tcp_in_range (diff_pct, below_target_tcp_slope)) - { - step_down_int = 0; - dprintf (6666, ("[CHP1-0] slope %.3f is flat and stcp is already close to target %.3f (%.3f%%), no change", - below_target_tcp_slope, smoothed_median_throughput_cost_percent, (diff_pct * 100.0))); - } - else - { - // If we adjusted last time and it was unsuccessful, we need to inc our failure count. - // If we have a non zero failure count, we don't want to adjust for a while if we continue to be in that same situation. - bool last_dec_p = (dynamic_heap_count_data.last_changed_gc_index > 0) && (dynamic_heap_count_data.last_changed_count < 0.0); - float last_dec_tcp_diff_pct = (last_dec_p ? - ((smoothed_median_throughput_cost_percent - dynamic_heap_count_data.last_changed_stcp) / dynamic_heap_count_data.last_changed_stcp) : 0.0f); - bool stable_p = last_dec_p && ((last_dec_tcp_diff_pct <= 0.2) && (last_dec_tcp_diff_pct >= -0.2)); - dprintf (6666, ("[CHP1] since last adjustment stcp changed %.3f->%.3f = %.3f%%, %s, dec_failure_count is %d", - dynamic_heap_count_data.last_changed_stcp, smoothed_median_throughput_cost_percent, (last_dec_tcp_diff_pct * 100.0), - (stable_p ? "stable" : "not stable"), dynamic_heap_count_data.dec_failure_count)); - - bool check_dec_p = true; - - if (stable_p) - { - if (dynamic_heap_count_data.dec_failure_count) - { - (dynamic_heap_count_data.dec_failure_count)++; - } - else - { - dynamic_heap_count_data.dec_failure_count = 1; - } - - if (dynamic_heap_count_data.dec_failure_count <= dynamic_heap_count_data.dec_failure_recheck_threshold) - { - check_dec_p = false; - dprintf (6666, ("[CHP1-1] dec was still unsuccessful, <= %d, no change", dynamic_heap_count_data.dec_failure_recheck_threshold)); - } - } - - if (check_dec_p) - { - dynamic_heap_count_data.dec_failure_count = 0; - - if (below_target_tcp_slope <= 0.0) - { - shrink_p = true; - } - else - { - // It's trending upwards, but if takes too many samples to get to target, we do want to shrink. - int num_samples_to_goal = (int)((target_tcp + below_target_tcp_slope - median_throughput_cost_percent) / below_target_tcp_slope); - bool far_below_goal_p = (num_samples_to_goal > (3 * dynamic_heap_count_data_t::sample_size)); - dprintf (6666, ("[CHP1] it'll take ((%.3f + %.3f - %.3f) / %.3f = %d) samples to get to target, %s", - target_tcp, below_target_tcp_slope, median_throughput_cost_percent, below_target_tcp_slope, - num_samples_to_goal, (far_below_goal_p ? "shrink" : "no change"))); - - if (far_below_goal_p) - { - // We could be in a situation where the slope changes directions but since we only compute one number, we take another look at - // the samples to make a better assessment by looking at the highest tcps and if their average is close to target, we don't shrink. - // - // TODO - we only check this when the slope is going up but since this includes the situation where the slope changes directions - // we should really be checking this regardless of the slope to handle that. - float highest_avg_tcp = 0.0; - int highest_count = dynamic_heap_count_data.highest_avg_recorded_tcp (below_target_tcp_count, avg_recorded_tcp, &highest_avg_tcp); - float highest_count_pct = (float)highest_count / (float)below_target_tcp_count; - - shrink_p = (highest_count_pct < 0.3) || (highest_avg_tcp < (target_tcp * 0.8)); - dprintf (6666, ("[CHP1-2] %d samples were above avg (%.3f%%), their avg is %.3f (%s)", - highest_count, (highest_count_pct * 100.0), highest_avg_tcp, (shrink_p ? "shrink" : "no change"))); - } - } - } - } - - if (shrink_p && step_down_int && (new_n_heaps > step_down_int)) - { - // TODO - if we see that it wants to shrink by 1 heap too many times, we do want to shrink. - if (step_down_int == 1) - { - step_down_int = 0; - dprintf (6666, ("[CHP1-3] don't shrink if it's just one heap. not worth it")); - } - - new_n_heaps -= step_down_int; - dprintf (6666, ("[CHP1] shrink by %d heaps -> %d", step_down_int, new_n_heaps)); - } - - // Always reinit the buffer as we want to look at the more recent history. - dynamic_heap_count_data.init_recorded_tcp (); - dynamic_heap_count_data.below_target_accumulation = 0; - } - - if (new_n_heaps < n_heaps) - { - dynamic_heap_count_data.last_changed_gc_index = current_gc_index; - dynamic_heap_count_data.last_changed_count = (float)(new_n_heaps - n_heaps); - dynamic_heap_count_data.last_changed_stcp = smoothed_median_throughput_cost_percent; - dprintf (6666, ("[CHP1] setting last changed gc index to %Id, count to %.3f, stcp to %.3f", - dynamic_heap_count_data.last_changed_gc_index, dynamic_heap_count_data.last_changed_count, dynamic_heap_count_data.last_changed_stcp)); - - if (dynamic_heap_count_data.inc_failure_count) - { - dprintf (6666, ("[CHP1] shrink, reset inc failure count (was %d)", dynamic_heap_count_data.inc_failure_count)); - dynamic_heap_count_data.inc_failure_count = 0; - } - } + dprintf (6666, ("[CHP2] tcp %.3f > 10, %d + %d = %d", median_gen2_tcp_percent, n_heaps, step_up, (n_heaps + step_up))); } + new_n_heaps += step_up; } - - if ((new_n_heaps == n_heaps) && !process_eph_samples_p && process_gen2_samples_p) + // if we can save at least 1% more in time than we spend in space, increase number of heaps + else if ((tcp_reduction_per_step_up - scp_increase_per_step_up) >= 1.0f) { - // The gen2 samples only serve as a backstop so this is quite crude. - if (median_gen2_tcp_percent > target_gen2_tcp) - { - float step_up_percent = log_with_base ((median_gen2_tcp_percent - target_gen2_tcp + log_base), log_base); - float step_up_float = (float)(step_up_percent / 100.0 * actual_n_max_heaps); - new_n_heaps += (int)step_up_float; - new_n_heaps = min (new_n_heaps, actual_n_max_heaps); - dprintf (6666, ("[CHP2-0] gen2 tcp: %.3f, inc by %.3f%% = %d, %d -> %d", median_gen2_tcp_percent, step_up_percent, (int)step_up_float, n_heaps, new_n_heaps)); - - if ((new_n_heaps < actual_n_max_heaps) && dynamic_heap_count_data.is_close_to_max (new_n_heaps, actual_n_max_heaps)) - { - dprintf (6666, ("[CHP2-1] %d is close to max heaps %d, grow to max", new_n_heaps, actual_n_max_heaps)); - new_n_heaps = actual_n_max_heaps; - } - } - else if ((dynamic_heap_count_data.last_processed_stcp < 1.0) && - (median_gen2_tcp_percent < (target_gen2_tcp / 2)) && - (scp_decrease_per_step_down - tcp_increase_per_step_down >= 1.0f)) - { - new_n_heaps -= step_down; - dprintf (6666, ("[CHP3-0] last eph stcp: %.3f, gen2 tcp: %.3f, dec by %d, %d -> %d", - dynamic_heap_count_data.last_processed_stcp, median_gen2_tcp_percent, step_down, n_heaps, new_n_heaps)); - } + dprintf (6666, ("[CHP3] % .3f - % .3f = % .3f, % d + % d = % d", + tcp_reduction_per_step_up, scp_increase_per_step_up, (tcp_reduction_per_step_up - scp_increase_per_step_up), + n_heaps, step_up, (n_heaps + step_up))); + new_n_heaps += step_up; + } + // if we can save at least 1% more in space than we spend in time, decrease number of heaps + else if ((smoothed_median_throughput_cost_percent < 1.0f) && + (median_gen2_tcp_percent < 5.0f) && + ((scp_decrease_per_step_down - tcp_increase_per_step_down) >= 1.0f)) + { + dprintf (6666, ("[CHP4] stcp %.3f tcp %.3f, %.3f - %.3f = %.3f, %d + %d = %d", + smoothed_median_throughput_cost_percent, median_gen2_tcp_percent, + scp_decrease_per_step_down, tcp_increase_per_step_down, (scp_decrease_per_step_down - tcp_increase_per_step_down), + n_heaps, step_up, (n_heaps + step_up))); + new_n_heaps -= step_down; } assert (new_n_heaps >= 1); - assert (new_n_heaps <= actual_n_max_heaps); - + assert (new_n_heaps <= n_max_heaps); #endif //STRESS_DYNAMIC_HEAP_COUNT // store data used for decision to emit in ETW event @@ -25719,28 +25350,13 @@ int gc_heap::calculate_new_heap_count () dynamic_heap_count_data.scp_decrease_per_step_down ); - if (process_eph_samples_p) - { - dprintf (6666, ("processed eph samples, updating processed %Id -> %Id", dynamic_heap_count_data.processed_samples_count, dynamic_heap_count_data.current_samples_count)); - dynamic_heap_count_data.processed_samples_count = dynamic_heap_count_data.current_samples_count; - } - - if (process_gen2_samples_p) - { - dprintf (6666, ("processed gen2 samples, updating processed %Id -> %Id", dynamic_heap_count_data.processed_gen2_samples_count, dynamic_heap_count_data.current_gen2_samples_count)); - dynamic_heap_count_data.processed_gen2_samples_count = dynamic_heap_count_data.current_gen2_samples_count; - } + dynamic_heap_count_data.prev_num_completed_gcs = num_completed_gcs; if (new_n_heaps != n_heaps) { - dprintf (6666, ("GC#%Id should change! %d->%d (%s)", - VolatileLoadWithoutBarrier (&settings.gc_index), n_heaps, new_n_heaps, ((n_heaps < new_n_heaps) ? "INC" : "DEC"))); + dprintf (6666, ("should change! %d->%d", n_heaps, new_n_heaps)); dynamic_heap_count_data.heap_count_to_change_to = new_n_heaps; dynamic_heap_count_data.should_change_heap_count = true; - dynamic_heap_count_data.init_recorded_tcp (); - dynamic_heap_count_data.below_target_accumulation = 0; - dynamic_heap_count_data.first_below_target_gc_index = current_gc_index; - dprintf (6666, ("CHANGING HC, resetting tcp index, below target")); } return new_n_heaps; @@ -25773,7 +25389,7 @@ void gc_heap::check_heap_count () if (dynamic_heap_count_data.new_n_heaps != n_heaps) { - dprintf (6666, ("prep to change from %d to %d at GC#%Id", n_heaps, dynamic_heap_count_data.new_n_heaps, VolatileLoadWithoutBarrier (&settings.gc_index))); + dprintf (6666, ("prep to change from %d to %d", n_heaps, dynamic_heap_count_data.new_n_heaps)); if (!prepare_to_change_heap_count (dynamic_heap_count_data.new_n_heaps)) { // we don't have sufficient resources - reset the new heap count @@ -25783,15 +25399,11 @@ void gc_heap::check_heap_count () if (dynamic_heap_count_data.new_n_heaps == n_heaps) { - dynamic_heap_count_data.last_changed_gc_index = 0; - dynamic_heap_count_data.last_changed_count = 0.0; - - dynamic_heap_count_data.processed_samples_count = dynamic_heap_count_data.current_samples_count; - dynamic_heap_count_data.processed_gen2_samples_count = dynamic_heap_count_data.current_gen2_samples_count; + // heap count stays the same, no work to do + dynamic_heap_count_data.prev_num_completed_gcs = get_num_completed_gcs (); dynamic_heap_count_data.should_change_heap_count = false; - dprintf (6666, ("heap count stays the same %d, no work to do, set processed sample count to %Id", - dynamic_heap_count_data.new_n_heaps, dynamic_heap_count_data.current_samples_count)); + dprintf (6666, ("heap count stays the same %d, no work to do, set prev completed to %Id", dynamic_heap_count_data.new_n_heaps, dynamic_heap_count_data.prev_num_completed_gcs)); return; } @@ -25831,14 +25443,17 @@ void gc_heap::check_heap_count () int old_n_heaps = n_heaps; + (dynamic_heap_count_data.heap_count_change_count)++; change_heap_count (dynamic_heap_count_data.new_n_heaps); GCToEEInterface::RestartEE(TRUE); dprintf (9999, ("h0 restarted EE")); - dynamic_heap_count_data.smoothed_median_throughput_cost_percent = 0.0; + // we made changes to the heap count that will change the overhead, + // so change the smoothed overhead to reflect that + dynamic_heap_count_data.smoothed_median_throughput_cost_percent = dynamic_heap_count_data.smoothed_median_throughput_cost_percent / n_heaps * old_n_heaps; - dprintf (6666, ("h0 finished changing, set should change to false!\n")); + dprintf (6666, ("h0 finished changing, set should change to false!")); dynamic_heap_count_data.should_change_heap_count = false; } @@ -25978,8 +25593,6 @@ bool gc_heap::prepare_to_change_heap_count (int new_n_heaps) bool gc_heap::change_heap_count (int new_n_heaps) { - uint64_t start_time = 0; - dprintf (9999, ("BEG heap%d changing %d->%d", heap_number, n_heaps, new_n_heaps)); // use this variable for clarity - n_heaps will change during the transition @@ -26004,9 +25617,11 @@ bool gc_heap::change_heap_count (int new_n_heaps) assert (dynamic_heap_count_data.new_n_heaps != old_n_heaps); + dprintf (9999, ("Waiting h0 heap%d changing %d->%d", heap_number, n_heaps, new_n_heaps)); + if (heap_number == 0) { - start_time = GetHighPrecisionTimeStamp (); + dprintf (3, ("switching heap count from %d to %d heaps", old_n_heaps, new_n_heaps)); // spread finalization data out to heaps coming into service // if this step fails, we can still continue @@ -26212,7 +25827,6 @@ bool gc_heap::change_heap_count (int new_n_heaps) gc_t_join.restart (); } } - #ifdef BACKGROUND_GC // there should be no items in the bgc_alloc_lock bgc_alloc_lock->check(); @@ -26223,31 +25837,23 @@ bool gc_heap::change_heap_count (int new_n_heaps) { // compute the total budget per generation over the old heaps // and figure out what the new budget per heap is - ptrdiff_t new_alloc_per_heap[total_generation_count]; - size_t desired_alloc_per_heap[total_generation_count]; + ptrdiff_t budget_per_heap[total_generation_count]; for (int gen_idx = 0; gen_idx < total_generation_count; gen_idx++) { - ptrdiff_t total_new_alloc = 0; - size_t total_desired_alloc = 0; + ptrdiff_t total_budget = 0; for (int i = 0; i < old_n_heaps; i++) { gc_heap* hp = g_heaps[i]; dynamic_data* dd = hp->dynamic_data_of (gen_idx); - total_new_alloc += dd_new_allocation (dd); - total_desired_alloc += dd_desired_allocation (dd); + total_budget += dd_new_allocation (dd); } // distribute the total budget for this generation over all new heaps if we are increasing heap count, // but keep the budget per heap if we are decreasing heap count int max_n_heaps = max (old_n_heaps, new_n_heaps); - new_alloc_per_heap[gen_idx] = Align (total_new_alloc / max_n_heaps, get_alignment_constant (gen_idx <= max_generation)); - desired_alloc_per_heap[gen_idx] = Align (total_desired_alloc / max_n_heaps, get_alignment_constant (gen_idx <= max_generation)); - size_t allocated_in_budget = total_desired_alloc - total_new_alloc; - dprintf (6666, ("g%d: total budget %zd (%zd / heap), left in budget: %zd (%zd / heap), (allocated %Id, %.3f%%), min %zd", - gen_idx, total_desired_alloc, desired_alloc_per_heap[gen_idx], - total_new_alloc, new_alloc_per_heap[gen_idx], - allocated_in_budget, ((double)allocated_in_budget * 100.0 / (double)total_desired_alloc), - dd_min_size (g_heaps[0]->dynamic_data_of (gen_idx)))); + budget_per_heap[gen_idx] = Align (total_budget/max_n_heaps, get_alignment_constant (gen_idx <= max_generation)); + + dprintf (6666, ("g%d: total budget: %zd budget per heap: %zd", gen_idx, total_budget, budget_per_heap[gen_idx])); } // distribute the new budget per heap over the new heaps @@ -26258,10 +25864,10 @@ bool gc_heap::change_heap_count (int new_n_heaps) for (int gen_idx = 0; gen_idx < total_generation_count; gen_idx++) { - // distribute the total leftover budget over all heaps. + // distribute the total budget over all heaps, but don't go below the min budget dynamic_data* dd = hp->dynamic_data_of (gen_idx); - dd_new_allocation (dd) = new_alloc_per_heap[gen_idx]; - dd_desired_allocation (dd) = max (desired_alloc_per_heap[gen_idx], dd_min_size (dd)); + dd_new_allocation (dd) = max (budget_per_heap[gen_idx], (ptrdiff_t)dd_min_size (dd)); + dd_desired_allocation (dd) = dd_new_allocation (dd); // recompute dd_fragmentation and dd_current_size generation* gen = hp->generation_of (gen_idx); @@ -26270,11 +25876,10 @@ bool gc_heap::change_heap_count (int new_n_heaps) assert (gen_size >= dd_fragmentation (dd)); dd_current_size (dd) = gen_size - dd_fragmentation (dd); - dprintf (3, ("h%d g%d: budget: %zd, left in budget: %zd, %zd generation_size: %zd fragmentation: %zd current_size: %zd", + dprintf (6666, ("h%d g%d: new allocation: %zd generation_size: %zd fragmentation: %zd current_size: %zd", i, gen_idx, - desired_alloc_per_heap[gen_idx], - new_alloc_per_heap[gen_idx], + dd_new_allocation (dd), gen_size, dd_fragmentation (dd), dd_current_size (dd))); @@ -26311,11 +25916,6 @@ bool gc_heap::change_heap_count (int new_n_heaps) } } - if (heap_number == 0) - { - change_heap_count_time = GetHighPrecisionTimeStamp() - start_time; - } - return true; } @@ -48805,10 +48405,6 @@ HRESULT GCHeap::Initialize() // This needs to be different from our initial heap count so we can make sure we wait for // the idle threads correctly in gc_thread_function. gc_heap::dynamic_heap_count_data.last_n_heaps = 0; - // This should be adjusted based on the target tcp. See comments in gcpriv.h - gc_heap::dynamic_heap_count_data.below_target_threshold = 10.0; - gc_heap::dynamic_heap_count_data.inc_recheck_threshold = 5; - gc_heap::dynamic_heap_count_data.dec_failure_recheck_threshold = 5; } #endif //DYNAMIC_HEAP_COUNT GCScan::GcRuntimeStructuresValid (TRUE); diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 788cbff9f5e507..71dc19b9f0c676 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -2556,6 +2556,8 @@ class gc_heap // re-initialize a heap in preparation to putting it back into service PER_HEAP_METHOD void recommission_heap(); + PER_HEAP_ISOLATED_METHOD size_t get_num_completed_gcs(); + PER_HEAP_ISOLATED_METHOD int calculate_new_heap_count(); // check if we should change the heap count @@ -4236,166 +4238,21 @@ class gc_heap struct dynamic_heap_count_data_t { static const int sample_size = 3; - static const int recorded_tcp_array_size = 64; struct sample { uint64_t elapsed_between_gcs; // time between gcs in microseconds (this should really be between_pauses) uint64_t gc_pause_time; // pause time for this GC uint64_t msl_wait_time; - size_t gc_survived_size; }; uint32_t sample_index; sample samples[sample_size]; - - size_t current_samples_count; - size_t processed_samples_count; - - // - // We need to observe the history of tcp's so record them in a small buffer. - // - float recorded_tcp_rearranged[recorded_tcp_array_size]; - float recorded_tcp[recorded_tcp_array_size]; - int recorded_tcp_index; - int total_recorded_tcp; - - int add_to_recorded_tcp (float tcp) - { - total_recorded_tcp++; - - recorded_tcp[recorded_tcp_index] = tcp; - recorded_tcp_index++; - if (recorded_tcp_index == recorded_tcp_array_size) - { - recorded_tcp_index = 0; - } - - return recorded_tcp_index; - } - - int rearrange_recorded_tcp () - { - int count = recorded_tcp_array_size; - int copied_count = 0; - - if (total_recorded_tcp >= recorded_tcp_array_size) - { - int earlier_entry_size = recorded_tcp_array_size - recorded_tcp_index; - memcpy (recorded_tcp_rearranged, (recorded_tcp + recorded_tcp_index), (earlier_entry_size * sizeof (float))); - - copied_count = earlier_entry_size; - } - - if (recorded_tcp_index) - { - memcpy ((recorded_tcp_rearranged + copied_count), recorded_tcp, (recorded_tcp_index * sizeof (float))); - copied_count += recorded_tcp_index; - } - - return copied_count; - } - - int highest_avg_recorded_tcp (int count, float avg, float* highest_avg) - { - float highest_sum = 0.0; - int highest_count = 0; - - for (int i = 0; i < count; i++) - { - if (recorded_tcp_rearranged[i] > avg) - { - highest_count++; - highest_sum += recorded_tcp_rearranged[i]; - } - } - - if (highest_count) - { - *highest_avg = highest_sum / highest_count; - } - - return highest_count; - } - - void init_recorded_tcp () - { - total_recorded_tcp = 0; - recorded_tcp_index = 0; - dprintf (6666, ("INIT tcp buffer")); - } - - int get_recorded_tcp_count () { return total_recorded_tcp; } - - // - // Maintain some info about last time we did change heap count. - // - size_t last_changed_gc_index; - // This is intentionally kept as a float for precision. - float last_changed_count; - float last_changed_stcp; - - // - // For tuning above/below target tcp. - // - // If we just increased the heap count and immediately need to grow again, that counts as a failure. - // The higher the failure count, the more aggressive we should grow. - int inc_failure_count; - - // If we are trending up and the tcp is already close enough to target, we need this many samples - // before we adjust. - int inc_recheck_threshold; - - // If we shrink and the stcp doesn't change much, that counts as a failure. For the below target case - // it's fine to stay here for a while. Either it'll naturally change and break out of this situation - // or we wait for a while before we re-evaluate. How long we wait is defined by dec_recheck_threshold - // each time our calculation tells us to shrink. - int dec_failure_count; - int dec_failure_recheck_threshold; - - // If we continue to be below target for an extended period of time, ie, we've accumulated more than - // below_target_threshold, we want to reduce the heap count. - float below_target_accumulation; - float below_target_threshold; - - // Currently only used for dprintf. - size_t first_below_target_gc_index; - - float get_range_upper (float t) - { - return (t * 1.2f); - } - - bool is_tcp_in_range (float diff_pct, float slope) - { - return ((diff_pct <= 0.2) && (diff_pct >= -0.2) && (slope <= 0.1) && (slope >= -0.1)); - } - - bool is_close_to_max (int new_n, int max) - { - return ((max - new_n) <= (max / 10)); - } - - // - // gen2 GCs are handled separately only as a backstop. - // - struct gen2_sample - { - // Recording the gen2 GC indices so we know how far apart they are. Currently unused - // but we should consider how much value there is if they are very far apart. - size_t gc_index; - // This is (gc_elapsed_time / time inbetween this and the last gen2 GC) - float gc_percent; - }; + size_t prev_num_completed_gcs; uint32_t gen2_sample_index; - gen2_sample gen2_samples[sample_size]; - - size_t current_gen2_samples_count; - size_t processed_gen2_samples_count; - - // This records the stcp last time we processed ephemeral samples. We use it - float last_processed_stcp; + // This is (gc_elapsed_time / time inbetween this and the last gen2 GC) + float gen2_gc_percents[sample_size]; float median_throughput_cost_percent; // estimated overhead of allocator + gc float smoothed_median_throughput_cost_percent; // exponentially smoothed version @@ -4414,13 +4271,14 @@ class gc_heap bool should_change_heap_count; int heap_count_to_change_to; + int heap_count_change_count; #ifdef STRESS_DYNAMIC_HEAP_COUNT int lowest_heap_with_msl_uoh; #endif //STRESS_DYNAMIC_HEAP_COUNT float get_median_gen2_gc_percent() { - return median_of_3 (gen2_samples[0].gc_percent, gen2_samples[1].gc_percent, gen2_samples[2].gc_percent); + return median_of_3 (gen2_gc_percents[0], gen2_gc_percents[1], gen2_gc_percents[2]); } }; PER_HEAP_ISOLATED_FIELD_MAINTAINED dynamic_heap_count_data_t dynamic_heap_count_data; @@ -4617,9 +4475,6 @@ class gc_heap // at the beginning of a BGC and the PM triggered full GCs // fall into this case. PER_HEAP_ISOLATED_FIELD_DIAG_ONLY uint64_t suspended_start_time; - // Right now this is diag only but may be used functionally later. - PER_HEAP_ISOLATED_FIELD_DIAG_ONLY uint64_t change_heap_count_time; - // TEMP END PER_HEAP_ISOLATED_FIELD_DIAG_ONLY uint64_t end_gc_time; PER_HEAP_ISOLATED_FIELD_DIAG_ONLY uint64_t total_suspended_time; PER_HEAP_ISOLATED_FIELD_DIAG_ONLY uint64_t process_start_time; diff --git a/src/coreclr/gc/unix/cgroup.cpp b/src/coreclr/gc/unix/cgroup.cpp index 3be77727363920..d2ad75bbf787b5 100644 --- a/src/coreclr/gc/unix/cgroup.cpp +++ b/src/coreclr/gc/unix/cgroup.cpp @@ -39,6 +39,7 @@ Module Name: #endif #define CGROUP2_SUPER_MAGIC 0x63677270 +#define TMPFS_MAGIC 0x01021994 #define PROC_MOUNTINFO_FILENAME "/proc/self/mountinfo" #define PROC_CGROUP_FILENAME "/proc/self/cgroup" @@ -128,16 +129,12 @@ class CGroup if (result != 0) return 0; - if (stats.f_type == CGROUP2_SUPER_MAGIC) + switch (stats.f_type) { - return 2; - } - else - { - // Assume that if /sys/fs/cgroup exists and the file system type is not cgroup2fs, - // it is cgroup v1. Typically the file system type is tmpfs, but other values have - // been seen in the wild. - return 1; + case TMPFS_MAGIC: return 1; + case CGROUP2_SUPER_MAGIC: return 2; + default: + return 0; } #endif } diff --git a/src/coreclr/ilasm/asmparse.y b/src/coreclr/ilasm/asmparse.y index c9861d58d79740..73ef9a892b5efb 100644 --- a/src/coreclr/ilasm/asmparse.y +++ b/src/coreclr/ilasm/asmparse.y @@ -486,7 +486,7 @@ typarAttrib : '+' { $$ = gpCovariant; | '-' { $$ = gpContravariant; } | CLASS_ { $$ = gpReferenceTypeConstraint; } | VALUETYPE_ { $$ = gpNotNullableValueTypeConstraint; } - | BYREFLIKE_ { $$ = gpAllowByRefLike; } + | BYREFLIKE_ { $$ = gpAcceptByRefLike; } | _CTOR { $$ = gpDefaultConstructorConstraint; } | FLAGS_ '(' int32 ')' { $$ = (CorGenericParamAttr)$3; } ; diff --git a/src/coreclr/ilasm/prebuilt/asmparse.cpp b/src/coreclr/ilasm/prebuilt/asmparse.cpp index 08f686f290187e..6bf91f56c57f44 100644 --- a/src/coreclr/ilasm/prebuilt/asmparse.cpp +++ b/src/coreclr/ilasm/prebuilt/asmparse.cpp @@ -2523,7 +2523,7 @@ case 152: { yyval.int32 = gpNotNullableValueTypeConstraint; } break; case 153: #line 489 "asmparse.y" -{ yyval.int32 = gpAllowByRefLike; } break; +{ yyval.int32 = gpAcceptByRefLike; } break; case 154: #line 490 "asmparse.y" { yyval.int32 = gpDefaultConstructorConstraint; } break; diff --git a/src/coreclr/ildasm/dasm.cpp b/src/coreclr/ildasm/dasm.cpp index 21dff99a381233..be95e36fa5d53f 100644 --- a/src/coreclr/ildasm/dasm.cpp +++ b/src/coreclr/ildasm/dasm.cpp @@ -3081,7 +3081,7 @@ char *DumpGenericPars(_Inout_updates_(SZSTRING_SIZE) char* szString, mdToken tok if ((attr & gpNotNullableValueTypeConstraint) != 0) szptr += sprintf_s(szptr,SZSTRING_REMAINING_SIZE(szptr), "valuetype "); CHECK_REMAINING_SIZE; - if ((attr & gpAllowByRefLike) != 0) + if ((attr & gpAcceptByRefLike) != 0) szptr += sprintf_s(szptr,SZSTRING_REMAINING_SIZE(szptr), "byreflike "); CHECK_REMAINING_SIZE; if ((attr & gpDefaultConstructorConstraint) != 0) diff --git a/src/coreclr/inc/CrstTypes.def b/src/coreclr/inc/CrstTypes.def index 7f94e9e0996a83..3bccb73e03a599 100644 --- a/src/coreclr/inc/CrstTypes.def +++ b/src/coreclr/inc/CrstTypes.def @@ -367,7 +367,7 @@ Crst PendingTypeLoadEntry DomainLocalBlock Exception ExecuteManRangeLock FuncPtrStubs FusionAppCtx GlobalStrLiteralMap HandleTable IbcProfile IJWFixupData IJWHash ISymUnmanagedReader Jit JumpStubCache LoaderHeap - Module ModuleLookupTable PEImage + Module ModuleLookupTable PEImage SecurityStackwalkCache SigConvert SingleUseLock StubDispatchCache StubUnwindInfoHeapSegments SyncBlockCache SystemDomain ThreadIdDispenser ThreadStore TypeIDMap UnresolvedClassLock SameLevelAs PendingTypeLoadEntry @@ -426,6 +426,9 @@ End Crst SaveModuleProfileData End +Crst SecurityStackwalkCache +End + Crst SigConvert AcquiredBefore LoaderHeap End diff --git a/src/coreclr/inc/bitvector.h b/src/coreclr/inc/bitvector.h index 0f17697dddce74..df06b4c75c66ec 100644 --- a/src/coreclr/inc/bitvector.h +++ b/src/coreclr/inc/bitvector.h @@ -32,9 +32,7 @@ #define UNDEF_ASSERTE #endif -#ifndef FEATURE_NATIVEAOT #define USE_BITVECTOR 1 -#endif #if USE_BITVECTOR /* The bitvector class is meant to be a drop in replacement for an integer diff --git a/src/coreclr/inc/check.h b/src/coreclr/inc/check.h index 6951e2a41837b6..c1ac08016d836f 100644 --- a/src/coreclr/inc/check.h +++ b/src/coreclr/inc/check.h @@ -684,9 +684,6 @@ CHECK CheckAligned(UINT value, UINT alignment); CHECK CheckAligned(ULONG value, UINT alignment); #endif CHECK CheckAligned(UINT64 value, UINT alignment); -#ifdef __APPLE__ -CHECK CheckAligned(SIZE_T value, UINT alignment); -#endif CHECK CheckAligned(const void *address, UINT alignment); CHECK CheckOverflow(UINT value1, UINT value2); @@ -694,9 +691,6 @@ CHECK CheckOverflow(UINT value1, UINT value2); CHECK CheckOverflow(ULONG value1, ULONG value2); #endif CHECK CheckOverflow(UINT64 value1, UINT64 value2); -#ifdef __APPLE__ -CHECK CheckOverflow(SIZE_T value1, SIZE_T value2); -#endif CHECK CheckOverflow(PTR_CVOID address, UINT offset); #if defined(_MSC_VER) CHECK CheckOverflow(const void *address, ULONG offset); @@ -708,17 +702,11 @@ CHECK CheckUnderflow(UINT value1, UINT value2); CHECK CheckUnderflow(ULONG value1, ULONG value2); #endif CHECK CheckUnderflow(UINT64 value1, UINT64 value2); -#ifdef __APPLE__ -CHECK CheckUnderflow(SIZE_T value1, SIZE_T value2); -#endif CHECK CheckUnderflow(const void *address, UINT offset); #if defined(_MSC_VER) CHECK CheckUnderflow(const void *address, ULONG offset); #endif CHECK CheckUnderflow(const void *address, UINT64 offset); -#ifdef __APPLE__ -CHECK CheckUnderflow(const void *address, SIZE_T offset); -#endif CHECK CheckUnderflow(const void *address, void *address2); CHECK CheckZeroedMemory(const void *memory, SIZE_T size); diff --git a/src/coreclr/inc/check.inl b/src/coreclr/inc/check.inl index 34a2956d1be6e2..9296c48f7a7a3d 100644 --- a/src/coreclr/inc/check.inl +++ b/src/coreclr/inc/check.inl @@ -156,15 +156,6 @@ inline CHECK CheckAligned(UINT64 value, UINT alignment) CHECK_OK; } -#ifdef __APPLE__ -inline CHECK CheckAligned(SIZE_T value, UINT alignment) -{ - STATIC_CONTRACT_WRAPPER; - CHECK(AlignmentTrim(value, alignment) == 0); - CHECK_OK; -} -#endif - inline CHECK CheckAligned(const void *address, UINT alignment) { STATIC_CONTRACT_WRAPPER; @@ -192,14 +183,6 @@ inline CHECK CheckOverflow(UINT64 value1, UINT64 value2) CHECK_OK; } -#ifdef __APPLE__ -inline CHECK CheckOverflow(SIZE_T value1, SIZE_T value2) -{ - CHECK(value1 + value2 >= value1); - CHECK_OK; -} -#endif - inline CHECK CheckOverflow(PTR_CVOID address, UINT offset) { TADDR targetAddr = dac_cast(address); @@ -271,15 +254,6 @@ inline CHECK CheckUnderflow(UINT64 value1, UINT64 value2) CHECK_OK; } -#ifdef __APPLE__ -inline CHECK CheckUnderflow(SIZE_T value1, SIZE_T value2) -{ - CHECK(value1 - value2 <= value1); - - CHECK_OK; -} -#endif - inline CHECK CheckUnderflow(const void *address, UINT offset) { #if POINTER_BITS == 32 @@ -316,20 +290,6 @@ inline CHECK CheckUnderflow(const void *address, UINT64 offset) CHECK_OK; } -#ifdef __APPLE__ -inline CHECK CheckUnderflow(const void *address, SIZE_T offset) -{ -#if POINTER_BITS == 32 - CHECK(offset >> 32 == 0); - CHECK((UINT) (SIZE_T) address - (UINT) offset <= (UINT) (SIZE_T) address); -#else - CHECK((UINT64) address - offset <= (UINT64) address); -#endif - - CHECK_OK; -} -#endif - inline CHECK CheckUnderflow(const void *address, void *address2) { #if POINTER_BITS == 32 diff --git a/src/coreclr/inc/clr_std/type_traits b/src/coreclr/inc/clr_std/type_traits index ba007c32d9fef2..12af99d5c4fee1 100644 --- a/src/coreclr/inc/clr_std/type_traits +++ b/src/coreclr/inc/clr_std/type_traits @@ -358,7 +358,7 @@ namespace std // On Unix 'long' is a 64-bit type (same as __int64) and the following two definitions // conflict with _Is_integral and _Is_integral. -#if !defined(HOST_UNIX) || defined(__APPLE__) +#ifndef HOST_UNIX template<> struct _Is_integral : true_type @@ -370,7 +370,7 @@ namespace std : true_type { // determine whether _Ty is integral }; -#endif /* !HOST_UNIX || __APPLE__ */ +#endif /* HOST_UNIX */ #if _HAS_CHAR16_T_LANGUAGE_SUPPORT template<> diff --git a/src/coreclr/inc/clrconfignocache.h b/src/coreclr/inc/clrconfignocache.h index 01675a24201d14..f75504a2289af0 100644 --- a/src/coreclr/inc/clrconfignocache.h +++ b/src/coreclr/inc/clrconfignocache.h @@ -46,8 +46,6 @@ class CLRConfigNoCache { return false; } - - result = (DWORD)rawResult; bool fSuccess = endPtr != _value; return fSuccess; } diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index d9571f0776456f..30956bf4a67418 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -259,7 +259,7 @@ RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_legacyCorruptedStateExceptionsPolicy, W("le CONFIG_DWORD_INFO(INTERNAL_SuppressLostExceptionTypeAssert, W("SuppressLostExceptionTypeAssert"), 0, "") RETAIL_CONFIG_DWORD_INFO(INTERNAL_UseEntryPointFilter, W("UseEntryPointFilter"), 0, "") RETAIL_CONFIG_DWORD_INFO(INTERNAL_Corhost_Swallow_Uncaught_Exceptions, W("Corhost_Swallow_Uncaught_Exceptions"), 0, "") -RETAIL_CONFIG_DWORD_INFO(EXTERNAL_LegacyExceptionHandling, W("LegacyExceptionHandling"), 1, "Enable legacy exception handling."); +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableNewExceptionHandling, W("EnableNewExceptionHandling"), 0, "Enable new exception handling."); /// diff --git a/src/coreclr/inc/clrnt.h b/src/coreclr/inc/clrnt.h index fb7d8102545625..56245ea46f25e7 100644 --- a/src/coreclr/inc/clrnt.h +++ b/src/coreclr/inc/clrnt.h @@ -184,17 +184,12 @@ RtlVirtualUnwind_Unsafe( #ifdef HOST_X86 typedef struct _RUNTIME_FUNCTION { DWORD BeginAddress; - // NOTE: R2R doesn't include EndAddress (see docs/design/coreclr/botr/readytorun-format.md). - // NativeAOT does include the EndAddress because the Microsoft linker expects it. In NativeAOT - // the info is generated in the managed ObjectWriter, so the structures don't have to match. - // DWORD EndAddress; DWORD UnwindData; } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; typedef struct _DISPATCHER_CONTEXT { _EXCEPTION_REGISTRATION_RECORD* RegistrationPointer; } DISPATCHER_CONTEXT, *PDISPATCHER_CONTEXT; - #endif // HOST_X86 #endif // !HOST_UNIX @@ -212,7 +207,7 @@ RtlpGetFunctionEndAddress ( _In_ TADDR ImageBase ) { - PUNWIND_INFO pUnwindInfo = (PUNWIND_INFO)(ImageBase + FunctionEntry->UnwindData); + PTR_UNWIND_INFO pUnwindInfo = (PTR_UNWIND_INFO)(ImageBase + FunctionEntry->UnwindData); return FunctionEntry->BeginAddress + pUnwindInfo->FunctionLength; } @@ -223,7 +218,10 @@ RtlpGetFunctionEndAddress ( #define RUNTIME_FUNCTION__SetUnwindInfoAddress(prf, addr) do { (prf)->UnwindData = (addr); } while(0) #ifdef HOST_X86 +EXTERN_C +NTSYSAPI PEXCEPTION_ROUTINE +NTAPI RtlVirtualUnwind ( _In_ DWORD HandlerType, _In_ DWORD ImageBase, diff --git a/src/coreclr/inc/clrtypes.h b/src/coreclr/inc/clrtypes.h index 9094e4932a2527..19e9720b34d90b 100644 --- a/src/coreclr/inc/clrtypes.h +++ b/src/coreclr/inc/clrtypes.h @@ -370,15 +370,6 @@ inline UINT64 AlignDown(UINT64 value, UINT alignment) return (value&~(UINT64)(alignment-1)); } -#ifdef __APPLE__ -inline SIZE_T AlignDown(SIZE_T value, UINT alignment) -{ - STATIC_CONTRACT_LEAF; - STATIC_CONTRACT_SUPPORTS_DAC; - return (value&~(SIZE_T)(alignment-1)); -} -#endif // __APPLE__ - inline UINT AlignmentPad(UINT value, UINT alignment) { STATIC_CONTRACT_WRAPPER; diff --git a/src/coreclr/inc/cordebuginfo.h b/src/coreclr/inc/cordebuginfo.h index b3125060f308eb..09fb19f4bb8734 100644 --- a/src/coreclr/inc/cordebuginfo.h +++ b/src/coreclr/inc/cordebuginfo.h @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // -// Keep in sync with src\coreclr\tools\Common\JitInterface\CorInfoTypes.VarInfo.cs +// Keep in sync with llvm/tools/objwriter/cordebuginfo.h in current objwriter branch in https://github.com/dotnet/llvm-project repo // /**********************************************************************************/ diff --git a/src/coreclr/inc/corhdr.h b/src/coreclr/inc/corhdr.h index c12c1cfdd4f710..3f67b33da9162a 100644 --- a/src/coreclr/inc/corhdr.h +++ b/src/coreclr/inc/corhdr.h @@ -847,7 +847,7 @@ typedef enum CorGenericParamAttr gpReferenceTypeConstraint = 0x0004, // type argument must be a reference type gpNotNullableValueTypeConstraint = 0x0008, // type argument must be a value type but not Nullable gpDefaultConstructorConstraint = 0x0010, // type argument must have a public default constructor - gpAllowByRefLike = 0x0020, // type argument can be ByRefLike + gpAcceptByRefLike = 0x0020, // type argument can be ByRefLike } CorGenericParamAttr; // structures and enums moved from COR.H diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index b9ebaa08546014..86680d6e20c91e 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -572,10 +572,7 @@ enum CorInfoHelpFunc CORINFO_HELP_INIT_PINVOKE_FRAME, // initialize an inlined PInvoke Frame for the JIT-compiler CORINFO_HELP_MEMSET, // Init block of memory - CORINFO_HELP_MEMZERO, // Init block of memory with zeroes CORINFO_HELP_MEMCPY, // Copy block of memory - CORINFO_HELP_NATIVE_MEMSET, // Init block of memory using native memset (not safe for pDst being null, - // not safe for unbounded size, does not trigger GC) CORINFO_HELP_RUNTIMEHANDLE_METHOD, // determine a type/field/method handle at run-time CORINFO_HELP_RUNTIMEHANDLE_METHOD_LOG, // determine a type/field/method handle at run-time, with IBC logging @@ -1982,16 +1979,6 @@ enum class GetTypeLayoutResult Failure, }; -#define MAX_SWIFT_LOWERED_ELEMENTS 4 - -struct CORINFO_SWIFT_LOWERING -{ - bool byReference; - CorInfoType loweredElements[MAX_SWIFT_LOWERED_ELEMENTS]; - uint32_t offsets[MAX_SWIFT_LOWERED_ELEMENTS]; - size_t numLoweredElements; -}; - #define SIZEOF__CORINFO_Object TARGET_POINTER_SIZE /* methTable */ #define CORINFO_Array_MaxLength 0x7FFFFFC7 @@ -2071,7 +2058,7 @@ class ICorStaticInfo // Example of a scenario addressed by notifyMethodInfoUsage: // 1) Crossgen (with --opt-cross-module=MyLib) attempts to inline a call from MyLib.dll into MyApp.dll // and realizes that the call always throws. - // 2) JIT aborts the inlining attempt and marks the call as no-return instead. The code that follows the call is + // 2) JIT aborts the inlining attempt and marks the call as no-return instead. The code that follows the call is // replaced with a breakpoint instruction that is expected to be unreachable. // 3) MyLib is updated to a new version so it's no longer within the same version bubble with MyApp.dll // and the new version of the call no longer throws and does some work. @@ -2231,7 +2218,6 @@ class ICorStaticInfo // should be looked up at runtime. virtual void expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN * pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT * pResult) = 0; // Is the given type in System.Private.Corelib and marked with IntrinsicAttribute? @@ -2648,7 +2634,6 @@ class ICorStaticInfo CORINFO_RESOLVED_TOKEN * pResolvedToken, CORINFO_LOOKUP_KIND * pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP * pLookup ) = 0; @@ -2656,7 +2641,6 @@ class ICorStaticInfo CORINFO_RESOLVED_TOKEN * pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP * pLookup ) = 0; @@ -2932,13 +2916,6 @@ class ICorStaticInfo uint32_t numMappings // [IN] Number of rich mappings ) = 0; - // Report back some metadata about the compilation to the EE -- for - // example, metrics about the compilation. - virtual void reportMetadata( - const char* key, - const void* value, - size_t length) = 0; - /*-------------------------- Misc ---------------------------------------*/ // Used to allocate memory that needs to handed to the EE. @@ -3083,9 +3060,6 @@ class ICorStaticInfo SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* structPassInRegDescPtr /* OUT */ ) = 0; - // Classifies a swift structure into primitives or an implicit byref for ABI purposes. - virtual void getSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering) = 0; - virtual uint32_t getLoongArch64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE cls) = 0; virtual uint32_t getRISCV64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE cls) = 0; }; @@ -3196,7 +3170,6 @@ class ICorDynamicInfo : public ICorStaticInfo virtual void embedGenericHandle( CORINFO_RESOLVED_TOKEN * pResolvedToken, bool fEmbedParent, // `true` - embeds parent type handle of the field/method handle - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT * pResult ) = 0; diff --git a/src/coreclr/inc/crsttypes_generated.h b/src/coreclr/inc/crsttypes_generated.h index 79864b97db018c..70847a5b367fcd 100644 --- a/src/coreclr/inc/crsttypes_generated.h +++ b/src/coreclr/inc/crsttypes_generated.h @@ -107,33 +107,34 @@ enum CrstType CrstRetThunkCache = 89, CrstSavedExceptionInfo = 90, CrstSaveModuleProfileData = 91, - CrstSigConvert = 92, - CrstSingleUseLock = 93, - CrstSpecialStatics = 94, - CrstStackSampler = 95, - CrstStaticBoxInit = 96, - CrstStressLog = 97, - CrstStubCache = 98, - CrstStubDispatchCache = 99, - CrstStubUnwindInfoHeapSegments = 100, - CrstSyncBlockCache = 101, - CrstSyncHashLock = 102, - CrstSystemBaseDomain = 103, - CrstSystemDomain = 104, - CrstSystemDomainDelayedUnloadList = 105, - CrstThreadIdDispenser = 106, - CrstThreadStore = 107, - CrstTieredCompilation = 108, - CrstTypeEquivalenceMap = 109, - CrstTypeIDMap = 110, - CrstUMEntryThunkCache = 111, - CrstUMEntryThunkFreeListLock = 112, - CrstUniqueStack = 113, - CrstUnresolvedClassLock = 114, - CrstUnwindInfoTableLock = 115, - CrstVSDIndirectionCellLock = 116, - CrstWrapperTemplate = 117, - kNumberOfCrstTypes = 118 + CrstSecurityStackwalkCache = 92, + CrstSigConvert = 93, + CrstSingleUseLock = 94, + CrstSpecialStatics = 95, + CrstStackSampler = 96, + CrstStaticBoxInit = 97, + CrstStressLog = 98, + CrstStubCache = 99, + CrstStubDispatchCache = 100, + CrstStubUnwindInfoHeapSegments = 101, + CrstSyncBlockCache = 102, + CrstSyncHashLock = 103, + CrstSystemBaseDomain = 104, + CrstSystemDomain = 105, + CrstSystemDomainDelayedUnloadList = 106, + CrstThreadIdDispenser = 107, + CrstThreadStore = 108, + CrstTieredCompilation = 109, + CrstTypeEquivalenceMap = 110, + CrstTypeIDMap = 111, + CrstUMEntryThunkCache = 112, + CrstUMEntryThunkFreeListLock = 113, + CrstUniqueStack = 114, + CrstUnresolvedClassLock = 115, + CrstUnwindInfoTableLock = 116, + CrstVSDIndirectionCellLock = 117, + CrstWrapperTemplate = 118, + kNumberOfCrstTypes = 119 }; #endif // __CRST_TYPES_INCLUDED @@ -236,6 +237,7 @@ int g_rgCrstLevelMap[] = 4, // CrstRetThunkCache 3, // CrstSavedExceptionInfo 0, // CrstSaveModuleProfileData + 0, // CrstSecurityStackwalkCache 4, // CrstSigConvert 5, // CrstSingleUseLock 0, // CrstSpecialStatics @@ -359,6 +361,7 @@ LPCSTR g_rgCrstNameMap[] = "CrstRetThunkCache", "CrstSavedExceptionInfo", "CrstSaveModuleProfileData", + "CrstSecurityStackwalkCache", "CrstSigConvert", "CrstSingleUseLock", "CrstSpecialStatics", diff --git a/src/coreclr/inc/daccess.h b/src/coreclr/inc/daccess.h index fcd5f5bbf1ff18..699947a02cdd42 100644 --- a/src/coreclr/inc/daccess.h +++ b/src/coreclr/inc/daccess.h @@ -614,7 +614,8 @@ struct DacTableHeader // Define TADDR as a non-pointer value so use of it as a pointer // will not work properly. Define it as unsigned so // pointer comparisons aren't affected by sign. -typedef uintptr_t TADDR; +// This requires special casting to ULONG64 to sign-extend if necessary. +typedef ULONG_PTR TADDR; // TSIZE_T used for counts or ranges that need to span the size of a // target pointer. For cross-plat, this may be different than SIZE_T @@ -806,6 +807,7 @@ struct COR_ILMETHOD* DacGetIlMethod(TADDR methAddr); struct _UNWIND_INFO * DacGetUnwindInfo(TADDR taUnwindInfo); // virtually unwind a CONTEXT out-of-process +struct _KNONVOLATILE_CONTEXT_POINTERS; BOOL DacUnwindStackFrame(T_CONTEXT * pContext, T_KNONVOLATILE_CONTEXT_POINTERS* pContextPointers); #endif // FEATURE_EH_FUNCLETS @@ -2126,7 +2128,7 @@ inline void DACCOP_IGNORE(DacCopWarningCode code, const char * szReasonString) // Declare TADDR as a non-pointer type so that arithmetic // can be done on it directly, as with the DACCESS_COMPILE definition. // This also helps expose pointer usage that may need to be changed. -typedef uintptr_t TADDR; +typedef ULONG_PTR TADDR; typedef void* PTR_VOID; typedef LPVOID* PTR_PTR_VOID; @@ -2373,7 +2375,6 @@ typedef DPTR(int32_t) PTR_int32_t; typedef DPTR(uint32_t) PTR_uint32_t; typedef DPTR(uint64_t) PTR_uint64_t; typedef DPTR(uintptr_t) PTR_uintptr_t; -typedef DPTR(TADDR) PTR_TADDR; #ifndef NATIVEAOT typedef ArrayDPTR(BYTE) PTR_BYTE; @@ -2395,6 +2396,7 @@ typedef DPTR(ULONG64) PTR_ULONG64; typedef DPTR(INT64) PTR_INT64; typedef DPTR(UINT64) PTR_UINT64; typedef DPTR(SIZE_T) PTR_SIZE_T; +typedef DPTR(TADDR) PTR_TADDR; typedef DPTR(int) PTR_int; typedef DPTR(BOOL) PTR_BOOL; typedef DPTR(unsigned) PTR_unsigned; @@ -2432,7 +2434,7 @@ typedef DPTR(IMAGE_TLS_DIRECTORY) PTR_IMAGE_TLS_DIRECTORY; #endif #ifndef NATIVEAOT -#if defined(TARGET_X86) && defined(FEATURE_EH_FUNCLETS) +#if defined(TARGET_X86) && defined(TARGET_UNIX) typedef DPTR(struct _UNWIND_INFO) PTR_UNWIND_INFO; #endif diff --git a/src/coreclr/inc/dacvars.h b/src/coreclr/inc/dacvars.h index 8f710c8fde1255..b632887e86d0f8 100644 --- a/src/coreclr/inc/dacvars.h +++ b/src/coreclr/inc/dacvars.h @@ -124,6 +124,8 @@ DEFINE_DACVAR(PTR_SString, SString__s_Empty, SString::s_Empty) DEFINE_DACVAR(INT32, ArrayBase__s_arrayBoundsZero, ArrayBase::s_arrayBoundsZero) +DEFINE_DACVAR(BOOL, StackwalkCache__s_Enabled, StackwalkCache::s_Enabled) + DEFINE_DACVAR(PTR_JITNotification, dac__g_pNotificationTable, ::g_pNotificationTable) DEFINE_DACVAR(ULONG32, dac__g_dacNotificationFlags, ::g_dacNotificationFlags) DEFINE_DACVAR(PTR_GcNotification, dac__g_pGcNotificationTable, ::g_pGcNotificationTable) diff --git a/src/coreclr/inc/eetwain.h b/src/coreclr/inc/eetwain.h index 71729a9182f3a1..9beca3f3729007 100644 --- a/src/coreclr/inc/eetwain.h +++ b/src/coreclr/inc/eetwain.h @@ -35,8 +35,8 @@ #define USE_GC_INFO_DECODER #endif -#ifdef TARGET_AMD64 -#define HAS_LIGHTUNWIND +#if (defined(TARGET_X86) && !defined(TARGET_UNIX)) || defined(TARGET_AMD64) +#define HAS_QUICKUNWIND #endif #define CHECK_APP_DOMAIN 0 @@ -103,8 +103,6 @@ enum ICodeManagerFlags NoReportUntracked = 0x0080, // EnumGCRefs/EnumerateLiveSlots should *not* include // any untracked slots - - LightUnwind = 0x0100, // Unwind just enough to get return addresses }; //***************************************************************************** @@ -203,7 +201,8 @@ virtual ULONG32 GetStackParameterSize(EECodeInfo* pCodeInfo) = 0; virtual bool UnwindStackFrame(PREGDISPLAY pContext, EECodeInfo *pCodeInfo, unsigned flags, - CodeManState *pState) = 0; + CodeManState *pState, + StackwalkCacheUnwindInfo *pUnwindInfo) = 0; /* Is the function currently at a "GC safe point" ? @@ -426,10 +425,11 @@ bool UnwindStackFrame( PREGDISPLAY pContext, EECodeInfo *pCodeInfo, unsigned flags, - CodeManState *pState); + CodeManState *pState, + StackwalkCacheUnwindInfo *pUnwindInfo); -#ifdef HAS_LIGHTUNWIND -enum LightUnwindFlag +#ifdef HAS_QUICKUNWIND +enum QuickUnwindFlag { UnwindCurrentStackFrame, EnsureCallerStackFrameIsValid @@ -441,11 +441,11 @@ enum LightUnwindFlag */ static -void LightUnwindStackFrame( +void QuickUnwindStackFrame( PREGDISPLAY pRD, - EECodeInfo *pCodeInfo, - LightUnwindFlag flag); -#endif // HAS_LIGHTUNWIND + StackwalkCacheEntry *pCacheEntry, + QuickUnwindFlag flag); +#endif // HAS_QUICKUNWIND /* Is the function currently at a "GC safe point" ? @@ -615,7 +615,7 @@ HRESULT FixContextForEnC(PCONTEXT pCtx, #endif // #ifndef DACCESS_COMPILE #ifdef FEATURE_EH_FUNCLETS - static void EnsureCallerContextIsValid( PREGDISPLAY pRD, EECodeInfo * pCodeInfo = NULL, unsigned flags = 0); + static void EnsureCallerContextIsValid( PREGDISPLAY pRD, StackwalkCacheEntry* pCacheEntry, EECodeInfo * pCodeInfo = NULL ); static size_t GetCallerSp( PREGDISPLAY pRD ); #ifdef TARGET_X86 static size_t GetResumeSp( PCONTEXT pContext ); @@ -629,7 +629,124 @@ HRESULT FixContextForEnC(PCONTEXT pCtx, }; #ifdef TARGET_X86 -#include "gc_unwind_x86.h" +bool UnwindStackFrame(PREGDISPLAY pContext, + EECodeInfo *pCodeInfo, + unsigned flags, + CodeManState *pState, + StackwalkCacheUnwindInfo *pUnwindInfo); + +size_t DecodeGCHdrInfo(GCInfoToken gcInfoToken, + unsigned curOffset, + hdrInfo * infoPtr); +#endif + +/***************************************************************************** + ToDo: Do we want to include JIT/IL/target.h? + */ + +enum regNum +{ + REGI_EAX, REGI_ECX, REGI_EDX, REGI_EBX, + REGI_ESP, REGI_EBP, REGI_ESI, REGI_EDI, + REGI_COUNT, + REGI_NA = REGI_COUNT +}; + +/***************************************************************************** + Register masks + */ + +enum RegMask +{ + RM_EAX = 0x01, + RM_ECX = 0x02, + RM_EDX = 0x04, + RM_EBX = 0x08, + RM_ESP = 0x10, + RM_EBP = 0x20, + RM_ESI = 0x40, + RM_EDI = 0x80, + + RM_NONE = 0x00, + RM_ALL = (RM_EAX|RM_ECX|RM_EDX|RM_EBX|RM_ESP|RM_EBP|RM_ESI|RM_EDI), + RM_CALLEE_SAVED = (RM_EBP|RM_EBX|RM_ESI|RM_EDI), + RM_CALLEE_TRASHED = (RM_ALL & ~RM_CALLEE_SAVED), +}; + +/***************************************************************************** + * + * Helper to extract basic info from a method info block. + */ + +struct hdrInfo +{ + unsigned int methodSize; // native code bytes + unsigned int argSize; // in bytes + unsigned int stackSize; // including callee saved registers + unsigned int rawStkSize; // excluding callee saved registers + ReturnKind returnKind; // The ReturnKind for this method. + + unsigned int prologSize; + + // Size of the epilogs in the method. + // For methods which use CEE_JMP, some epilogs may end with a "ret" instruction + // and some may end with a "jmp". The epilogSize reported should be for the + // epilog with the smallest size. + unsigned int epilogSize; + + unsigned char epilogCnt; + bool epilogEnd; // is the epilog at the end of the method + + bool ebpFrame; // locals and arguments addressed relative to EBP + bool doubleAlign; // is the stack double-aligned? locals addressed relative to ESP, and arguments relative to EBP + bool interruptible; // intr. at all times (excluding prolog/epilog), not just call sites + + bool handlers; // has callable handlers + bool localloc; // uses localloc + bool editNcontinue; // has been compiled in EnC mode + bool varargs; // is this a varargs routine + bool profCallbacks; // does the method have Enter-Leave callbacks + bool genericsContext;// has a reported generic context parameter + bool genericsContextIsMethodDesc;// reported generic context parameter is methoddesc + bool isSpeculativeStackWalk; // is the stackwalk seeded by an untrusted source (e.g., sampling profiler)? + + // These always includes EBP for EBP-frames and double-aligned-frames + RegMask savedRegMask:8; // which callee-saved regs are saved on stack + + // Count of the callee-saved registers, excluding the frame pointer. + // This does not include EBP for EBP-frames and double-aligned-frames. + unsigned int savedRegsCountExclFP; + + unsigned int untrackedCnt; + unsigned int varPtrTableSize; + unsigned int argTabOffset; // INVALID_ARGTAB_OFFSET if argtab must be reached by stepping through ptr tables + unsigned int gsCookieOffset; // INVALID_GS_COOKIE_OFFSET if there is no GuardStack cookie + + unsigned int syncStartOffset; // start/end code offset of the protected region in synchronized methods. + unsigned int syncEndOffset; // INVALID_SYNC_OFFSET if there not synchronized method + unsigned int syncEpilogStart; // The start of the epilog. Synchronized methods are guaranteed to have no more than one epilog. + unsigned int revPInvokeOffset; // INVALID_REV_PINVOKE_OFFSET if there is no Reverse PInvoke frame + + enum { NOT_IN_PROLOG = -1, NOT_IN_EPILOG = -1 }; + + int prologOffs; // NOT_IN_PROLOG if not in prolog + int epilogOffs; // NOT_IN_EPILOG if not in epilog. It is never 0 + + // + // Results passed back from scanArgRegTable + // + regNum thisPtrResult; // register holding "this" + RegMask regMaskResult; // registers currently holding GC ptrs + RegMask iregMaskResult; // iptr qualifier for regMaskResult + unsigned argHnumResult; + PTR_CBYTE argTabResult; // Table of encoded offsets of pending ptr args + unsigned argTabBytes; // Number of bytes in argTabResult[] + + // These next two are now large structs (i.e 132 bytes each) + + ptrArgTP argMaskResult; // pending arguments mask + ptrArgTP iargMaskResult; // iptr qualifier for argMaskResult +}; /***************************************************************************** How the stackwalkers buffer will be interpreted @@ -640,9 +757,6 @@ struct CodeManStateBuf DWORD hdrInfoSize; hdrInfo hdrInfoBody; }; - -#endif - //***************************************************************************** #endif // _EETWAIN_H //***************************************************************************** diff --git a/src/coreclr/inc/gc_unwind_x86.h b/src/coreclr/inc/gc_unwind_x86.h deleted file mode 100644 index e5be6b2e4aa43f..00000000000000 --- a/src/coreclr/inc/gc_unwind_x86.h +++ /dev/null @@ -1,138 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#ifndef _UNWIND_X86_H -#define _UNWIND_X86_H - -// This file is shared between CoreCLR and NativeAOT. Some of the differences are handled -// with the FEATURE_NATIVEAOT and FEATURE_EH_FUNCLETS defines. There are three main methods -// that are used by both runtimes - DecodeGCHdrInfo, UnwindStackFrameX86, and EnumGcRefsX86. -// -// The IN_EH_FUNCLETS and IN_EH_FUNCLETS_COMMA macros are used to specify some parameters -// for the above methods that are specific for a certain runtime or configuration. -#ifdef FEATURE_EH_FUNCLETS -#define IN_EH_FUNCLETS(a) a -#define IN_EH_FUNCLETS_COMMA(a) a, -#else -#define IN_EH_FUNCLETS(a) -#define IN_EH_FUNCLETS_COMMA(a) -#endif - -enum regNum -{ - REGI_EAX, REGI_ECX, REGI_EDX, REGI_EBX, - REGI_ESP, REGI_EBP, REGI_ESI, REGI_EDI, - REGI_COUNT, - REGI_NA = REGI_COUNT -}; - -/***************************************************************************** - Register masks - */ - -enum RegMask -{ - RM_EAX = 0x01, - RM_ECX = 0x02, - RM_EDX = 0x04, - RM_EBX = 0x08, - RM_ESP = 0x10, - RM_EBP = 0x20, - RM_ESI = 0x40, - RM_EDI = 0x80, - - RM_NONE = 0x00, - RM_ALL = (RM_EAX|RM_ECX|RM_EDX|RM_EBX|RM_ESP|RM_EBP|RM_ESI|RM_EDI), - RM_CALLEE_SAVED = (RM_EBP|RM_EBX|RM_ESI|RM_EDI), - RM_CALLEE_TRASHED = (RM_ALL & ~RM_CALLEE_SAVED), -}; - -/***************************************************************************** - * - * Helper to extract basic info from a method info block. - */ - -struct hdrInfo -{ - unsigned int methodSize; // native code bytes - unsigned int argSize; // in bytes - unsigned int stackSize; // including callee saved registers - unsigned int rawStkSize; // excluding callee saved registers - ReturnKind returnKind; // The ReturnKind for this method. - - unsigned int prologSize; - - // Size of the epilogs in the method. - // For methods which use CEE_JMP, some epilogs may end with a "ret" instruction - // and some may end with a "jmp". The epilogSize reported should be for the - // epilog with the smallest size. - unsigned int epilogSize; - - unsigned char epilogCnt; - bool epilogEnd; // is the epilog at the end of the method - - bool ebpFrame; // locals and arguments addressed relative to EBP - bool doubleAlign; // is the stack double-aligned? locals addressed relative to ESP, and arguments relative to EBP - bool interruptible; // intr. at all times (excluding prolog/epilog), not just call sites - - bool handlers; // has callable handlers - bool localloc; // uses localloc - bool editNcontinue; // has been compiled in EnC mode - bool varargs; // is this a varargs routine - bool profCallbacks; // does the method have Enter-Leave callbacks - bool genericsContext;// has a reported generic context parameter - bool genericsContextIsMethodDesc;// reported generic context parameter is methoddesc - bool isSpeculativeStackWalk; // is the stackwalk seeded by an untrusted source (e.g., sampling profiler)? - - // These always includes EBP for EBP-frames and double-aligned-frames - RegMask savedRegMask:8; // which callee-saved regs are saved on stack - - // Count of the callee-saved registers, excluding the frame pointer. - // This does not include EBP for EBP-frames and double-aligned-frames. - unsigned int savedRegsCountExclFP; - - unsigned int untrackedCnt; - unsigned int varPtrTableSize; - unsigned int argTabOffset; // INVALID_ARGTAB_OFFSET if argtab must be reached by stepping through ptr tables - unsigned int gsCookieOffset; // INVALID_GS_COOKIE_OFFSET if there is no GuardStack cookie - - unsigned int syncStartOffset; // start/end code offset of the protected region in synchronized methods. - unsigned int syncEndOffset; // INVALID_SYNC_OFFSET if there not synchronized method - unsigned int syncEpilogStart; // The start of the epilog. Synchronized methods are guaranteed to have no more than one epilog. - unsigned int revPInvokeOffset; // INVALID_REV_PINVOKE_OFFSET if there is no Reverse PInvoke frame - - enum { NOT_IN_PROLOG = -1, NOT_IN_EPILOG = -1 }; - - int prologOffs; // NOT_IN_PROLOG if not in prolog - int epilogOffs; // NOT_IN_EPILOG if not in epilog. It is never 0 - - // - // Results passed back from scanArgRegTable - // - regNum thisPtrResult; // register holding "this" - RegMask regMaskResult; // registers currently holding GC ptrs - RegMask iregMaskResult; // iptr qualifier for regMaskResult - unsigned argHnumResult; - PTR_CBYTE argTabResult; // Table of encoded offsets of pending ptr args - unsigned argTabBytes; // Number of bytes in argTabResult[] - - // These next two are now large structs (i.e 132 bytes each) - - ptrArgTP argMaskResult; // pending arguments mask - ptrArgTP iargMaskResult; // iptr qualifier for argMaskResult -}; - -bool UnwindStackFrameX86(PREGDISPLAY pContext, - PTR_CBYTE methodStart, - DWORD curOffs, - hdrInfo * info, - PTR_CBYTE table, - IN_EH_FUNCLETS_COMMA(PTR_CBYTE funcletStart) - IN_EH_FUNCLETS_COMMA(bool isFunclet) - bool updateAllRegs); - -size_t DecodeGCHdrInfo(GCInfoToken gcInfoToken, - unsigned curOffset, - hdrInfo * infoPtr); - -#endif // _UNWIND_X86_H diff --git a/src/coreclr/inc/gcinfo.h b/src/coreclr/inc/gcinfo.h index f334b099f2578e..66933b10f0445a 100644 --- a/src/coreclr/inc/gcinfo.h +++ b/src/coreclr/inc/gcinfo.h @@ -13,6 +13,9 @@ /*****************************************************************************/ #include "daccess.h" +#include "windef.h" // For BYTE + +// Some declarations in this file are used on non-x86 platforms, but most are x86-specific. // Use the lower 2 bits of the offsets stored in the tables // to encode properties @@ -20,15 +23,14 @@ const unsigned OFFSET_MASK = 0x3; // mask to access the low 2 bits // +// Note for untracked locals the flags allowed are "pinned" and "byref" +// and for tracked locals the flags allowed are "this" and "byref" // Note that these definitions should also match the definitions of // GC_CALL_INTERIOR and GC_CALL_PINNED in VM/gc.h // const unsigned byref_OFFSET_FLAG = 0x1; // the offset is an interior ptr const unsigned pinned_OFFSET_FLAG = 0x2; // the offset is a pinned ptr -#if defined(TARGET_X86) && !defined(FEATURE_EH_FUNCLETS) -// JIT32_ENCODER has additional restriction on x86 without funclets: -// - for untracked locals the flags allowed are "pinned" and "byref" -// - for tracked locals the flags allowed are "this" and "byref" +#if !defined(TARGET_X86) || !defined(FEATURE_EH_FUNCLETS) const unsigned this_OFFSET_FLAG = 0x2; // the offset is "this" #endif @@ -55,17 +57,9 @@ const unsigned this_OFFSET_FLAG = 0x2; // the offset is "this" struct GCInfoToken { PTR_VOID Info; - uint32_t Version; - -#ifdef FEATURE_NATIVEAOT - GCInfoToken(PTR_VOID info) - { - Info = info; - Version = GCINFO_VERSION; - } -#endif + UINT32 Version; - static uint32_t ReadyToRunVersionToGcInfoVersion(uint32_t readyToRunMajorVersion) + static UINT32 ReadyToRunVersionToGcInfoVersion(UINT32 readyToRunMajorVersion) { // GcInfo version is current from ReadyToRun version 2.0 return GCINFO_VERSION; diff --git a/src/coreclr/inc/gcinfodecoder.h b/src/coreclr/inc/gcinfodecoder.h index eb60728af5b1f7..34af8c53055687 100644 --- a/src/coreclr/inc/gcinfodecoder.h +++ b/src/coreclr/inc/gcinfodecoder.h @@ -31,17 +31,7 @@ #ifdef FEATURE_NATIVEAOT -#include "gcinfo.h" - typedef ArrayDPTR(const uint8_t) PTR_CBYTE; -#ifdef TARGET_X86 -// Bridge few additional pointer types used in x86 unwinding code -typedef DPTR(DWORD) PTR_DWORD; -typedef DPTR(WORD) PTR_WORD; -typedef DPTR(BYTE) PTR_BYTE; -typedef DPTR(signed char) PTR_SBYTE; -typedef DPTR(INT32) PTR_INT32; -#endif #define LIMITED_METHOD_CONTRACT #define SUPPORTS_DAC @@ -60,12 +50,22 @@ typedef DPTR(INT32) PTR_INT32; #define SSIZE_T intptr_t #define LPVOID void* -#define CHECK_APP_DOMAIN 0 - typedef void * OBJECTREF; #define GET_CALLER_SP(pREGDISPLAY) ((TADDR)0) +struct GCInfoToken +{ + PTR_VOID Info; + UINT32 Version; + + GCInfoToken(PTR_VOID info) + { + Info = info; + Version = 2; + } +}; + #else // FEATURE_NATIVEAOT // Stuff from cgencpu.h: @@ -179,7 +179,6 @@ enum ICodeManagerFlags ExecutionAborted = 0x0002, // execution of this function has been aborted // (i.e. it will not continue execution at the // current location) - AbortingCall = 0x0004, // The current call will never return ParentOfFuncletStackFrame = 0x0040, // A funclet for this frame was previously reported diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h index 2348162d948545..8dd993f5b47829 100644 --- a/src/coreclr/inc/icorjitinfoimpl_generated.h +++ b/src/coreclr/inc/icorjitinfoimpl_generated.h @@ -104,7 +104,6 @@ CORINFO_CLASS_HANDLE getDefaultEqualityComparerClass( void expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) override; bool isIntrinsicType( @@ -298,14 +297,12 @@ bool getReadyToRunHelper( CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) override; void getReadyToRunDelegateCtorHelper( CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) override; CorInfoInitClassResult initClass( @@ -441,11 +438,6 @@ void reportRichMappings( ICorDebugInfo::RichOffsetMapping* mappings, uint32_t numMappings) override; -void reportMetadata( - const char* key, - const void* value, - size_t length) override; - void* allocateArray( size_t cBytes) override; @@ -507,10 +499,6 @@ bool getSystemVAmd64PassStructInRegisterDescriptor( CORINFO_CLASS_HANDLE structHnd, SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* structPassInRegDescPtr) override; -void getSwiftLowering( - CORINFO_CLASS_HANDLE structHnd, - CORINFO_SWIFT_LOWERING* pLowering) override; - uint32_t getLoongArch64PassStructInRegisterFlags( CORINFO_CLASS_HANDLE structHnd) override; @@ -563,7 +551,6 @@ CORINFO_FIELD_HANDLE embedFieldHandle( void embedGenericHandle( CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) override; void getLocationOfThisType( diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 41e58ca24537b8..6355fc20dd0fd5 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID; #define GUID_DEFINED #endif // !GUID_DEFINED -constexpr GUID JITEEVersionIdentifier = { /* 6fd660c7-96be-4832-a84c-4200141f7d08 */ - 0x6fd660c7, - 0x96be, - 0x4832, - {0xa8, 0x4c, 0x42, 0x00, 0x14, 0x1f, 0x7d, 0x08} +constexpr GUID JITEEVersionIdentifier = { /* 0fb71692-0ee6-4914-88a8-6446e45f23e8 */ + 0x0fb71692, + 0x0ee6, + 0x4914, + {0x88, 0xa8, 0x64, 0x46, 0xe4, 0x5f, 0x23, 0xe8} }; ////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index a0982f3ac6520f..65167abd6a4dd6 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -235,10 +235,13 @@ DYNAMICJITHELPER(CORINFO_HELP_INIT_PINVOKE_FRAME, NULL, CORINFO_HELP_SIG_REG_ONLY) #endif - DYNAMICJITHELPER(CORINFO_HELP_MEMSET, NULL, CORINFO_HELP_SIG_REG_ONLY) - DYNAMICJITHELPER(CORINFO_HELP_MEMZERO, NULL, CORINFO_HELP_SIG_REG_ONLY) - DYNAMICJITHELPER(CORINFO_HELP_MEMCPY, NULL, CORINFO_HELP_SIG_REG_ONLY) - JITHELPER(CORINFO_HELP_NATIVE_MEMSET, Jit_NativeMemSet, CORINFO_HELP_SIG_REG_ONLY) +#ifdef TARGET_X86 + JITHELPER(CORINFO_HELP_MEMSET, NULL, CORINFO_HELP_SIG_CANNOT_USE_ALIGN_STUB) + JITHELPER(CORINFO_HELP_MEMCPY, NULL, CORINFO_HELP_SIG_CANNOT_USE_ALIGN_STUB) +#else + JITHELPER(CORINFO_HELP_MEMSET, JIT_MemSet, CORINFO_HELP_SIG_REG_ONLY) + JITHELPER(CORINFO_HELP_MEMCPY, JIT_MemCpy, CORINFO_HELP_SIG_REG_ONLY) +#endif // Generics JITHELPER(CORINFO_HELP_RUNTIMEHANDLE_METHOD, JIT_GenericHandleMethod, CORINFO_HELP_SIG_REG_ONLY) diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 41a4aa251fa742..b3128cb00e4b73 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -20,7 +20,7 @@ // If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION` // and handle pending work. #define READYTORUN_MAJOR_VERSION 0x0009 -#define READYTORUN_MINOR_VERSION 0x0002 +#define READYTORUN_MINOR_VERSION 0x0001 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x009 @@ -33,8 +33,6 @@ // R2R Version 8.0 Changes the alignment of the Int128 type // R2R Version 9.0 adds support for the Vector512 type // R2R Version 9.1 adds new helpers to allocate objects on frozen segments -// R2R Version 9.2 adds MemZero and NativeMemSet helpers - struct READYTORUN_CORE_HEADER { @@ -327,9 +325,7 @@ enum ReadyToRunHelper READYTORUN_HELPER_Stelem_Ref = 0x38, READYTORUN_HELPER_Ldelema_Ref = 0x39, - READYTORUN_HELPER_MemZero = 0x3E, - READYTORUN_HELPER_MemSet = 0x3F, - READYTORUN_HELPER_NativeMemSet = 0x40, + READYTORUN_HELPER_MemSet = 0x40, READYTORUN_HELPER_MemCpy = 0x41, // PInvoke helpers @@ -445,6 +441,10 @@ enum ReadyToRunHelper READYTORUN_HELPER_StackProbe = 0x111, READYTORUN_HELPER_GetCurrentManagedThreadId = 0x112, + + // Array helpers for use with native ints + READYTORUN_HELPER_Stelem_Ref_I = 0x113, + READYTORUN_HELPER_Ldelema_Ref_I = 0x114, }; #include "readytoruninstructionset.h" diff --git a/src/coreclr/inc/readytorunhelpers.h b/src/coreclr/inc/readytorunhelpers.h index bbb586e8eb4a30..8691f9b9cb8c0c 100644 --- a/src/coreclr/inc/readytorunhelpers.h +++ b/src/coreclr/inc/readytorunhelpers.h @@ -29,8 +29,6 @@ HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST, HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, ) HELPER(READYTORUN_HELPER_MemSet, CORINFO_HELP_MEMSET, ) -HELPER(READYTORUN_HELPER_MemZero, CORINFO_HELP_MEMZERO, ) -HELPER(READYTORUN_HELPER_NativeMemSet, CORINFO_HELP_NATIVE_MEMSET, ) HELPER(READYTORUN_HELPER_MemCpy, CORINFO_HELP_MEMCPY, ) HELPER(READYTORUN_HELPER_LogMethodEnter, CORINFO_HELP_BBT_FCN_ENTER, ) diff --git a/src/coreclr/inc/regdisp.h b/src/coreclr/inc/regdisp.h index ec47b9019dbc02..4832791ebfa5dc 100644 --- a/src/coreclr/inc/regdisp.h +++ b/src/coreclr/inc/regdisp.h @@ -131,12 +131,6 @@ inline LPVOID GetRegdisplayFPAddress(REGDISPLAY *display) { return (LPVOID)display->GetEbpLocation(); } -inline void SetRegdisplayPCTAddr(REGDISPLAY *display, TADDR addr) -{ - display->PCTAddr = addr; - display->ControlPC = *PTR_PCODE(addr); -} - // This function tells us if the given stack pointer is in one of the frames of the functions called by the given frame inline BOOL IsInCalleesFrames(REGDISPLAY *display, LPVOID stackPointer) { @@ -324,7 +318,7 @@ struct REGDISPLAY : public REGDISPLAY_BASE { memset(this, 0, sizeof(REGDISPLAY)); // Setup the pointer to ControlPC field - pPC = (DWORD *)&ControlPC; + pPC = &ControlPC; } }; @@ -453,7 +447,7 @@ inline void FillContextPointers(PT_KNONVOLATILE_CONTEXT_POINTERS pCtxPtrs, PT_CO } #endif // FEATURE_EH_FUNCLETS -inline void FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, PT_CONTEXT pCallerCtx = NULL, bool fLightUnwind = false) +inline void FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, PT_CONTEXT pCallerCtx = NULL) { WRAPPER_NO_CONTRACT; @@ -503,16 +497,6 @@ inline void FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, PT_CONTEXT pC pRD->IsCallerSPValid = TRUE; // Don't add usage of this field. This is only temporary. } -#ifdef DEBUG_REGDISPLAY - pRD->_pThread = NULL; -#endif // DEBUG_REGDISPLAY - - // This will setup the PC and SP - SyncRegDisplayToCurrentContext(pRD); - - if (fLightUnwind) - return; - FillContextPointers(&pRD->ctxPtrsOne, pctx); #if defined(TARGET_ARM) @@ -566,6 +550,12 @@ inline void FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, PT_CONTEXT pC pRD->volatileCurrContextPointers.T6 = &pctx->T6; #endif // TARGET_RISCV64 +#ifdef DEBUG_REGDISPLAY + pRD->_pThread = NULL; +#endif // DEBUG_REGDISPLAY + + // This will setup the PC and SP + SyncRegDisplayToCurrentContext(pRD); #endif // !FEATURE_EH_FUNCLETS } diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index 5ba50306d1b72a..aa660321075890 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -23,8 +23,6 @@ function(create_standalone_jit) if(TARGETDETAILS_OS STREQUAL "unix_osx" OR TARGETDETAILS_OS STREQUAL "unix_anyos") set(JIT_ARCH_LINK_LIBRARIES gcinfo_unix_${TARGETDETAILS_ARCH}) - elseif(TARGETDETAILS_OS STREQUAL "win_aot") - set(JIT_ARCH_LINK_LIBRARIES gcinfo_win_${TARGETDETAILS_ARCH}) else() set(JIT_ARCH_LINK_LIBRARIES gcinfo_${TARGETDETAILS_OS}_${TARGETDETAILS_ARCH}) endif() @@ -96,6 +94,7 @@ set( JIT_SOURCES bitset.cpp block.cpp buildstring.cpp + layout.cpp codegencommon.cpp codegenlinear.cpp compiler.cpp @@ -124,23 +123,20 @@ set( JIT_SOURCES gentree.cpp gschecks.cpp hashbv.cpp - helperexpansion.cpp - hostallocator.cpp hwintrinsic.cpp + hostallocator.cpp ifconversion.cpp - importer.cpp + helperexpansion.cpp + indirectcalltransformer.cpp importercalls.cpp + importer.cpp importervectorization.cpp - indirectcalltransformer.cpp - inductionvariableopts.cpp inline.cpp inlinepolicy.cpp instr.cpp jitconfig.cpp jiteh.cpp jithashtable.cpp - jitmetadata.cpp - layout.cpp lclmorph.cpp lclvars.cpp likelyclass.cpp @@ -155,6 +151,7 @@ set( JIT_SOURCES objectalloc.cpp optcse.cpp optimizebools.cpp + switchrecognition.cpp optimizer.cpp patchpoint.cpp phase.cpp @@ -167,7 +164,6 @@ set( JIT_SOURCES regalloc.cpp registerargconvention.cpp regset.cpp - scev.cpp scopeinfo.cpp sideeffects.cpp sm.cpp @@ -176,7 +172,6 @@ set( JIT_SOURCES ssabuilder.cpp ssarenamestate.cpp stacklevelsetter.cpp - switchrecognition.cpp treelifeupdater.cpp unwind.cpp utils.cpp @@ -339,8 +334,6 @@ set( JIT_HEADERS jitexpandarray.h jitgcinfo.h jithashtable.h - jitmetadata.h - jitmetadatalist.h jitpch.h jitstd.h lir.h @@ -363,7 +356,6 @@ set( JIT_HEADERS registerargconvention.h register.h regset.h - scev.h sideeffects.h simd.h simdashwintrinsic.h @@ -654,7 +646,6 @@ else() create_standalone_jit(TARGET clrjit_universal_arm_${ARCH_HOST_NAME} OS universal ARCH arm DESTINATIONS .) target_compile_definitions(clrjit_universal_arm_${ARCH_HOST_NAME} PRIVATE ARM_SOFTFP CONFIGURABLE_ARM_ABI) create_standalone_jit(TARGET clrjit_win_x86_${ARCH_HOST_NAME} OS win ARCH x86 DESTINATIONS .) - create_standalone_jit(TARGET clrjit_win_aot_x86_${ARCH_HOST_NAME} OS win_aot ARCH x86 DESTINATIONS .) endif (CLR_CMAKE_TARGET_ARCH_RISCV64) if (CLR_CMAKE_TARGET_ARCH_I386 AND CLR_CMAKE_TARGET_UNIX) diff --git a/src/coreclr/jit/ICorJitInfo_names_generated.h b/src/coreclr/jit/ICorJitInfo_names_generated.h index 30c499518e007c..5fe1f716d474b8 100644 --- a/src/coreclr/jit/ICorJitInfo_names_generated.h +++ b/src/coreclr/jit/ICorJitInfo_names_generated.h @@ -108,7 +108,6 @@ DEF_CLR_API(setBoundaries) DEF_CLR_API(getVars) DEF_CLR_API(setVars) DEF_CLR_API(reportRichMappings) -DEF_CLR_API(reportMetadata) DEF_CLR_API(allocateArray) DEF_CLR_API(freeArray) DEF_CLR_API(getArgNext) @@ -125,7 +124,6 @@ DEF_CLR_API(printMethodName) DEF_CLR_API(getMethodNameFromMetadata) DEF_CLR_API(getMethodHash) DEF_CLR_API(getSystemVAmd64PassStructInRegisterDescriptor) -DEF_CLR_API(getSwiftLowering) DEF_CLR_API(getLoongArch64PassStructInRegisterFlags) DEF_CLR_API(getRISCV64PassStructInRegisterFlags) DEF_CLR_API(getThreadTLSIndex) diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp index 77af720739ecee..cae9b5d7b39e59 100644 --- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp +++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp @@ -202,11 +202,10 @@ CORINFO_CLASS_HANDLE WrapICorJitInfo::getDefaultEqualityComparerClass( void WrapICorJitInfo::expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { API_ENTER(expandRawHandleIntrinsic); - wrapHnd->expandRawHandleIntrinsic(pResolvedToken, callerHandle, pResult); + wrapHnd->expandRawHandleIntrinsic(pResolvedToken, pResult); API_LEAVE(expandRawHandleIntrinsic); } @@ -688,11 +687,10 @@ bool WrapICorJitInfo::getReadyToRunHelper( CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { API_ENTER(getReadyToRunHelper); - bool temp = wrapHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup); + bool temp = wrapHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, pLookup); API_LEAVE(getReadyToRunHelper); return temp; } @@ -701,11 +699,10 @@ void WrapICorJitInfo::getReadyToRunDelegateCtorHelper( CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { API_ENTER(getReadyToRunDelegateCtorHelper); - wrapHnd->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); + wrapHnd->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, pLookup); API_LEAVE(getReadyToRunDelegateCtorHelper); } @@ -1031,16 +1028,6 @@ void WrapICorJitInfo::reportRichMappings( API_LEAVE(reportRichMappings); } -void WrapICorJitInfo::reportMetadata( - const char* key, - const void* value, - size_t length) -{ - API_ENTER(reportMetadata); - wrapHnd->reportMetadata(key, value, length); - API_LEAVE(reportMetadata); -} - void* WrapICorJitInfo::allocateArray( size_t cBytes) { @@ -1196,15 +1183,6 @@ bool WrapICorJitInfo::getSystemVAmd64PassStructInRegisterDescriptor( return temp; } -void WrapICorJitInfo::getSwiftLowering( - CORINFO_CLASS_HANDLE structHnd, - CORINFO_SWIFT_LOWERING* pLowering) -{ - API_ENTER(getSwiftLowering); - wrapHnd->getSwiftLowering(structHnd, pLowering); - API_LEAVE(getSwiftLowering); -} - uint32_t WrapICorJitInfo::getLoongArch64PassStructInRegisterFlags( CORINFO_CLASS_HANDLE structHnd) { @@ -1333,11 +1311,10 @@ CORINFO_FIELD_HANDLE WrapICorJitInfo::embedFieldHandle( void WrapICorJitInfo::embedGenericHandle( CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { API_ENTER(embedGenericHandle); - wrapHnd->embedGenericHandle(pResolvedToken, fEmbedParent, callerHandle, pResult); + wrapHnd->embedGenericHandle(pResolvedToken, fEmbedParent, pResult); API_LEAVE(embedGenericHandle); } diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index 68a01a1ab7fcd1..fb317faea42c13 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -857,27 +857,12 @@ void Compiler::optPrintAssertion(AssertionDsc* curAssertion, AssertionIndex asse if (curAssertion->op1.kind == O1K_EXACT_TYPE) { ssize_t iconVal = curAssertion->op2.u1.iconVal; - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf("Exact Type MT(0x%p)", dspPtr(iconVal)); - } - else - { - printf("Exact Type MT(0x%p %s)", dspPtr(iconVal), - eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); - } + printf("Exact Type MT(0x%p %s)", dspPtr(iconVal), eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); } else if (curAssertion->op1.kind == O1K_SUBTYPE) { ssize_t iconVal = curAssertion->op2.u1.iconVal; - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf("MT(0x%p)", dspPtr(iconVal)); - } - else - { - printf("MT(0x%p %s)", dspPtr(iconVal), eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); - } + printf("MT(0x%p %s)", dspPtr(iconVal), eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); assert(curAssertion->op2.HasIconFlag()); } else if ((curAssertion->op1.kind == O1K_BOUND_OPER_BND) || @@ -2637,20 +2622,28 @@ AssertionIndex Compiler::optAssertionIsSubtype(GenTree* tree, GenTree* methodTab { AssertionIndex const index = GetAssertionIndex(bvIndex); AssertionDsc* curAssertion = optGetAssertion(index); - if ((curAssertion->assertionKind != OAK_EQUAL) || - ((curAssertion->op1.kind != O1K_SUBTYPE) && (curAssertion->op1.kind != O1K_EXACT_TYPE))) - { - // TODO-CQ: We might benefit from OAK_NOT_EQUAL assertion as well, e.g.: - // if (obj is not MyClass) // obj is known to be never of MyClass class - // { - // if (obj is MyClass) // can be folded to false - // { - // + if (curAssertion->assertionKind != OAK_EQUAL || + (curAssertion->op1.kind != O1K_SUBTYPE && curAssertion->op1.kind != O1K_EXACT_TYPE)) + { continue; } - if ((curAssertion->op1.vn != vnStore->VNConservativeNormalValue(tree->gtVNPair) || - (curAssertion->op2.kind != O2K_CONST_INT))) + // If local assertion prop use "lcl" based comparison, if global assertion prop use vn based comparison. + if ((optLocalAssertionProp) ? (curAssertion->op1.lcl.lclNum != tree->AsLclVarCommon()->GetLclNum()) + : (curAssertion->op1.vn != vnStore->VNConservativeNormalValue(tree->gtVNPair))) + { + continue; + } + + if (curAssertion->op2.kind == O2K_IND_CNS_INT) + { + if (methodTableArg->gtOper != GT_IND) + { + continue; + } + methodTableArg = methodTableArg->AsOp()->gtOp1; + } + else if (curAssertion->op2.kind != O2K_CONST_INT) { continue; } @@ -2664,8 +2657,6 @@ AssertionIndex Compiler::optAssertionIsSubtype(GenTree* tree, GenTree* methodTab if (curAssertion->op2.u1.iconVal == methodTableVal) { - // TODO-CQ: if they don't match, we might still be able to prove that the result is foldable via - // compareTypesForCast. return index; } } @@ -2687,6 +2678,10 @@ GenTree* Compiler::optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, G { switch (call->GetHelperNum()) { + // + // Fold "CAST(IsInstanceOf(obj, cls), cls)" to "IsInstanceOf(obj, cls)" + // where CAST is either ISINST or CASTCLASS. + // case CORINFO_HELP_CHKCASTARRAY: case CORINFO_HELP_CHKCASTANY: case CORINFO_HELP_CHKCASTINTERFACE: @@ -2696,8 +2691,10 @@ GenTree* Compiler::optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, G case CORINFO_HELP_ISINSTANCEOFANY: case CORINFO_HELP_ISINSTANCEOFINTERFACE: { - GenTree* castClsArg = call->gtArgs.GetUserArgByIndex(0)->GetNode(); - GenTree* castObjArg = call->gtArgs.GetUserArgByIndex(1)->GetNode(); + GenTree* castClsArg = call->gtArgs.GetUserArgByIndex(0)->GetNode(); + GenTree* castObjArg = call->gtArgs.GetUserArgByIndex(1)->GetNode(); + ValueNum castClsArgVN = castClsArg->gtVNPair.GetConservative(); + ValueNum castObjArgVN = castObjArg->gtVNPair.GetConservative(); if ((castObjArg->gtFlags & GTF_ALL_EFFECT) != 0) { @@ -2707,26 +2704,17 @@ GenTree* Compiler::optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, G return nullptr; } - // If object has the same VN as the cast, then the cast is effectively a no-op. - // - if (castObjArg->gtVNPair == call->gtVNPair) - { - return gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); - } - - // Let's see if gtGetClassHandle may help us to fold the cast (since VNForCast did not). - if (castClsArg->IsIconHandle(GTF_ICON_CLASS_HDL)) + VNFuncApp funcApp; + if (vnStore->GetVNFunc(castObjArgVN, &funcApp) && (funcApp.m_func == VNF_IsInstanceOf)) { - bool isExact; - bool isNonNull; - CORINFO_CLASS_HANDLE castFrom = gtGetClassHandle(castObjArg, &isExact, &isNonNull); - if (castFrom != NO_CLASS_HANDLE) + ValueNum innerCastClsVN = funcApp.m_args[0]; + if (innerCastClsVN == castClsArgVN) { - CORINFO_CLASS_HANDLE castTo = gtGetHelperArgClassHandle(castClsArg); - if (info.compCompHnd->compareTypesForCast(castFrom, castTo) == TypeCompareState::Must) - { - return gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); - } + // The outer cast is redundant, remove it and preserve its side effects + // We do ignoreRoot here because the actual cast node never throws any exceptions. + GenTree* result = gtWrapWithSideEffects(castObjArg, call, GTF_ALL_EFFECT, true); + fgSetTreeSeq(result); + return result; } } } @@ -3042,18 +3030,6 @@ GenTree* Compiler::optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, G break; } break; - - case TYP_MASK: - { - simdmask_t value = vnStore->ConstantValue(vnCns); - - GenTreeVecCon* vecCon = gtNewVconNode(tree->TypeGet()); - memcpy(&vecCon->gtSimdVal, &value, sizeof(simdmask_t)); - - conValTree = vecCon; - break; - } - break; #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -4222,20 +4198,20 @@ AssertionIndex Compiler::optGlobalAssertionIsEqualOrNotEqual(ASSERT_VALARG_TP as return assertionIndex; } - // Look for matching exact type assertions based on vtable accesses. E.g.: - // - // op1: VNF_InvariantLoad(myObj) or in other words: a vtable access - // op2: 'MyType' class handle - // Assertion: 'myObj's type is exactly MyType - // + // Look for matching exact type assertions based on vtable accesses if ((curAssertion->assertionKind == OAK_EQUAL) && (curAssertion->op1.kind == O1K_EXACT_TYPE) && - (curAssertion->op2.vn == vnStore->VNConservativeNormalValue(op2->gtVNPair)) && op1->TypeIs(TYP_I_IMPL)) + op1->OperIs(GT_IND)) { - VNFuncApp funcApp; - if (vnStore->GetVNFunc(vnStore->VNConservativeNormalValue(op1->gtVNPair), &funcApp) && - (funcApp.m_func == VNF_InvariantLoad) && (curAssertion->op1.vn == funcApp.m_args[0])) + GenTree* indirAddr = op1->AsIndir()->Addr(); + + if (indirAddr->OperIs(GT_LCL_VAR) && (indirAddr->TypeGet() == TYP_REF)) { - return assertionIndex; + // op1 is accessing vtable of a ref type local var + if ((curAssertion->op1.vn == vnStore->VNConservativeNormalValue(indirAddr->gtVNPair)) && + (curAssertion->op2.vn == vnStore->VNConservativeNormalValue(op2->gtVNPair))) + { + return assertionIndex; + } } } } @@ -5210,8 +5186,7 @@ GenTree* Compiler::optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCal { return optAssertionProp_Update(call, call, stmt); } - - if (!optLocalAssertionProp && call->IsHelperCall()) + else if (!optLocalAssertionProp && call->IsHelperCall()) { const CorInfoHelpFunc helper = eeGetHelperNum(call->gtCallMethHnd); if ((helper == CORINFO_HELP_ISINSTANCEOFINTERFACE) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY) || @@ -5220,21 +5195,22 @@ GenTree* Compiler::optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCal (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTANY) || (helper == CORINFO_HELP_CHKCASTCLASS_SPECIAL)) { - GenTree* castToArg = call->gtArgs.GetArgByIndex(0)->GetNode(); - GenTree* objArg = call->gtArgs.GetArgByIndex(1)->GetNode(); + GenTree* arg1 = call->gtArgs.GetArgByIndex(1)->GetNode(); + if (arg1->gtOper != GT_LCL_VAR) + { + return nullptr; + } - // We require objArg to be side effect free due to limitations in gtWrapWithSideEffects - if ((objArg->gtFlags & GTF_ALL_EFFECT) == 0) + GenTree* arg2 = call->gtArgs.GetArgByIndex(0)->GetNode(); + + unsigned index = optAssertionIsSubtype(arg1, arg2, assertions); + if (index != NO_ASSERTION_INDEX) { - const unsigned index = optAssertionIsSubtype(objArg, castToArg, assertions); - if (index != NO_ASSERTION_INDEX) - { - JITDUMP("\nDid VN based subtype prop for index #%02u in " FMT_BB ":\n", index, compCurBB->bbNum); - DISPTREE(call); + JITDUMP("\nDid VN based subtype prop for index #%02u in " FMT_BB ":\n", index, compCurBB->bbNum); + DISPTREE(call); - objArg = gtWrapWithSideEffects(objArg, call, GTF_SIDE_EFFECT, true); - return optAssertionProp_Update(objArg, call, stmt); - } + arg1 = gtWrapWithSideEffects(arg1, call, GTF_SIDE_EFFECT, true); + return optAssertionProp_Update(arg1, call, stmt); } // Leave a hint for fgLateCastExpansion that obj is never null. @@ -5242,7 +5218,7 @@ GenTree* Compiler::optAssertionProp_Call(ASSERT_VALARG_TP assertions, GenTreeCal INDEBUG(bool vnBased = false); // GTF_CALL_M_CAST_CAN_BE_EXPANDED check is to improve TP if (((call->gtCallMoreFlags & GTF_CALL_M_CAST_CAN_BE_EXPANDED) != 0) && - optAssertionIsNonNull(objArg, assertions DEBUGARG(&vnBased) DEBUGARG(&nonNullIdx))) + optAssertionIsNonNull(arg1, assertions DEBUGARG(&vnBased) DEBUGARG(&nonNullIdx))) { call->gtCallMoreFlags |= GTF_CALL_M_CAST_OBJ_NONNULL; return optAssertionProp_Update(call, call, stmt); @@ -5499,6 +5475,7 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, case GT_IND: case GT_STOREIND: case GT_NULLCHECK: + case GT_STORE_DYN_BLK: return optAssertionProp_Ind(assertions, tree, stmt); case GT_BOUNDS_CHECK: @@ -6152,7 +6129,7 @@ ASSERT_TP* Compiler::optComputeAssertionGen() AssertionIndex valueAssertionIndex; AssertionIndex jumpDestAssertionIndex; - if (info.AssertionHoldsOnFalseEdge()) + if (info.IsNextEdgeAssertion()) { valueAssertionIndex = info.GetAssertionIndex(); jumpDestAssertionIndex = optFindComplementary(info.GetAssertionIndex()); diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 1e7750997a103a..58b91b739fe424 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -68,68 +68,6 @@ unsigned SsaStressHashHelper() } #endif -//------------------------------------------------------------------------ -// setLikelihood: set the likelihood of a flow edge -// -// Arguments: -// likelihood -- value in range [0.0, 1.0] indicating how likely -// the source block is to transfer control along this edge. -// -void FlowEdge::setLikelihood(weight_t likelihood) -{ - assert(likelihood >= 0.0); - assert(likelihood <= 1.0); - - if (m_likelihoodSet) - { - JITDUMP("setting likelihood of " FMT_BB " -> " FMT_BB " from " FMT_WT " to " FMT_WT "\n", m_sourceBlock->bbNum, - m_destBlock->bbNum, m_likelihood, likelihood); - } - else - { - JITDUMP("setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT "\n", m_sourceBlock->bbNum, - m_destBlock->bbNum, likelihood); - } - - m_likelihoodSet = true; - m_likelihood = likelihood; -} - -//------------------------------------------------------------------------ -// addLikelihood: adjust the likelihood of a flow edge -// -// Arguments: -// addedLikelihood -- value in range [-likelihood, 1.0 - likelihood] -// to add to current likelihood. -// -void FlowEdge::addLikelihood(weight_t addedLikelihood) -{ - assert(m_likelihoodSet); - - weight_t newLikelihood = m_likelihood + addedLikelihood; - - // Tolerate slight overflow or underflow - // - const weight_t eps = 0.0001; - - if ((newLikelihood < 0) && (newLikelihood > -eps)) - { - newLikelihood = 0.0; - } - else if ((newLikelihood > 1) && (newLikelihood < 1 + eps)) - { - newLikelihood = 1.0; - } - - assert(newLikelihood >= 0.0); - assert(newLikelihood <= 1.0); - - JITDUMP("updating likelihood of " FMT_BB " -> " FMT_BB " from " FMT_WT " to " FMT_WT "\n", m_sourceBlock->bbNum, - m_destBlock->bbNum, m_likelihood, newLikelihood); - - m_likelihood = newLikelihood; -} - //------------------------------------------------------------------------ // AllSuccessorEnumerator: Construct an instance of the enumerator. // @@ -688,33 +626,20 @@ void BasicBlock::dspSuccs(Compiler* compiler) // things strictly. void BasicBlock::dspKind() const { - auto dspBlockNum = [](const FlowEdge* e) -> const char* { + auto dspBlockNum = [](const BasicBlock* b) -> const char* { static char buffers[3][64]; // static array of 3 to allow 3 concurrent calls in one printf() static int nextBufferIndex = 0; - auto& buffer = buffers[nextBufferIndex]; - nextBufferIndex = (nextBufferIndex + 1) % ArrLen(buffers); - const size_t sizeOfBuffer = ArrLen(buffer); - int written; + auto& buffer = buffers[nextBufferIndex]; + nextBufferIndex = (nextBufferIndex + 1) % ArrLen(buffers); - const BasicBlock* b = e->getDestinationBlock(); if (b == nullptr) { - written = _snprintf_s(buffer, sizeOfBuffer, sizeOfBuffer, "NULL"); + _snprintf_s(buffer, ArrLen(buffer), ArrLen(buffer), "NULL"); } else { - written = _snprintf_s(buffer, sizeOfBuffer, sizeOfBuffer, FMT_BB, b->bbNum); - } - - const bool printEdgeLikelihoods = true; // TODO: parameterize this? - if (printEdgeLikelihoods) - { - if (e->hasLikelihood()) - { - written = _snprintf_s(buffer + written, sizeOfBuffer - written, sizeOfBuffer - written, "(" FMT_WT ")", - e->getLikelihood()); - } + _snprintf_s(buffer, ArrLen(buffer), ArrLen(buffer), FMT_BB, b->bbNum); } return buffer; @@ -738,7 +663,7 @@ void BasicBlock::dspKind() const for (unsigned i = 0; i < jumpCnt; i++) { - printf("%c%s", (i == 0) ? ' ' : ',', dspBlockNum(jumpTab[i])); + printf("%c%s", (i == 0) ? ' ' : ',', dspBlockNum(jumpTab[i]->getDestinationBlock())); } } @@ -751,11 +676,11 @@ void BasicBlock::dspKind() const break; case BBJ_EHFILTERRET: - printf(" -> %s (fltret)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (fltret)", dspBlockNum(bbTarget)); break; case BBJ_EHCATCHRET: - printf(" -> %s (cret)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (cret)", dspBlockNum(bbTarget)); break; case BBJ_THROW: @@ -769,36 +694,36 @@ void BasicBlock::dspKind() const case BBJ_ALWAYS: if (HasFlag(BBF_KEEP_BBJ_ALWAYS)) { - printf(" -> %s (ALWAYS)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (ALWAYS)", dspBlockNum(bbTarget)); } else { - printf(" -> %s (always)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (always)", dspBlockNum(bbTarget)); } break; case BBJ_LEAVE: - printf(" -> %s (leave)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (leave)", dspBlockNum(bbTarget)); break; case BBJ_CALLFINALLY: - printf(" -> %s (callf)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (callf)", dspBlockNum(bbTarget)); break; case BBJ_CALLFINALLYRET: - printf(" -> %s (callfr)", dspBlockNum(GetTargetEdge())); + printf(" -> %s (callfr)", dspBlockNum(bbTarget)); break; case BBJ_COND: - printf(" -> %s,%s (cond)", dspBlockNum(GetTrueEdge()), dspBlockNum(GetFalseEdge())); + printf(" -> %s,%s (cond)", dspBlockNum(bbTrueTarget), dspBlockNum(bbFalseTarget)); break; case BBJ_SWITCH: { printf(" ->"); - const unsigned jumpCnt = bbSwtTargets->bbsCount; - FlowEdge** const jumpTab = bbSwtTargets->bbsDstTab; + const unsigned jumpCnt = bbSwtTargets->bbsCount; + BasicBlock** const jumpTab = bbSwtTargets->bbsDstTab; for (unsigned i = 0; i < jumpCnt; i++) { @@ -932,16 +857,11 @@ void BasicBlock::TransferTarget(BasicBlock* from) SetEhf(from->GetEhfTargets()); from->bbEhfTargets = nullptr; // Make sure nobody uses the descriptor after this. break; - - // TransferTarget may be called after setting the source block of `from`'s - // successor edges to this block. - // This means calling GetTarget/GetTrueTarget/GetFalseTarget would trigger asserts. - // Avoid this by accessing the edges directly. case BBJ_COND: - SetCond(from->bbTrueEdge, from->bbFalseEdge); + SetCond(from->GetTrueTarget(), from->GetFalseTarget()); break; case BBJ_ALWAYS: - SetKindAndTargetEdge(BBJ_ALWAYS, from->bbTargetEdge); + SetKindAndTarget(from->GetKind(), from->GetTarget()); CopyFlags(from, BBF_NONE_QUIRK); break; case BBJ_CALLFINALLY: @@ -949,10 +869,10 @@ void BasicBlock::TransferTarget(BasicBlock* from) case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: - SetKindAndTargetEdge(from->GetKind(), from->bbTargetEdge); + SetKindAndTarget(from->GetKind(), from->GetTarget()); break; default: - SetKindAndTargetEdge(from->GetKind()); // Clear the target + SetKindAndTarget(from->GetKind()); // Clear the target break; } assert(KindIs(from->GetKind())); @@ -1065,7 +985,7 @@ BasicBlock* BasicBlock::GetUniquePred(Compiler* compiler) const // BasicBlock* BasicBlock::GetUniqueSucc() const { - return KindIs(BBJ_ALWAYS) ? GetTarget() : nullptr; + return KindIs(BBJ_ALWAYS) ? bbTarget : nullptr; } // Static vars. @@ -1225,7 +1145,7 @@ unsigned BasicBlock::NumSucc() const return 1; case BBJ_COND: - if (bbTrueEdge == bbFalseEdge) + if (bbTrueTarget == bbFalseTarget) { return 1; } @@ -1260,15 +1180,15 @@ unsigned BasicBlock::NumSucc() const } //------------------------------------------------------------------------ -// GetSucc: Returns the requested successor edge. See the declaration comment for details. +// GetSucc: Returns the requested block successor. See the declaration comment for details. // // Arguments: // i - index of successor to return. 0 <= i <= NumSucc(). // // Return Value: -// Requested successor edge +// Requested successor block // -FlowEdge* BasicBlock::GetSuccEdge(unsigned i) const +BasicBlock* BasicBlock::GetSucc(unsigned i) const { assert(i < NumSucc()); // Index bounds check. switch (bbKind) @@ -1279,22 +1199,22 @@ FlowEdge* BasicBlock::GetSuccEdge(unsigned i) const case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: - return GetTargetEdge(); + return bbTarget; case BBJ_COND: if (i == 0) { - return GetFalseEdge(); + return bbFalseTarget; } else { assert(i == 1); - assert(bbTrueEdge != bbFalseEdge); - return GetTrueEdge(); + assert(bbFalseTarget != bbTrueTarget); + return bbTrueTarget; } case BBJ_EHFINALLYRET: - return bbEhfTargets->bbeSuccs[i]; + return bbEhfTargets->bbeSuccs[i]->getDestinationBlock(); case BBJ_SWITCH: return bbSwtTargets->bbsDstTab[i]; @@ -1304,20 +1224,6 @@ FlowEdge* BasicBlock::GetSuccEdge(unsigned i) const } } -//------------------------------------------------------------------------ -// GetSucc: Returns the requested block successor. See the declaration comment for details. -// -// Arguments: -// i - index of successor to return. 0 <= i <= NumSucc(). -// -// Return Value: -// Requested successor block -// -BasicBlock* BasicBlock::GetSucc(unsigned i) const -{ - return GetSuccEdge(i)->getDestinationBlock(); -} - //------------------------------------------------------------------------ // NumSucc: Returns the count of block successors. See the declaration comment for details. // @@ -1364,7 +1270,7 @@ unsigned BasicBlock::NumSucc(Compiler* comp) return 1; case BBJ_COND: - if (bbTrueEdge == bbFalseEdge) + if (bbTrueTarget == bbFalseTarget) { return 1; } @@ -1385,16 +1291,16 @@ unsigned BasicBlock::NumSucc(Compiler* comp) } //------------------------------------------------------------------------ -// GetSucc: Returns the requested successor edge. See the declaration comment for details. +// GetSucc: Returns the requested block successor. See the declaration comment for details. // // Arguments: // i - index of successor to return. 0 <= i <= NumSucc(comp). // comp - Compiler instance // // Return Value: -// Requested successor edge +// Requested successor block // -FlowEdge* BasicBlock::GetSuccEdge(unsigned i, Compiler* comp) +BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) { assert(comp != nullptr); @@ -1403,31 +1309,31 @@ FlowEdge* BasicBlock::GetSuccEdge(unsigned i, Compiler* comp) { case BBJ_EHFILTERRET: // Handler is the (sole) normal successor of the filter. - assert(comp->fgFirstBlockOfHandler(this) == GetTarget()); - return GetTargetEdge(); + assert(comp->fgFirstBlockOfHandler(this) == bbTarget); + return bbTarget; case BBJ_EHFINALLYRET: assert(bbEhfTargets != nullptr); assert(i < bbEhfTargets->bbeCount); - return bbEhfTargets->bbeSuccs[i]; + return bbEhfTargets->bbeSuccs[i]->getDestinationBlock(); case BBJ_CALLFINALLY: case BBJ_CALLFINALLYRET: case BBJ_ALWAYS: case BBJ_EHCATCHRET: case BBJ_LEAVE: - return GetTargetEdge(); + return bbTarget; case BBJ_COND: if (i == 0) { - return GetFalseEdge(); + return bbFalseTarget; } else { assert(i == 1); - assert(bbTrueEdge != bbFalseEdge); - return GetTrueEdge(); + assert(bbFalseTarget != bbTrueTarget); + return bbTrueTarget; } case BBJ_SWITCH: @@ -1442,21 +1348,6 @@ FlowEdge* BasicBlock::GetSuccEdge(unsigned i, Compiler* comp) } } -//------------------------------------------------------------------------ -// GetSucc: Returns the requested block successor. See the declaration comment for details. -// -// Arguments: -// i - index of successor to return. 0 <= i <= NumSucc(comp). -// comp - Compiler instance -// -// Return Value: -// Requested successor block -// -BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) -{ - return GetSuccEdge(i, comp)->getDestinationBlock(); -} - void BasicBlock::InitVarSets(Compiler* comp) { VarSetOps::AssignNoCopy(comp, bbVarUse, VarSetOps::MakeEmpty(comp)); @@ -1694,10 +1585,15 @@ BasicBlock* BasicBlock::New(Compiler* compiler) return block; } -BasicBlock* BasicBlock::New(Compiler* compiler, BBKinds kind) +BasicBlock* BasicBlock::New(Compiler* compiler, BBKinds kind, BasicBlock* target /* = nullptr */) { BasicBlock* block = BasicBlock::New(compiler); - block->bbKind = kind; + + // In some cases, we don't know a block's jump target during initialization, so don't check the jump kind/target + // yet. + // The checks will be done any time the jump kind/target is read or written to after initialization. + block->bbKind = kind; + block->bbTarget = target; if (block->KindIs(BBJ_THROW)) { @@ -1878,7 +1774,7 @@ BBswtDesc::BBswtDesc(Compiler* comp, const BBswtDesc* other) { // Allocate and fill in a new dst tab // - bbsDstTab = new (comp, CMK_FlowEdge) FlowEdge*[bbsCount]; + bbsDstTab = new (comp, CMK_BasicBlock) BasicBlock*[bbsCount]; for (unsigned i = 0; i < bbsCount; i++) { bbsDstTab[i] = other->bbsDstTab[i]; diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index ae881d99f7361b..09fc1c71c35638 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -46,13 +46,9 @@ typedef BitVec_ValRet_T ASSERT_VALRET_TP; // Use this format for loop indices #define FMT_LP "L%02u" -// Use this format for profile weights +// And this format for profile weights #define FMT_WT "%.7g" -// Use this format for profile weights where we want to conserve horizontal space, at the expense of displaying -// less precision. -#define FMT_WT_NARROW "%.3g" - /***************************************************************************** * * Each basic block ends with a jump which is described as a value @@ -262,9 +258,7 @@ class PredEdgeList // PredBlockList: adapter class for forward iteration of the predecessor edge linked list yielding // predecessor blocks, using range-based `for`, normally used via BasicBlock::PredBlocks(), e.g.: // for (BasicBlock* const predBlock : block->PredBlocks()) ... -// allowEdits controls whether the iterator should be resilient to changes to the predecessor list. // -template class PredBlockList { FlowEdge* m_begin; @@ -276,12 +270,13 @@ class PredBlockList { FlowEdge* m_pred; - // When allowEdits=false, try to guard against the user of the iterator from modifying the predecessor list - // being traversed: cache the edge we think should be next, then check it when we actually do the `++` +#ifdef DEBUG + // Try to guard against the user of the iterator from making changes to the IR that would invalidate + // the iterator: cache the edge we think should be next, then check it when we actually do the `++` // operation. This is a bit conservative, but attempts to protect against callers assuming too much about // this iterator implementation. - // When allowEdits=true, m_next is always used to update m_pred, so changes to m_pred don't break the iterator. FlowEdge* m_next; +#endif public: iterator(FlowEdge* pred); @@ -312,67 +307,44 @@ class PredBlockList } }; -// BBArrayIterator: forward iterator for an array of BasicBlock*. +// BBArrayIterator: forward iterator for an array of BasicBlock*, such as the BBswtDesc->bbsDstTab. // It is an error (with assert) to yield a nullptr BasicBlock* in this array. -// `m_edgeEntry` can be nullptr, but it only makes sense if both the begin and end of an iteration range are nullptr +// `m_bbEntry` can be nullptr, but it only makes sense if both the begin and end of an iteration range are nullptr // (meaning, no actual iteration will happen). // class BBArrayIterator { - FlowEdge* const* m_edgeEntry; - -public: - BBArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry) - { - } - - BasicBlock* operator*() const; - - BBArrayIterator& operator++() - { - assert(m_edgeEntry != nullptr); - ++m_edgeEntry; - return *this; - } - - bool operator!=(const BBArrayIterator& i) const - { - return m_edgeEntry != i.m_edgeEntry; - } -}; + // Quirk: Some BasicBlock kinds refer to their successors with BasicBlock pointers, + // while others use FlowEdge pointers. Eventually, every type will use FlowEdge pointers. + // For now, support iterating with both types. + union { + BasicBlock* const* m_bbEntry; + FlowEdge* const* m_edgeEntry; + }; -// FlowEdgeArrayIterator: forward iterator for an array of FlowEdge*, such as the BBswtDesc->bbsDstTab. -// It is an error (with assert) to yield a nullptr FlowEdge* in this array. -// `m_edgeEntry` can be nullptr, but it only makes sense if both the begin and end of an iteration range are nullptr -// (meaning, no actual iteration will happen). -// -class FlowEdgeArrayIterator -{ - FlowEdge* const* m_edgeEntry; + bool iterateEdges; public: - FlowEdgeArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry) + BBArrayIterator(BasicBlock* const* bbEntry) : m_bbEntry(bbEntry), iterateEdges(false) { } - FlowEdge* operator*() const + BBArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry), iterateEdges(true) { - assert(m_edgeEntry != nullptr); - FlowEdge* const edge = *m_edgeEntry; - assert(edge != nullptr); - return edge; } - FlowEdgeArrayIterator& operator++() + BasicBlock* operator*() const; + + BBArrayIterator& operator++() { - assert(m_edgeEntry != nullptr); - ++m_edgeEntry; + assert(m_bbEntry != nullptr); + ++m_bbEntry; return *this; } - bool operator!=(const FlowEdgeArrayIterator& i) const + bool operator!=(const BBArrayIterator& i) const { - return m_edgeEntry != i.m_edgeEntry; + return m_bbEntry != i.m_bbEntry; } }; @@ -534,180 +506,6 @@ enum class BasicBlockVisit // clang-format on -//------------------------------------------------------------------------- -// FlowEdge -- control flow edge -// -// In compiler terminology the control flow between two BasicBlocks -// is typically referred to as an "edge". Most well known are the -// backward branches for loops, which are often called "back-edges". -// -// "struct FlowEdge" is the type that represents our control flow edges. -// This type is a linked list of zero or more "edges". -// (The list of zero edges is represented by NULL.) -// Every BasicBlock has a field called bbPreds of this type. This field -// represents the list of "edges" that flow into this BasicBlock. -// The FlowEdge type only stores the BasicBlock* of the source for the -// control flow edge. The destination block for the control flow edge -// is implied to be the block which contained the bbPreds field. -// -// For a switch branch target there may be multiple "edges" that have -// the same source block (and destination block). We need to count the -// number of these edges so that during optimization we will know when -// we have zero of them. Rather than have extra FlowEdge entries we -// track this via the DupCount property. -// -// When we have Profile weight for the BasicBlocks we can usually compute -// the number of times each edge was executed by examining the adjacent -// BasicBlock weights. As we are doing for BasicBlocks, we call the number -// of times that a control flow edge was executed the "edge weight". -// In order to compute the edge weights we need to use a bounded range -// for every edge weight. These two fields, 'flEdgeWeightMin' and 'flEdgeWeightMax' -// are used to hold a bounded range. Most often these will converge such -// that both values are the same and that value is the exact edge weight. -// Sometimes we are left with a rage of possible values between [Min..Max] -// which represents an inexact edge weight. -// -// The bbPreds list is initially created by Compiler::fgLinkBasicBlocks() -// and is incrementally kept up to date. -// -// The edge weight are computed by Compiler::fgComputeEdgeWeights() -// the edge weights are used to straighten conditional branches -// by Compiler::fgReorderBlocks() -// -struct FlowEdge -{ -private: - // The next predecessor edge in the list, nullptr for end of list. - FlowEdge* m_nextPredEdge; - - // The source of the control flow - BasicBlock* m_sourceBlock; - - // The destination of the control flow - BasicBlock* m_destBlock; - - // Edge weights - weight_t m_edgeWeightMin; - weight_t m_edgeWeightMax; - - // Likelihood that m_sourceBlock transfers control along this edge. - // Values in range [0..1] - weight_t m_likelihood; - - // The count of duplicate "edges" (used for switch stmts or degenerate branches) - unsigned m_dupCount; - - // True if likelihood has been set - bool m_likelihoodSet; - -public: - FlowEdge(BasicBlock* sourceBlock, BasicBlock* destBlock, FlowEdge* rest) - : m_nextPredEdge(rest) - , m_sourceBlock(sourceBlock) - , m_destBlock(destBlock) - , m_edgeWeightMin(0) - , m_edgeWeightMax(0) - , m_likelihood(0) - , m_dupCount(0) - , m_likelihoodSet(false) - { - } - - FlowEdge* getNextPredEdge() const - { - return m_nextPredEdge; - } - - FlowEdge** getNextPredEdgeRef() - { - return &m_nextPredEdge; - } - - void setNextPredEdge(FlowEdge* newEdge) - { - m_nextPredEdge = newEdge; - } - - BasicBlock* getSourceBlock() const - { - assert(m_sourceBlock != nullptr); - return m_sourceBlock; - } - - void setSourceBlock(BasicBlock* newBlock) - { - assert(newBlock != nullptr); - m_sourceBlock = newBlock; - } - - BasicBlock* getDestinationBlock() const - { - assert(m_destBlock != nullptr); - return m_destBlock; - } - - void setDestinationBlock(BasicBlock* newBlock) - { - assert(newBlock != nullptr); - m_destBlock = newBlock; - } - - weight_t edgeWeightMin() const - { - return m_edgeWeightMin; - } - - weight_t edgeWeightMax() const - { - return m_edgeWeightMax; - } - - // These two methods are used to set new values for edge weights. - // They return false if the newWeight is not between the current [min..max] - // when slop is non-zero we allow for the case where our weights might be off by 'slop' - // - bool setEdgeWeightMinChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); - bool setEdgeWeightMaxChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); - void setEdgeWeights(weight_t newMinWeight, weight_t newMaxWeight, BasicBlock* bDst); - - weight_t getLikelihood() const - { - return m_likelihood; - } - - void setLikelihood(weight_t likelihood); - void addLikelihood(weight_t addedLikelihod); - - void clearLikelihood() - { - m_likelihood = 0.0; - m_likelihoodSet = false; - } - - bool hasLikelihood() const - { - return m_likelihoodSet; - } - - weight_t getLikelyWeight() const; - - unsigned getDupCount() const - { - return m_dupCount; - } - - void incrementDupCount() - { - m_dupCount++; - } - - void decrementDupCount() - { - assert(m_dupCount >= 1); - m_dupCount--; - } -}; - //------------------------------------------------------------------------ // BasicBlock: describes a basic block in the flowgraph. // @@ -727,19 +525,19 @@ struct BasicBlock : private LIR::Range /* The following union describes the jump target(s) of this block */ union { - unsigned bbTargetOffs; // PC offset (temporary only) - FlowEdge* bbTargetEdge; // successor edge for block kinds with only one successor (BBJ_ALWAYS, etc) - FlowEdge* bbTrueEdge; // BBJ_COND successor edge when its condition is true (alias for bbTargetEdge) - BBswtDesc* bbSwtTargets; // switch descriptor - BBehfDesc* bbEhfTargets; // BBJ_EHFINALLYRET descriptor + unsigned bbTargetOffs; // PC offset (temporary only) + BasicBlock* bbTarget; // basic block + BasicBlock* bbTrueTarget; // BBJ_COND jump target when its condition is true (alias for bbTarget) + BBswtDesc* bbSwtTargets; // switch descriptor + BBehfDesc* bbEhfTargets; // BBJ_EHFINALLYRET descriptor }; - // Successor edge of a BBJ_COND block if bbTrueEdge is not taken - FlowEdge* bbFalseEdge; + // Points to the successor of a BBJ_COND block if bbTrueTarget is not taken + BasicBlock* bbFalseTarget; public: static BasicBlock* New(Compiler* compiler); - static BasicBlock* New(Compiler* compiler, BBKinds kind); + static BasicBlock* New(Compiler* compiler, BBKinds kind, BasicBlock* target = nullptr); static BasicBlock* New(Compiler* compiler, BBehfDesc* ehfTargets); static BasicBlock* New(Compiler* compiler, BBswtDesc* swtTargets); static BasicBlock* New(Compiler* compiler, BBKinds kind, unsigned targetOffs); @@ -825,145 +623,100 @@ struct BasicBlock : private LIR::Range return bbTargetOffs; } - bool HasTarget() const + void SetKindAndTarget(BBKinds kind, unsigned targetOffs) { - // These block types should always have bbTargetEdge set - return KindIs(BBJ_ALWAYS, BBJ_CALLFINALLY, BBJ_CALLFINALLYRET, BBJ_EHCATCHRET, BBJ_EHFILTERRET, BBJ_LEAVE); + bbKind = kind; + bbTargetOffs = targetOffs; + assert(KindIs(BBJ_ALWAYS, BBJ_COND, BBJ_LEAVE)); } - BasicBlock* GetTarget() const + bool HasTarget() const { - return GetTargetEdge()->getDestinationBlock(); + // These block types should always have bbTarget set + return KindIs(BBJ_ALWAYS, BBJ_CALLFINALLY, BBJ_CALLFINALLYRET, BBJ_EHCATCHRET, BBJ_EHFILTERRET, BBJ_LEAVE); } - FlowEdge* GetTargetEdge() const + BasicBlock* GetTarget() const { - // Only block kinds that use `bbTargetEdge` can access it, and it must be non-null. + // Only block kinds that use `bbTarget` can access it, and it must be non-null. assert(HasInitializedTarget()); - assert(bbTargetEdge->getSourceBlock() == this); - assert(bbTargetEdge->getDestinationBlock() != nullptr); - return bbTargetEdge; + return bbTarget; } - void SetTargetEdge(FlowEdge* targetEdge) + void SetTarget(BasicBlock* target) { // SetKindAndTarget() nulls target for non-jump kinds, - // so don't use SetTargetEdge() to null bbTargetEdge without updating bbKind. - bbTargetEdge = targetEdge; + // so don't use SetTarget() to null bbTarget without updating bbKind. + bbTarget = target; assert(HasInitializedTarget()); - assert(bbTargetEdge->getSourceBlock() == this); - assert(bbTargetEdge->getDestinationBlock() != nullptr); - - // This is the only successor edge for this block, so likelihood should be 1.0 - bbTargetEdge->setLikelihood(1.0); } BasicBlock* GetTrueTarget() const - { - return GetTrueEdge()->getDestinationBlock(); - } - - FlowEdge* GetTrueEdge() const { assert(KindIs(BBJ_COND)); - assert(bbTrueEdge != nullptr); - assert(bbTrueEdge->getSourceBlock() == this); - assert(bbTrueEdge->getDestinationBlock() != nullptr); - return bbTrueEdge; + assert(bbTrueTarget != nullptr); + return bbTrueTarget; } - void SetTrueEdge(FlowEdge* trueEdge) + void SetTrueTarget(BasicBlock* target) { assert(KindIs(BBJ_COND)); - bbTrueEdge = trueEdge; - assert(bbTrueEdge != nullptr); - assert(bbTrueEdge->getSourceBlock() == this); - assert(bbTrueEdge->getDestinationBlock() != nullptr); + assert(target != nullptr); + bbTrueTarget = target; } bool TrueTargetIs(const BasicBlock* target) const { - return (GetTrueTarget() == target); - } - - bool TrueEdgeIs(const FlowEdge* targetEdge) const - { - return (GetTrueEdge() == targetEdge); + assert(KindIs(BBJ_COND)); + assert(bbTrueTarget != nullptr); + return (bbTrueTarget == target); } BasicBlock* GetFalseTarget() const - { - return GetFalseEdge()->getDestinationBlock(); - } - - FlowEdge* GetFalseEdge() const { assert(KindIs(BBJ_COND)); - assert(bbFalseEdge != nullptr); - assert(bbFalseEdge->getSourceBlock() == this); - assert(bbFalseEdge->getDestinationBlock() != nullptr); - return bbFalseEdge; + assert(bbFalseTarget != nullptr); + return bbFalseTarget; } - void SetFalseEdge(FlowEdge* falseEdge) + void SetFalseTarget(BasicBlock* target) { assert(KindIs(BBJ_COND)); - bbFalseEdge = falseEdge; - assert(bbFalseEdge != nullptr); - assert(bbFalseEdge->getSourceBlock() == this); - assert(bbFalseEdge->getDestinationBlock() != nullptr); + assert(target != nullptr); + bbFalseTarget = target; } bool FalseTargetIs(const BasicBlock* target) const { - return (GetFalseTarget() == target); - } - - bool FalseEdgeIs(const FlowEdge* targetEdge) const - { - return (GetFalseEdge() == targetEdge); - } - - void SetCond(FlowEdge* trueEdge, FlowEdge* falseEdge) - { - bbKind = BBJ_COND; - SetTrueEdge(trueEdge); - SetFalseEdge(falseEdge); + assert(KindIs(BBJ_COND)); + assert(bbFalseTarget != nullptr); + return (bbFalseTarget == target); } - // In most cases, a block's true and false targets are known by the time SetCond is called. - // To simplify the few cases where the false target isn't available until later, - // overload SetCond to initialize only the true target. - // This simplifies, for example, lowering switch blocks into jump sequences. - void SetCond(FlowEdge* trueEdge) + void SetCond(BasicBlock* trueTarget, BasicBlock* falseTarget) { - bbKind = BBJ_COND; - SetTrueEdge(trueEdge); + assert(trueTarget != nullptr); + bbKind = BBJ_COND; + bbTrueTarget = trueTarget; + bbFalseTarget = falseTarget; } - // Set both the block kind and target edge. - void SetKindAndTargetEdge(BBKinds kind, FlowEdge* targetEdge) + // Set both the block kind and target. This can clear `bbTarget` when setting + // block kinds that don't use `bbTarget`. + void SetKindAndTarget(BBKinds kind, BasicBlock* target = nullptr) { - bbKind = kind; - bbTargetEdge = targetEdge; - assert(HasInitializedTarget()); - - // This is the only successor edge for this block, so likelihood should be 1.0 - bbTargetEdge->setLikelihood(1.0); - } + bbKind = kind; + bbTarget = target; - // Set the block kind, and clear bbTargetEdge. - void SetKindAndTargetEdge(BBKinds kind) - { - bbKind = kind; - bbTargetEdge = nullptr; - assert(!HasTarget()); + // If bbKind indicates this block has a jump, bbTarget cannot be null. + // You shouldn't use this to set a BBJ_COND, BBJ_SWITCH, or BBJ_EHFINALLYRET. + assert(HasTarget() ? HasInitializedTarget() : (bbTarget == nullptr)); } bool HasInitializedTarget() const { assert(HasTarget()); - return (bbTargetEdge != nullptr); + return (bbTarget != nullptr); } bool TargetIs(const BasicBlock* target) const @@ -1009,13 +762,19 @@ struct BasicBlock : private LIR::Range bbEhfTargets = ehfTarget; } - // BBJ_CALLFINALLYRET uses the `bbTargetEdge` field. However, also treat it specially: + // BBJ_CALLFINALLYRET uses the `bbTarget` field. However, also treat it specially: // for callers that know they want a continuation, use this function instead of the // general `GetTarget()` to allow asserting on the block kind. BasicBlock* GetFinallyContinuation() const { assert(KindIs(BBJ_CALLFINALLYRET)); - return GetTarget(); + return bbTarget; + } + + void SetFinallyContinuation(BasicBlock* finallyContinuation) + { + assert(KindIs(BBJ_CALLFINALLYRET)); + bbTarget = finallyContinuation; } #ifdef DEBUG @@ -1024,42 +783,21 @@ struct BasicBlock : private LIR::Range BasicBlock* GetTargetRaw() const { assert(HasTarget()); - return (bbTargetEdge == nullptr) ? nullptr : bbTargetEdge->getDestinationBlock(); + return bbTarget; } // Return the BBJ_COND true target; it might be null. Only used during dumping. BasicBlock* GetTrueTargetRaw() const { assert(KindIs(BBJ_COND)); - return (bbTrueEdge == nullptr) ? nullptr : bbTrueEdge->getDestinationBlock(); + return bbTrueTarget; } // Return the BBJ_COND false target; it might be null. Only used during dumping. BasicBlock* GetFalseTargetRaw() const { assert(KindIs(BBJ_COND)); - return (bbFalseEdge == nullptr) ? nullptr : bbFalseEdge->getDestinationBlock(); - } - - // Return the target edge; it might be null. Only used during dumping. - FlowEdge* GetTargetEdgeRaw() const - { - assert(HasTarget()); - return bbTargetEdge; - } - - // Return the BBJ_COND true target edge; it might be null. Only used during dumping. - FlowEdge* GetTrueEdgeRaw() const - { - assert(KindIs(BBJ_COND)); - return bbTrueEdge; - } - - // Return the BBJ_COND false target edge; it might be null. Only used during dumping. - FlowEdge* GetFalseEdgeRaw() const - { - assert(KindIs(BBJ_COND)); - return bbFalseEdge; + return bbFalseTarget; } #endif // DEBUG @@ -1349,11 +1087,7 @@ struct BasicBlock : private LIR::Range unsigned NumSucc() const; unsigned NumSucc(Compiler* comp); - // GetSuccEdge: Returns the "i"th successor edge. Requires (0 <= i < NumSucc()). - FlowEdge* GetSuccEdge(unsigned i) const; - FlowEdge* GetSuccEdge(unsigned i, Compiler* comp); - - // GetSucc: Returns the "i"th successor block. Requires (0 <= i < NumSucc()). + // GetSucc: Returns the "i"th successor. Requires (0 <= i < NumSucc()). BasicBlock* GetSucc(unsigned i) const; BasicBlock* GetSucc(unsigned i, Compiler* comp); @@ -1556,18 +1290,9 @@ struct BasicBlock : private LIR::Range // PredBlocks: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: // for (BasicBlock* const predBlock : block->PredBlocks()) ... // - PredBlockList PredBlocks() const - { - return PredBlockList(bbPreds); - } - - // PredBlocksEditing: convenience method for enabling range-based `for` iteration over predecessor blocks, e.g.: - // for (BasicBlock* const predBlock : block->PredBlocksEditing()) ... - // This iterator tolerates modifications to bbPreds. - // - PredBlockList PredBlocksEditing() const + PredBlockList PredBlocks() const { - return PredBlockList(bbPreds); + return PredBlockList(bbPreds); } // Pred list maintenance @@ -1841,64 +1566,37 @@ struct BasicBlock : private LIR::Range bool HasPotentialEHSuccs(Compiler* comp); - // Base class for Successor block/edge iterators. + // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, + // normally used via BasicBlock::Succs(), e.g.: + // for (BasicBlock* const target : block->Succs()) ... // - class SuccList + class BBSuccList { - protected: // For one or two successors, pre-compute and stash the successors inline, in m_succs[], so we don't // need to call a function or execute another `switch` to get them. Also, pre-compute the begin and end // points of the iteration, for use by BBArrayIterator. `m_begin` and `m_end` will either point at // `m_succs` or at the switch table successor array. - FlowEdge* m_succs[2]; - FlowEdge* const* m_begin; - FlowEdge* const* m_end; - - SuccList(const BasicBlock* block); - }; + BasicBlock* m_succs[2]; + + // Quirk: Some BasicBlock kinds refer to their successors with BasicBlock pointers, + // while others use FlowEdge pointers. Eventually, every type will use FlowEdge pointers. + // For now, support iterating with both types. + union { + BasicBlock* const* m_begin; + FlowEdge* const* m_beginEdge; + }; - // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, - // normally used via BasicBlock::Succs(), e.g.: - // for (BasicBlock* const target : block->Succs()) ... - // - class BBSuccList : private SuccList - { - public: - BBSuccList(const BasicBlock* block) : SuccList(block) - { - } + union { + BasicBlock* const* m_end; + FlowEdge* const* m_endEdge; + }; - BBArrayIterator begin() const - { - return BBArrayIterator(m_begin); - } + bool iterateEdges; - BBArrayIterator end() const - { - return BBArrayIterator(m_end); - } - }; - - // BBSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, - // normally used via BasicBlock::SuccEdges(), e.g.: - // for (FlowEdge* const succEdge : block->SuccEdges()) ... - // - class BBSuccEdgeList : private SuccList - { public: - BBSuccEdgeList(const BasicBlock* block) : SuccList(block) - { - } - - FlowEdgeArrayIterator begin() const - { - return FlowEdgeArrayIterator(m_begin); - } - - FlowEdgeArrayIterator end() const - { - return FlowEdgeArrayIterator(m_end); - } + BBSuccList(const BasicBlock* block); + BBArrayIterator begin() const; + BBArrayIterator end() const; }; // BBCompilerSuccList: adapter class for forward iteration of block successors, using range-based `for`, @@ -1912,7 +1610,7 @@ struct BasicBlock : private LIR::Range Compiler* m_comp; BasicBlock* m_block; - // iterator: forward iterator for an array of BasicBlock* + // iterator: forward iterator for an array of BasicBlock*, such as the BBswtDesc->bbsDstTab. // class iterator { @@ -1962,67 +1660,6 @@ struct BasicBlock : private LIR::Range } }; - // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, - // normally used via BasicBlock::SuccEdges(), e.g.: - // for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ... - // - // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation - // of the implications of this versus the version that does not take `Compiler*`. - class BBCompilerSuccEdgeList - { - Compiler* m_comp; - BasicBlock* m_block; - - // iterator: forward iterator for an array of BasicBlock* - // - class iterator - { - Compiler* m_comp; - BasicBlock* m_block; - unsigned m_succNum; - - public: - iterator(Compiler* comp, BasicBlock* block, unsigned succNum) - : m_comp(comp), m_block(block), m_succNum(succNum) - { - } - - FlowEdge* operator*() const - { - assert(m_block != nullptr); - FlowEdge* succEdge = m_block->GetSuccEdge(m_succNum, m_comp); - assert(succEdge != nullptr); - return succEdge; - } - - iterator& operator++() - { - ++m_succNum; - return *this; - } - - bool operator!=(const iterator& i) const - { - return m_succNum != i.m_succNum; - } - }; - - public: - BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) : m_comp(comp), m_block(block) - { - } - - iterator begin() const - { - return iterator(m_comp, m_block, 0); - } - - iterator end() const - { - return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); - } - }; - // Succs: convenience methods for enabling range-based `for` iteration over a block's successors, e.g.: // for (BasicBlock* const succ : block->Succs()) ... // @@ -2039,16 +1676,6 @@ struct BasicBlock : private LIR::Range return BBCompilerSuccList(comp, this); } - BBSuccEdgeList SuccEdges() - { - return BBSuccEdgeList(this); - } - - BBCompilerSuccEdgeList SuccEdges(Compiler* comp) - { - return BBCompilerSuccEdgeList(comp, this); - } - // Clone block state and statements from `from` block to `to` block (which must be new/empty) static void CloneBlockState(Compiler* compiler, BasicBlock* to, const BasicBlock* from); @@ -2213,8 +1840,8 @@ class BasicBlockRangeList // struct BBswtDesc { - FlowEdge** bbsDstTab; // case label table address - unsigned bbsCount; // count of cases (includes 'default' if bbsHasDefault) + BasicBlock** bbsDstTab; // case label table address + unsigned bbsCount; // count of cases (includes 'default' if bbsHasDefault) // Case number and likelihood of most likely case // (only known with PGO, only valid if bbsHasDominantCase is true) @@ -2240,7 +1867,7 @@ struct BBswtDesc bbsCount--; } - FlowEdge* getDefault() + BasicBlock* getDefault() { assert(bbsHasDefault); assert(bbsCount > 0); @@ -2300,11 +1927,12 @@ inline BBArrayIterator BBEhfSuccList::end() const return BBArrayIterator(m_bbeDesc->bbeSuccs + m_bbeDesc->bbeCount); } -// SuccList out-of-class-declaration implementations +// BBSuccList out-of-class-declaration implementations // -inline BasicBlock::SuccList::SuccList(const BasicBlock* block) +inline BasicBlock::BBSuccList::BBSuccList(const BasicBlock* block) { assert(block != nullptr); + iterateEdges = false; switch (block->bbKind) { @@ -2322,24 +1950,24 @@ inline BasicBlock::SuccList::SuccList(const BasicBlock* block) case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: - m_succs[0] = block->GetTargetEdge(); + m_succs[0] = block->bbTarget; m_begin = &m_succs[0]; m_end = &m_succs[1]; break; case BBJ_COND: - m_succs[0] = block->GetFalseEdge(); + m_succs[0] = block->bbFalseTarget; m_begin = &m_succs[0]; // If both fall-through and branch successors are identical, then only include // them once in the iteration (this is the same behavior as NumSucc()/GetSucc()). - if (block->TrueEdgeIs(block->GetFalseEdge())) + if (block->TrueTargetIs(block->GetFalseTarget())) { m_end = &m_succs[1]; } else { - m_succs[1] = block->GetTrueEdge(); + m_succs[1] = block->bbTrueTarget; m_end = &m_succs[2]; } break; @@ -2350,14 +1978,16 @@ inline BasicBlock::SuccList::SuccList(const BasicBlock* block) // been computed. if (block->GetEhfTargets() == nullptr) { - m_begin = nullptr; - m_end = nullptr; + m_beginEdge = nullptr; + m_endEdge = nullptr; } else { - m_begin = block->GetEhfTargets()->bbeSuccs; - m_end = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; + m_beginEdge = block->GetEhfTargets()->bbeSuccs; + m_endEdge = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; } + + iterateEdges = true; break; case BBJ_SWITCH: @@ -2375,6 +2005,16 @@ inline BasicBlock::SuccList::SuccList(const BasicBlock* block) assert(m_end >= m_begin); } +inline BBArrayIterator BasicBlock::BBSuccList::begin() const +{ + return (iterateEdges ? BBArrayIterator(m_beginEdge) : BBArrayIterator(m_begin)); +} + +inline BBArrayIterator BasicBlock::BBSuccList::end() const +{ + return (iterateEdges ? BBArrayIterator(m_endEdge) : BBArrayIterator(m_end)); +} + // We have a simpler struct, BasicBlockList, which is simply a singly-linked // list of blocks. @@ -2392,23 +2032,206 @@ struct BasicBlockList } }; -// FlowEdge implementations (that are required to be defined after the declaration of BasicBlock) - -inline weight_t FlowEdge::getLikelyWeight() const +//------------------------------------------------------------------------- +// FlowEdge -- control flow edge +// +// In compiler terminology the control flow between two BasicBlocks +// is typically referred to as an "edge". Most well known are the +// backward branches for loops, which are often called "back-edges". +// +// "struct FlowEdge" is the type that represents our control flow edges. +// This type is a linked list of zero or more "edges". +// (The list of zero edges is represented by NULL.) +// Every BasicBlock has a field called bbPreds of this type. This field +// represents the list of "edges" that flow into this BasicBlock. +// The FlowEdge type only stores the BasicBlock* of the source for the +// control flow edge. The destination block for the control flow edge +// is implied to be the block which contained the bbPreds field. +// +// For a switch branch target there may be multiple "edges" that have +// the same source block (and destination block). We need to count the +// number of these edges so that during optimization we will know when +// we have zero of them. Rather than have extra FlowEdge entries we +// track this via the DupCount property. +// +// When we have Profile weight for the BasicBlocks we can usually compute +// the number of times each edge was executed by examining the adjacent +// BasicBlock weights. As we are doing for BasicBlocks, we call the number +// of times that a control flow edge was executed the "edge weight". +// In order to compute the edge weights we need to use a bounded range +// for every edge weight. These two fields, 'flEdgeWeightMin' and 'flEdgeWeightMax' +// are used to hold a bounded range. Most often these will converge such +// that both values are the same and that value is the exact edge weight. +// Sometimes we are left with a rage of possible values between [Min..Max] +// which represents an inexact edge weight. +// +// The bbPreds list is initially created by Compiler::fgLinkBasicBlocks() +// and is incrementally kept up to date. +// +// The edge weight are computed by Compiler::fgComputeEdgeWeights() +// the edge weights are used to straighten conditional branches +// by Compiler::fgReorderBlocks() +// +struct FlowEdge { - assert(m_likelihoodSet); - return m_likelihood * m_sourceBlock->bbWeight; -} +private: + // The next predecessor edge in the list, nullptr for end of list. + FlowEdge* m_nextPredEdge; + + // The source of the control flow + BasicBlock* m_sourceBlock; + + // The destination of the control flow + BasicBlock* m_destBlock; + + // Edge weights + weight_t m_edgeWeightMin; + weight_t m_edgeWeightMax; + + // Likelihood that m_sourceBlock transfers control along this edge. + // Values in range [0..1] + weight_t m_likelihood; + + // The count of duplicate "edges" (used for switch stmts or degenerate branches) + unsigned m_dupCount; + + // True if likelihood has been set + bool m_likelihoodSet; + +public: + FlowEdge(BasicBlock* sourceBlock, BasicBlock* destBlock, FlowEdge* rest) + : m_nextPredEdge(rest) + , m_sourceBlock(sourceBlock) + , m_destBlock(destBlock) + , m_edgeWeightMin(0) + , m_edgeWeightMax(0) + , m_likelihood(0) + , m_dupCount(0) + , m_likelihoodSet(false) + { + } + + FlowEdge* getNextPredEdge() const + { + return m_nextPredEdge; + } + + FlowEdge** getNextPredEdgeRef() + { + return &m_nextPredEdge; + } + + void setNextPredEdge(FlowEdge* newEdge) + { + m_nextPredEdge = newEdge; + } + + BasicBlock* getSourceBlock() const + { + assert(m_sourceBlock != nullptr); + return m_sourceBlock; + } + + void setSourceBlock(BasicBlock* newBlock) + { + assert(newBlock != nullptr); + m_sourceBlock = newBlock; + } + + BasicBlock* getDestinationBlock() const + { + assert(m_destBlock != nullptr); + return m_destBlock; + } + + void setDestinationBlock(BasicBlock* newBlock) + { + assert(newBlock != nullptr); + m_destBlock = newBlock; + } + + weight_t edgeWeightMin() const + { + return m_edgeWeightMin; + } + + weight_t edgeWeightMax() const + { + return m_edgeWeightMax; + } + + // These two methods are used to set new values for edge weights. + // They return false if the newWeight is not between the current [min..max] + // when slop is non-zero we allow for the case where our weights might be off by 'slop' + // + bool setEdgeWeightMinChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); + bool setEdgeWeightMaxChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); + void setEdgeWeights(weight_t newMinWeight, weight_t newMaxWeight, BasicBlock* bDst); + + weight_t getLikelihood() const + { + return m_likelihood; + } + + void setLikelihood(weight_t likelihood) + { + assert(likelihood >= 0.0); + assert(likelihood <= 1.0); + m_likelihoodSet = true; + m_likelihood = likelihood; + } + + void clearLikelihood() + { + m_likelihood = 0.0; + m_likelihoodSet = false; + } + + bool hasLikelihood() const + { + return m_likelihoodSet; + } + + weight_t getLikelyWeight() const + { + assert(m_likelihoodSet); + return m_likelihood * m_sourceBlock->bbWeight; + } + + unsigned getDupCount() const + { + return m_dupCount; + } + + void incrementDupCount() + { + m_dupCount++; + } + + void decrementDupCount() + { + assert(m_dupCount >= 1); + m_dupCount--; + } +}; // BasicBlock iterator implementations (that are required to be defined after the declaration of FlowEdge) inline BasicBlock* BBArrayIterator::operator*() const { - assert(m_edgeEntry != nullptr); - FlowEdge* edgeTarget = *m_edgeEntry; - assert(edgeTarget != nullptr); - assert(edgeTarget->getDestinationBlock() != nullptr); - return edgeTarget->getDestinationBlock(); + if (iterateEdges) + { + assert(m_edgeEntry != nullptr); + FlowEdge* edgeTarget = *m_edgeEntry; + assert(edgeTarget != nullptr); + assert(edgeTarget->getDestinationBlock() != nullptr); + return edgeTarget->getDestinationBlock(); + } + + assert(m_bbEntry != nullptr); + BasicBlock* bTarget = *m_bbEntry; + assert(bTarget != nullptr); + return bTarget; } // Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and FlowEdge) @@ -2434,45 +2257,29 @@ inline PredEdgeList::iterator& PredEdgeList::iterator::operator++() return *this; } -template -inline PredBlockList::iterator::iterator(FlowEdge* pred) : m_pred(pred) +inline PredBlockList::iterator::iterator(FlowEdge* pred) : m_pred(pred) { - bool initNextPointer = allowEdits; - INDEBUG(initNextPointer = true); - if (initNextPointer) - { - m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); - } +#ifdef DEBUG + m_next = (m_pred == nullptr) ? nullptr : m_pred->getNextPredEdge(); +#endif } -template -inline BasicBlock* PredBlockList::iterator::operator*() const +inline BasicBlock* PredBlockList::iterator::operator*() const { return m_pred->getSourceBlock(); } -template -inline typename PredBlockList::iterator& PredBlockList::iterator::operator++() +inline PredBlockList::iterator& PredBlockList::iterator::operator++() { - if (allowEdits) - { - // For editing iterators, m_next is always used and maintained - m_pred = m_next; - m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); - } - else - { - FlowEdge* next = m_pred->getNextPredEdge(); + FlowEdge* next = m_pred->getNextPredEdge(); #ifdef DEBUG - // If allowEdits=false, check that the next block is the one we expect to see. - assert(next == m_next); - m_next = (m_next == nullptr) ? nullptr : m_next->getNextPredEdge(); + // Check that the next block is the one we expect to see. + assert(next == m_next); + m_next = (next == nullptr) ? nullptr : next->getNextPredEdge(); #endif // DEBUG - m_pred = next; - } - + m_pred = next; return *this; } diff --git a/src/coreclr/jit/clrjit.natvis b/src/coreclr/jit/clrjit.natvis index cfbc6a181e9743..95dd3dc305689b 100644 --- a/src/coreclr/jit/clrjit.natvis +++ b/src/coreclr/jit/clrjit.natvis @@ -8,8 +8,9 @@ The .NET Foundation licenses this file to you under the MIT license. @@ -20,17 +21,12 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- - BB{bbNum,d}->BB{bbTargetEdge->m_destBlock->bbNum,d}; {bbKind,en} + BB{bbNum,d}->BB{bbTarget->bbNum,d}; {bbKind,en} BB{bbNum,d}; {bbKind,en}; {bbSwtTargets->bbsCount} cases BB{bbNum,d}; {bbKind,en}; {bbEhfTargets->bbeCount} succs BB{bbNum,d}; {bbKind,en} - - BB{m_sourceBlock->bbNum,d}->BB{m_destBlock->bbNum,d} ({m_likelihood,g}) (dup {m_dupCount,d}) - BB{m_sourceBlock->bbNum,d}->BB{m_destBlock->bbNum,d} ({m_likelihood,g}) - - REMOVED [BB{lpTop->bbNum,d}..BB{lpBottom->bbNum,d}] pre-h:BB{lpHead->bbNum,d} e:BB{lpEntry->bbNum,d} {lpFlags,en} @@ -90,11 +86,6 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- {gtTreeID, d}: [{gtOper,en}, {gtType,en} V{((GenTreeLclFld*)this)->_gtLclNum,u}[+{((GenTreeLclFld*)this)->m_lclOffs,u}]] - - - [{Oper,en}, {Type,en}] - - LinearScan @@ -178,7 +169,6 @@ Documentation for VS debugger format specifiers: https://learn.microsoft.com/en- - [U{this->relatedInterval->varNum,d}, #{this->intervalIndex, d}, reg={(regNumber)physReg, en}] [V{this->varNum,d}, #{this->intervalIndex, d}, reg={(regNumber)physReg, en}] [C{this->intervalIndex, d}, reg={(regNumber)physReg, en}] [I{this->intervalIndex, d}, reg={(regNumber)physReg, en}] diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 7a2359c9fb5fc9..afd9e42a9d2cdd 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1182,10 +1182,10 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCodeForCpObj(GenTreeBlk* cpObjNode); void genCodeForCpBlkRepMovs(GenTreeBlk* cpBlkNode); void genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode); +#ifndef TARGET_X86 + void genCodeForCpBlkHelper(GenTreeBlk* cpBlkNode); +#endif void genCodeForPhysReg(GenTreePhysReg* tree); -#ifdef SWIFT_SUPPORT - void genCodeForSwiftErrorReg(GenTree* tree); -#endif // SWIFT_SUPPORT void genCodeForNullCheck(GenTreeIndir* tree); void genCodeForCmpXchg(GenTreeCmpXchg* tree); void genCodeForReuseVal(GenTree* treeNode); @@ -1257,10 +1257,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif // FEATURE_PUT_STRUCT_ARG_STK void genCodeForStoreBlk(GenTreeBlk* storeBlkNode); +#ifndef TARGET_X86 + void genCodeForInitBlkHelper(GenTreeBlk* initBlkNode); +#endif void genCodeForInitBlkLoop(GenTreeBlk* initBlkNode); void genCodeForInitBlkRepStos(GenTreeBlk* initBlkNode); void genCodeForInitBlkUnroll(GenTreeBlk* initBlkNode); - unsigned genEmitJumpTable(GenTree* treeNode, bool relativeAddr); void genJumpTable(GenTree* tree); void genTableBasedSwitch(GenTree* tree); #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) diff --git a/src/coreclr/jit/codegenarm.cpp b/src/coreclr/jit/codegenarm.cpp index 8cf3ac32b3a329..4a8c08a89858e8 100644 --- a/src/coreclr/jit/codegenarm.cpp +++ b/src/coreclr/jit/codegenarm.cpp @@ -647,7 +647,29 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) // void CodeGen::genJumpTable(GenTree* treeNode) { - unsigned jmpTabBase = genEmitJumpTable(treeNode, false); + noway_assert(compiler->compCurBB->KindIs(BBJ_SWITCH)); + assert(treeNode->OperGet() == GT_JMPTABLE); + + unsigned jumpCount = compiler->compCurBB->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTable = compiler->compCurBB->GetSwitchTargets()->bbsDstTab; + unsigned jmpTabBase; + + jmpTabBase = GetEmitter()->emitBBTableDataGenBeg(jumpCount, false); + + JITDUMP("\n J_M%03u_DS%02u LABEL DWORD\n", compiler->compMethodID, jmpTabBase); + + for (unsigned i = 0; i < jumpCount; i++) + { + BasicBlock* target = *jumpTable++; + noway_assert(target->HasFlag(BBF_HAS_LABEL)); + + JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); + + GetEmitter()->emitDataGenData(i, target); + } + + GetEmitter()->emitDataGenEnd(); + genMov32RelocatableDataLabel(jmpTabBase, treeNode->GetRegNum()); genProduceReg(treeNode); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 81370e6413835f..3883b20118ad8a 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -3626,7 +3626,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) unsigned slots = layout->GetSlotCount(); // Temp register(s) used to perform the sequence of loads and stores. - regNumber tmpReg = cpObjNode->ExtractTempReg(RBM_ALLINT); + regNumber tmpReg = cpObjNode->ExtractTempReg(); regNumber tmpReg2 = REG_NA; assert(genIsValidIntReg(tmpReg)); @@ -3635,7 +3635,7 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) if (slots > 1) { - tmpReg2 = cpObjNode->ExtractTempReg(RBM_ALLINT); + tmpReg2 = cpObjNode->GetSingleTempReg(); assert(tmpReg2 != tmpReg); assert(genIsValidIntReg(tmpReg2)); assert(tmpReg2 != REG_WRITE_BARRIER_DST_BYREF); @@ -3682,60 +3682,26 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) { unsigned gcPtrCount = cpObjNode->GetLayout()->GetGCPtrCount(); - // We might also need SIMD regs if we have 4 or more continuous non-gc slots - // On ARM64, SIMD loads/stores provide 8-byte atomicity guarantees when aligned to 8 bytes. - regNumber tmpSimdReg1 = REG_NA; - regNumber tmpSimdReg2 = REG_NA; - if ((slots >= 4) && compiler->IsBaselineSimdIsaSupported()) - { - tmpSimdReg1 = cpObjNode->ExtractTempReg(RBM_ALLFLOAT); - tmpSimdReg2 = cpObjNode->ExtractTempReg(RBM_ALLFLOAT); - } - unsigned i = 0; while (i < slots) { if (!layout->IsGCPtr(i)) { - // How many continuous non-gc slots do we have? - unsigned nonGcSlots = 0; - do + // Check if the next slot's type is also TYP_GC_NONE and use ldp/stp + if ((i + 1 < slots) && !layout->IsGCPtr(i + 1)) { - nonGcSlots++; - i++; - } while ((i < slots) && !layout->IsGCPtr(i)); - - const regNumber srcReg = REG_WRITE_BARRIER_SRC_BYREF; - const regNumber dstReg = REG_WRITE_BARRIER_DST_BYREF; - while (nonGcSlots > 0) + emit->emitIns_R_R_R_I(INS_ldp, EA_8BYTE, tmpReg, tmpReg2, REG_WRITE_BARRIER_SRC_BYREF, + 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); + emit->emitIns_R_R_R_I(INS_stp, EA_8BYTE, tmpReg, tmpReg2, REG_WRITE_BARRIER_DST_BYREF, + 2 * TARGET_POINTER_SIZE, INS_OPTS_POST_INDEX); + ++i; // extra increment of i, since we are copying two items + } + else { - regNumber tmp1 = tmpReg; - regNumber tmp2 = tmpReg2; - emitAttr size = EA_8BYTE; - insOpts opts = INS_OPTS_POST_INDEX; - - // Copy at least two slots at a time - if (nonGcSlots >= 2) - { - // Do 4 slots at a time if SIMD is supported - if ((nonGcSlots >= 4) && compiler->IsBaselineSimdIsaSupported()) - { - // We need SIMD temp regs now - tmp1 = tmpSimdReg1; - tmp2 = tmpSimdReg2; - size = EA_16BYTE; - nonGcSlots -= 2; - } - nonGcSlots -= 2; - emit->emitIns_R_R_R_I(INS_ldp, size, tmp1, tmp2, srcReg, EA_SIZE(size) * 2, opts); - emit->emitIns_R_R_R_I(INS_stp, size, tmp1, tmp2, dstReg, EA_SIZE(size) * 2, opts); - } - else - { - nonGcSlots--; - emit->emitIns_R_R_I(INS_ldr, EA_8BYTE, tmp1, srcReg, EA_SIZE(size), opts); - emit->emitIns_R_R_I(INS_str, EA_8BYTE, tmp1, dstReg, EA_SIZE(size), opts); - } + emit->emitIns_R_R_I(INS_ldr, EA_8BYTE, tmpReg, REG_WRITE_BARRIER_SRC_BYREF, TARGET_POINTER_SIZE, + INS_OPTS_POST_INDEX); + emit->emitIns_R_R_I(INS_str, EA_8BYTE, tmpReg, REG_WRITE_BARRIER_DST_BYREF, TARGET_POINTER_SIZE, + INS_OPTS_POST_INDEX); } } else @@ -3743,8 +3709,8 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) // In the case of a GC-Pointer we'll call the ByRef write barrier helper genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE); gcPtrCount--; - i++; } + ++i; } assert(gcPtrCount == 0); } @@ -3784,7 +3750,32 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) // emits the table and an instruction to get the address of the first element void CodeGen::genJumpTable(GenTree* treeNode) { - unsigned jmpTabBase = genEmitJumpTable(treeNode, true); + noway_assert(compiler->compCurBB->KindIs(BBJ_SWITCH)); + assert(treeNode->OperGet() == GT_JMPTABLE); + + unsigned jumpCount = compiler->compCurBB->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTable = compiler->compCurBB->GetSwitchTargets()->bbsDstTab; + unsigned jmpTabOffs; + unsigned jmpTabBase; + + jmpTabBase = GetEmitter()->emitBBTableDataGenBeg(jumpCount, true); + + jmpTabOffs = 0; + + JITDUMP("\n J_M%03u_DS%02u LABEL DWORD\n", compiler->compMethodID, jmpTabBase); + + for (unsigned i = 0; i < jumpCount; i++) + { + BasicBlock* target = *jumpTable++; + noway_assert(target->HasFlag(BBF_HAS_LABEL)); + + JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); + + GetEmitter()->emitDataGenData(i, target); + }; + + GetEmitter()->emitDataGenEnd(); + // Access to inline data is 'abstracted' by a special type of static member // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference // to constant data, not a real static field. diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 3c02c968fe919d..47cc0fb6bfd194 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -4581,14 +4581,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_subr, EA_SCALABLE, REG_V2, REG_P0, REG_V13, INS_OPTS_SCALABLE_S); // SUBR ., /M, ., . -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_AB_3B - theEmitter->emitIns_R_R_R(INS_sve_addpt, EA_SCALABLE, REG_V0, REG_P1, REG_V2, - INS_OPTS_SCALABLE_D); // ADDPT .D, /M, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_subpt, EA_SCALABLE, REG_V0, REG_P1, REG_V2, - INS_OPTS_SCALABLE_D); // SUBPT .D, /M, .D, .D -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_AC_3A theEmitter->emitIns_R_R_R(INS_sve_sdiv, EA_SCALABLE, REG_V3, REG_P2, REG_V9, INS_OPTS_SCALABLE_S); // SDIV ., /M, ., . @@ -4733,54 +4725,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_P0, REG_V0, INS_OPTS_SCALABLE_S, INS_SCALABLE_OPTS_WIDE); // LSR ., /M, ., .D - // IF_SVE_CE_2A - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P2, REG_V12, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .B, - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V2, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [0] - - // IF_SVE_CE_2B - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P15, REG_V7, 7, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .D, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V16, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .D, [] - - // IF_SVE_CE_2C - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P0, REG_V31, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V1, REG_P1, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] - - // IF_SVE_CE_2D - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P3, REG_V9, 3, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .S, [] - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P10, REG_V4, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .S, [] - - // IF_SVE_CF_2A - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V11, REG_P12, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV , .B - theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V2, REG_P7, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [0], .S - - // IF_SVE_CF_2B - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V6, REG_P8, 7, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .D - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V9, REG_P7, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .D - - // IF_SVE_CF_2C - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V8, REG_P4, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .H - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V5, REG_P9, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .H - - // IF_SVE_CF_2D - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V14, REG_P2, 3, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .S - theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V3, REG_P15, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .S - // IF_SVE_CJ_2A theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_P1, REG_P2, INS_OPTS_SCALABLE_B); // REV ., . @@ -5157,106 +5101,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_fsubr, EA_SCALABLE, REG_V6, REG_P4, REG_V29, INS_OPTS_SCALABLE_D); // FSUBR ., /M, ., . - // IF_SVE_HL_3B - theEmitter->emitIns_R_R_R(INS_sve_bfadd, EA_SCALABLE, REG_V0, REG_P0, REG_V1, - INS_OPTS_SCALABLE_H); // BFADD .H, /M, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmax, EA_SCALABLE, REG_V2, REG_P1, REG_V3, - INS_OPTS_SCALABLE_H); // BFMAX .H, /M, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmaxnm, EA_SCALABLE, REG_V4, REG_P2, REG_V5, - INS_OPTS_SCALABLE_H); // BFMAXNM .H, /M, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmin, EA_SCALABLE, REG_V6, REG_P3, REG_V7, - INS_OPTS_SCALABLE_H); // BFMIN .H, /M, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfminnm, EA_SCALABLE, REG_V8, REG_P4, REG_V9, - INS_OPTS_SCALABLE_H); // BFMINNM .H, /M, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmul, EA_SCALABLE, REG_V10, REG_P5, REG_V11, - INS_OPTS_SCALABLE_H); // BFMUL .H, /M, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfsub, EA_SCALABLE, REG_V12, REG_P6, REG_V13, - INS_OPTS_SCALABLE_H); // BFSUB .H, /M, .H, .H - - // IF_SVE_HO_3A - theEmitter->emitIns_R_R_R(INS_sve_bfcvt, EA_SCALABLE, REG_V3, REG_P2, REG_V9, - INS_OPTS_S_TO_H); // BFCVT .H, /M, .S - - // IF_SVE_HO_3B - theEmitter->emitIns_R_R_R(INS_sve_fcvt, EA_SCALABLE, REG_V7, REG_P7, REG_V1, - INS_OPTS_S_TO_D); // FCVT .D, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_fcvt, EA_SCALABLE, REG_V29, REG_P3, REG_V12, - INS_OPTS_D_TO_S); // FCVT .S, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_fcvt, EA_SCALABLE, REG_V0, REG_P4, REG_V13, - INS_OPTS_D_TO_H); // FCVT .H, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_fcvt, EA_SCALABLE, REG_V1, REG_P5, REG_V14, - INS_OPTS_H_TO_D); // FCVT .D, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_fcvt, EA_SCALABLE, REG_V2, REG_P6, REG_V15, - INS_OPTS_S_TO_H); // FCVT .H, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_fcvt, EA_SCALABLE, REG_V3, REG_P7, REG_V16, - INS_OPTS_H_TO_S); // FCVT .S, /M, .H - - // IF_SVE_HO_3C - theEmitter->emitIns_R_R_R(INS_sve_fcvtx, EA_SCALABLE, REG_V2, REG_P0, REG_V6, - INS_OPTS_D_TO_S); // FCVTX .S, /M, .D - - // IF_SVE_HP_3B - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V9, REG_P1, REG_V3, - INS_OPTS_SCALABLE_S); // FCVTZS .S, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V5, REG_P0, REG_V24, - INS_OPTS_S_TO_D); // FCVTZS .D, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V12, REG_P3, REG_V6, - INS_OPTS_D_TO_S); // FCVTZS .S, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V2, REG_P1, REG_V17, - INS_OPTS_SCALABLE_D); // FCVTZS .D, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V3, REG_P2, REG_V18, - INS_OPTS_SCALABLE_H); // FCVTZS .H, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V4, REG_P3, REG_V19, - INS_OPTS_H_TO_S); // FCVTZS .S, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_fcvtzs, EA_SCALABLE, REG_V5, REG_P4, REG_V20, - INS_OPTS_H_TO_D); // FCVTZS .D, /M, .H - - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V3, REG_P2, REG_V10, - INS_OPTS_SCALABLE_S); // FCVTZU .S, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V10, REG_P7, REG_V1, - INS_OPTS_S_TO_D); // FCVTZU .D, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V4, REG_P3, REG_V13, - INS_OPTS_D_TO_S); // FCVTZU .S, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V22, REG_P6, REG_V4, - INS_OPTS_SCALABLE_D); // FCVTZU .D, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V23, REG_P7, REG_V5, - INS_OPTS_SCALABLE_H); // FCVTZU .H, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V24, REG_P0, REG_V6, - INS_OPTS_H_TO_S); // FCVTZU .S, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_fcvtzu, EA_SCALABLE, REG_V25, REG_P1, REG_V7, - INS_OPTS_H_TO_D); // FCVTZU .D, /M, .H - - // IF_SVE_HS_3A - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V19, REG_P2, REG_V8, - INS_OPTS_SCALABLE_S); // SCVTF .S, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V1, REG_P5, REG_V19, - INS_OPTS_S_TO_D); // SCVTF .D, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V4, REG_P0, REG_V14, - INS_OPTS_D_TO_S); // SCVTF .S, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V0, REG_P0, REG_V0, - INS_OPTS_SCALABLE_D); // SCVTF .D, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V12, REG_P5, REG_V14, - INS_OPTS_SCALABLE_H); // SCVTF .H, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V14, REG_P7, REG_V16, - INS_OPTS_S_TO_H); // SCVTF .H, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_scvtf, EA_SCALABLE, REG_V16, REG_P1, REG_V18, - INS_OPTS_D_TO_H); // SCVTF .H, /M, .D - - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V17, REG_P6, REG_V11, - INS_OPTS_SCALABLE_S); // UCVTF .S, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V3, REG_P3, REG_V20, - INS_OPTS_S_TO_D); // UCVTF .D, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V8, REG_P1, REG_V7, - INS_OPTS_D_TO_S); // UCVTF .S, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V8, REG_P4, REG_V9, - INS_OPTS_SCALABLE_D); // UCVTF .D, /M, .D - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V13, REG_P6, REG_V15, - INS_OPTS_SCALABLE_H); // UCVTF .H, /M, .H - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V15, REG_P0, REG_V17, - INS_OPTS_S_TO_H); // UCVTF .H, /M, .S - theEmitter->emitIns_R_R_R(INS_sve_ucvtf, EA_SCALABLE, REG_V17, REG_P2, REG_V19, - INS_OPTS_D_TO_H); // UCVTF .H, /M, .D - // IF_SVE_HT_4A theEmitter->emitIns_R_R_R_R(INS_sve_facge, EA_SCALABLE, REG_P0, REG_P0, REG_V10, REG_V31, INS_OPTS_SCALABLE_H); // FACGE ., /Z, ., . @@ -5281,16 +5125,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R_R(INS_sve_fcmuo, EA_SCALABLE, REG_P5, REG_P2, REG_V31, REG_V20, INS_OPTS_SCALABLE_S); // FCMUO ., /Z, ., . - // IF_SVE_HU_4A - theEmitter->emitIns_R_R_R_R(INS_sve_fmla, EA_SCALABLE, REG_V0, REG_P0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // FMLA ., /M, ., . - theEmitter->emitIns_R_R_R_R(INS_sve_fmls, EA_SCALABLE, REG_V3, REG_P2, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // FMLS ., /M, ., . - theEmitter->emitIns_R_R_R_R(INS_sve_fnmla, EA_SCALABLE, REG_V6, REG_P4, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // FNMLA ., /M, ., . - theEmitter->emitIns_R_R_R_R(INS_sve_fnmls, EA_SCALABLE, REG_V9, REG_P6, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // FNMLS ., /M, ., . - // IF_SVE_AF_3A theEmitter->emitIns_R_R_R(INS_sve_andv, EA_1BYTE, REG_V0, REG_P0, REG_V0, INS_OPTS_SCALABLE_B); // ANDV , , . @@ -5435,10 +5269,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_umulh, EA_SCALABLE, REG_V31, REG_V5, REG_V0, INS_OPTS_SCALABLE_D, INS_SCALABLE_OPTS_UNPREDICATED); // UMULH ., ., . - // IF_SVE_BD_3B - theEmitter->emitIns_R_R_R(INS_sve_pmul, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // PMUL .B, .B, .B - // IF_SVE_BE_3A theEmitter->emitIns_R_R_R(INS_sve_sqdmulh, EA_SCALABLE, REG_V7, REG_V28, REG_V0, INS_OPTS_SCALABLE_B); // SQDMULH ., ., . @@ -5453,590 +5283,10 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_lsr, EA_SCALABLE, REG_V29, REG_V10, REG_V22, INS_OPTS_SCALABLE_S, INS_SCALABLE_OPTS_UNPREDICATED_WIDE); // LSR ., ., .D - // IF_SVE_BH_3A - theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V4, REG_V2, REG_V0, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_LSL_N); // ADR ., [., .{, }] - theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V29, REG_V1, REG_V10, 1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_LSL_N); // ADR ., [., .{, }] - - // IF_SVE_BH_3B - theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V9, REG_V7, REG_V9, 0, - INS_OPTS_SCALABLE_D_SXTW); // ADR .D, [.D, .D, SXTW{}] - theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V12, REG_V3, REG_V5, 2, - INS_OPTS_SCALABLE_D_SXTW); // ADR .D, [.D, .D, SXTW{}] - - // IF_SVE_BH_3B_A - theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V9, REG_V10, REG_V14, 0, - INS_OPTS_SCALABLE_D_UXTW); // ADR .D, [.D, .D, UXTW{}] - theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V3, REG_V15, REG_V11, 3, - INS_OPTS_SCALABLE_D_UXTW); // ADR .D, [.D, .D, UXTW{}] - // IF_SVE_BK_3A theEmitter->emitIns_R_R_R(INS_sve_ftssel, EA_SCALABLE, REG_V17, REG_V16, REG_V15, INS_OPTS_SCALABLE_D); // FTSSEL ., ., . - // IF_SVE_BR_3A - theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V18, REG_V19, REG_V20, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V21, REG_V22, REG_V23, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V24, REG_V25, REG_V26, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V27, REG_V28, REG_V29, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V30, REG_V31, REG_V0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V1, REG_V2, REG_V3, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP2 ., ., . - - // IF_SVE_BR_3B - theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN1 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_trn2, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // TRN2 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_uzp1, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP1 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_uzp2, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // UZP2 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_zip1, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP1 .Q, .Q, .Q - theEmitter->emitIns_R_R_R(INS_sve_zip2, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_UNPREDICATED); // ZIP2 .Q, .Q, .Q - - // IF_SVE_BS_1A - theEmitter->emitIns_R_I(INS_sve_and, EA_SCALABLE, REG_V0, 0x00000000000000AA, - INS_OPTS_SCALABLE_B); // AND ., ., # - theEmitter->emitIns_R_I(INS_sve_bic, EA_SCALABLE, REG_V1, 0xFFFFFFFFFFFFFF55, - INS_OPTS_SCALABLE_B); // BIC ., ., # - theEmitter->emitIns_R_I(INS_sve_and, EA_SCALABLE, REG_V2, 0x000000000000FF00, - INS_OPTS_SCALABLE_H); // AND ., ., # - theEmitter->emitIns_R_I(INS_sve_bic, EA_SCALABLE, REG_V3, 0xFFFFFFFFFFFF00FF, - INS_OPTS_SCALABLE_H); // BIC ., ., # - theEmitter->emitIns_R_I(INS_sve_eor, EA_SCALABLE, REG_V4, 0x0000000003FFC000, - INS_OPTS_SCALABLE_S); // EOR ., ., # - theEmitter->emitIns_R_I(INS_sve_eon, EA_SCALABLE, REG_V5, 0xFFFFFFFFFC003FFF, - INS_OPTS_SCALABLE_S); // EON ., ., # - theEmitter->emitIns_R_I(INS_sve_orr, EA_SCALABLE, REG_V6, 0x00FFFFF000000000, - INS_OPTS_SCALABLE_D); // ORR ., ., # - theEmitter->emitIns_R_I(INS_sve_orn, EA_SCALABLE, REG_V7, 0xFF00000FFFFFFFFF, - INS_OPTS_SCALABLE_D); // ORN ., ., # - - // IF_SVE_BT_1A - theEmitter->emitIns_R_I(INS_sve_dupm, EA_SCALABLE, REG_V0, 0x0000000000000070, - INS_OPTS_SCALABLE_B); // DUPM ., # - theEmitter->emitIns_R_I(INS_sve_dupm, EA_SCALABLE, REG_V1, 0x0000000000003FFC, - INS_OPTS_SCALABLE_H); // DUPM ., # - theEmitter->emitIns_R_I(INS_sve_dupm, EA_SCALABLE, REG_V2, 0x0000000000007000, - INS_OPTS_SCALABLE_S); // DUPM ., # - theEmitter->emitIns_R_I(INS_sve_dupm, EA_SCALABLE, REG_V3, 0xFFFFFFFFFFFF0000, - INS_OPTS_SCALABLE_D); // DUPM ., # - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V4, 0x000000000000003F, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_IMM_BITMASK); // MOV ., # - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V5, 0x0000000000000700, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_IMM_BITMASK); // MOV ., # - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V6, 0x0000000000FFFFF0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_IMM_BITMASK); // MOV ., # - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V7, 0xFFFFF00000FFFFFF, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_IMM_BITMASK); // MOV ., # - - // IF_SVE_BV_2A - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V15, REG_P5, 0, - INS_OPTS_SCALABLE_B); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V5, REG_P15, 27, - INS_OPTS_SCALABLE_B); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V31, REG_P0, -128, - INS_OPTS_SCALABLE_B); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V0, REG_P5, 127, - INS_OPTS_SCALABLE_B); // MOV ., /Z, #{, } - - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V15, REG_P5, 0, - INS_OPTS_SCALABLE_H); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V23, REG_P12, 10, - INS_OPTS_SCALABLE_S); // MOV ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V4, REG_P0, -128, - INS_OPTS_SCALABLE_D); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V19, REG_P15, 127, - INS_OPTS_SCALABLE_H); // MOV ., /Z, #{, } - - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V1, REG_P0, 256, - INS_OPTS_SCALABLE_S); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V1, REG_P0, 3072, - INS_OPTS_SCALABLE_D); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V1, REG_P0, -3072, - INS_OPTS_SCALABLE_H); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V1, REG_P0, -32768, - INS_OPTS_SCALABLE_S); // CPY ., /Z, #{, } - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_P0, REG_V0, 32512, - INS_OPTS_SCALABLE_D); // MOV ., /Z, #{, } - - // IF_SVE_BV_2A_A - theEmitter->emitIns_R_R_I(INS_sve_cpy, EA_SCALABLE, REG_V1, REG_P12, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_PREDICATE_MERGE); // CPY ., /M, #{, } - - // IF_SVE_BV_2A_J - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V27, REG_P13, 5632, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_PREDICATE_MERGE); // MOV ., /M, #{, } - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V27, REG_P13, -5632, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_PREDICATE_MERGE); // MOV ., /M, #{, } - - // IF_SVE_BV_2B - theEmitter->emitIns_R_R(INS_sve_fmov, EA_SCALABLE, REG_V0, REG_P1, - INS_OPTS_SCALABLE_H); // FMOV ., /M, #0.0 - theEmitter->emitIns_R_R(INS_sve_fmov, EA_SCALABLE, REG_V2, REG_P3, - INS_OPTS_SCALABLE_S); // FMOV ., /M, #0.0 - theEmitter->emitIns_R_R(INS_sve_fmov, EA_SCALABLE, REG_V4, REG_P5, - INS_OPTS_SCALABLE_D); // FMOV ., /M, #0.0 - - // IF_SVE_BW_2A - // MOV should be preferred alias when emitting DUP - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V4, REG_V12, 63, - INS_OPTS_SCALABLE_B); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V8, REG_V9, 31, - INS_OPTS_SCALABLE_H); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V11, REG_V28, 15, - INS_OPTS_SCALABLE_S); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V21, REG_V12, 7, - INS_OPTS_SCALABLE_D); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V14, REG_V7, 3, - INS_OPTS_SCALABLE_Q); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V13, REG_V8, 0, - INS_OPTS_SCALABLE_B); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V2, REG_V0, 0, - INS_OPTS_SCALABLE_H); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V15, REG_V31, 0, - INS_OPTS_SCALABLE_S); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V23, REG_V27, 0, - INS_OPTS_SCALABLE_D); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_dup, EA_SCALABLE, REG_V4, REG_V3, 0, - INS_OPTS_SCALABLE_Q); // MOV ., - - // MOV implementation should produce same output as DUP implementation with same parameters - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V1, REG_V16, 63, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V17, REG_V18, 31, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V11, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V2, REG_V3, 7, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V3, REG_V8, 3, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., .[] - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V13, REG_V9, 0, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V12, REG_V6, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V2, REG_V7, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - theEmitter->emitIns_R_R_I(INS_sve_mov, EA_SCALABLE, REG_V10, REG_V20, 0, INS_OPTS_SCALABLE_Q, - INS_SCALABLE_OPTS_BROADCAST); // MOV ., - - // IF_SVE_BZ_3A - theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // TBL ., {.}, . - theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // TBL ., {.}, . - theEmitter->emitIns_R_R_R(INS_sve_tbx, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // TBX ., ., . - theEmitter->emitIns_R_R_R(INS_sve_tbx, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // TBX ., ., . - - // IF_SVE_BZ_3A_A - theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // TBL ., {., .}, . - theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // TBL ., {., .}, . - theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // TBL ., {., .}, . - theEmitter->emitIns_R_R_R(INS_sve_tbl, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // TBL ., {., .}, . - - // IF_SVE_CA_3A - theEmitter->emitIns_R_R_R(INS_sve_tbxq, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // TBXQ ., ., . - theEmitter->emitIns_R_R_R(INS_sve_tbxq, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // TBXQ ., ., . - theEmitter->emitIns_R_R_R(INS_sve_tbxq, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // TBXQ ., ., . - theEmitter->emitIns_R_R_R(INS_sve_tbxq, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // TBXQ ., ., . - - // IF_SVE_EH_3A - theEmitter->emitIns_R_R_R(INS_sve_sdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_S); // SDOT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sdot, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_D); // SDOT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_udot, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // UDOT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_udot, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // UDOT ., ., . - - // IF_SVE_EL_3A - theEmitter->emitIns_R_R_R(INS_sve_smlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SMLALB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_smlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SMLALT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_smlslb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // SMLSLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_smlslt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // SMLSLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umlalb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_S); // UMLALB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umlalt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_D); // UMLALT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umlslb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, - INS_OPTS_SCALABLE_H); // UMLSLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umlslt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, - INS_OPTS_SCALABLE_S); // UMLSLT ., ., . - - // IF_SVE_EM_3A - theEmitter->emitIns_R_R_R(INS_sve_sqrdmlah, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // SQRDMLAH ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqrdmlah, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // SQRDMLAH ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqrdmlsh, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // SQRDMLSH ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqrdmlsh, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // SQRDMLSH ., ., . - - // IF_SVE_EN_3A - theEmitter->emitIns_R_R_R(INS_sve_sqdmlalbt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SQDMLALBT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmlslbt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SQDMLSLBT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmlslbt, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // SQDMLSLBT ., ., . - - // IF_SVE_EO_3A - theEmitter->emitIns_R_R_R(INS_sve_sqdmlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SQDMLALB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SQDMLALT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmlslb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // SQDMLSLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmlslt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // SQDMLSLT ., ., . - - // IF_SVE_EV_3A - theEmitter->emitIns_R_R_R(INS_sve_sclamp, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // SCLAMP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sclamp, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // SCLAMP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uclamp, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // UCLAMP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uclamp, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // UCLAMP ., ., . - - // IF_SVE_EX_3A - theEmitter->emitIns_R_R_R(INS_sve_tblq, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // TBLQ ., {.}, . - theEmitter->emitIns_R_R_R(INS_sve_uzpq1, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // UZPQ1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uzpq2, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // UZPQ2 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zipq1, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // ZIPQ1 ., ., . - theEmitter->emitIns_R_R_R(INS_sve_zipq2, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_B); // ZIPQ2 ., ., . - - // IF_SVE_FL_3A - theEmitter->emitIns_R_R_R(INS_sve_sabdlb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SABDLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sabdlt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SABDLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_saddlb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // SADDLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_saddlt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // SADDLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ssublb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_S); // SSUBLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ssublt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_D); // SSUBLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uabdlb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, - INS_OPTS_SCALABLE_H); // UABDLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uabdlt, EA_SCALABLE, REG_V21, REG_V22, REG_V24, - INS_OPTS_SCALABLE_S); // UABDLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uaddlb, EA_SCALABLE, REG_V24, REG_V25, REG_V26, - INS_OPTS_SCALABLE_D); // UADDLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uaddlt, EA_SCALABLE, REG_V27, REG_V28, REG_V29, - INS_OPTS_SCALABLE_H); // UADDLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_usublb, EA_SCALABLE, REG_V30, REG_V31, REG_V0, - INS_OPTS_SCALABLE_S); // USUBLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_usublt, EA_SCALABLE, REG_V1, REG_V2, REG_V3, - INS_OPTS_SCALABLE_D); // USUBLT ., ., . - - // IF_SVE_FM_3A - theEmitter->emitIns_R_R_R(INS_sve_saddwb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SADDWB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_saddwt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SADDWT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ssubwb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // SSUBWB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ssubwt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // SSUBWT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uaddwb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_S); // UADDWB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uaddwt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_D); // UADDWT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_usubwb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, - INS_OPTS_SCALABLE_H); // USUBWB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_usubwt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, - INS_OPTS_SCALABLE_S); // USUBWT ., ., . - - // IF_SVE_FN_3A - theEmitter->emitIns_R_R_R(INS_sve_pmullb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // PMULLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_pmullt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_D); // PMULLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_smullb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_H); // SMULLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_smullt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // SMULLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmullb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_H); // SQDMULLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sqdmullt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_D); // SQDMULLT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umullb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, - INS_OPTS_SCALABLE_H); // UMULLB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_umullt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, - INS_OPTS_SCALABLE_D); // UMULLT ., ., . - - // IF_SVE_FN_3B - theEmitter->emitIns_R_R_R(INS_sve_pmullb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_Q); // PMULLB .Q, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_pmullt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_Q); // PMULLT .Q, .D, .D - - // IF_SVE_FO_3A - theEmitter->emitIns_R_R_R(INS_sve_smmla, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_S); // SMMLA .S, .B, .B - theEmitter->emitIns_R_R_R(INS_sve_ummla, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // UMMLA .S, .B, .B - theEmitter->emitIns_R_R_R(INS_sve_usmmla, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // USMMLA .S, .B, .B - - // IF_SVE_FP_3A - theEmitter->emitIns_R_R_R(INS_sve_eorbt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // EORBT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_eorbt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // EORBT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_eortb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // EORTB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_eortb, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // EORTB ., ., . - - // IF_SVE_FQ_3A - theEmitter->emitIns_R_R_R(INS_sve_bdep, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // BDEP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_bext, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // BEXT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_bgrp, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // BGRP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_bgrp, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // BGRP ., ., . - - // IF_SVE_FS_3A - theEmitter->emitIns_R_R_R(INS_sve_saddlbt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SADDLBT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ssublbt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SSUBLBT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ssubltb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // SSUBLTB ., ., . - - // IF_SVE_FW_3A - theEmitter->emitIns_R_R_R(INS_sve_saba, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // SABA ., ., . - theEmitter->emitIns_R_R_R(INS_sve_saba, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // SABA ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uaba, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // UABA ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uaba, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // UABA ., ., . - - // IF_SVE_FX_3A - theEmitter->emitIns_R_R_R(INS_sve_sabalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SABALB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_sabalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SABALT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uabalb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // UABALB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_uabalt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // UABALT ., ., . - -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_GC_3A - theEmitter->emitIns_R_R_R(INS_sve_addhnb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // ADDHNB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_addhnt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // ADDHNT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_raddhnb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_S); // RADDHNB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_raddhnt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_B); // RADDHNT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_rsubhnb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_H); // RSUBHNB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_rsubhnt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_S); // RSUBHNT ., ., . - theEmitter->emitIns_R_R_R(INS_sve_subhnb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, - INS_OPTS_SCALABLE_B); // SUBHNB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_subhnt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, - INS_OPTS_SCALABLE_H); // SUBHNT ., ., . -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - - // IF_SVE_GF_3A - theEmitter->emitIns_R_R_R(INS_sve_histseg, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // HISTSEG .B, .B, .B - - // IF_SVE_GW_3A - theEmitter->emitIns_R_R_R(INS_sve_fclamp, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // FCLAMP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fclamp, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // FCLAMP ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fclamp, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // FCLAMP ., ., . - - // IF_SVE_GW_3B - theEmitter->emitIns_R_R_R(INS_sve_bfclamp, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // BFCLAMP .H, .H, .H - - // IF_SVE_HK_3A - theEmitter->emitIns_R_R_R(INS_sve_fadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // FADD ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // FMUL ., ., . - theEmitter->emitIns_R_R_R(INS_sve_frecps, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // FRECPS ., ., . - theEmitter->emitIns_R_R_R(INS_sve_frsqrts, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // FRSQRTS ., ., . - theEmitter->emitIns_R_R_R(INS_sve_fsub, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // FSUB ., ., . - theEmitter->emitIns_R_R_R(INS_sve_ftsmul, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // FTSMUL ., ., . - - // IF_SVE_HK_3B - theEmitter->emitIns_R_R_R(INS_sve_bfadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // BFADD .H, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // BFMUL .H, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfsub, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); // BFSUB .H, .H, .H - -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_AT_3B - theEmitter->emitIns_R_R_R(INS_sve_addpt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // ADDPT .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_subpt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // SUBPT .D, .D, .D -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - - // IF_SVE_AU_3A - theEmitter->emitIns_R_R_R(INS_sve_and, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // AND .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_bic, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // BIC .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_eor, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // EOR .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // MOV .D, .D - theEmitter->emitIns_R_R_R(INS_sve_orr, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // ORR .D, .D, .D - - // IF_SVE_AV_3A - theEmitter->emitIns_R_R_R(INS_sve_bcax, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_D); // BCAX .D, .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_bsl, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_D); // BSL .D, .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_bsl1n, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_D); // BSL1N .D, .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_bsl2n, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_D); // BSL2N .D, .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_eor3, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_D); // EOR3 .D, .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_nbsl, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_D); // NBSL .D, .D, .D, .D - - // IF_SVE_AW_2A - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V0, REG_V1, 1, - INS_OPTS_SCALABLE_B); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V2, REG_V3, 8, - INS_OPTS_SCALABLE_B); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V4, REG_V5, 2, - INS_OPTS_SCALABLE_H); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V6, REG_V7, 16, - INS_OPTS_SCALABLE_H); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V8, REG_V9, 3, - INS_OPTS_SCALABLE_S); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V10, REG_V11, 32, - INS_OPTS_SCALABLE_S); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V12, REG_V13, 4, - INS_OPTS_SCALABLE_D); // XAR ., ., ., # - theEmitter->emitIns_R_R_I(INS_sve_xar, EA_SCALABLE, REG_V14, REG_V15, 64, - INS_OPTS_SCALABLE_D); // XAR ., ., ., # - - // IF_SVE_AX_1A - theEmitter->emitIns_R_I_I(INS_sve_index, EA_SCALABLE, REG_V0, -16, 15, - INS_OPTS_SCALABLE_B); // INDEX ., #, # - theEmitter->emitIns_R_I_I(INS_sve_index, EA_SCALABLE, REG_V1, 15, -16, - INS_OPTS_SCALABLE_H); // INDEX ., #, # - theEmitter->emitIns_R_I_I(INS_sve_index, EA_SCALABLE, REG_V2, 0, 0, - INS_OPTS_SCALABLE_S); // INDEX ., #, # - theEmitter->emitIns_R_I_I(INS_sve_index, EA_SCALABLE, REG_V3, -5, 5, - INS_OPTS_SCALABLE_D); // INDEX ., #, # - - // IF_SVE_AY_2A - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V0, REG_R0, -16, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_IMM_FIRST); // INDEX ., #, - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V1, REG_R1, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_IMM_FIRST); // INDEX ., #, - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V2, REG_R2, 5, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_IMM_FIRST); // INDEX ., #, - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V3, REG_R3, 10, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_IMM_FIRST); // INDEX ., #, - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V4, REG_ZR, -16, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_IMM_FIRST); // INDEX ., #, - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V5, REG_ZR, 15, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_IMM_FIRST); // INDEX ., #, - - // IF_SVE_AZ_2A - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V0, REG_R0, -16, - INS_OPTS_SCALABLE_B); // INDEX ., , # - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V1, REG_R1, 0, - INS_OPTS_SCALABLE_H); // INDEX ., , # - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V2, REG_R2, 5, - INS_OPTS_SCALABLE_S); // INDEX ., , # - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V3, REG_R3, 10, - INS_OPTS_SCALABLE_D); // INDEX ., , # - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V4, REG_ZR, -16, - INS_OPTS_SCALABLE_B); // INDEX ., , # - theEmitter->emitIns_R_R_I(INS_sve_index, EA_SCALABLE, REG_V5, REG_ZR, 15, - INS_OPTS_SCALABLE_D); // INDEX ., , # - - // IF_SVE_BB_2A - theEmitter->emitIns_R_R_I(INS_sve_addpl, EA_8BYTE, REG_R0, REG_R1, -32); // ADDPL , , # - theEmitter->emitIns_R_R_I(INS_sve_addpl, EA_8BYTE, REG_R2, REG_SP, 0); // ADDPL , , # - theEmitter->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_R3, REG_R4, 5); // ADDVL , , # - theEmitter->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_SP, REG_R5, 31); // ADDVL , , # - theEmitter->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_SP, REG_SP, 0); // ADDVL , , # - - // IF_SVE_BC_1A - theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R0, -32); // RDVL , # - theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R5, 0); // RDVL , # - theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R10, 5); // RDVL , # - theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R15, 31); // RDVL , # - // IF_SVE_BL_1A theEmitter->emitIns_R_PATTERN_I(INS_sve_cntb, EA_8BYTE, REG_R0, SVE_PATTERN_POW2, 1); // CNTB {, {, MUL #}} @@ -6053,154 +5303,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_PATTERN_I(INS_sve_cnth, EA_8BYTE, REG_R5, SVE_PATTERN_ALL, 13); // CNTH {, {, MUL #}} - // IF_SVE_BM_1A - theEmitter->emitIns_R_PATTERN_I(INS_sve_decb, EA_8BYTE, REG_R0, SVE_PATTERN_POW2, - 1); // DECB {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_decd, EA_8BYTE, REG_R1, SVE_PATTERN_VL16, - 3); // DECD {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_dech, EA_8BYTE, REG_R2, SVE_PATTERN_VL32, - 5); // DECH {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_decw, EA_8BYTE, REG_R3, SVE_PATTERN_VL64, - 7); // DECW {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_incb, EA_8BYTE, REG_R4, SVE_PATTERN_VL128, - 9); // INCB {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_incd, EA_8BYTE, REG_R5, SVE_PATTERN_MUL3, - 10); // INCD {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_inch, EA_8BYTE, REG_R6, SVE_PATTERN_MUL4, - 13); // INCH {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_incw, EA_8BYTE, REG_R7, SVE_PATTERN_ALL, - 16); // INCW {, {, MUL #}} - - // IF_SVE_BN_1A - theEmitter->emitIns_R_PATTERN_I(INS_sve_decd, EA_SCALABLE, REG_V0, SVE_PATTERN_POW2, 1, - INS_OPTS_SCALABLE_D); // DECD .D{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_dech, EA_SCALABLE, REG_V1, SVE_PATTERN_VL2, 2, - INS_OPTS_SCALABLE_H); // DECH .H{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_decw, EA_SCALABLE, REG_V2, SVE_PATTERN_VL3, 4, - INS_OPTS_SCALABLE_S); // DECW .S{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_incd, EA_SCALABLE, REG_V3, SVE_PATTERN_VL4, 8, - INS_OPTS_SCALABLE_D); // INCD .D{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_inch, EA_SCALABLE, REG_V4, SVE_PATTERN_VL5, 12, - INS_OPTS_SCALABLE_H); // INCH .H{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_incw, EA_SCALABLE, REG_V5, SVE_PATTERN_VL6, 16, - INS_OPTS_SCALABLE_S); // INCW .S{, {, MUL #}} - - // IF_SVE_BO_1A - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdecb, EA_4BYTE, REG_R0, SVE_PATTERN_POW2, - 1); // SQDECB , {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdecd, EA_8BYTE, REG_R1, SVE_PATTERN_VL1, - 2); // SQDECD {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdech, EA_4BYTE, REG_R2, SVE_PATTERN_VL2, - 3); // SQDECH , {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdecw, EA_8BYTE, REG_R3, SVE_PATTERN_VL3, - 4); // SQDECW {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqincb, EA_4BYTE, REG_R4, SVE_PATTERN_VL4, - 5); // SQINCB , {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqincd, EA_8BYTE, REG_R5, SVE_PATTERN_VL5, - 6); // SQINCD {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqinch, EA_4BYTE, REG_R6, SVE_PATTERN_VL6, - 7); // SQINCH , {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqincw, EA_8BYTE, REG_R7, SVE_PATTERN_VL7, - 8); // SQINCW {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdecb, EA_4BYTE, REG_R8, SVE_PATTERN_VL8, - 9); // UQDECB {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdecd, EA_8BYTE, REG_R9, SVE_PATTERN_VL16, - 10); // UQDECD {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdech, EA_4BYTE, REG_R10, SVE_PATTERN_VL32, - 11); // UQDECH {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdecw, EA_8BYTE, REG_R11, SVE_PATTERN_VL64, - 12); // UQDECW {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqincb, EA_4BYTE, REG_R12, SVE_PATTERN_VL128, - 13); // UQINCB {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqincd, EA_8BYTE, REG_R13, SVE_PATTERN_VL256, - 14); // UQINCD {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqinch, EA_4BYTE, REG_R14, SVE_PATTERN_MUL4, - 15); // UQINCH {, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqincw, EA_8BYTE, REG_R15, SVE_PATTERN_ALL, - 16); // UQINCW {, {, MUL #}} - - // IF_SVE_BP_1A - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdecd, EA_SCALABLE, REG_V0, SVE_PATTERN_VL1, 1, - INS_OPTS_SCALABLE_D); // SQDECD .D{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdech, EA_SCALABLE, REG_V1, SVE_PATTERN_VL2, 2, - INS_OPTS_SCALABLE_H); // SQDECH .H{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqdecw, EA_SCALABLE, REG_V2, SVE_PATTERN_VL3, 3, - INS_OPTS_SCALABLE_S); // SQDECW .S{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqincd, EA_SCALABLE, REG_V3, SVE_PATTERN_VL4, 4, - INS_OPTS_SCALABLE_D); // SQINCD .D{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqinch, EA_SCALABLE, REG_V4, SVE_PATTERN_VL5, 5, - INS_OPTS_SCALABLE_H); // SQINCH .H{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_sqincw, EA_SCALABLE, REG_V5, SVE_PATTERN_VL6, 6, - INS_OPTS_SCALABLE_S); // SQINCW .S{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdecd, EA_SCALABLE, REG_V6, SVE_PATTERN_VL7, 7, - INS_OPTS_SCALABLE_D); // UQDECD .D{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdech, EA_SCALABLE, REG_V7, SVE_PATTERN_VL8, 8, - INS_OPTS_SCALABLE_H); // UQDECH .H{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqdecw, EA_SCALABLE, REG_V8, SVE_PATTERN_VL16, 9, - INS_OPTS_SCALABLE_S); // UQDECW .S{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqincd, EA_SCALABLE, REG_V9, SVE_PATTERN_VL32, 10, - INS_OPTS_SCALABLE_D); // UQINCD .D{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqinch, EA_SCALABLE, REG_V10, SVE_PATTERN_POW2, 11, - INS_OPTS_SCALABLE_H); // UQINCH .H{, {, MUL #}} - theEmitter->emitIns_R_PATTERN_I(INS_sve_uqincw, EA_SCALABLE, REG_V11, SVE_PATTERN_ALL, 16, - INS_OPTS_SCALABLE_S); // UQINCW .S{, {, MUL #}} - - // IF_SVE_BQ_2A - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V0, REG_V1, 0, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // EXT .B, {.B, .B }, # - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V2, REG_V3, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // EXT .B, {.B, .B }, # - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V4, REG_V5, 128, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // EXT .B, {.B, .B }, # - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V6, REG_FP_LAST, 255, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); // EXT .B, {.B, .B }, # - - // IF_SVE_BQ_2B - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V0, REG_V1, 0, - INS_OPTS_SCALABLE_B); // EXT .B, .B, .B, # - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V2, REG_V3, 31, - INS_OPTS_SCALABLE_B); // EXT .B, .B, .B, # - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V4, REG_V5, 64, - INS_OPTS_SCALABLE_B); // EXT .B, .B, .B, # - theEmitter->emitIns_R_R_I(INS_sve_ext, EA_SCALABLE, REG_V6, REG_V7, 255, - INS_OPTS_SCALABLE_B); // EXT .B, .B, .B, # - - // IF_SVE_BU_2A - theEmitter->emitIns_R_R_F(INS_sve_fcpy, EA_SCALABLE, REG_V0, REG_P1, 2.0, - INS_OPTS_SCALABLE_H); // FCPY ., /M, # - theEmitter->emitIns_R_R_F(INS_sve_fcpy, EA_SCALABLE, REG_V2, REG_P3, 1.0, - INS_OPTS_SCALABLE_S); // FCPY ., /M, # - theEmitter->emitIns_R_R_F(INS_sve_fcpy, EA_SCALABLE, REG_V4, REG_P5, -10.0, - INS_OPTS_SCALABLE_D); // FCPY ., /M, # - theEmitter->emitIns_R_R_F(INS_sve_fmov, EA_SCALABLE, REG_V6, REG_P7, -0.125, - INS_OPTS_SCALABLE_H); // FMOV ., /M, # - theEmitter->emitIns_R_R_F(INS_sve_fmov, EA_SCALABLE, REG_V8, REG_P9, 31.0, - INS_OPTS_SCALABLE_S); // FMOV ., /M, # - theEmitter->emitIns_R_R_F(INS_sve_fmov, EA_SCALABLE, REG_V10, REG_P11, 0.5, - INS_OPTS_SCALABLE_D); // FMOV ., /M, # - - // IF_SVE_CC_2A - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V0, REG_V13, - INS_OPTS_SCALABLE_B); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V29, REG_V0, - INS_OPTS_SCALABLE_H); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V4, REG_V15, - INS_OPTS_SCALABLE_S); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V8, REG_V2, - INS_OPTS_SCALABLE_D); // INSR ., - - // IF_SVE_CD_2A - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V4, REG_R23, - INS_OPTS_SCALABLE_B); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V11, REG_R1, - INS_OPTS_SCALABLE_H); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V14, REG_R9, - INS_OPTS_SCALABLE_S); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V19, REG_R0, - INS_OPTS_SCALABLE_D); // INSR ., - theEmitter->emitIns_R_R(INS_sve_insr, EA_SCALABLE, REG_V29, REG_ZR, - INS_OPTS_SCALABLE_D); // INSR ., - // IF_SVE_CI_3A theEmitter->emitIns_R_R_R(INS_sve_trn1, EA_SCALABLE, REG_P1, REG_P3, REG_P4, INS_OPTS_SCALABLE_B); // TRN1 ., ., . @@ -6297,14 +5399,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V2, REG_P6, REG_V28, INS_OPTS_SCALABLE_S); // SPLICE ., , ., . - // IF_SVE_CW_4A - theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V0, REG_P0, REG_V30, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_PREDICATE_MERGE); // MOV ., /M, . - theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V29, REG_P15, REG_V28, REG_V4, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); // SEL ., , ., . - theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V5, REG_P13, REG_V27, REG_V5, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); // SEL ., , ., . - // IF_SVE_EQ_3A // Note: Scalable size is the size of the destination , not the source . theEmitter->emitIns_R_R_R(INS_sve_sadalp, EA_SCALABLE, REG_V26, REG_P3, REG_V8, @@ -6574,24 +5668,8 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R(INS_sve_aesimc, EA_SCALABLE, REG_V0); // AESIMC .B, .B theEmitter->emitIns_R(INS_sve_aesmc, EA_SCALABLE, REG_V5); // AESMC .B, .B +// IF_SVE_GS_3A #ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_GN_3A - theEmitter->emitIns_R_R_R(INS_sve_fmlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // FMLALB .H, .B, .B - theEmitter->emitIns_R_R_R(INS_sve_fmlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_B); // FMLALT .H, .B, .B - - // IF_SVE_GO_3A - theEmitter->emitIns_R_R_R(INS_sve_fmlallbb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // FMLALLBB .S, .B, .B - theEmitter->emitIns_R_R_R(INS_sve_fmlallbt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_B); // FMLALLBT .S, .B, .B - theEmitter->emitIns_R_R_R(INS_sve_fmlalltb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_B); // FMLALLTB .S, .B, .B - theEmitter->emitIns_R_R_R(INS_sve_fmlalltt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_B); // FMLALLTT .S, .B, .B - - // IF_SVE_GS_3A theEmitter->emitIns_R_R_R(INS_sve_faddqv, EA_8BYTE, REG_V16, REG_P0, REG_V12, INS_OPTS_SCALABLE_H); // FADDQV ., , . theEmitter->emitIns_R_R_R(INS_sve_fmaxnmqv, EA_8BYTE, REG_V17, REG_P1, REG_V11, @@ -6724,49 +5802,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R_I(INS_sve_fmlslt, EA_SCALABLE, REG_V14, REG_V15, REG_V7, 7, INS_OPTS_SCALABLE_H); // FMLSLT .S, .H, .H[] - // IF_SVE_HA_3A - theEmitter->emitIns_R_R_R(INS_sve_bfdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // BFDOT .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_fdot, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // FDOT .S, .H, .H - -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_HA_3A_E - theEmitter->emitIns_R_R_R(INS_sve_fdot, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_B); // FDOT .H, .B, .B - - // IF_SVE_HA_3A_F - theEmitter->emitIns_R_R_R(INS_sve_fdot, EA_SCALABLE, REG_V9, REG_V10, REG_V11); // FDOT .S, .B, .B -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - - // IF_SVE_HB_3A - theEmitter->emitIns_R_R_R(INS_sve_bfmlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // BFMLALB .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // BFMLALT .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmlslb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, - INS_OPTS_SCALABLE_H); // BFMLSLB .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_bfmlslt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, - INS_OPTS_SCALABLE_H); // BFMLSLT .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_fmlalb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, - INS_OPTS_SCALABLE_H); // FMLALB .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_fmlalt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, - INS_OPTS_SCALABLE_H); // FMLALT .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_fmlslb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, - INS_OPTS_SCALABLE_H); // FMLSLB .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_fmlslt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, - INS_OPTS_SCALABLE_H); // FMLSLT .S, .H, .H - - // IF_SVE_HD_3A - theEmitter->emitIns_R_R_R(INS_sve_bfmmla, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // BFMMLA .S, .H, .H - -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_HD_3A_A - theEmitter->emitIns_R_R_R(INS_sve_fmmla, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_D); // FMMLA .D, .D, .D -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_HE_3A theEmitter->emitIns_R_R_R(INS_sve_faddv, EA_2BYTE, REG_V21, REG_P7, REG_V7, INS_OPTS_SCALABLE_H); // FADDV , , . @@ -6955,20 +5990,20 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_EB_1A theEmitter->emitIns_R_I(INS_sve_dup, EA_SCALABLE, REG_V0, -128, INS_OPTS_SCALABLE_B); // DUP ., #{, } - theEmitter->emitIns_R_I(INS_sve_dup, EA_SCALABLE, REG_V1, 0, - INS_OPTS_SCALABLE_H); // DUP ., #{, } + theEmitter->emitIns_R_I(INS_sve_dup, EA_SCALABLE, REG_V1, 0, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_SHIFT); // DUP ., #{, } theEmitter->emitIns_R_I(INS_sve_dup, EA_SCALABLE, REG_V2, 5, INS_OPTS_SCALABLE_S); // DUP ., #{, } theEmitter->emitIns_R_I(INS_sve_dup, EA_SCALABLE, REG_V3, 127, INS_OPTS_SCALABLE_D); // DUP ., #{, } - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V4, 256, - INS_OPTS_SCALABLE_D); // MOV ., #{, } - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V5, -32768, - INS_OPTS_SCALABLE_H); // MOV ., #{, } - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V6, 1280, - INS_OPTS_SCALABLE_S); // MOV ., #{, } - theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V7, 32512, - INS_OPTS_SCALABLE_D); // MOV ., #{, } + theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V4, 0, + INS_OPTS_SCALABLE_B); // MOV ., #{, } + theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V5, -128, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_SHIFT); // MOV ., #{, } + theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V6, 5, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_SHIFT); // MOV ., #{, } + theEmitter->emitIns_R_I(INS_sve_mov, EA_SCALABLE, REG_V7, 127, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_SHIFT); // MOV ., #{, } // IF_SVE_EB_1B theEmitter->emitIns_R(INS_sve_fmov, EA_SCALABLE, REG_V0, INS_OPTS_SCALABLE_B); // FMOV ., #0.0 @@ -6979,18 +6014,18 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_EC_1A theEmitter->emitIns_R_I(INS_sve_add, EA_SCALABLE, REG_V0, 0, INS_OPTS_SCALABLE_B); // ADD ., ., #{, } - theEmitter->emitIns_R_I(INS_sve_sqadd, EA_SCALABLE, REG_V1, 5, - INS_OPTS_SCALABLE_H); // SQADD ., ., #{, } - theEmitter->emitIns_R_I(INS_sve_sqsub, EA_SCALABLE, REG_V2, 128, + theEmitter->emitIns_R_I(INS_sve_sqadd, EA_SCALABLE, REG_V1, 0, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_SHIFT); // SQADD ., ., #{, } + theEmitter->emitIns_R_I(INS_sve_sqsub, EA_SCALABLE, REG_V2, 1, INS_OPTS_SCALABLE_S); // SQSUB ., ., #{, } - theEmitter->emitIns_R_I(INS_sve_sub, EA_SCALABLE, REG_V3, 255, + theEmitter->emitIns_R_I(INS_sve_sub, EA_SCALABLE, REG_V3, 128, INS_OPTS_SCALABLE_D); // SUB ., ., #{, } - theEmitter->emitIns_R_I(INS_sve_subr, EA_SCALABLE, REG_V4, 256, - INS_OPTS_SCALABLE_D); // SUBR ., ., #{, } - theEmitter->emitIns_R_I(INS_sve_uqadd, EA_SCALABLE, REG_V5, 1280, - INS_OPTS_SCALABLE_H); // UQADD ., ., #{, } - theEmitter->emitIns_R_I(INS_sve_uqsub, EA_SCALABLE, REG_V6, 65280, - INS_OPTS_SCALABLE_S); // UQSUB ., ., #{, } + theEmitter->emitIns_R_I(INS_sve_subr, EA_SCALABLE, REG_V4, 255, + INS_OPTS_SCALABLE_B); // SUBR ., ., #{, } + theEmitter->emitIns_R_I(INS_sve_uqadd, EA_SCALABLE, REG_V5, 5, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_SHIFT); // UQADD ., ., #{, } + theEmitter->emitIns_R_I(INS_sve_uqsub, EA_SCALABLE, REG_V6, 255, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_SHIFT); // UQSUB ., ., #{, } // IF_SVE_EG_3A theEmitter->emitIns_R_R_R_I(INS_sve_sdot, EA_SCALABLE, REG_V1, REG_V2, REG_V0, 0, @@ -7390,16 +6425,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_I(INS_sve_mul, EA_SCALABLE, REG_V3, 127, INS_OPTS_SCALABLE_D); // MUL ., ., # - // IF_SVE_EF_3A - theEmitter->emitIns_R_R_R(INS_sve_sdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_H); // SDOT .S, .H, .H - theEmitter->emitIns_R_R_R(INS_sve_udot, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_H); // UDOT .S, .H, .H - - // IF_SVE_EI_3A - theEmitter->emitIns_R_R_R(INS_sve_usdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_B); // USDOT .S, .B, .B - // IF_SVE_FA_3A theEmitter->emitIns_R_R_R_I_I(INS_sve_cdot, EA_SCALABLE, REG_V0, REG_V7, REG_V1, 3, 0, INS_OPTS_SCALABLE_B); // CDOT .S, .B, .B[], @@ -8457,18 +7482,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R_R_I(INS_sve_fcmla, EA_SCALABLE, REG_V2, REG_P3, REG_V0, REG_V6, 270, INS_OPTS_SCALABLE_D); // FCMLA ., /M, ., ., - // IF_SVE_GI_4A - theEmitter->emitIns_R_R_R_R(INS_sve_histcnt, EA_SCALABLE, REG_V0, REG_P0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_S); // HISTCNT ., /Z, ., . - theEmitter->emitIns_R_R_R_R(INS_sve_histcnt, EA_SCALABLE, REG_V3, REG_P7, REG_V4, REG_V5, - INS_OPTS_SCALABLE_D); // HISTCNT ., /Z, ., . - - // IF_SVE_GJ_3A - theEmitter->emitIns_R_R_R(INS_sve_rax1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, - INS_OPTS_SCALABLE_D); // RAX1 .D, .D, .D - theEmitter->emitIns_R_R_R(INS_sve_sm4ekey, EA_SCALABLE, REG_V3, REG_V4, REG_V5, - INS_OPTS_SCALABLE_S); // SM4EKEY .S, .S, .S - // IF_SVE_HI_3A theEmitter->emitIns_R_R_R(INS_sve_fcmeq, EA_SCALABLE, REG_P2, REG_P3, REG_V4, INS_OPTS_SCALABLE_H); // FCMEQ ., /Z, ., #0.0 @@ -8586,189 +7599,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 255, INS_OPTS_NONE, INS_SCALABLE_OPTS_UNPREDICATED); -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_GG_3A - // LUTI2 .B, {.B }, [] - // luti2 z0.b, {z0.b}, z0[0] // 01000101-00100000-10110000-00000000 - // CHECK-INST: luti2 z0.b, { z0.b }, z0[0] - // CHECK-ENCODING: [0x00,0xb0,0x20,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); - // luti2 z21.b, {z10.b}, z21[1] // 01000101-01110101-10110001-01010101 - // CHECK-INST: luti2 z21.b, { z10.b }, z21[1] - // CHECK-ENCODING: [0x55,0xb1,0x75,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 1, INS_OPTS_SCALABLE_B); - - // IF_SVE_GH_3B - // LUTI4 .H, {.H, .H }, [] - // luti4 z0.h, {z0.h, z1.h}, z0[0] // 01000101-00100000-10110100-00000000 - // CHECK-INST: luti4 z0.h, { z0.h, z1.h }, z0[0] - // CHECK-ENCODING: [0x00,0xb4,0x20,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H, EA_UNKNOWN, - INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); - // luti4 z21.h, {z10.h, z11.h}, z21[1] // 01000101-01110101-10110101-01010101 - // CHECK-INST: luti4 z21.h, { z10.h, z11.h }, z21[1] - // CHECK-ENCODING: [0x55,0xb5,0x75,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 1, INS_OPTS_SCALABLE_H, - EA_UNKNOWN, INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); - // luti4 z31.h, {z31.h, z0.h}, z31[3] // 01000101-11111111-10110111-11111111 - // CHECK-INST: luti4 z31.h, { z31.h, z0.h }, z31[3] - // CHECK-ENCODING: [0xff,0xb7,0xff,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 3, INS_OPTS_SCALABLE_H, - EA_UNKNOWN, INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); - - // IF_SVE_GH_3B_B - // LUTI4 .H, {.H }, [] - // luti4 z0.h, {z0.h}, z0[0] // 01000101-00100000-10111100-00000000 - // CHECK-INST: luti4 z0.h, { z0.h }, z0[0] - // CHECK-ENCODING: [0x00,0xbc,0x20,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); - // luti4 z21.h, {z10.h}, z21[1] // 01000101-01110101-10111101-01010101 - // CHECK-INST: luti4 z21.h, { z10.h }, z21[1] - // CHECK-ENCODING: [0x55,0xbd,0x75,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 1, INS_OPTS_SCALABLE_H); - // luti4 z31.h, {z31.h}, z31[3] // 01000101-11111111-10111111-11111111 - // CHECK-INST: luti4 z31.h, { z31.h }, z31[3] - // CHECK-ENCODING: [0xff,0xbf,0xff,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 3, INS_OPTS_SCALABLE_H); - - // IF_SVE_GG_3B - // LUTI2 .H, {.H }, [] - // luti2 z0.h, {z0.h}, z0[0] // 01000101-00100000-10101000-00000000 - // CHECK-INST: luti2 z0.h, { z0.h }, z0[0] - // CHECK-ENCODING: [0x00,0xa8,0x20,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); - // luti2 z21.h, {z10.h}, z21[3] // 01000101-01110101-10111001-01010101 - // CHECK-INST: luti2 z21.h, { z10.h }, z21[3] - // CHECK-ENCODING: [0x55,0xb9,0x75,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 3, INS_OPTS_SCALABLE_H); - // luti2 z31.h, {z31.h}, z31[7] // 01000101-11111111-10111011-11111111 - // CHECK-INST: luti2 z31.h, { z31.h }, z31[7] - // CHECK-ENCODING: [0xff,0xbb,0xff,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_H); - - // IF_SVE_GH_3A - // LUTI4 .B, {.B }, [] - // luti4 z0.b, {z0.b}, z0[0] // 01000101-01100000-10100100-00000000 - // CHECK-INST: luti4 z0.b, { z0.b }, z0[0] - // CHECK-ENCODING: [0x00,0xa4,0x60,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); - // luti4 z31.b, {z31.b}, z31[1] // 01000101-11111111-10100111-11111111 - // CHECK-INST: luti4 z31.b, { z31.b }, z31[1] - // CHECK-ENCODING: [0xff,0xa7,0xff,0x45] - theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 1, INS_OPTS_SCALABLE_B); -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - - // IF_SVE_HY_3A - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P6, REG_R5, REG_V4, - INS_OPTS_SCALABLE_S_UXTW, - INS_SCALABLE_OPTS_MOD_N); // PRFD , , [, .S, #3] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_R8, REG_V9, - INS_OPTS_SCALABLE_S_SXTW, - INS_SCALABLE_OPTS_MOD_N); // PRFH , , [, .S, #1] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_R2, REG_V1, - INS_OPTS_SCALABLE_S_UXTW, - INS_SCALABLE_OPTS_MOD_N); // PRFW , , [, .S, #2] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1STRM, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL2KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL2STRM, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL3KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL3STRM, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PSTL1KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PSTL1STRM, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PSTL2KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PSTL2STRM, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PSTL3KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PSTL3STRM, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_CONST6, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_SXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_CONST7, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_SXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_CONST14, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_CONST15, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] - - // IF_SVE_HY_3A_A - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P1, REG_R2, REG_V3, - INS_OPTS_SCALABLE_D_UXTW); // PRFB , , [, .D, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P6, REG_R5, REG_V4, - INS_OPTS_SCALABLE_D_UXTW, - INS_SCALABLE_OPTS_MOD_N); // PRFD , , [, .D, #3] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_R8, REG_V9, - INS_OPTS_SCALABLE_D_SXTW, - INS_SCALABLE_OPTS_MOD_N); // PRFH , , [, .D, #1] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_R2, REG_V1, - INS_OPTS_SCALABLE_D_UXTW, - INS_SCALABLE_OPTS_MOD_N); // PRFW , , [, .D, #2] - - // IF_SVE_HY_3B - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_R1, REG_V2, - INS_OPTS_SCALABLE_D); // PRFB , , [, .D] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_R4, REG_V3, - INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_LSL_N); // PRFD , , [, .D, LSL #3] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P6, REG_R5, REG_V4, - INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_LSL_N); // PRFH , , [, .D, LSL #1] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P3, REG_R2, REG_V1, - INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_LSL_N); // PRFW , , [, .D, LSL #2] - - // IF_SVE_HZ_2A_B - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P1, REG_V2, 0, - INS_OPTS_SCALABLE_S); // PRFB , , [.S{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P4, REG_V3, 248, - INS_OPTS_SCALABLE_S); // PRFD , , [.S{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_V4, 62, - INS_OPTS_SCALABLE_S); // PRFH , , [.S{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_V5, 124, - INS_OPTS_SCALABLE_S); // PRFW , , [.S{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P1, REG_V2, 31, - INS_OPTS_SCALABLE_D); // PRFB , , [.D{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P4, REG_V3, 248, - INS_OPTS_SCALABLE_D); // PRFD , , [.D{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_V4, 62, - INS_OPTS_SCALABLE_D); // PRFH , , [.D{, #}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_V5, 124, - INS_OPTS_SCALABLE_D); // PRFW , , [.D{, #}] - - // IF_SVE_IA_2A - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P2, REG_R3, - -32); // PRFB , , [{, #, MUL VL}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_R4, - 31); // PRFD , , [{, #, MUL VL}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P1, REG_R2, - 0); // PRFH , , [{, #, MUL VL}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_R5, - -32); // PRFW , , [{, #, MUL VL}] - theEmitter->emitIns_PRFOP_R_R_I(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P2, REG_R3, - 17); // PRFB , , [{, #, MUL VL}] - - // IF_SVE_IB_3A - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P0, REG_R1, - REG_R2); // PRFB , , [, ] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfd, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P5, REG_R4, REG_R3, - INS_OPTS_NONE, - INS_SCALABLE_OPTS_LSL_N); // PRFD , , [, , LSL #3] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfh, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P6, REG_R7, REG_R8, - INS_OPTS_NONE, - INS_SCALABLE_OPTS_LSL_N); // PRFH , , [, , LSL #1] - theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfw, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P7, REG_R1, REG_R9, - INS_OPTS_NONE, - INS_SCALABLE_OPTS_LSL_N); // PRFW , , [, , LSL #2] // IF_SVE_HX_3A_B theEmitter->emitIns_R_R_R_I(INS_sve_ld1b, EA_SCALABLE, REG_V0, REG_P0, REG_V1, 0, INS_OPTS_SCALABLE_S); // LD1B {.S }, /Z, [.S{, #}] @@ -8894,319 +7724,6 @@ void CodeGen::genArm64EmitterUnitTestsSve() INS_OPTS_SCALABLE_S); // LD1RB {.S }, /Z, [{, #}] theEmitter->emitIns_R_R_R_I(INS_sve_ld1rb, EA_SCALABLE, REG_V1, REG_P0, REG_R9, 63, INS_OPTS_SCALABLE_B); // LD1RB {.D }, /Z, [{, #}] - - // IF_SVE_HF_2A - // FRECPE ., . - theEmitter->emitIns_R_R(INS_sve_frecpe, EA_SCALABLE, REG_V0, REG_V2, INS_OPTS_SCALABLE_H); - // FRSQRTE ., . - theEmitter->emitIns_R_R(INS_sve_frsqrte, EA_SCALABLE, REG_V5, REG_V3, INS_OPTS_SCALABLE_S); - // FRSQRTE ., . - theEmitter->emitIns_R_R(INS_sve_frsqrte, EA_SCALABLE, REG_V9, REG_V5, INS_OPTS_SCALABLE_D); - - // IF_SVE_CH_2A - // SUNPKHI ., . - theEmitter->emitIns_R_R(INS_sve_sunpkhi, EA_SCALABLE, REG_V2, REG_V4, INS_OPTS_SCALABLE_H); - // SUNPKLO ., . - theEmitter->emitIns_R_R(INS_sve_sunpklo, EA_SCALABLE, REG_V1, REG_V5, INS_OPTS_SCALABLE_S); - // UUNPKHI ., . - theEmitter->emitIns_R_R(INS_sve_uunpkhi, EA_SCALABLE, REG_V5, REG_V1, INS_OPTS_SCALABLE_D); - // UUNPKLO ., . - theEmitter->emitIns_R_R(INS_sve_uunpklo, EA_SCALABLE, REG_V8, REG_V6, INS_OPTS_SCALABLE_S); - - // IF_SVE_CG_2A - // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V3, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V4, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V7, REG_V1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - // REV ., . - theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_V2, REG_V5, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - - // IF_SVE_CB_2A - // Note: EA_4BYTE used for B and H (source register is W) - // DUP ., - theEmitter->emitIns_R_R(INS_sve_dup, EA_4BYTE, REG_V0, REG_R1, INS_OPTS_SCALABLE_B); - // DUP ., - theEmitter->emitIns_R_R(INS_sve_dup, EA_4BYTE, REG_V2, REG_R3, INS_OPTS_SCALABLE_H); - // DUP ., - theEmitter->emitIns_R_R(INS_sve_dup, EA_4BYTE, REG_V1, REG_R5, INS_OPTS_SCALABLE_S); - // DUP ., - theEmitter->emitIns_R_R(INS_sve_dup, EA_8BYTE, REG_V4, REG_SP, INS_OPTS_SCALABLE_D); - // MOV ., - theEmitter->emitIns_R_R(INS_sve_mov, EA_4BYTE, REG_V4, REG_R2, INS_OPTS_SCALABLE_B); - // MOV ., - theEmitter->emitIns_R_R(INS_sve_mov, EA_4BYTE, REG_V4, REG_R2, INS_OPTS_SCALABLE_H); - // MOV ., - theEmitter->emitIns_R_R(INS_sve_mov, EA_4BYTE, REG_V1, REG_R3, INS_OPTS_SCALABLE_S); - // MOV ., - theEmitter->emitIns_R_R(INS_sve_mov, EA_8BYTE, REG_V5, REG_SP, INS_OPTS_SCALABLE_D); - // MOV ., - theEmitter->emitIns_R_R(INS_sve_mov, EA_8BYTE, REG_V2, REG_R9, INS_OPTS_SCALABLE_D); - - // IF_SVE_BJ_2A - // FEXPA ., . - theEmitter->emitIns_R_R(INS_sve_fexpa, EA_SCALABLE, REG_V0, REG_V1, INS_OPTS_SCALABLE_H); - // FEXPA ., . - theEmitter->emitIns_R_R(INS_sve_fexpa, EA_SCALABLE, REG_V3, REG_V0, INS_OPTS_SCALABLE_S); - // FEXPA ., . - theEmitter->emitIns_R_R(INS_sve_fexpa, EA_SCALABLE, REG_V1, REG_V0, INS_OPTS_SCALABLE_D); - -#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - // IF_SVE_HH_2A - // BF1CVT .H, .B - theEmitter->emitIns_R_R(INS_sve_bf1cvt, EA_SCALABLE, REG_V2, REG_V3, INS_OPTS_SCALABLE_H); - // BF1CVTLT .H, .B - theEmitter->emitIns_R_R(INS_sve_bf1cvtlt, EA_SCALABLE, REG_V1, REG_V5, INS_OPTS_SCALABLE_H); - // BF2CVT .H, .B - theEmitter->emitIns_R_R(INS_sve_bf2cvt, EA_SCALABLE, REG_V6, REG_V2, INS_OPTS_SCALABLE_H); - // BF2CVTLT .H, .B - theEmitter->emitIns_R_R(INS_sve_bf2cvtlt, EA_SCALABLE, REG_V3, REG_V1, INS_OPTS_SCALABLE_H); - // F1CVT .H, .B - theEmitter->emitIns_R_R(INS_sve_f1cvt, EA_SCALABLE, REG_V6, REG_V7, INS_OPTS_SCALABLE_H); - // F1CVTLT .H, .B - theEmitter->emitIns_R_R(INS_sve_f1cvtlt, EA_SCALABLE, REG_V1, REG_V8, INS_OPTS_SCALABLE_H); - // F2CVT .H, .B - theEmitter->emitIns_R_R(INS_sve_f2cvt, EA_SCALABLE, REG_V3, REG_V4, INS_OPTS_SCALABLE_H); - // F2CVTLT .H, .B - theEmitter->emitIns_R_R(INS_sve_f2cvtlt, EA_SCALABLE, REG_V1, REG_V2, INS_OPTS_SCALABLE_H); -#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED - - // IF_SVE_BI_2A - // MOVPRFX , - theEmitter->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, REG_V3, REG_V5); - - // IF_SVE_BF_2A - // ASR ., ., # - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - // LSL ., ., #emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 31, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 63, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - // LSR ., ., #emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, - INS_SCALABLE_OPTS_UNPREDICATED); - - // IF_SVE_FT_2A - // SLI ., ., # - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 31, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 63, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); - // SRI ., ., # - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); - - // IF_SVE_FU_2A - // SRSRA ., ., # - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); - // SSRA ., ., # - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); - // URSRA ., ., # - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); - // USRA ., ., # - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); - - // IF_SVE_BX_2A - // DUPQ ., .[] - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V21, REG_V10, 10, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V21, REG_V10, 5, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_H); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V21, REG_V10, 2, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V31, REG_V31, 3, INS_OPTS_SCALABLE_S); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D); - theEmitter->emitIns_R_R_I(INS_sve_dupq, EA_SCALABLE, REG_V31, REG_V31, 1, INS_OPTS_SCALABLE_D); - - // IF_SVE_BY_2A - // EXTQ .B, .B, .B, # - theEmitter->emitIns_R_R_I(INS_sve_extq, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); - theEmitter->emitIns_R_R_I(INS_sve_extq, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_B); } #endif // defined(TARGET_ARM64) && defined(DEBUG) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 08eb2367a8fba1..92e99471956f0d 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -441,12 +441,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; #endif // TARGET_ARM64 -#ifdef SWIFT_SUPPORT - case GT_SWIFT_ERROR: - genCodeForSwiftErrorReg(treeNode); - break; -#endif // SWIFT_SUPPORT - case GT_RELOAD: // do nothing - reload is just a marker. // The parent node will call genConsumeReg on this which will trigger the unspill of this node's child @@ -513,6 +507,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #endif break; + case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -1930,6 +1925,37 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree) genProduceReg(tree); } +//---------------------------------------------------------------------------------- +// genCodeForCpBlkHelper - Generate code for a CpBlk node by the means of the VM memcpy helper call +// +// Arguments: +// cpBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForCpBlkHelper(GenTreeBlk* cpBlkNode) +{ + // Destination address goes in arg0, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(cpBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + if (cpBlkNode->IsVolatile()) + { + // issue a full memory barrier before a volatile CpBlk operation + instGen_MemoryBarrier(); + } + + genEmitHelperCall(CORINFO_HELP_MEMCPY, 0, EA_UNKNOWN); + + if (cpBlkNode->IsVolatile()) + { + // issue a load barrier after a volatile CpBlk operation + instGen_MemoryBarrier(BARRIER_LOAD_ONLY); + } +} + #ifdef TARGET_ARM64 // The following classes @@ -3192,6 +3218,31 @@ void CodeGen::genCodeForMemmove(GenTreeBlk* tree) #endif } +//------------------------------------------------------------------------ +// genCodeForInitBlkHelper - Generate code for an InitBlk node by the means of the VM memcpy helper call +// +// Arguments: +// initBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode) +{ + // Size goes in arg2, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(initBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + if (initBlkNode->IsVolatile()) + { + // issue a full memory barrier before a volatile initBlock Operation + instGen_MemoryBarrier(); + } + + genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN); +} + //------------------------------------------------------------------------ // genCodeForInitBlkLoop - Generate code for an InitBlk using an inlined for-loop. // It's needed for cases when size is too big to unroll and we're not allowed @@ -3410,7 +3461,7 @@ void CodeGen::genCall(GenTreeCall* call) for (unsigned i = 0; i < regCount; ++i) { var_types regType = pRetTypeDesc->GetReturnRegType(i); - returnReg = pRetTypeDesc->GetABIReturnReg(i, call->GetUnmanagedCallConv()); + returnReg = pRetTypeDesc->GetABIReturnReg(i); regNumber allocatedReg = call->GetRegNumByIdx(i); inst_Mov(regType, allocatedReg, returnReg, /* canSkip */ true); } @@ -4556,14 +4607,14 @@ void CodeGen::inst_JMP(emitJumpKind jmp, BasicBlock* tgtBlock) } //------------------------------------------------------------------------ -// genCodeForStoreBlk: Produce code for a GT_STORE_BLK node. +// genCodeForStoreBlk: Produce code for a GT_STORE_DYN_BLK/GT_STORE_BLK node. // // Arguments: // tree - the node // void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) { - assert(blkOp->OperIs(GT_STORE_BLK)); + assert(blkOp->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); bool isCopyBlk = blkOp->OperIsCopyBlkOp(); @@ -4579,6 +4630,18 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) genCodeForInitBlkLoop(blkOp); break; + case GenTreeBlk::BlkOpKindHelper: + assert(!blkOp->gtBlkOpGcUnsafe); + if (isCopyBlk) + { + genCodeForCpBlkHelper(blkOp); + } + else + { + genCodeForInitBlkHelper(blkOp); + } + break; + case GenTreeBlk::BlkOpKindUnroll: case GenTreeBlk::BlkOpKindUnrollMemmove: if (isCopyBlk) @@ -4828,7 +4891,7 @@ void CodeGen::genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc) for (unsigned i = 0; i < regCount; ++i) { var_types type = retTypeDesc->GetReturnRegType(i); - regNumber reg = retTypeDesc->GetABIReturnReg(i, compiler->info.compCallConv); + regNumber reg = retTypeDesc->GetABIReturnReg(i); if (varTypeIsFloating(type)) { // If the register piece is to be passed in a floating point register diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index dde13812a2284d..825837fe45ef50 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -315,8 +315,10 @@ void CodeGen::genPrepForCompiler() } } VarSetOps::AssignNoCopy(compiler, genLastLiveSet, VarSetOps::MakeEmpty(compiler)); - genLastLiveMask = RBM_NONE; - compiler->Metrics.BasicBlocksAtCodegen = compiler->fgBBcount; + genLastLiveMask = RBM_NONE; +#ifdef DEBUG + compiler->fgBBcountAtCodegen = compiler->fgBBcount; +#endif } //------------------------------------------------------------------------ @@ -595,6 +597,28 @@ void CodeGenInterface::genUpdateRegLife(const LclVarDsc* varDsc, bool isBorn, bo // compHelperCallKillSet: Gets a register mask that represents the kill set for a helper call. // Not all JIT Helper calls follow the standard ABI on the target architecture. // +// TODO-CQ: Currently this list is incomplete (not all helpers calls are +// enumerated) and not 100% accurate (some killsets are bigger than +// what they really are). +// There's some work to be done in several places in the JIT to +// accurately track the registers that are getting killed by +// helper calls: +// a) LSRA needs several changes to accommodate more precise killsets +// for every helper call it sees (both explicitly [easy] and +// implicitly [hard]) +// b) Currently for AMD64, when we generate code for a helper call +// we're independently over-pessimizing the killsets of the call +// (independently from LSRA) and this needs changes +// both in CodeGenAmd64.cpp and emitx86.cpp. +// +// The best solution for this problem would be to try to centralize +// the killset information in a single place but then make the +// corresponding changes so every code generation phase is in sync +// about this. +// +// The interim solution is to only add known helper calls that don't +// follow the AMD64 ABI and actually trash registers that are supposed to be non-volatile. +// // Arguments: // helper - The helper being inquired about // @@ -605,12 +629,6 @@ regMaskTP Compiler::compHelperCallKillSet(CorInfoHelpFunc helper) { switch (helper) { - // Most of the helpers are written in C++ and C# and we can't make - // any additional assumptions beyond the standard ABI. However, some are written in raw assembly, - // so we can narrow down the kill sets. - // - // TODO-CQ: Inspect all asm helpers and narrow down the kill sets for them. - // case CORINFO_HELP_ASSIGN_REF: case CORINFO_HELP_CHECKED_ASSIGN_REF: return RBM_CALLEE_TRASH_WRITEBARRIER; @@ -2024,7 +2042,7 @@ void CodeGen::genEmitMachineCode() printf("; Total bytes of code %d, prolog size %d, PerfScore %.2f, instruction count %d, allocated bytes for " "code %d", - codeSize, prologSize, compiler->Metrics.PerfScore, instrCount, + codeSize, prologSize, compiler->info.compPerfScore, instrCount, GetEmitter()->emitTotalHotCodeSize + GetEmitter()->emitTotalColdCodeSize); if (dspMetrics) @@ -3779,7 +3797,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere regNumber begRegNum = genMapRegArgNumToRegNum(begReg, destMemType); GetEmitter()->emitIns_Mov(insCopy, size, xtraReg, begRegNum, /* canSkip */ false); - assert(!genIsValidIntReg(xtraReg) || !genIsValidFloatReg(begRegNum)); regSet.verifyRegUsed(xtraReg); @@ -3794,7 +3811,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere regNumber srcRegNum = genMapRegArgNumToRegNum(srcReg, destMemType); GetEmitter()->emitIns_Mov(insCopy, size, destRegNum, srcRegNum, /* canSkip */ false); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(srcRegNum)); regSet.verifyRegUsed(destRegNum); @@ -3846,7 +3862,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere regNumber destRegNum = genMapRegArgNumToRegNum(destReg, destMemType); GetEmitter()->emitIns_Mov(insCopy, size, destRegNum, xtraReg, /* canSkip */ false); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(xtraReg)); regSet.verifyRegUsed(destRegNum); /* mark the beginning register as processed */ @@ -3921,7 +3936,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere // todo -- suppress self move GetEmitter()->emitIns_R_R_I_I(INS_mov, EA_4BYTE, destRegNum, regNum, regArgTab[currentArgNum].slot - 1, 0); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(regNum)); regArgTab[currentArgNum].processed = true; regArgMaskLive &= ~genRegMask(regNum); } @@ -4095,7 +4109,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere } #endif inst_Mov(destMemType, destRegNum, regNum, /* canSkip */ false, size); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(regNum)); } /* mark the argument as processed */ @@ -4121,7 +4134,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere // Emit a shufpd with a 0 immediate, which preserves the 0th element of the dest reg // and moves the 0th element of the src reg into the 1st element of the dest reg. GetEmitter()->emitIns_R_R_I(INS_shufpd, emitActualTypeSize(varRegType), destRegNum, nextRegNum, 0); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(nextRegNum)); // Set destRegNum to regNum so that we skip the setting of the register below, // but mark argNum as processed and clear regNum from the live mask. destRegNum = regNum; @@ -4149,7 +4161,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere noway_assert(genIsValidFloatReg(nextRegNum)); noway_assert(genIsValidFloatReg(destRegNum)); GetEmitter()->emitIns_Mov(INS_mov, EA_8BYTE, destRegNum, nextRegNum, /* canSkip */ false); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(nextRegNum)); } } #if defined(TARGET_ARM64) && defined(FEATURE_SIMD) @@ -4170,7 +4181,6 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere noway_assert(genIsValidFloatReg(nextRegNum)); noway_assert(genIsValidFloatReg(destRegNum)); GetEmitter()->emitIns_R_R_I_I(INS_mov, EA_4BYTE, destRegNum, nextRegNum, i, 0); - assert(!genIsValidIntReg(destRegNum) || !genIsValidFloatReg(nextRegNum)); } } #endif // defined(TARGET_ARM64) && defined(FEATURE_SIMD) @@ -6444,47 +6454,6 @@ void CodeGen::genFnProlog() #pragma warning(pop) #endif -//---------------------------------------------------------------------------------- -// genEmitJumpTable: emit jump table and return its base offset -// -// Arguments: -// treeNode - the GT_JMPTABLE node -// relativeAddr - if true, references are treated as 4-byte relative addresses, -// otherwise they are absolute pointers -// -// Return Value: -// base offset to jump table -// -// Assumption: -// The current basic block in process ends with a switch statement -// -unsigned CodeGen::genEmitJumpTable(GenTree* treeNode, bool relativeAddr) -{ - noway_assert(compiler->compCurBB->KindIs(BBJ_SWITCH)); - assert(treeNode->OperGet() == GT_JMPTABLE); - - emitter* emit = GetEmitter(); - const unsigned jumpCount = compiler->compCurBB->GetSwitchTargets()->bbsCount; - FlowEdge** jumpTable = compiler->compCurBB->GetSwitchTargets()->bbsDstTab; - const unsigned jmpTabBase = emit->emitBBTableDataGenBeg(jumpCount, relativeAddr); - - JITDUMP("\n J_M%03u_DS%02u LABEL DWORD\n", compiler->compMethodID, jmpTabBase); - - for (unsigned i = 0; i < jumpCount; i++) - { - BasicBlock* target = (*jumpTable)->getDestinationBlock(); - jumpTable++; - noway_assert(target->HasFlag(BBF_HAS_LABEL)); - - JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); - - emit->emitDataGenData(i, target); - }; - - emit->emitDataGenEnd(); - return jmpTabBase; -} - //------------------------------------------------------------------------ // getCallTarget - Get the node that evaluates to the call target // @@ -6579,19 +6548,10 @@ void CodeGen::genDefinePendingCallLabel(GenTreeCall* call) // For certain indirect calls we may introduce helper calls before that we need to skip: // - CFG may introduce a call to the validator first // - Generic virtual methods may compute the target dynamically through a separate helper call - // - memset/memcpy helper calls emitted for GT_STORE_BLK - if (call->IsHelperCall()) + if (call->IsHelperCall(compiler, CORINFO_HELP_VALIDATE_INDIRECT_CALL) || + call->IsHelperCall(compiler, CORINFO_HELP_VIRTUAL_FUNC_PTR)) { - switch (compiler->eeGetHelperNum(call->gtCallMethHnd)) - { - case CORINFO_HELP_VALIDATE_INDIRECT_CALL: - case CORINFO_HELP_VIRTUAL_FUNC_PTR: - case CORINFO_HELP_MEMSET: - case CORINFO_HELP_MEMCPY: - return; - default: - break; - } + return; } genDefineInlineTempLabel(genPendingCallLabel); @@ -7396,26 +7356,6 @@ void CodeGen::genReportRichDebugInfoToFile() #endif -//------------------------------------------------------------------------ -// SuccessfulSibling: -// Find the next sibling inline context that was successfully inlined. -// -// Parameters: -// context - the inline context. Can be nullptr in which case nullptr is returned. -// -// Returns: -// The sibling, or nullptr if there is no succesful sibling. -// -static InlineContext* SuccessfulSibling(InlineContext* context) -{ - while ((context != nullptr) && !context->IsSuccess()) - { - context = context->GetSibling(); - } - - return context; -} - //------------------------------------------------------------------------ // genRecordRichDebugInfoInlineTree: // Recursively process a context in the inline tree and record information @@ -7427,28 +7367,26 @@ static InlineContext* SuccessfulSibling(InlineContext* context) // void CodeGen::genRecordRichDebugInfoInlineTree(InlineContext* context, ICorDebugInfo::InlineTreeNode* nodes) { - assert(context->IsSuccess()); - - // We expect 1 + NumInlines unique ordinals - assert(context->GetOrdinal() <= compiler->m_inlineStrategy->GetInlineCount()); - - InlineContext* successfulChild = SuccessfulSibling(context->GetChild()); - InlineContext* successfulSibling = SuccessfulSibling(context->GetSibling()); + if (context->IsSuccess()) + { + // We expect 1 + NumInlines unique ordinals + assert(context->GetOrdinal() <= compiler->m_inlineStrategy->GetInlineCount()); - ICorDebugInfo::InlineTreeNode* node = &nodes[context->GetOrdinal()]; - node->Method = context->GetCallee(); - node->ILOffset = context->GetActualCallOffset(); - node->Child = successfulChild == nullptr ? 0 : successfulChild->GetOrdinal(); - node->Sibling = successfulSibling == nullptr ? 0 : successfulSibling->GetOrdinal(); + ICorDebugInfo::InlineTreeNode* node = &nodes[context->GetOrdinal()]; + node->Method = context->GetCallee(); + node->ILOffset = context->GetActualCallOffset(); + node->Child = context->GetChild() == nullptr ? 0 : context->GetChild()->GetOrdinal(); + node->Sibling = context->GetSibling() == nullptr ? 0 : context->GetSibling()->GetOrdinal(); + } - if (successfulSibling != nullptr) + if (context->GetSibling() != nullptr) { - genRecordRichDebugInfoInlineTree(successfulSibling, nodes); + genRecordRichDebugInfoInlineTree(context->GetSibling(), nodes); } - if (successfulChild != nullptr) + if (context->GetChild() != nullptr) { - genRecordRichDebugInfoInlineTree(successfulChild, nodes); + genRecordRichDebugInfoInlineTree(context->GetChild(), nodes); } } @@ -7498,28 +7436,6 @@ void CodeGen::genReportRichDebugInfo() mappingIndex++; } -#ifdef DEBUG - if (verbose) - { - printf("Reported inline tree:\n"); - for (unsigned i = 0; i < numContexts; i++) - { - printf(" [#%d] %s @ %d, child = %d, sibling = %d\n", i, - compiler->eeGetMethodFullName(inlineTree[i].Method), inlineTree[i].ILOffset, inlineTree[i].Child, - inlineTree[i].Sibling); - } - - printf("\nReported rich mappings:\n"); - for (size_t i = 0; i < mappingIndex; i++) - { - printf(" [%zu] 0x%x <-> IL %d in #%d\n", i, mappings[i].NativeOffset, mappings[i].ILOffset, - mappings[i].Inlinee); - } - - printf("\n"); - } -#endif - compiler->info.compCompHnd->reportRichMappings(inlineTree, numContexts, mappings, numRichMappings); } @@ -7770,8 +7686,7 @@ void CodeGen::genReturn(GenTree* treeNode) { if (varTypeIsGC(retTypeDesc.GetReturnRegType(i))) { - gcInfo.gcMarkRegPtrVal(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv), - retTypeDesc.GetReturnRegType(i)); + gcInfo.gcMarkRegPtrVal(retTypeDesc.GetABIReturnReg(i), retTypeDesc.GetReturnRegType(i)); } } } @@ -7788,7 +7703,7 @@ void CodeGen::genReturn(GenTree* treeNode) { if (varTypeIsGC(retTypeDesc.GetReturnRegType(i))) { - gcInfo.gcMarkRegSetNpt(genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv))); + gcInfo.gcMarkRegSetNpt(genRegMask(retTypeDesc.GetABIReturnReg(i))); } } } @@ -7895,7 +7810,7 @@ void CodeGen::genStructReturn(GenTree* treeNode) // On LoongArch64, for a struct like "{ int, double }", "retTypeDesc" will be "{ TYP_INT, TYP_DOUBLE }", // i. e. not include the padding for the first field, and so the general loop below won't work. var_types type = retTypeDesc.GetReturnRegType(0); - regNumber toReg = retTypeDesc.GetABIReturnReg(0, compiler->info.compCallConv); + regNumber toReg = retTypeDesc.GetABIReturnReg(0); GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), toReg, lclNode->GetLclNum(), 0); if (regCount > 1) { @@ -7903,7 +7818,7 @@ void CodeGen::genStructReturn(GenTree* treeNode) int offset = genTypeSize(type); type = retTypeDesc.GetReturnRegType(1); offset = (int)((unsigned int)offset < genTypeSize(type) ? genTypeSize(type) : offset); - toReg = retTypeDesc.GetABIReturnReg(1, compiler->info.compCallConv); + toReg = retTypeDesc.GetABIReturnReg(1); GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), toReg, lclNode->GetLclNum(), offset); } #else // !TARGET_LOONGARCH64 && !TARGET_RISCV64 @@ -7911,7 +7826,7 @@ void CodeGen::genStructReturn(GenTree* treeNode) for (unsigned i = 0; i < regCount; ++i) { var_types type = retTypeDesc.GetReturnRegType(i); - regNumber toReg = retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv); + regNumber toReg = retTypeDesc.GetABIReturnReg(i); GetEmitter()->emitIns_R_S(ins_Load(type), emitTypeSize(type), toReg, lclNode->GetLclNum(), offset); offset += genTypeSize(type); } @@ -7922,7 +7837,7 @@ void CodeGen::genStructReturn(GenTree* treeNode) for (unsigned i = 0; i < regCount; ++i) { var_types type = retTypeDesc.GetReturnRegType(i); - regNumber toReg = retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv); + regNumber toReg = retTypeDesc.GetABIReturnReg(i); regNumber fromReg = op1->GetRegByIndex(i); if ((fromReg == REG_NA) && op1->OperIs(GT_COPY)) { @@ -8046,16 +7961,6 @@ void CodeGen::genMultiRegStoreToLocal(GenTreeLclVar* lclNode) assert(regCount == varDsc->lvFieldCnt); } -#ifdef SWIFT_SUPPORT - const uint32_t* offsets = nullptr; - if (op1->IsCall() && (op1->AsCall()->GetUnmanagedCallConv() == CorInfoCallConvExtension::Swift)) - { - const CORINFO_SWIFT_LOWERING* lowering = compiler->GetSwiftLowering(op1->AsCall()->gtRetClsHnd); - assert(!lowering->byReference && (regCount == lowering->numLoweredElements)); - offsets = lowering->offsets; - } -#endif - for (unsigned i = 0; i < regCount; ++i) { regNumber reg = genConsumeReg(op1, i); @@ -8098,12 +8003,6 @@ void CodeGen::genMultiRegStoreToLocal(GenTreeLclVar* lclNode) #if defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) // should consider the padding field within a struct. offset = (offset % genTypeSize(srcType)) ? AlignUp(offset, genTypeSize(srcType)) : offset; -#endif -#ifdef SWIFT_SUPPORT - if (offsets != nullptr) - { - offset = offsets[i]; - } #endif // Several fields could be passed in one register, copy using the register type. // It could rewrite memory outside of the fields but local on the stack are rounded to POINTER_SIZE so @@ -8453,9 +8352,7 @@ void CodeGen::genPoisonFrame(regMaskTP regLiveIn) GetEmitter()->emitIns_R_S(INS_lea, EA_PTRSIZE, REG_ARG_0, (int)varNum, 0); instGen_Set_Reg_To_Imm(EA_4BYTE, REG_ARG_1, static_cast(poisonVal)); instGen_Set_Reg_To_Imm(EA_PTRSIZE, REG_ARG_2, size); - - // Call non-managed memset - genEmitHelperCall(CORINFO_HELP_NATIVE_MEMSET, 0, EA_UNKNOWN); + genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN); // May kill REG_SCRATCH, so we need to reload it. hasPoisonImm = false; #endif @@ -8619,31 +8516,3 @@ void CodeGen::genCodeForReuseVal(GenTree* treeNode) genDefineTempLabel(genCreateTempLabel()); } } - -#ifdef SWIFT_SUPPORT -//--------------------------------------------------------------------- -// genCodeForSwiftErrorReg - generate code for a GT_SWIFT_ERROR node -// -// Arguments -// tree - the GT_SWIFT_ERROR node -// -// Return value: -// None -// -void CodeGen::genCodeForSwiftErrorReg(GenTree* tree) -{ - assert(tree->OperIs(GT_SWIFT_ERROR)); - - var_types targetType = tree->TypeGet(); - regNumber targetReg = tree->GetRegNum(); - - // LSRA should have picked REG_SWIFT_ERROR as the destination register, too - // (see LinearScan::BuildNode for an explanation of why we want this) - assert(targetReg == REG_SWIFT_ERROR); - - inst_Mov(targetType, targetReg, REG_SWIFT_ERROR, /* canSkip */ true); - genTransferRegGCState(targetReg, REG_SWIFT_ERROR); - - genProduceReg(tree); -} -#endif // SWIFT_SUPPORT diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 78df10811a4c24..913f3a47002a51 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1935,9 +1935,18 @@ void CodeGen::genSetBlockSize(GenTreeBlk* blkNode, regNumber sizeReg) { if (sizeReg != REG_NA) { - assert((blkNode->gtRsvdRegs & genRegMask(sizeReg)) != 0); - // This can go via helper which takes the size as a native uint. - instGen_Set_Reg_To_Imm(EA_PTRSIZE, sizeReg, blkNode->Size()); + unsigned blockSize = blkNode->Size(); + if (!blkNode->OperIs(GT_STORE_DYN_BLK)) + { + assert((blkNode->gtRsvdRegs & genRegMask(sizeReg)) != 0); + // This can go via helper which takes the size as a native uint. + instGen_Set_Reg_To_Imm(EA_PTRSIZE, sizeReg, blockSize); + } + else + { + GenTree* sizeNode = blkNode->AsStoreDynBlk()->gtDynamicSize; + inst_Mov(sizeNode->TypeGet(), sizeReg, sizeNode->GetRegNum(), /* canSkip */ true); + } } } @@ -2043,6 +2052,12 @@ void CodeGen::genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber genConsumeReg(dstAddr); // The source may be a local or in a register; 'genConsumeBlockSrc' will check that. genConsumeBlockSrc(blkNode); + // 'genSetBlockSize' (called below) will ensure that a register has been reserved as needed + // in the case where the size is a constant (i.e. it is not GT_STORE_DYN_BLK). + if (blkNode->OperGet() == GT_STORE_DYN_BLK) + { + genConsumeReg(blkNode->AsStoreDynBlk()->gtDynamicSize); + } // Next, perform any necessary moves. genCopyRegIfNeeded(dstAddr, dstReg); diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index 9d84acbcca3a1b..41266917205a52 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -2418,7 +2418,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) // Floating point divide never raises an exception assert(varTypeIsFloating(tree->gtOp1)); assert(varTypeIsFloating(tree->gtOp2)); - assert(tree->OperIs(GT_DIV)); + assert(tree->gtOper == GT_DIV); instruction ins = genGetInsForOper(tree); emit->emitIns_R_R_R(ins, emitActualTypeSize(targetType), tree->GetRegNum(), tree->gtOp1->GetRegNum(), @@ -2480,7 +2480,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) } // check (MinInt / -1) => ArithmeticException - if (tree->OperIs(GT_DIV, GT_MOD)) + if (tree->gtOper == GT_DIV || tree->gtOper == GT_MOD) { if ((exSetFlags & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None) { @@ -2514,20 +2514,55 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) // Generate the sdiv instruction if (size == EA_4BYTE) { - ins = tree->OperIs(GT_DIV) ? INS_div_w : INS_mod_w; + if (tree->OperGet() == GT_DIV) + { + ins = INS_div_w; + } + else + { + ins = INS_mod_w; + } } else { - ins = tree->OperIs(GT_DIV) ? INS_div_d : INS_mod_d; + if (tree->OperGet() == GT_DIV) + { + ins = INS_div_d; + } + else + { + ins = INS_mod_d; + } } emit->emitIns_R_R_R(ins, size, tree->GetRegNum(), Reg1, divisorReg); } - else // tree->OperIs(GT_UDIV, GT_UMOD) + else // if (tree->gtOper == GT_UDIV) GT_UMOD { + // Only one possible exception + // (AnyVal / 0) => DivideByZeroException + // + // Note that division by the constant 0 was already checked for above by the + // op2->IsIntegralConst(0) check + // + + if (!divisorOp->IsCnsIntOrI()) + { + // divisorOp is not a constant, so it could be zero + // + genJumpToThrowHlpBlk_la(SCK_DIV_BY_ZERO, INS_beq, divisorReg); + } + if (size == EA_4BYTE) { - ins = tree->OperIs(GT_UDIV) ? INS_div_wu : INS_mod_wu; + if (tree->OperGet() == GT_UDIV) + { + ins = INS_div_wu; + } + else + { + ins = INS_mod_wu; + } // TODO-LOONGARCH64: here is just for signed-extension ? emit->emitIns_R_R_I(INS_slli_w, EA_4BYTE, Reg1, Reg1, 0); @@ -2535,7 +2570,14 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) } else { - ins = tree->OperIs(GT_UDIV) ? INS_div_du : INS_mod_du; + if (tree->OperGet() == GT_UDIV) + { + ins = INS_div_du; + } + else + { + ins = INS_mod_du; + } } emit->emitIns_R_R_R(ins, size, tree->GetRegNum(), Reg1, divisorReg); @@ -2885,7 +2927,32 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) // emits the table and an instruction to get the address of the first element void CodeGen::genJumpTable(GenTree* treeNode) { - unsigned jmpTabBase = genEmitJumpTable(treeNode, true); + noway_assert(compiler->compCurBB->KindIs(BBJ_SWITCH)); + assert(treeNode->OperGet() == GT_JMPTABLE); + + unsigned jumpCount = compiler->compCurBB->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTable = compiler->compCurBB->GetSwitchTargets()->bbsDstTab; + unsigned jmpTabOffs; + unsigned jmpTabBase; + + jmpTabBase = GetEmitter()->emitBBTableDataGenBeg(jumpCount, true); + + jmpTabOffs = 0; + + JITDUMP("\n J_M%03u_DS%02u LABEL DWORD\n", compiler->compMethodID, jmpTabBase); + + for (unsigned i = 0; i < jumpCount; i++) + { + BasicBlock* target = *jumpTable++; + noway_assert(target->HasFlag(BBF_HAS_LABEL)); + + JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); + + GetEmitter()->emitDataGenData(i, target); + }; + + GetEmitter()->emitDataGenEnd(); + // Access to inline data is 'abstracted' by a special type of static member // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference // to constant data, not a real static field. @@ -4954,6 +5021,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_ld_d, EA_PTRSIZE, genPendingCallLabel, targetReg); break; + case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -5248,7 +5316,7 @@ void CodeGen::genPutArgStk(GenTreePutArgStk* treeNode) // addrNode can either be a GT_LCL_ADDR<0> or an address expression // - if (addrNode->isContained() && addrNode->IsLclVarAddr()) + if (addrNode->IsLclVarAddr()) { // We have a GT_BLK(GT_LCL_ADDR<0>) // @@ -5521,7 +5589,7 @@ void CodeGen::genPutArgSplit(GenTreePutArgSplit* treeNode) // addrNode can either be a GT_LCL_ADDR<0> or an address expression // - if (addrNode->isContained() && addrNode->IsLclVarAddr()) + if (addrNode->IsLclVarAddr()) { // We have a GT_BLK(GT_LCL_ADDR<0>) // @@ -6062,6 +6130,37 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree) genProduceReg(tree); } +//---------------------------------------------------------------------------------- +// genCodeForCpBlkHelper - Generate code for a CpBlk node by the means of the VM memcpy helper call +// +// Arguments: +// cpBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForCpBlkHelper(GenTreeBlk* cpBlkNode) +{ + // Destination address goes in arg0, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(cpBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + if (cpBlkNode->IsVolatile()) + { + // issue a full memory barrier before a volatile CpBlk operation + instGen_MemoryBarrier(); + } + + genEmitHelperCall(CORINFO_HELP_MEMCPY, 0, EA_UNKNOWN); + + if (cpBlkNode->IsVolatile()) + { + // issue a INS_BARRIER_RMB after a volatile CpBlk operation + instGen_MemoryBarrier(BARRIER_FULL); + } +} + //---------------------------------------------------------------------------------- // genCodeForCpBlkUnroll: Generates CpBlk code by performing a loop unroll // @@ -6244,6 +6343,31 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode) } } +//------------------------------------------------------------------------ +// genCodeForInitBlkHelper - Generate code for an InitBlk node by the means of the VM memcpy helper call +// +// Arguments: +// initBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode) +{ + // Size goes in arg2, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(initBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + if (initBlkNode->IsVolatile()) + { + // issue a full memory barrier before a volatile initBlock Operation + instGen_MemoryBarrier(); + } + + genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN); +} + //------------------------------------------------------------------------ // genCodeForInitBlkLoop - Generate code for an InitBlk using an inlined for-loop. // It's needed for cases when size is too big to unroll and we're not allowed @@ -6292,6 +6416,27 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) } } +// Generate code for a load from some address + offset +// base: tree node which can be either a local address or arbitrary node +// offset: distance from the base from which to load +void CodeGen::genCodeForLoadOffset(instruction ins, emitAttr size, regNumber dst, GenTree* base, unsigned offset) +{ + emitter* emit = GetEmitter(); + + if (base->OperIs(GT_LCL_ADDR)) + { + if (base->gtOper == GT_LCL_ADDR) + { + offset += base->AsLclFld()->GetLclOffs(); + } + emit->emitIns_R_S(ins, size, dst, base->AsLclVarCommon()->GetLclNum(), offset); + } + else + { + emit->emitIns_R_R_I(ins, size, dst, base->GetRegNum(), offset); + } +} + //------------------------------------------------------------------------ // genCall: Produce code for a GT_CALL node // @@ -6436,7 +6581,7 @@ void CodeGen::genCall(GenTreeCall* call) for (unsigned i = 0; i < regCount; ++i) { var_types regType = pRetTypeDesc->GetReturnRegType(i); - returnReg = pRetTypeDesc->GetABIReturnReg(i, call->GetUnmanagedCallConv()); + returnReg = pRetTypeDesc->GetABIReturnReg(i); regNumber allocatedReg = call->GetRegNumByIdx(i); inst_Mov(regType, allocatedReg, returnReg, /* canSkip */ true); } @@ -7159,14 +7304,14 @@ void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, } //------------------------------------------------------------------------ -// genCodeForStoreBlk: Produce code for a GT_STORE_BLK node. +// genCodeForStoreBlk: Produce code for a GT_STORE_DYN_BLK/GT_STORE_BLK node. // // Arguments: // tree - the node // void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) { - assert(blkOp->OperIs(GT_STORE_BLK)); + assert(blkOp->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); if (blkOp->gtBlkOpGcUnsafe) { @@ -7187,6 +7332,17 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) genCodeForInitBlkLoop(blkOp); break; + case GenTreeBlk::BlkOpKindHelper: + if (isCopyBlk) + { + genCodeForCpBlkHelper(blkOp); + } + else + { + genCodeForInitBlkHelper(blkOp); + } + break; + case GenTreeBlk::BlkOpKindUnroll: if (isCopyBlk) { diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index f3f0ae76d2f352..a468c026c22cbd 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -2482,6 +2482,18 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) } else // if (tree->OperIs(GT_UDIV, GT_UMOD)) { + // Only one possible exception + // (AnyVal / 0) => DivideByZeroException + // + // Note that division by the constant 0 was already checked for above by the + // op2->IsIntegralConst(0) check + + if (!divisorOp->IsCnsIntOrI()) + { + // divisorOp is not a constant, so it could be zero + genJumpToThrowHlpBlk_la(SCK_DIV_BY_ZERO, INS_beq, divisorReg); + } + if (tree->OperIs(GT_UDIV)) { ins = is4 ? INS_divuw : INS_divu; @@ -2499,7 +2511,7 @@ void CodeGen::genCodeForDivMod(GenTreeOp* tree) // Generate code for InitBlk by performing a loop unroll // Preconditions: // a) Both the size and fill byte value are integer constants. -// b) The size of the struct to initialize is smaller than getUnrollThreshold() bytes. +// b) The size of the struct to initialize is smaller than INITBLK_UNROLL_LIMIT bytes. void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node) { assert(node->OperIs(GT_STORE_BLK)); @@ -2837,7 +2849,32 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) // emits the table and an instruction to get the address of the first element void CodeGen::genJumpTable(GenTree* treeNode) { - unsigned jmpTabBase = genEmitJumpTable(treeNode, true); + noway_assert(compiler->compCurBB->KindIs(BBJ_SWITCH)); + assert(treeNode->OperGet() == GT_JMPTABLE); + + unsigned jumpCount = compiler->compCurBB->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTable = compiler->compCurBB->GetSwitchTargets()->bbsDstTab; + unsigned jmpTabOffs; + unsigned jmpTabBase; + + jmpTabBase = GetEmitter()->emitBBTableDataGenBeg(jumpCount, true); + + jmpTabOffs = 0; + + JITDUMP("\n J_M%03u_DS%02u LABEL DWORD\n", compiler->compMethodID, jmpTabBase); + + for (unsigned i = 0; i < jumpCount; i++) + { + BasicBlock* target = *jumpTable++; + noway_assert(target->HasFlag(BBF_HAS_LABEL)); + + JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); + + GetEmitter()->emitDataGenData(i, target); + }; + + GetEmitter()->emitDataGenEnd(); + // Access to inline data is 'abstracted' by a special type of static member // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference // to constant data, not a real static field. @@ -5064,6 +5101,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_ld, EA_PTRSIZE, genPendingCallLabel, targetReg); break; + case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -6128,6 +6166,37 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree) genProduceReg(tree); } +//---------------------------------------------------------------------------------- +// genCodeForCpBlkHelper - Generate code for a CpBlk node by the means of the VM memcpy helper call +// +// Arguments: +// cpBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForCpBlkHelper(GenTreeBlk* cpBlkNode) +{ + // Destination address goes in arg0, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(cpBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + if (cpBlkNode->IsVolatile()) + { + // issue a full memory barrier before a volatile CpBlk operation + instGen_MemoryBarrier(); + } + + genEmitHelperCall(CORINFO_HELP_MEMCPY, 0, EA_UNKNOWN); + + if (cpBlkNode->IsVolatile()) + { + // issue a INS_BARRIER_RMB after a volatile CpBlk operation + instGen_MemoryBarrier(BARRIER_FULL); + } +} + //---------------------------------------------------------------------------------- // genCodeForCpBlkUnroll: Generates CpBlk code by performing a loop unroll // @@ -6138,7 +6207,7 @@ void CodeGen::genCodeForIndir(GenTreeIndir* tree) // None // // Assumption: -// The size argument of the CpBlk node is a constant and <= getUnrollThreshold() bytes. +// The size argument of the CpBlk node is a constant and <= CPBLK_UNROLL_LIMIT bytes. // void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode) { @@ -6365,6 +6434,31 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) } } +//------------------------------------------------------------------------ +// genCodeForInitBlkHelper - Generate code for an InitBlk node by the means of the VM memcpy helper call +// +// Arguments: +// initBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode) +{ + // Size goes in arg2, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(initBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + if (initBlkNode->IsVolatile()) + { + // issue a full memory barrier before a volatile initBlock Operation + instGen_MemoryBarrier(); + } + + genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN); +} + //------------------------------------------------------------------------ // genCall: Produce code for a GT_CALL node // @@ -6509,7 +6603,7 @@ void CodeGen::genCall(GenTreeCall* call) for (unsigned i = 0; i < regCount; ++i) { var_types regType = pRetTypeDesc->GetReturnRegType(i); - returnReg = pRetTypeDesc->GetABIReturnReg(i, call->GetUnmanagedCallConv()); + returnReg = pRetTypeDesc->GetABIReturnReg(i); regNumber allocatedReg = call->GetRegNumByIdx(i); inst_Mov(regType, allocatedReg, returnReg, /* canSkip */ true); } @@ -7207,14 +7301,14 @@ void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, } //------------------------------------------------------------------------ -// genCodeForStoreBlk: Produce code for a GT_STORE_BLK node. +// genCodeForStoreBlk: Produce code for a GT_STORE_DYN_BLK/GT_STORE_BLK node. // // Arguments: // tree - the node // void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) { - assert(blkOp->OperIs(GT_STORE_BLK)); + assert(blkOp->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); if (blkOp->gtBlkOpGcUnsafe) { @@ -7234,6 +7328,17 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) genCodeForInitBlkLoop(blkOp); break; + case GenTreeBlk::BlkOpKindHelper: + if (isCopyBlk) + { + genCodeForCpBlkHelper(blkOp); + } + else + { + genCodeForInitBlkHelper(blkOp); + } + break; + case GenTreeBlk::BlkOpKindUnroll: if (isCopyBlk) { @@ -7782,11 +7887,6 @@ void CodeGen::genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroe { assert(compiler->compGeneratingProlog); - // The 'initReg' could have been calculated as one of the callee-saved registers (let's say T0, T1 and T2 are in - // use, so the next possible register is S1, which should be callee-save register). This is fine, as long as we - // save callee-saved registers before using 'initReg' for the first time. Instead, we can use REG_SCRATCH - // beforehand. We don't care if REG_SCRATCH will be overwritten, so we'll skip 'RegZeroed check'. - // // Unlike on x86/x64, we can also push float registers to stack regMaskTP rsPushRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED; @@ -7899,11 +7999,11 @@ void CodeGen::genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroe calleeSaveSPDelta = AlignUp((UINT)offset, STACK_ALIGN); offset = calleeSaveSPDelta - offset; - genStackPointerAdjustment(-calleeSaveSPDelta, REG_SCRATCH, nullptr, /* reportUnwindData */ true); + genStackPointerAdjustment(-calleeSaveSPDelta, initReg, pInitRegZeroed, /* reportUnwindData */ true); } else { - genStackPointerAdjustment(-totalFrameSize, REG_SCRATCH, nullptr, /* reportUnwindData */ true); + genStackPointerAdjustment(-totalFrameSize, initReg, pInitRegZeroed, /* reportUnwindData */ true); } } @@ -7912,8 +8012,6 @@ void CodeGen::genPushCalleeSavedRegisters(regNumber initReg, bool* pInitRegZeroe genSaveCalleeSavedRegistersHelp(rsPushRegs, offset, 0); offset += (int)(genCountBits(rsPushRegs) << 3); // each reg has 8 bytes - // From now on, we can safely use initReg. - emit->emitIns_R_R_I(INS_sd, EA_PTRSIZE, REG_RA, REG_SPBASE, offset); compiler->unwindSaveReg(REG_RA, offset); @@ -7983,7 +8081,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) unsigned(compiler->lvaOutgoingArgSpaceSize), totalFrameSize, compiler->compCalleeRegsPushed, dspBool(compiler->compLocallocUsed)); - if ((compiler->lvaOutgoingArgSpaceSize + (compiler->compCalleeRegsPushed << 3)) > 2047) + if ((compiler->lvaOutgoingArgSpaceSize + (compiler->compCalleeRegsPushed << 3)) >= 2040) { calleeSaveSPOffset = compiler->lvaOutgoingArgSpaceSize & 0xfffffff0; @@ -7995,8 +8093,8 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog) { genStackPointerAdjustment(calleeSaveSPOffset, REG_RA, nullptr, /* reportUnwindData */ true); } - remainingSPSize = totalFrameSize - calleeSaveSPOffset; calleeSaveSPOffset = compiler->lvaOutgoingArgSpaceSize - calleeSaveSPOffset; + remainingSPSize = remainingSPSize - calleeSaveSPOffset; } else { diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 8384d2b337daf5..b877262b1583d8 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -121,8 +121,7 @@ void CodeGen::genEmitGSCookieCheck(bool pushReg) for (unsigned i = 0; i < regCount; ++i) { - gcInfo.gcMarkRegPtrVal(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv), - retTypeDesc.GetReturnRegType(i)); + gcInfo.gcMarkRegPtrVal(retTypeDesc.GetABIReturnReg(i), retTypeDesc.GetReturnRegType(i)); } } } @@ -333,7 +332,7 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) #endif // !FEATURE_EH_FUNCLETS // The BBJ_CALLFINALLYRET is used because the BBJ_CALLFINALLY can't point to the - // jump target using bbTargetEdge - that is already used to point + // jump target using bbTarget - that is already used to point // to the finally block. So just skip past the BBJ_CALLFINALLYRET unless the // block is RETLESS. if (!block->HasFlag(BBF_RETLESS_CALL)) @@ -1319,8 +1318,8 @@ void CodeGen::genSIMDSplitReturn(GenTree* src, ReturnTypeDesc* retTypeDesc) // This is a case of operand is in a single reg and needs to be // returned in multiple ABI return registers. regNumber opReg = src->GetRegNum(); - regNumber reg0 = retTypeDesc->GetABIReturnReg(0, compiler->info.compCallConv); - regNumber reg1 = retTypeDesc->GetABIReturnReg(1, compiler->info.compCallConv); + regNumber reg0 = retTypeDesc->GetABIReturnReg(0); + regNumber reg1 = retTypeDesc->GetABIReturnReg(1); assert((reg0 != REG_NA) && (reg1 != REG_NA) && (opReg != REG_NA)); @@ -2108,12 +2107,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_NOP: break; -#ifdef SWIFT_SUPPORT - case GT_SWIFT_ERROR: - genCodeForSwiftErrorReg(treeNode); - break; -#endif // SWIFT_SUPPORT - case GT_KEEPALIVE: genConsumeRegs(treeNode->AsOp()->gtOp1); break; @@ -2184,6 +2177,7 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_lea, EA_PTR_DSP_RELOC, genPendingCallLabel, treeNode->GetRegNum()); break; + case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -2248,6 +2242,7 @@ void CodeGen::genMultiRegStoreToSIMDLocal(GenTreeLclVar* lclNode) // This case is always a call (AsCall() will assert if it is not). GenTreeCall* call = actualOp1->AsCall(); const ReturnTypeDesc* retTypeDesc = call->GetReturnTypeDesc(); + assert(retTypeDesc->GetReturnRegCount() == MAX_RET_REG_COUNT); assert(regCount == 2); regNumber targetReg = lclNode->GetRegNum(); @@ -3056,7 +3051,7 @@ void CodeGen::genLclHeap(GenTree* tree) void CodeGen::genCodeForStoreBlk(GenTreeBlk* storeBlkNode) { - assert(storeBlkNode->OperIs(GT_STORE_BLK)); + assert(storeBlkNode->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); bool isCopyBlk = storeBlkNode->OperIsCopyBlkOp(); @@ -3075,6 +3070,19 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* storeBlkNode) genCodeForInitBlkLoop(storeBlkNode); break; +#ifdef TARGET_AMD64 + case GenTreeBlk::BlkOpKindHelper: + assert(!storeBlkNode->gtBlkOpGcUnsafe); + if (isCopyBlk) + { + genCodeForCpBlkHelper(storeBlkNode); + } + else + { + genCodeForInitBlkHelper(storeBlkNode); + } + break; +#endif // TARGET_AMD64 case GenTreeBlk::BlkOpKindRepInstr: #ifndef JIT32_GCENCODER assert(!storeBlkNode->gtBlkOpGcUnsafe); @@ -3394,6 +3402,27 @@ void CodeGen::genCodeForInitBlkLoop(GenTreeBlk* initBlkNode) } } +#ifdef TARGET_AMD64 +//------------------------------------------------------------------------ +// genCodeForInitBlkHelper - Generate code for an InitBlk node by the means of the VM memcpy helper call +// +// Arguments: +// initBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForInitBlkHelper(GenTreeBlk* initBlkNode) +{ + // Destination address goes in arg0, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(initBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + genEmitHelperCall(CORINFO_HELP_MEMSET, 0, EA_UNKNOWN); +} +#endif // TARGET_AMD64 + #ifdef FEATURE_PUT_STRUCT_ARG_STK // Generate code for a load from some address + offset // base: tree node which can be either a local or an indir @@ -4281,6 +4310,27 @@ void CodeGen::genCodeForCpObj(GenTreeBlk* cpObjNode) gcInfo.gcMarkRegSetNpt(RBM_RDI); } +#ifdef TARGET_AMD64 +//---------------------------------------------------------------------------------- +// genCodeForCpBlkHelper - Generate code for a CpBlk node by the means of the VM memcpy helper call +// +// Arguments: +// cpBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK] +// +// Preconditions: +// The register assignments have been set appropriately. +// This is validated by genConsumeBlockOp(). +// +void CodeGen::genCodeForCpBlkHelper(GenTreeBlk* cpBlkNode) +{ + // Destination address goes in arg0, source address goes in arg1, and size goes in arg2. + // genConsumeBlockOp takes care of this for us. + genConsumeBlockOp(cpBlkNode, REG_ARG_0, REG_ARG_1, REG_ARG_2); + + genEmitHelperCall(CORINFO_HELP_MEMCPY, 0, EA_UNKNOWN); +} +#endif // TARGET_AMD64 + // generate code do a switch statement based on a table of ip-relative offsets void CodeGen::genTableBasedSwitch(GenTree* treeNode) { @@ -4303,7 +4353,32 @@ void CodeGen::genTableBasedSwitch(GenTree* treeNode) // emits the table and an instruction to get the address of the first element void CodeGen::genJumpTable(GenTree* treeNode) { - unsigned jmpTabBase = genEmitJumpTable(treeNode, true); + noway_assert(compiler->compCurBB->KindIs(BBJ_SWITCH)); + assert(treeNode->OperGet() == GT_JMPTABLE); + + unsigned jumpCount = compiler->compCurBB->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTable = compiler->compCurBB->GetSwitchTargets()->bbsDstTab; + unsigned jmpTabOffs; + unsigned jmpTabBase; + + jmpTabBase = GetEmitter()->emitBBTableDataGenBeg(jumpCount, true); + + jmpTabOffs = 0; + + JITDUMP("\n J_M%03u_DS%02u LABEL DWORD\n", compiler->compMethodID, jmpTabBase); + + for (unsigned i = 0; i < jumpCount; i++) + { + BasicBlock* target = *jumpTable++; + noway_assert(target->HasFlag(BBF_HAS_LABEL)); + + JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); + + GetEmitter()->emitDataGenData(i, target); + }; + + GetEmitter()->emitDataGenEnd(); + // Access to inline data is 'abstracted' by a special type of static member // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference // to constant data, not a real static field. @@ -6063,7 +6138,7 @@ void CodeGen::genCall(GenTreeCall* call) for (unsigned i = 0; i < regCount; ++i) { var_types regType = retTypeDesc->GetReturnRegType(i); - returnReg = retTypeDesc->GetABIReturnReg(i, call->GetUnmanagedCallConv()); + returnReg = retTypeDesc->GetABIReturnReg(i); regNumber allocatedReg = call->GetRegNumByIdx(i); inst_Mov(regType, allocatedReg, returnReg, /* canSkip */ true); } @@ -6074,7 +6149,7 @@ void CodeGen::genCall(GenTreeCall* call) // the native compiler doesn't guarantee it. if (call->IsUnmanaged() && (returnType == TYP_SIMD12)) { - returnReg = retTypeDesc->GetABIReturnReg(1, call->GetUnmanagedCallConv()); + returnReg = retTypeDesc->GetABIReturnReg(1); genSimd12UpperClear(returnReg); } #endif // FEATURE_SIMD @@ -10768,12 +10843,9 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindEndProlog(); // TODO We may need EBP restore sequence here if we introduce PSPSym - CLANG_FORMAT_COMMENT_ANCHOR; -#ifdef UNIX_X86_ABI // Add a padding for 16-byte alignment inst_RV_IV(INS_sub, REG_SPBASE, 12, EA_PTRSIZE); -#endif } /***************************************************************************** @@ -10792,10 +10864,8 @@ void CodeGen::genFuncletEpilog() ScopedSetVariable _setGeneratingEpilog(&compiler->compGeneratingEpilog, true); -#ifdef UNIX_X86_ABI // Revert a padding that was added for 16-byte alignment inst_RV_IV(INS_add, REG_SPBASE, 12, EA_PTRSIZE); -#endif instGen_Return(0); } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 676be4aa0aa208..c351419687fbf3 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -854,42 +854,6 @@ var_types Compiler::getReturnTypeForStruct(CORINFO_CLASS_HANDLE clsHnd, } assert(structSize > 0); -#ifdef SWIFT_SUPPORT - if (callConv == CorInfoCallConvExtension::Swift) - { - const CORINFO_SWIFT_LOWERING* lowering = GetSwiftLowering(clsHnd); - if (lowering->byReference) - { - howToReturnStruct = SPK_ByReference; - useType = TYP_UNKNOWN; - } - else if (lowering->numLoweredElements == 1) - { - useType = JITtype2varType(lowering->loweredElements[0]); - if (genTypeSize(useType) == structSize) - { - howToReturnStruct = SPK_PrimitiveType; - } - else - { - howToReturnStruct = SPK_EnclosingType; - } - } - else - { - howToReturnStruct = SPK_ByValue; - useType = TYP_STRUCT; - } - - if (wbReturnStruct != nullptr) - { - *wbReturnStruct = howToReturnStruct; - } - - return useType; - } -#endif - #ifdef UNIX_AMD64_ABI // An 8-byte struct may need to be returned in a floating point register // So we always consult the struct "Classifier" routine @@ -1829,13 +1793,9 @@ void Compiler::compInit(ArenaAllocator* pAlloc, info.compMethodName = eeGetMethodName(methodHnd); info.compClassName = eeGetClassName(info.compClassHnd); info.compFullName = eeGetMethodFullName(methodHnd); + info.compPerfScore = 0.0; info.compMethodSuperPMIIndex = g_jitHost->getIntConfigValue(W("SuperPMIMethodContextNumber"), -1); - - if (!compIsForInlining()) - { - JitMetadata::report(this, JitMetadata::MethodFullName, info.compFullName, strlen(info.compFullName)); - } #endif // defined(DEBUG) || defined(LATE_DISASM) || DUMP_FLOWGRAPHS #if defined(DEBUG) @@ -1903,7 +1863,9 @@ void Compiler::compInit(ArenaAllocator* pAlloc, // // Initialize all the per-method statistics gathering data structures. // - CLANG_FORMAT_COMMENT_ANCHOR; + + optLoopsCloned = 0; + #if LOOP_HOIST_STATS m_loopsConsidered = 0; m_curLoopHasHoistedExpression = false; @@ -1986,10 +1948,6 @@ void Compiler::compInit(ArenaAllocator* pAlloc, fgSsaValid = false; fgVNPassesCompleted = 0; -#ifdef SWIFT_SUPPORT - m_swiftLoweringCache = nullptr; -#endif - // check that HelperCallProperties are initialized assert(s_helperCallProperties.IsPure(CORINFO_HELP_GETSHARED_GCSTATIC_BASE)); @@ -2007,8 +1965,6 @@ void Compiler::compInit(ArenaAllocator* pAlloc, compUsesThrowHelper = false; m_preferredInitCctor = CORINFO_HELP_UNDEF; - - new (&Metrics, jitstd::placement_t()) JitMetrics(); } /***************************************************************************** @@ -4650,6 +4606,11 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_MORPH_ADD_INTERNAL, &Compiler::fgAddInternal); + // Disable profile checks now. + // Over time we will move this further and further back in the phase list, as we fix issues. + // + activePhaseChecks &= ~PhaseChecks::CHECK_PROFILE; + // Remove empty try regions // DoPhase(this, PHASE_EMPTY_TRY, &Compiler::fgRemoveEmptyTry); @@ -4864,19 +4825,19 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // DoPhase(this, PHASE_CANONICALIZE_ENTRY, &Compiler::fgCanonicalizeFirstBB); - // Compute DFS tree and remove all unreachable blocks. + // Compute reachability sets and dominators. // - DoPhase(this, PHASE_DFS_BLOCKS2, &Compiler::fgDfsBlocksAndRemove); + DoPhase(this, PHASE_COMPUTE_REACHABILITY, &Compiler::fgComputeReachability); + + // Scale block weights and mark run rarely blocks. + // + DoPhase(this, PHASE_SET_BLOCK_WEIGHTS, &Compiler::optSetBlockWeights); // Discover and classify natural loops (e.g. mark iterative loops as such). Also marks loop blocks // and sets bbWeight to the loop nesting levels. // DoPhase(this, PHASE_FIND_LOOPS, &Compiler::optFindLoopsPhase); - // Scale block weights and mark run rarely blocks. - // - DoPhase(this, PHASE_SET_BLOCK_WEIGHTS, &Compiler::optSetBlockWeights); - // Clone loops with optimization opportunities, and choose one based on dynamic condition evaluation. // DoPhase(this, PHASE_CLONE_LOOPS, &Compiler::optCloneLoops); @@ -4928,7 +4889,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl bool doValueNum = true; bool doLoopHoisting = true; bool doCopyProp = true; - bool doOptimizeIVs = true; bool doBranchOpt = true; bool doCse = true; bool doAssertionProp = true; @@ -4941,7 +4901,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl doSsa = (JitConfig.JitDoSsa() != 0); doEarlyProp = doSsa && (JitConfig.JitDoEarlyProp() != 0); doValueNum = doSsa && (JitConfig.JitDoValueNumber() != 0); - doOptimizeIVs = doSsa && (JitConfig.JitDoOptimizeIVs() != 0); doLoopHoisting = doValueNum && (JitConfig.JitDoLoopHoisting() != 0); doCopyProp = doValueNum && (JitConfig.JitDoCopyProp() != 0); doBranchOpt = doValueNum && (JitConfig.JitDoRedundantBranchOpts() != 0); @@ -5042,13 +5001,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl DoPhase(this, PHASE_OPTIMIZE_INDEX_CHECKS, &Compiler::rangeCheckPhase); } - if (doOptimizeIVs) - { - // Simplify and optimize induction variables used in natural loops - // - DoPhase(this, PHASE_OPTIMIZE_INDUCTION_VARIABLES, &Compiler::optInductionVariables); - } - if (doVNBasedDeadStoreRemoval) { // Note: this invalidates SSA and value numbers on tree nodes. @@ -5056,9 +5008,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl DoPhase(this, PHASE_VN_BASED_DEAD_STORE_REMOVAL, &Compiler::optVNBasedDeadStoreRemoval); } - // Conservatively mark all VNs as stale - vnStore = nullptr; - if (fgModified) { // update the flowgraph if we modified it during the optimization phase @@ -5093,9 +5042,6 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl DoPhase(this, PHASE_STRESS_SPLIT_TREE, &Compiler::StressSplitTree); #endif - // Expand casts - DoPhase(this, PHASE_EXPAND_CASTS, &Compiler::fgLateCastExpansion); - // Expand runtime lookups (an optimization but we'd better run it in tier0 too) DoPhase(this, PHASE_EXPAND_RTLOOKUPS, &Compiler::fgExpandRuntimeLookups); @@ -5105,6 +5051,9 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl // Expand thread local access DoPhase(this, PHASE_EXPAND_TLS, &Compiler::fgExpandThreadLocalAccess); + // Expand casts + DoPhase(this, PHASE_EXPAND_CASTS, &Compiler::fgLateCastExpansion); + // Insert GC Polls DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls); @@ -5263,7 +5212,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl #ifdef DEBUG if (JitConfig.JitMetrics() > 0) { - sprintf_s(metricPart, 128, ", perfScore=%.2f, numCse=%u", Metrics.PerfScore, optCSEcount); + sprintf_s(metricPart, 128, ", perfScore=%.2f, numCse=%u", info.compPerfScore, optCSEcount); } #endif @@ -5285,7 +5234,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl #elif FEATURE_SIMD fprintf(compJitFuncInfoFile, " %s\n", eeGetMethodFullName(info.compMethodHnd)); #endif - fflush(compJitFuncInfoFile); + fprintf(compJitFuncInfoFile, ""); // in our logic this causes a flush } #endif // FUNC_INFO_LOGGING } @@ -5467,7 +5416,7 @@ PhaseStatus Compiler::placeLoopAlignInstructions() { block->SetFlags(BBF_LOOP_ALIGN); BitVecOps::AddElemD(&loopTraits, alignedLoops, loop->GetIndex()); - Metrics.LoopAlignmentCandidates++; + INDEBUG(loopAlignCandidates++); BasicBlock* prev = block->Prev(); // shouldAlignLoop should have guaranteed these properties. @@ -5516,7 +5465,7 @@ PhaseStatus Compiler::placeLoopAlignInstructions() } } - JITDUMP("Found %d candidates for loop alignment\n", Metrics.LoopAlignmentCandidates); + JITDUMP("Found %u candidates for loop alignment\n", loopAlignCandidates); return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; } @@ -5886,20 +5835,19 @@ void Compiler::RecomputeFlowGraphAnnotations() // Recompute reachability sets, dominators, and loops. optResetLoopInfo(); - fgRenumberBlocks(); + fgComputeReachability(); + optSetBlockWeights(); + fgInvalidateDfsTree(); - fgDfsBlocksAndRemove(); + m_dfsTree = fgComputeDfs(); optFindLoops(); - // Should we call this using the phase method: - // DoPhase(this, PHASE_SET_BLOCK_WEIGHTS, &Compiler::optSetBlockWeights); - // ? It could be called multiple times. - optSetBlockWeights(); - - if (m_domTree == nullptr) + if (fgHasLoops) { - m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); + optFindAndScaleGeneralLoopBlocks(); } + + m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); } /*****************************************************************************/ @@ -6490,8 +6438,6 @@ void Compiler::compCompileFinish() compArenaAllocator->finishMemStats(); memAllocHist.record((unsigned)((compArenaAllocator->getTotalBytesAllocated() + 1023) / 1024)); memUsedHist.record((unsigned)((compArenaAllocator->getTotalBytesUsed() + 1023) / 1024)); - - Metrics.BytesAllocated = (int64_t)compArenaAllocator->getTotalBytesUsed(); } #ifdef DEBUG @@ -6675,7 +6621,7 @@ void Compiler::compCompileFinish() printf(" %3d |", optCallCount); printf(" %3d |", optIndirectCallCount); - printf(" %3d |", Metrics.BasicBlocksAtCodegen); + printf(" %3d |", fgBBcountAtCodegen); printf(" %3d |", lvaCount); if (opts.MinOpts()) @@ -6688,13 +6634,13 @@ void Compiler::compCompileFinish() printf(" %3d |", optCSEcount); } - if (Metrics.PerfScore < 9999.995) + if (info.compPerfScore < 9999.995) { - printf(" %7.2f |", Metrics.PerfScore); + printf(" %7.2f |", info.compPerfScore); } else { - printf(" %7.0f |", Metrics.PerfScore); + printf(" %7.0f |", info.compPerfScore); } printf(" %4d |", info.compMethodInfo->ILCodeSize); @@ -6705,13 +6651,9 @@ void Compiler::compCompileFinish() printf(""); // in our logic this causes a flush } - JITDUMP("Final metrics:\n"); - Metrics.report(this); - DBEXEC(verbose, Metrics.dump()); - if (verbose) { - printf("\n****** DONE compiling %s\n", info.compFullName); + printf("****** DONE compiling %s\n", info.compFullName); printf(""); // in our logic this causes a flush } @@ -7204,13 +7146,6 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr, opts.disAsm = false; } -#ifdef DEBUG - { - const char* tieringName = compGetTieringName(true); - JitMetadata::report(this, JitMetadata::TieringName, tieringName, strlen(tieringName)); - } -#endif - #if COUNT_BASIC_BLOCKS bbCntTable.record(fgBBcount); @@ -9139,12 +9074,12 @@ void JitTimer::PrintCsvMethodStats(Compiler* comp) fprintf(s_csvFile, "%u,", comp->info.compILCodeSize); fprintf(s_csvFile, "%u,", comp->fgBBcount); fprintf(s_csvFile, "%u,", comp->opts.MinOpts()); - fprintf(s_csvFile, "%d,", comp->Metrics.LoopsFoundDuringOpts); - fprintf(s_csvFile, "%d,", comp->Metrics.LoopsCloned); + fprintf(s_csvFile, "%u,", comp->optNumNaturalLoopsFound); + fprintf(s_csvFile, "%u,", comp->optLoopsCloned); #if FEATURE_LOOP_ALIGN #ifdef DEBUG - fprintf(s_csvFile, "%d,", comp->Metrics.LoopAlignmentCandidates); - fprintf(s_csvFile, "%d,", comp->Metrics.LoopsAligned); + fprintf(s_csvFile, "%u,", comp->loopAlignCandidates); + fprintf(s_csvFile, "%u,", comp->loopsAligned); #endif // DEBUG #endif // FEATURE_LOOP_ALIGN unsigned __int64 totCycles = 0; @@ -9454,7 +9389,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:cLoops") #pragma comment(linker, "/include:cLoopsA") #pragma comment(linker, "/include:cLoop") -#pragma comment(linker, "/include:cScev") #pragma comment(linker, "/include:cTreeFlags") #pragma comment(linker, "/include:cVN") @@ -9480,7 +9414,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:dCVarSet") #pragma comment(linker, "/include:dLoop") #pragma comment(linker, "/include:dLoops") -#pragma comment(linker, "/include:dScev") #pragma comment(linker, "/include:dTreeFlags") #pragma comment(linker, "/include:dVN") @@ -9724,39 +9657,24 @@ JITDBGAPI void __cdecl cCVarSet(Compiler* comp, VARSET_VALARG_TP vars) JITDBGAPI void __cdecl cLoops(Compiler* comp) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *Loops %u\n", sequenceNumber++); + printf("===================================================================== *NewLoops %u\n", sequenceNumber++); FlowGraphNaturalLoops::Dump(comp->m_loops); } JITDBGAPI void __cdecl cLoopsA(Compiler* comp, FlowGraphNaturalLoops* loops) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *LoopsA %u\n", sequenceNumber++); + printf("===================================================================== *NewLoopsA %u\n", sequenceNumber++); FlowGraphNaturalLoops::Dump(loops); } JITDBGAPI void __cdecl cLoop(Compiler* comp, FlowGraphNaturalLoop* loop) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *Loop %u\n", sequenceNumber++); + printf("===================================================================== *NewLoop %u\n", sequenceNumber++); FlowGraphNaturalLoop::Dump(loop); } -JITDBGAPI void __cdecl cScev(Compiler* comp, Scev* scev) -{ - static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *Scev %u\n", sequenceNumber++); - if (scev == nullptr) - { - printf(" NULL\n"); - } - else - { - scev->Dump(comp); - printf("\n"); - } -} - JITDBGAPI void __cdecl cTreeFlags(Compiler* comp, GenTree* tree) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called @@ -9950,6 +9868,14 @@ JITDBGAPI void __cdecl cTreeFlags(Compiler* comp, GenTree* tree) } break; + case GT_QMARK: + + if (tree->gtFlags & GTF_QMARK_CAST_INSTOF) + { + chars += printf("[QMARK_CAST_INSTOF]"); + } + break; + case GT_BOX: if (tree->gtFlags & GTF_BOX_VALUE) @@ -9982,6 +9908,7 @@ JITDBGAPI void __cdecl cTreeFlags(Compiler* comp, GenTree* tree) case GT_BLK: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: if (tree->gtFlags & GTF_IND_VOLATILE) { @@ -10346,11 +10273,6 @@ JITDBGAPI void __cdecl dLoop(FlowGraphNaturalLoop* loop) cLoop(JitTls::GetCompiler(), loop); } -JITDBGAPI void __cdecl dScev(Scev* scev) -{ - cScev(JitTls::GetCompiler(), scev); -} - JITDBGAPI void __cdecl dTreeFlags(GenTree* tree) { cTreeFlags(JitTls::GetCompiler(), tree); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 2bf36d481ea479..c850b8f76176a2 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -42,7 +42,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "jitexpandarray.h" #include "tinyarray.h" #include "valuenum.h" -#include "scev.h" #include "namedintrinsiclist.h" #ifdef LATE_DISASM #include "disasm.h" @@ -66,8 +65,6 @@ inline var_types genActualType(T value); #include "simd.h" #include "simdashwintrinsic.h" -#include "jitmetadata.h" - /***************************************************************************** * Forward declarations */ @@ -1584,10 +1581,9 @@ enum class ProfileChecks : unsigned int CHECK_NONE = 0, CHECK_CLASSIC = 1 << 0, // check "classic" jit weights CHECK_HASLIKELIHOOD = 1 << 1, // check all FlowEdges for hasLikelihood - CHECK_LIKELIHOODSUM = 1 << 2, // check block successor likelihoods sum to 1 - CHECK_LIKELY = 1 << 3, // fully check likelihood based weights - RAISE_ASSERT = 1 << 4, // assert on check failure - CHECK_ALL_BLOCKS = 1 << 5, // check blocks even if bbHasProfileWeight is false + CHECK_LIKELY = 1 << 2, // fully check likelihood based weights + RAISE_ASSERT = 1 << 3, // assert on check failure + CHECK_ALL_BLOCKS = 1 << 4, // check blocks even if bbHasProfileWeight is false }; inline constexpr ProfileChecks operator ~(ProfileChecks a) @@ -1951,10 +1947,6 @@ class FlowGraphDfsTree return m_hasCycle; } -#ifdef DEBUG - void Dump() const; -#endif // DEBUG - bool Contains(BasicBlock* block) const; bool IsAncestor(BasicBlock* ancestor, BasicBlock* descendant) const; }; @@ -2234,7 +2226,7 @@ class FlowGraphNaturalLoops return m_dfsTree; } - size_t NumLoops() const + size_t NumLoops() { return m_loops.size(); } @@ -2326,13 +2318,7 @@ class FlowGraphDominatorTree } static BasicBlock* IntersectDom(BasicBlock* block1, BasicBlock* block2); - public: - const FlowGraphDfsTree* GetDfsTree() - { - return m_dfsTree; - } - BasicBlock* Intersect(BasicBlock* block, BasicBlock* block2); bool Dominates(BasicBlock* dominator, BasicBlock* dominated); @@ -2360,10 +2346,6 @@ class BlockToNaturalLoopMap FlowGraphNaturalLoop* GetLoop(BasicBlock* block); static BlockToNaturalLoopMap* Build(FlowGraphNaturalLoops* loops); - -#ifdef DEBUG - void Dump() const; -#endif // DEBUG }; // Represents a data structure that can answer A -> B reachability queries in @@ -2371,28 +2353,23 @@ class BlockToNaturalLoopMap // exceptional flow, then CanReach returns false. class BlockReachabilitySets { - const FlowGraphDfsTree* m_dfsTree; + FlowGraphDfsTree* m_dfsTree; BitVec* m_reachabilitySets; - BlockReachabilitySets(const FlowGraphDfsTree* dfsTree, BitVec* reachabilitySets) + BlockReachabilitySets(FlowGraphDfsTree* dfsTree, BitVec* reachabilitySets) : m_dfsTree(dfsTree) , m_reachabilitySets(reachabilitySets) { } public: - const FlowGraphDfsTree* GetDfsTree() - { - return m_dfsTree; - } - bool CanReach(BasicBlock* from, BasicBlock* to); #ifdef DEBUG void Dump(); #endif - static BlockReachabilitySets* Build(const FlowGraphDfsTree* dfsTree); + static BlockReachabilitySets* Build(FlowGraphDfsTree* dfsTree); }; enum class FieldKindForVN @@ -2513,7 +2490,6 @@ class Compiler friend class CSE_HeuristicRandom; friend class CSE_HeuristicReplay; friend class CSE_HeuristicRL; - friend class CSE_HeuristicParameterized; friend class CSE_Heuristic; friend class CodeGenInterface; friend class CodeGen; @@ -3335,8 +3311,6 @@ class Compiler #endif #endif // FEATURE_HW_INTRINSICS - GenTree* gtNewMemoryBarrier(bool loadOnly = false); - GenTree* gtNewMustThrowException(unsigned helper, var_types type, CORINFO_CLASS_HANDLE clsHnd); GenTreeLclFld* gtNewLclFldNode(unsigned lnum, var_types type, unsigned offset); @@ -3385,6 +3359,9 @@ class Compiler GenTreeBlk* gtNewStoreBlkNode( ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); + GenTreeStoreDynBlk* gtNewStoreDynBlkNode( + GenTree* addr, GenTree* data, GenTree* dynamicSize, GenTreeFlags indirFlags = GTF_EMPTY); + GenTreeStoreInd* gtNewStoreIndNode( var_types type, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); @@ -3449,7 +3426,7 @@ class Compiler GenTreeAllocObj* gtNewAllocObjNode( unsigned int helper, bool helperHasSideEffects, CORINFO_CLASS_HANDLE clsHnd, var_types type, GenTree* op1); - GenTreeAllocObj* gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, bool useParent); + GenTreeAllocObj* gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool useParent); GenTree* gtNewRuntimeLookup(CORINFO_GENERIC_HANDLE hnd, CorInfoGenericHandleType hndTyp, GenTree* lookupTree); @@ -4386,13 +4363,7 @@ class Compiler void impCheckForPInvokeCall( GenTreeCall* call, CORINFO_METHOD_HANDLE methHnd, CORINFO_SIG_INFO* sig, unsigned mflags, BasicBlock* block); GenTreeCall* impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugInfo& di = DebugInfo()); - void impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, CallArg** swiftErrorArg); - void impPopArgsForSwiftCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, CallArg** swiftErrorArg); - void impRetypeUnmanagedCallArgs(GenTreeCall* call); - -#ifdef SWIFT_SUPPORT - void impAppendSwiftErrorStore(GenTreeCall* call, CallArg* const swiftErrorArg); -#endif // SWIFT_SUPPORT + void impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig); void impInsertHelperCall(CORINFO_HELPER_DESC* helperCall); void impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall); @@ -4447,23 +4418,17 @@ class Compiler Ordinal = 4, OrdinalIgnoreCase = 5 }; - enum class StringComparisonJoint + enum StringComparisonJoint { Eq, // (d1 == cns1) && (s2 == cns2) Xor, // (d1 ^ cns1) | (s2 ^ cns2) }; - enum class StringComparisonKind - { - Equals, - StartsWith, - EndsWith - }; - GenTree* impUtf16StringComparison(StringComparisonKind kind, CORINFO_SIG_INFO* sig, unsigned methodFlags); - GenTree* impUtf16SpanComparison(StringComparisonKind kind, CORINFO_SIG_INFO* sig, unsigned methodFlags); + GenTree* impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags); + GenTree* impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags); GenTree* impExpandHalfConstEquals(GenTreeLclVarCommon* data, GenTree* lengthFld, bool checkForNull, - StringComparisonKind kind, + bool startsWith, WCHAR* cnsData, int len, int dataOffset, @@ -4473,7 +4438,7 @@ class Compiler ssize_t offset, ssize_t value, StringComparison ignoreCase, - StringComparisonJoint joint = StringComparisonJoint::Eq); + StringComparisonJoint joint = Eq); GenTree* impExpandHalfConstEqualsSWAR( GenTreeLclVarCommon* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode); GenTree* impExpandHalfConstEqualsSIMD( @@ -4565,11 +4530,6 @@ class Compiler NamedIntrinsic intrinsic, GenTree* immOp, bool mustExpand, int immLowerBound, int immUpperBound); GenTree* addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound, int immUpperBound); -#if defined(TARGET_ARM64) - GenTree* convertHWIntrinsicToMask(var_types type, GenTree* node, CorInfoType simdBaseJitType, unsigned simdSize); - GenTree* convertHWIntrinsicFromMask(GenTreeHWIntrinsic* node, var_types type); -#endif - #endif // FEATURE_HW_INTRINSICS GenTree* impArrayAccessIntrinsic(CORINFO_CLASS_HANDLE clsHnd, CORINFO_SIG_INFO* sig, @@ -4932,7 +4892,7 @@ class Compiler unsigned impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reason)); - GenTree* impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& lclInfo); + GenTree* impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclTypeInfo); bool impInlineIsThis(GenTree* tree, InlArgInfo* inlArgInfo); @@ -5000,9 +4960,10 @@ class Compiler unsigned fgEdgeCount; // # of control flow edges between the BBs unsigned fgBBcount; // # of BBs in the method (in the linked list that starts with fgFirstBB) #ifdef DEBUG + unsigned fgBBcountAtCodegen; // # of BBs in the method at the start of codegen jitstd::vector* fgBBOrder; // ordered vector of BBs #endif - // Used as a quick check for whether phases downstream of loop finding should look for natural loops. + // Used as a quick check for whether loop alignment should look for natural loops. // If true: there may or may not be any natural loops in the flow graph, so try to find them // If false: there's definitely not any natural loops in the flow graph bool fgMightHaveNaturalLoops; @@ -5028,6 +4989,7 @@ class Compiler // 2. All loop exits where bbIsHandlerBeg(exit) is false have only loop predecessors. // bool optLoopsCanonical; + unsigned optNumNaturalLoopsFound; // Number of natural loops found in the loop finding phase bool fgBBVarSetsInited; @@ -5108,31 +5070,34 @@ class Compiler void fgExtendEHRegionBefore(BasicBlock* block); void fgExtendEHRegionAfter(BasicBlock* block); - BasicBlock* fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool extendRegion); + BasicBlock* fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool extendRegion, BasicBlock* jumpDest = nullptr); - BasicBlock* fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool extendRegion); + BasicBlock* fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool extendRegion, BasicBlock* jumpDest = nullptr); - BasicBlock* fgNewBBFromTreeAfter(BBKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, bool updateSideEffects = false); + BasicBlock* fgNewBBFromTreeAfter(BBKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, BasicBlock* jumpDest = nullptr, bool updateSideEffects = false); BasicBlock* fgNewBBinRegion(BBKinds jumpKind, unsigned tryIndex, unsigned hndIndex, BasicBlock* nearBlk, + BasicBlock* jumpDest = nullptr, bool putInFilter = false, bool runRarely = false, bool insertAtEnd = false); BasicBlock* fgNewBBinRegion(BBKinds jumpKind, BasicBlock* srcBlk, + BasicBlock* jumpDest = nullptr, bool runRarely = false, bool insertAtEnd = false); - BasicBlock* fgNewBBinRegion(BBKinds jumpKind); + BasicBlock* fgNewBBinRegion(BBKinds jumpKind, BasicBlock* jumpDest = nullptr); BasicBlock* fgNewBBinRegionWorker(BBKinds jumpKind, BasicBlock* afterBlk, unsigned xcptnIndex, - bool putInTryRegion); + bool putInTryRegion, + BasicBlock* jumpDest = nullptr); void fgInsertBBbefore(BasicBlock* insertBeforeBlk, BasicBlock* newBlk); void fgInsertBBafter(BasicBlock* insertAfterBlk, BasicBlock* newBlk); @@ -5145,6 +5110,7 @@ class Compiler bool fgModified; // True if the flow graph has been modified recently bool fgPredsComputed; // Have we computed the bbPreds list + bool fgReturnBlocksComputed; // Have we computed the return blocks list? bool fgOptimizedFinally; // Did we optimize any try-finallys? bool fgCanonicalizedFirstBB; // TODO-Quirk: did we end up canonicalizing first BB? @@ -5244,6 +5210,14 @@ class Compiler void fgCleanupContinuation(BasicBlock* continuation); PhaseStatus fgTailMergeThrows(); + void fgTailMergeThrowsFallThroughHelper(BasicBlock* predBlock, + BasicBlock* nonCanonicalBlock, + BasicBlock* canonicalBlock, + FlowEdge* predEdge); + void fgTailMergeThrowsJumpToHelper(BasicBlock* predBlock, + BasicBlock* nonCanonicalBlock, + BasicBlock* canonicalBlock, + FlowEdge* predEdge); bool fgRetargetBranchesToCanonicalCallFinally(BasicBlock* block, BasicBlock* handler, @@ -5374,6 +5348,7 @@ class Compiler Statement* fgNewStmtFromTree(GenTree* tree, const DebugInfo& di); GenTreeQmark* fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst = nullptr); + bool fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt); bool fgExpandQmarkStmt(BasicBlock* block, Statement* stmt); void fgExpandQmarkNodes(); @@ -5513,12 +5488,6 @@ class Compiler return m_signatureToLookupInfoMap; } -#ifdef SWIFT_SUPPORT - typedef JitHashTable, CORINFO_SWIFT_LOWERING*> SwiftLoweringMap; - SwiftLoweringMap* m_swiftLoweringCache; - const CORINFO_SWIFT_LOWERING* GetSwiftLowering(CORINFO_CLASS_HANDLE clsHnd); -#endif - void optRecordLoopMemoryDependence(GenTree* tree, BasicBlock* block, ValueNum memoryVN); void optCopyLoopMemoryDependence(GenTree* fromTree, GenTree* toTree); @@ -5800,6 +5769,8 @@ class Compiler template bool fgRemoveUnreachableBlocks(CanRemoveBlockBody canRemoveBlock); + PhaseStatus fgComputeReachability(); // Perform flow graph node reachability analysis. + PhaseStatus fgComputeDominators(); // Compute dominators bool fgRemoveDeadBlocks(); // Identify and remove dead blocks. @@ -5848,15 +5819,21 @@ class Compiler public: // For many purposes, it is desirable to be able to enumerate the *distinct* targets of a switch statement, // skipping duplicate targets. (E.g., in flow analyses that are only interested in the set of possible targets.) - // SwitchUniqueSuccSet contains the non-duplicated switch successor edges. - // Code that modifies the flowgraph (such as by renumbering blocks) must call Compiler::InvalidateUniqueSwitchSuccMap, - // and code that modifies the targets of a switch block must call Compiler::fgInvalidateSwitchDescMapEntry. - // If the unique targets of a switch block are needed later, they will be recomputed, ensuring they're up-to-date. + // SwitchUniqueSuccSet contains the non-duplicated switch targets. + // (Code that modifies the jump table of a switch has an obligation to call Compiler::UpdateSwitchTableTarget, + // which in turn will call the "UpdateTarget" method of this type if a SwitchUniqueSuccSet has already + // been computed for the switch block. If a switch block is deleted or is transformed into a non-switch, + // we leave the entry associated with the block, but it will no longer be accessed.) struct SwitchUniqueSuccSet { - unsigned numDistinctSuccs; // Number of distinct targets of the switch. - FlowEdge** nonDuplicates; // Array of "numDistinctSuccs", containing all the distinct switch target - // successor edges. + unsigned numDistinctSuccs; // Number of distinct targets of the switch. + BasicBlock** nonDuplicates; // Array of "numDistinctSuccs", containing all the distinct switch target + // successors. + + // The switch block "switchBlk" just had an entry with value "from" modified to the value "to". + // Update "this" as necessary: if "from" is no longer an element of the jump table of "switchBlk", + // remove it from "this", and ensure that "to" is a member. Use "alloc" to do any required allocation. + void UpdateTarget(CompAllocator alloc, BasicBlock* switchBlk, BasicBlock* from, BasicBlock* to); }; typedef JitHashTable, SwitchUniqueSuccSet> BlockToSwitchDescMap; @@ -5888,6 +5865,11 @@ class Compiler // the corresponding SwitchUniqueSuccSet. SwitchUniqueSuccSet GetDescriptorForSwitch(BasicBlock* switchBlk); + // The switch block "switchBlk" just had an entry with value "from" modified to the value "to". + // Update "this" as necessary: if "from" is no longer an element of the jump table of "switchBlk", + // remove it from "this", and ensure that "to" is a member. + void UpdateSwitchTableTarget(BasicBlock* switchBlk, BasicBlock* from, BasicBlock* to); + // Remove the "SwitchUniqueSuccSet" of "switchBlk" in the BlockToSwitchDescMap. void fgInvalidateSwitchDescMapEntry(BasicBlock* switchBlk); @@ -5899,6 +5881,8 @@ class Compiler FlowEdge* fgGetPredForBlock(BasicBlock* block, BasicBlock* blockPred, FlowEdge*** ptrToPred); + FlowEdge* fgRemoveRefPred(BasicBlock* block, BasicBlock* blockPred); + void fgRemoveRefPred(FlowEdge* edge); FlowEdge* fgRemoveAllRefPreds(BasicBlock* block, BasicBlock* blockPred); @@ -5917,22 +5901,14 @@ class Compiler void fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, BasicBlock* newTarget); + void fgReplacePred(BasicBlock* block, BasicBlock* oldPred, BasicBlock* newPred); + void fgReplacePred(FlowEdge* edge, BasicBlock* const newPred); // initializingPreds is only 'true' when we are computing preds in fgLinkBasicBlocks() template FlowEdge* fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowEdge* oldEdge = nullptr); -private: - FlowEdge** fgGetPredInsertPoint(BasicBlock* blockPred, BasicBlock* newTarget); - -public: - void fgRedirectTargetEdge(BasicBlock* block, BasicBlock* newTarget); - - void fgRedirectTrueEdge(BasicBlock* block, BasicBlock* newTarget); - - void fgRedirectFalseEdge(BasicBlock* block, BasicBlock* newTarget); - void fgFindBasicBlocks(); bool fgCheckEHCanInsertAfterBlock(BasicBlock* blk, unsigned regionIndex, bool putInTryRegion); @@ -6091,11 +6067,7 @@ class Compiler void fgDispBBLiveness(BasicBlock* block); void fgDispBBLiveness(); - void fgTableDispBasicBlock(const BasicBlock* block, - const BasicBlock* nextBlock = nullptr, - bool printEdgeLikelihoods = true, - int blockTargetFieldWidth = 21, - int ibcColWidth = 0); + void fgTableDispBasicBlock(const BasicBlock* block, const BasicBlock* nextBlock = nullptr, int blockTargetFieldWidth = 21, int ibcColWidth = 0); void fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, bool dumpTrees); void fgDispBasicBlocks(bool dumpTrees = false); void fgDumpStmtTree(const BasicBlock* block, Statement* stmt); @@ -6130,7 +6102,7 @@ class Compiler bool fgDebugCheckIncomingProfileData(BasicBlock* block, ProfileChecks checks); bool fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks checks); - void fgDebugCheckFlowGraphAnnotations(); + void fgDebugCheckDfsTree(); #endif // DEBUG @@ -6490,6 +6462,7 @@ class Compiler public: GenTree* fgMorphInitBlock(GenTree* tree); GenTree* fgMorphCopyBlock(GenTree* tree); + GenTree* fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree); private: GenTree* fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optAssertionPropDone = nullptr); void fgTryReplaceStructLocalWithField(GenTree* tree); @@ -6648,7 +6621,6 @@ class Compiler void fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* result, InlineContext** createdContext); void fgInsertInlineeBlocks(InlineInfo* pInlineInfo); - void fgInsertInlineeArgument(const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI); Statement* fgInlinePrependStatements(InlineInfo* inlineInfo); void fgInlineAppendStatements(InlineInfo* inlineInfo, BasicBlock* block, Statement* stmt); @@ -6711,8 +6683,6 @@ class Compiler public: bool fgIsBigOffset(size_t offset); - bool IsValidLclAddr(unsigned lclNum, unsigned offset); - private: bool fgNeedReturnSpillTemp(); @@ -6830,7 +6800,7 @@ class Compiler public: PhaseStatus optOptimizeBools(); PhaseStatus optSwitchRecognition(); - bool optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, weight_t falseLikelihood, GenTree* nodeToTest); + bool optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, GenTree* nodeToTest); bool optSwitchDetectAndConvert(BasicBlock* firstBlock); PhaseStatus optInvertLoops(); // Invert loops so they're entered at top and tested at bottom. @@ -6864,11 +6834,16 @@ class Compiler public: bool fgHasLoops; +#ifdef DEBUG + unsigned loopAlignCandidates; // number of candidates identified by placeLoopAlignInstructions + unsigned loopsAligned; // number of loops actually aligned +#endif // DEBUG protected: unsigned optCallCount; // number of calls made in the method unsigned optIndirectCallCount; // number of virtual, interface and indirect calls made in the method unsigned optNativeCallCount; // number of Pinvoke/Native calls made in the method + unsigned optLoopsCloned; // number of loops cloned in the current method. #ifdef DEBUG void optCheckPreds(); @@ -6887,7 +6862,7 @@ class Compiler bool optExtractInitTestIncr( BasicBlock** pInitBlock, BasicBlock* bottom, BasicBlock* top, GenTree** ppInit, GenTree** ppTest, GenTree** ppIncr); - void optSetMappedBlockTargets(BasicBlock* blk, + void optRedirectBlock(BasicBlock* blk, BasicBlock* newBlk, BlockToBlockMap* redirectMap); @@ -7090,7 +7065,6 @@ class Compiler unsigned optCSEstart; // The first local variable number that is a CSE unsigned optCSEattempt; // The number of CSEs attempted so far. unsigned optCSEcount; // The total count of CSEs introduced. - unsigned optCSEunmarks; // Number of CSE trees unmarked weight_t optCSEweight; // The weight of the current block when we are doing PerformCSE CSE_HeuristicCommon* optCSEheuristic; // CSE Heuristic to use for this method @@ -7435,18 +7409,6 @@ class Compiler BasicBlock* basicBlock); #endif - PhaseStatus optInductionVariables(); - bool optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop); - bool optIsIVWideningProfitable(unsigned lclNum, - BasicBlock* initBlock, - bool initedToConstant, - FlowGraphNaturalLoop* loop, - ArrayStack& ivUses); - void optBestEffortReplaceNarrowIVUses( - unsigned lclNum, unsigned ssaNum, unsigned newLclNum, BasicBlock* block, Statement* firstStmt); - void optReplaceWidenedIV(unsigned lclNum, unsigned ssaNum, unsigned newLclNum, Statement* stmt); - void optSinkWidenedIV(unsigned lclNum, unsigned newLclNum, FlowGraphNaturalLoop* loop); - // Redundant branch opts // PhaseStatus optRedundantBranches(); @@ -9257,7 +9219,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // | arm64 | 256 | 128 | ldp/stp (2x128bit) // | arm | 32 | 16 | no SIMD support // | loongarch64 | 64 | 32 | no SIMD support - // | riscv64 | 64 | 32 | no SIMD support // // We might want to use a different multiplier for truly hot/cold blocks based on PGO data // @@ -10187,6 +10148,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX const char* compMethodName; const char* compClassName; const char* compFullName; + double compPerfScore; int compMethodSuperPMIIndex; // useful when debugging under SuperPMI #endif // defined(DEBUG) || defined(LATE_DISASM) || DUMP_FLOWGRAPHS @@ -10237,9 +10199,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX unsigned compArgStackSize; // Incoming argument stack size in bytes #endif // FEATURE_FASTTAILCALL - unsigned compRetBuffArg; // position of hidden return param var (0, 1) (BAD_VAR_NUM means not present); - unsigned compTypeCtxtArg; // position of hidden param for type context for generic code - // (CORINFO_CALLCONV_PARAMTYPE) + unsigned compRetBuffArg; // position of hidden return param var (0, 1) (BAD_VAR_NUM means not present); + int compTypeCtxtArg; // position of hidden param for type context for generic code (CORINFO_CALLCONV_PARAMTYPE) unsigned compThisArg; // position of implicit this pointer param (not to be confused with lvaArg0Var) unsigned compILlocalsCount; // Number of vars : args + locals (incl. implicit but not hidden) unsigned compLocalsCount; // Number of vars : args + locals (incl. implicit and hidden) @@ -10648,8 +10609,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX static EnregisterStats s_enregisterStats; #endif // TRACK_ENREG_STATS - JitMetrics Metrics; - bool compIsForInlining() const; bool compDonotInline(); @@ -11362,7 +11321,6 @@ class GenTreeVisitor case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: - case GT_SWIFT_ERROR: break; // Lclvar unary operators @@ -11490,6 +11448,28 @@ class GenTreeVisitor break; } + case GT_STORE_DYN_BLK: + { + GenTreeStoreDynBlk* const dynBlock = node->AsStoreDynBlk(); + + result = WalkTree(&dynBlock->gtOp1, dynBlock); + if (result == fgWalkResult::WALK_ABORT) + { + return result; + } + result = WalkTree(&dynBlock->gtOp2, dynBlock); + if (result == fgWalkResult::WALK_ABORT) + { + return result; + } + result = WalkTree(&dynBlock->gtDynamicSize, dynBlock); + if (result == fgWalkResult::WALK_ABORT) + { + return result; + } + break; + } + case GT_CALL: { GenTreeCall* const call = node->AsCall(); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 359adc030f8083..67f5c59d93265e 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -664,27 +664,27 @@ BasicBlockVisit BasicBlock::VisitAllSuccs(Compiler* comp, TFunc func) return VisitEHSuccs(comp, func); case BBJ_CALLFINALLY: - RETURN_ON_ABORT(func(GetTarget())); + RETURN_ON_ABORT(func(bbTarget)); return ::VisitEHSuccs(comp, this, func); case BBJ_CALLFINALLYRET: // These are "pseudo-blocks" and control never actually flows into them // (codegen directly jumps to its successor after finally calls). - return func(GetTarget()); + return func(bbTarget); case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: case BBJ_ALWAYS: - RETURN_ON_ABORT(func(GetTarget())); + RETURN_ON_ABORT(func(bbTarget)); return VisitEHSuccs(comp, func); case BBJ_COND: - RETURN_ON_ABORT(func(GetFalseTarget())); + RETURN_ON_ABORT(func(bbFalseTarget)); - if (!TrueEdgeIs(GetFalseEdge())) + if (bbTrueTarget != bbFalseTarget) { - RETURN_ON_ABORT(func(GetTrueTarget())); + RETURN_ON_ABORT(func(bbTrueTarget)); } return VisitEHSuccs(comp, func); @@ -694,7 +694,7 @@ BasicBlockVisit BasicBlock::VisitAllSuccs(Compiler* comp, TFunc func) Compiler::SwitchUniqueSuccSet sd = comp->GetDescriptorForSwitch(this); for (unsigned i = 0; i < sd.numDistinctSuccs; i++) { - RETURN_ON_ABORT(func(sd.nonDuplicates[i]->getDestinationBlock())); + RETURN_ON_ABORT(func(sd.nonDuplicates[i])); } return VisitEHSuccs(comp, func); @@ -744,14 +744,14 @@ BasicBlockVisit BasicBlock::VisitRegularSuccs(Compiler* comp, TFunc func) case BBJ_EHFILTERRET: case BBJ_LEAVE: case BBJ_ALWAYS: - return func(GetTarget()); + return func(bbTarget); case BBJ_COND: - RETURN_ON_ABORT(func(GetFalseTarget())); + RETURN_ON_ABORT(func(bbFalseTarget)); - if (!TrueEdgeIs(GetFalseEdge())) + if (bbTrueTarget != bbFalseTarget) { - RETURN_ON_ABORT(func(GetTrueTarget())); + RETURN_ON_ABORT(func(bbTrueTarget)); } return BasicBlockVisit::Continue; @@ -761,7 +761,7 @@ BasicBlockVisit BasicBlock::VisitRegularSuccs(Compiler* comp, TFunc func) Compiler::SwitchUniqueSuccSet sd = comp->GetDescriptorForSwitch(this); for (unsigned i = 0; i < sd.numDistinctSuccs; i++) { - RETURN_ON_ABORT(func(sd.nonDuplicates[i]->getDestinationBlock())); + RETURN_ON_ABORT(func(sd.nonDuplicates[i])); } return BasicBlockVisit::Continue; @@ -2443,7 +2443,7 @@ inline bool Compiler::lvaReportParamTypeArg() { if (info.compMethodInfo->options & (CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE)) { - assert(info.compTypeCtxtArg != BAD_VAR_NUM); + assert(info.compTypeCtxtArg != -1); // If the VM requires us to keep the generics context alive and report it (for example, if any catch // clause catches a type that uses a generic parameter of this method) this flag will be set. @@ -2773,13 +2773,13 @@ inline unsigned Compiler::compMapILargNum(unsigned ILargNum) assert(ILargNum < info.compLocalsCount); // compLocals count already adjusted. } - if (ILargNum >= info.compTypeCtxtArg) + if (ILargNum >= (unsigned)info.compTypeCtxtArg) { ILargNum++; assert(ILargNum < info.compLocalsCount); // compLocals count already adjusted. } - if (ILargNum >= lvaVarargsHandleArg) + if (ILargNum >= (unsigned)lvaVarargsHandleArg) { ILargNum++; assert(ILargNum < info.compLocalsCount); // compLocals count already adjusted. @@ -3165,24 +3165,6 @@ inline bool Compiler::fgIsBigOffset(size_t offset) return (offset > compMaxUncheckedOffsetForNullObject); } -//------------------------------------------------------------------------ -// IsValidLclAddr: Can the given local address be represented as "LCL_FLD_ADDR"? -// -// Local address nodes cannot point beyond the local and can only store -// 16 bits worth of offset. -// -// Arguments: -// lclNum - The local's number -// offset - The address' offset -// -// Return Value: -// Whether "LCL_FLD_ADDR [+offset]" would be valid IR. -// -inline bool Compiler::IsValidLclAddr(unsigned lclNum, unsigned offset) -{ - return (offset < UINT16_MAX) && (offset < lvaLclExactSize(lclNum)); -} - /* XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -4264,7 +4246,6 @@ void GenTree::VisitOperands(TVisitor visitor) case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: - case GT_SWIFT_ERROR: return; // Unary operators with an optional operand @@ -4380,6 +4361,21 @@ void GenTree::VisitOperands(TVisitor visitor) return; } + case GT_STORE_DYN_BLK: + { + GenTreeStoreDynBlk* const dynBlock = this->AsStoreDynBlk(); + if (visitor(dynBlock->gtOp1) == VisitResult::Abort) + { + return; + } + if (visitor(dynBlock->gtOp2) == VisitResult::Abort) + { + return; + } + visitor(dynBlock->gtDynamicSize); + return; + } + case GT_CALL: { GenTreeCall* const call = this->AsCall(); diff --git a/src/coreclr/jit/compmemkind.h b/src/coreclr/jit/compmemkind.h index e986682894c3b6..835d85f798d29b 100644 --- a/src/coreclr/jit/compmemkind.h +++ b/src/coreclr/jit/compmemkind.h @@ -50,7 +50,6 @@ CompMemKindMacro(LoopOpt) CompMemKindMacro(LoopClone) CompMemKindMacro(LoopUnroll) CompMemKindMacro(LoopHoist) -CompMemKindMacro(LoopIVOpts) CompMemKindMacro(Unknown) CompMemKindMacro(RangeCheck) CompMemKindMacro(CopyProp) diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index ff6dd70d3e72fb..23930985319769 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -42,7 +42,6 @@ CompPhaseNameMacro(PHASE_CLONE_FINALLY, "Clone finally", CompPhaseNameMacro(PHASE_UPDATE_FINALLY_FLAGS, "Update finally target flags", false, -1, false) CompPhaseNameMacro(PHASE_EARLY_UPDATE_FLOW_GRAPH, "Update flow graph early pass", false, -1, false) CompPhaseNameMacro(PHASE_DFS_BLOCKS, "DFS blocks and remove dead code",false, -1, false) -CompPhaseNameMacro(PHASE_DFS_BLOCKS2, "DFS blocks and remove dead code 2",false, -1, false) CompPhaseNameMacro(PHASE_STR_ADRLCL, "Morph - Structs/AddrExp", false, -1, false) CompPhaseNameMacro(PHASE_EARLY_LIVENESS, "Early liveness", false, -1, false) CompPhaseNameMacro(PHASE_PHYSICAL_PROMOTION, "Physical promotion", false, -1, false) @@ -64,6 +63,7 @@ CompPhaseNameMacro(PHASE_INVERT_LOOPS, "Invert loops", CompPhaseNameMacro(PHASE_HEAD_TAIL_MERGE2, "Post-morph head and tail merge", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_FLOW, "Optimize control flow", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_LAYOUT, "Optimize layout", false, -1, false) +CompPhaseNameMacro(PHASE_COMPUTE_REACHABILITY, "Compute blocks reachability", false, -1, false) CompPhaseNameMacro(PHASE_COMPUTE_DOMINATORS, "Compute dominators", false, -1, false) CompPhaseNameMacro(PHASE_CANONICALIZE_ENTRY, "Canonicalize entry", false, -1, false) CompPhaseNameMacro(PHASE_SET_BLOCK_WEIGHTS, "Set block weights", false, -1, false) @@ -84,7 +84,6 @@ CompPhaseNameMacro(PHASE_BUILD_SSA_DF, "SSA: DF", CompPhaseNameMacro(PHASE_BUILD_SSA_INSERT_PHIS, "SSA: insert phis", false, PHASE_BUILD_SSA, false) CompPhaseNameMacro(PHASE_BUILD_SSA_RENAME, "SSA: rename", false, PHASE_BUILD_SSA, false) CompPhaseNameMacro(PHASE_EARLY_PROP, "Early Value Propagation", false, -1, false) -CompPhaseNameMacro(PHASE_OPTIMIZE_INDUCTION_VARIABLES, "Optimize Induction Variables", false, -1, false) CompPhaseNameMacro(PHASE_VALUE_NUMBER, "Do value numbering", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_INDEX_CHECKS, "Optimize index checks", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_VALNUM_CSES, "Optimize Valnum CSEs", false, -1, false) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 778caa9b57e75e..d3ac84e7919a1d 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -4393,7 +4393,7 @@ size_t emitter::emitIssue1Instr(insGroup* ig, instrDesc* id, BYTE** dp) float insExeCost = insEvaluateExecutionCost(id); // All compPerfScore calculations must be performed using doubles double insPerfScore = (double)(ig->igWeight / (double)BB_UNITY_WEIGHT) * insExeCost; - emitComp->Metrics.PerfScore += insPerfScore; + emitComp->info.compPerfScore += insPerfScore; ig->igPerfScore += insPerfScore; #endif // defined(DEBUG) || defined(LATE_DISASM) @@ -8154,13 +8154,13 @@ CORINFO_FIELD_HANDLE emitter::emitFltOrDblConst(double constValue, emitAttr attr // Return Value: // A field handle representing the data offset to access the constant. // -// Note: -// Access to inline data is 'abstracted' by a special type of static member -// (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference -// to constant data, not a real static field. -// CORINFO_FIELD_HANDLE emitter::emitSimd8Const(simd8_t constValue) { + // Access to inline data is 'abstracted' by a special type of static member + // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference + // to constant data, not a real static field. + CLANG_FORMAT_COMMENT_ANCHOR; + unsigned cnsSize = 8; unsigned cnsAlign = cnsSize; @@ -8177,6 +8177,11 @@ CORINFO_FIELD_HANDLE emitter::emitSimd8Const(simd8_t constValue) CORINFO_FIELD_HANDLE emitter::emitSimd16Const(simd16_t constValue) { + // Access to inline data is 'abstracted' by a special type of static member + // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference + // to constant data, not a real static field. + CLANG_FORMAT_COMMENT_ANCHOR; + unsigned cnsSize = 16; unsigned cnsAlign = cnsSize; @@ -8194,6 +8199,11 @@ CORINFO_FIELD_HANDLE emitter::emitSimd16Const(simd16_t constValue) #if defined(TARGET_XARCH) CORINFO_FIELD_HANDLE emitter::emitSimd32Const(simd32_t constValue) { + // Access to inline data is 'abstracted' by a special type of static member + // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference + // to constant data, not a real static field. + CLANG_FORMAT_COMMENT_ANCHOR; + unsigned cnsSize = 32; unsigned cnsAlign = cnsSize; @@ -8208,6 +8218,11 @@ CORINFO_FIELD_HANDLE emitter::emitSimd32Const(simd32_t constValue) CORINFO_FIELD_HANDLE emitter::emitSimd64Const(simd64_t constValue) { + // Access to inline data is 'abstracted' by a special type of static member + // (produced by eeFindJitDataOffs) which the emitter recognizes as being a reference + // to constant data, not a real static field. + CLANG_FORMAT_COMMENT_ANCHOR; + unsigned cnsSize = 64; unsigned cnsAlign = cnsSize; @@ -8219,22 +8234,6 @@ CORINFO_FIELD_HANDLE emitter::emitSimd64Const(simd64_t constValue) UNATIVE_OFFSET cnum = emitDataConst(&constValue, cnsSize, cnsAlign, TYP_SIMD64); return emitComp->eeFindJitDataOffs(cnum); } - -CORINFO_FIELD_HANDLE emitter::emitSimdMaskConst(simdmask_t constValue) -{ - unsigned cnsSize = 8; - unsigned cnsAlign = cnsSize; - -#ifdef TARGET_XARCH - if (emitComp->compCodeOpt() == Compiler::SMALL_CODE) - { - cnsAlign = dataSection::MIN_DATA_ALIGN; - } -#endif // TARGET_XARCH - - UNATIVE_OFFSET cnum = emitDataConst(&constValue, cnsSize, cnsAlign, TYP_MASK); - return emitComp->eeFindJitDataOffs(cnum); -} #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -8610,10 +8609,9 @@ void emitter::emitGCvarLiveSet(int offs, GCtype gcType, BYTE* addr, ssize_t disp desc->vpdNext = nullptr; +#if !defined(JIT32_GCENCODER) || !defined(FEATURE_EH_FUNCLETS) /* the lower 2 bits encode props about the stk ptr */ - CLANG_FORMAT_COMMENT_ANCHOR; -#if defined(JIT32_GCENCODER) && !defined(FEATURE_EH_FUNCLETS) if (offs == emitSyncThisObjOffs) { desc->vpdVarNum |= this_OFFSET_FLAG; diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 54f37632cc930a..45be55272f661f 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -1458,36 +1458,25 @@ class emitter assert(!idIsSmallDsc()); idAddr()->_idRegBit = val ? 1 : 0; } - insSvePattern idSvePattern() const + bool idOptionalShift() const { assert(!idIsSmallDsc()); - return (idAddr()->_idSvePattern); + return (idAddr()->_idRegBit == 1); } - void idSvePattern(insSvePattern idSvePattern) + void idOptionalShift(bool val) { assert(!idIsSmallDsc()); - idAddr()->_idSvePattern = idSvePattern; + idAddr()->_idRegBit = val ? 1 : 0; } - insSvePrfop idSvePrfop() const + insSvePattern idSvePattern() const { assert(!idIsSmallDsc()); - return (insSvePrfop)(idAddr()->_idReg4); + return (idAddr()->_idSvePattern); } - void idSvePrfop(insSvePrfop idSvePrfop) + void idSvePattern(insSvePattern idSvePattern) { assert(!idIsSmallDsc()); - idAddr()->_idReg4 = (regNumber)idSvePrfop; - } - bool idHasShift() const - { - return !idIsSmallDsc() && (idAddr()->_idRegBit == 1); - } - void idHasShift(bool val) - { - if (!idIsSmallDsc()) - { - idAddr()->_idRegBit = val ? 1 : 0; - } + idAddr()->_idSvePattern = idSvePattern; } #endif // TARGET_ARM64 @@ -2506,7 +2495,6 @@ class emitter #if defined(TARGET_XARCH) CORINFO_FIELD_HANDLE emitSimd32Const(simd32_t constValue); CORINFO_FIELD_HANDLE emitSimd64Const(simd64_t constValue); - CORINFO_FIELD_HANDLE emitSimdMaskConst(simdmask_t constValue); #endif // TARGET_XARCH #endif // FEATURE_SIMD regNumber emitInsBinary(instruction ins, emitAttr attr, GenTree* dst, GenTree* src); diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 8af972f628fef1..dea80c05e6b8be 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -243,7 +243,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isIntegerRegister(id->idReg1()) || // ZR isVectorRegister(id->idReg1())); assert(isIntegerRegister(id->idReg2())); // SP - assert(isValidUimm<12>(emitGetInsSC(id))); + assert(isValidUimm12(emitGetInsSC(id))); assert(insOptsNone(id->idInsOpt())); break; @@ -358,7 +358,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_DI_1A: // DI_1A X.......shiiiiii iiiiiinnnnn..... Rn imm(i12,sh) assert(isValidGeneralDatasize(id->idOpSize())); assert(isGeneralRegister(id->idReg1())); - assert(isValidUimm<12>(emitGetInsSC(id))); + assert(isValidUimm12(emitGetInsSC(id))); assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); break; @@ -394,7 +394,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isValidGeneralDatasize(id->idOpSize())); assert(isIntegerRegister(id->idReg1())); // SP assert(isIntegerRegister(id->idReg2())); // SP - assert(isValidUimm<12>(emitGetInsSC(id))); + assert(isValidUimm12(emitGetInsSC(id))); assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); break; @@ -574,7 +574,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) elemsize = id->idOpSize(); assert(isValidVectorElemsizeFloat(elemsize)); assert(isVectorRegister(id->idReg1())); - assert(isValidUimm<8>(emitGetInsSC(id))); + assert(isValidUimm8(emitGetInsSC(id))); break; case IF_DV_1B: // DV_1B .QX..........iii cmod..iiiiiddddd Vd imm8 (immediate vector) @@ -613,7 +613,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) } } assert(isVectorRegister(id->idReg1())); - assert(isValidUimm<8>(imm)); + assert(isValidUimm8(imm)); break; case IF_DV_1C: // DV_1C .........X...... ......nnnnn..... Vn #0.0 (fcmp - with zero) @@ -1059,8 +1059,6 @@ void emitter::emitInsSanityCheck(instrDesc* id) // (predicated) case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand // (predicated) - case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) - case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend elemsize = id->idOpSize(); assert(insOptsScalableStandard(id->idInsOpt())); // xx assert(isVectorRegister(id->idReg1())); // ddddd @@ -1071,35 +1069,12 @@ void emitter::emitInsSanityCheck(instrDesc* id) break; // Scalable, unpredicated - case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high - // (unpredicated) - case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) - case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient - case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) - case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long - case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high - case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long - case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) - case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long - case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide - case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved - case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute - case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long - case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate - case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long - case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part - case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) - case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high + // (unpredicated) + case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) + case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient elemsize = id->idOpSize(); assert(insOptsScalableStandard(id->idInsOpt())); // xx assert(isVectorRegister(id->idReg1())); // ddddd @@ -1119,150 +1094,12 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isValidScalarDatasize(elemsize)); break; - case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation - assert(id->idInsOpt() == INS_OPTS_SCALABLE_S || id->idInsOpt() == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isValidUimm<2>(emitGetInsSC(id))); // hh - break; - - case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - assert(id->idInsOpt() == INS_OPTS_SCALABLE_D_SXTW || id->idInsOpt() == INS_OPTS_SCALABLE_D_UXTW); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isValidUimm<2>(emitGetInsSC(id))); // hh - break; - case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count - case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count elemsize = id->idOpSize(); assert(id->idInsOpt() == INS_OPTS_NONE); assert(isGeneralRegister(id->idReg1())); assert(elemsize == EA_8BYTE); - assert(isValidUimmFrom1<4>(emitGetInsSC(id))); - break; - - case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count - case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count - elemsize = id->idOpSize(); - assert(insOptsScalableAtLeastHalf(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isScalableVectorSize(elemsize)); - assert(isValidUimmFrom1<4>(emitGetInsSC(id))); - break; - - case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) - case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate - imm = emitGetInsSC(id); - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidImmNRS(imm, optGetSveElemsize(id->idInsOpt()))); - break; - - case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count - elemsize = id->idOpSize(); - assert(id->idInsOpt() == INS_OPTS_NONE); - assert(isGeneralRegister(id->idReg1())); - assert(isValidGeneralDatasize(elemsize)); - assert(isValidUimmFrom1<4>(emitGetInsSC(id))); - break; - - case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) - case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) - assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isValidUimm<8>(emitGetInsSC(id))); // iiiii iii - break; - - case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) - { - imm = emitGetInsSC(id); - floatImm8 fpImm; - fpImm.immFPIVal = (unsigned)imm; - assert(insOptsScalableAtLeastHalf(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidSimm<8>((ssize_t)emitDecodeFloatImm8(fpImm))); // iiiiiiii - assert(isPredicateRegister(id->idReg2())); // gggg - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - break; - } - - case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - imm = emitGetInsSC(id); - assert(insOptsScalableStandard(id->idInsOpt())); // xx - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // gggg - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - assert(isValidSimm<8>(imm)); // iiiiiiii - break; - - case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) - assert(insOptsScalableAtLeastHalf(id->idInsOpt())); // xx - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // gggg - break; - - case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector - assert(isPredicateRegister(id->idReg1())); // DDDD - assert(isVectorRegister(id->idReg2())); // nnnnn - break; - - case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector - assert(isPredicateRegister(id->idReg1())); // DDDD - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isValidUimm<3>(emitGetInsSC(id))); - break; - - case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector - assert(isPredicateRegister(id->idReg1())); // DDDD - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isValidUimm<1>(emitGetInsSC(id))); // i - break; - - case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector - assert(isPredicateRegister(id->idReg1())); // DDDD - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isValidUimm<3>(emitGetInsSC(id))); // ii - break; - - case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // NNNN - break; - - case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // NNNN - assert(isValidUimm<3>(emitGetInsSC(id))); - break; - - case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // NNNN - assert(isValidUimm<1>(emitGetInsSC(id))); // i - break; - - case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // NNNN - assert(isValidUimm<2>(emitGetInsSC(id))); // ii - break; - - case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // mmmmm - break; - - case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isGeneralRegisterOrZR(id->idReg2())); // mmmmm + assert(isValidUimm4From1(emitGetInsSC(id))); break; case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements @@ -1315,7 +1152,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isPredicateRegister(id->idReg1())); // DDDD assert(isLowPredicateRegister(id->idReg2())); // ggg assert(isVectorRegister(id->idReg3())); // nnnnn - assert(isValidSimm<5>(emitGetInsSC(id))); // iiiii + assert(isValidSimm5(emitGetInsSC(id))); // iiiii break; case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate @@ -1325,41 +1162,15 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isPredicateRegister(id->idReg1())); // DDDD assert(isLowPredicateRegister(id->idReg2())); // ggg assert(isVectorRegister(id->idReg3())); // nnnnn - assert(isValidUimm<7>(emitGetInsSC(id))); // iiiii - break; - - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate - case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) - case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product - case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product - case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations - case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long - case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long - case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long - case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations - assert(insOptsScalable(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn/mmmmm - assert(isVectorRegister(id->idReg3())); // mmmmm/aaaaa + assert(isValidUimm7(emitGetInsSC(id))); // iiiii break; - case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - assert(insOptsNone(id->idInsOpt())); + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) + assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn/aaaaa - assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isVectorRegister(id->idReg2())); // nnnnn/mmmmm + assert(isVectorRegister(id->idReg3())); // mmmmm/aaaaa break; case IF_SVE_EG_3A: // ...........iimmm ......nnnnnddddd -- SVE two-way dot product (indexed) @@ -1378,7 +1189,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg2())); // nnnnn assert(isVectorRegister(id->idReg3())); // mmm assert((REG_V0 <= id->idReg3()) && (id->idReg3() <= REG_V7)); - assert(isValidUimm<2>(emitGetInsSC(id))); // ii + assert(isValidUimm2(emitGetInsSC(id))); // ii break; case IF_SVE_FD_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) @@ -1398,7 +1209,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg2())); // nnnnn assert(isVectorRegister(id->idReg3())); // mmm assert((REG_V0 <= id->idReg3()) && (id->idReg3() <= REG_V7)); - assert(isValidUimm<3>(emitGetInsSC(id))); // iii + assert(isValidUimm3(emitGetInsSC(id))); // iii break; case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) @@ -1409,7 +1220,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg1())); // ddddd assert(isVectorRegister(id->idReg2())); // nnnnn assert(isLowVectorRegister(id->idReg3())); // mmmm - assert(isValidUimm<2>(emitGetInsSC(id))); // ii + assert(isValidUimm2(emitGetInsSC(id))); // ii break; case IF_SVE_EY_3B: // ...........immmm ......nnnnnddddd -- SVE integer dot product (indexed) @@ -1423,7 +1234,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg1())); // ddddd assert(isVectorRegister(id->idReg2())); // nnnnn assert(isLowVectorRegister(id->idReg3())); // mmmm - assert(isValidUimm<1>(emitGetInsSC(id))); // i + assert(isValidImm1(emitGetInsSC(id))); // i break; case IF_SVE_CZ_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations @@ -1562,45 +1373,6 @@ void emitter::emitInsSanityCheck(instrDesc* id) } break; - case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - assert(id->idInsOpt() == INS_OPTS_S_TO_H); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isLowPredicateRegister(id->idReg2())); // ggg - assert(isVectorRegister(id->idReg3())); // nnnnn - break; - - case IF_SVE_HO_3B: - assert(insOptsConvertFloatToFloat(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isLowPredicateRegister(id->idReg2())); // ggg - assert(isVectorRegister(id->idReg3())); // nnnnn - break; - - case IF_SVE_HO_3C: - assert(id->idInsOpt() == INS_OPTS_D_TO_S); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isLowPredicateRegister(id->idReg2())); // ggg - assert(isVectorRegister(id->idReg3())); // nnnnn - break; - - case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer - assert(insOptsScalableFloat(id->idInsOpt()) || id->idInsOpt() == INS_OPTS_H_TO_S || - id->idInsOpt() == INS_OPTS_H_TO_D || id->idInsOpt() == INS_OPTS_S_TO_D || - id->idInsOpt() == INS_OPTS_D_TO_S); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isLowPredicateRegister(id->idReg2())); // ggg - assert(isVectorRegister(id->idReg3())); // nnnnn - break; - - case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point - assert(insOptsScalableAtLeastHalf(id->idInsOpt()) || id->idInsOpt() == INS_OPTS_S_TO_H || - id->idInsOpt() == INS_OPTS_S_TO_D || id->idInsOpt() == INS_OPTS_D_TO_H || - id->idInsOpt() == INS_OPTS_D_TO_S); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isLowPredicateRegister(id->idReg2())); // ggg - assert(isVectorRegister(id->idReg3())); // nnnnn - break; - case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors elemsize = id->idOpSize(); assert(isScalableVectorSize(elemsize)); @@ -1623,16 +1395,6 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isScalableVectorSize(elemsize)); break; - case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - elemsize = id->idOpSize(); - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isLowPredicateRegister(id->idReg2())); // ggg - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isScalableVectorSize(elemsize)); - break; - // Scalable to Simd Vector. case IF_SVE_AG_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (quadwords) case IF_SVE_AJ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (quadwords) @@ -1729,19 +1491,6 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg3())); // nnnnn break; - case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) - elemsize = id->idOpSize(); - assert(isScalableVectorSize(elemsize)); // xx - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isPredicateRegister(id->idReg2())); // VVVV - assert(isVectorRegister(id->idReg3())); // nnnnn - if (id->idIns() == INS_sve_sel) - { - assert(isVectorRegister(id->idReg4())); // mmmmm - } - break; - // Scalable from general scalar (possibly SP) case IF_SVE_CQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy general register to vector (predicated) elemsize = id->idOpSize(); @@ -1847,79 +1596,6 @@ void emitter::emitInsSanityCheck(instrDesc* id) // x break; - case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment - assert(insOptsNone(id->idInsOpt())); - assert(id->idOpSize() == EA_8BYTE); - assert(isGeneralRegisterOrZR(id->idReg1())); // ddddd - assert(isGeneralRegisterOrZR(id->idReg2())); // nnnnn - assert(isValidSimm<6>(emitGetInsSC(id))); // iiiiii - break; - - case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size - assert(insOptsNone(id->idInsOpt())); - assert(id->idOpSize() == EA_8BYTE); - assert(isGeneralRegister(id->idReg1())); // ddddd - assert(isValidSimm<6>(emitGetInsSC(id))); // iiiiii - break; - - case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm - { - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx xx - const ssize_t imm = emitGetInsSC(id); - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); // iii - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); // xiii - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); // xxiii - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimmFrom1<6>(imm)); // xx xiii - break; - - default: - unreached(); - break; - } - break; - } - - case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate - // increment) - { - ssize_t imm1; - ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidSimm<5>(imm1)); // iiiii - assert(isValidSimm<5>(imm2)); // iiiii - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - break; - } - - case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register - // increment) - case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate - // increment) - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidSimm<5>(emitGetInsSC(id))); // iiiii - assert(isIntegerRegister(id->idReg2())); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - break; - case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long { assert(insOptsScalableWide(id->idInsOpt())); @@ -1931,15 +1607,15 @@ void emitter::emitInsSanityCheck(instrDesc* id) switch (id->idInsOpt()) { case INS_OPTS_SCALABLE_B: - assert(isValidUimm<3>(imm)); // iii + assert(isValidUimm3(imm)); // iii break; case INS_OPTS_SCALABLE_H: - assert(isValidUimm<4>(imm)); // x iii + assert(isValidUimm4(imm)); // x iii break; case INS_OPTS_SCALABLE_S: - assert(isValidUimm<5>(imm)); // xx iii + assert(isValidUimm5(imm)); // xx iii break; default: @@ -1960,15 +1636,15 @@ void emitter::emitInsSanityCheck(instrDesc* id) switch (id->idInsOpt()) { case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); // iii + assert(isValidUimm3From1(imm)); // iii break; case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); // x iii + assert(isValidUimm4From1(imm)); // x iii break; case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); // xx iii + assert(isValidUimm5From1(imm)); // xx iii break; default: @@ -2043,19 +1719,19 @@ void emitter::emitInsSanityCheck(instrDesc* id) switch (id->idInsOpt()) { case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); + assert(isValidUimm4(imm)); break; case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); + assert(isValidUimm3(imm)); break; case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); + assert(isValidUimm2(imm)); break; case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); + assert(isValidImm1(imm)); break; default: @@ -2067,14 +1743,14 @@ void emitter::emitInsSanityCheck(instrDesc* id) } case IF_SVE_DW_2B: // ........xx...... .......iNNN.DDDD -- SVE extract mask predicate from predicate-as-counter - assert(isValidUimm<1>(emitGetInsSC(id))); // i + assert(isValidImm1(emitGetInsSC(id))); // i FALLTHROUGH; case IF_SVE_DW_2A: // ........xx...... ......iiNNN.DDDD -- SVE extract mask predicate from predicate-as-counter assert(insOptsScalableStandard(id->idInsOpt())); assert(isPredicateRegister(id->idReg1())); // DDDD assert(isHighPredicateRegister(id->idReg2())); // NNN - assert(isValidUimm<2>(emitGetInsSC(id))); // ii + assert(isValidUimm2(emitGetInsSC(id))); // ii assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx break; @@ -2105,24 +1781,26 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_SVE_EA_1A: // ........xx...... ...iiiiiiiiddddd -- SVE broadcast floating-point immediate (unpredicated) assert(insOptsScalableAtLeastHalf(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidUimm<8>(emitGetInsSC(id))); // iiiiiiii + assert(isValidUimm8(emitGetInsSC(id))); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx break; case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) - imm = emitGetInsSC(id); assert(insOptsScalableStandard(id->idInsOpt())); + // Size specifier must be able to fit left-shifted immediate + assert(insOptsScalableAtLeastHalf(id->idInsOpt()) || !id->idOptionalShift()); assert(isVectorRegister(id->idReg1())); // ddddd + assert(isValidSimm8(emitGetInsSC(id))); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - assert(isValidSimm<8>(imm)); // iiiiiiii break; case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) - imm = emitGetInsSC(id); assert(insOptsScalableStandard(id->idInsOpt())); + // Size specifier must be able to fit left-shifted immediate + assert(insOptsScalableAtLeastHalf(id->idInsOpt()) || !id->idOptionalShift()); assert(isVectorRegister(id->idReg1())); // ddddd + assert(isValidUimm8(emitGetInsSC(id))); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx - assert(isValidUimm<8>(imm)); // iiiiiiii break; case IF_SVE_EB_1B: // ........xx...... ...........ddddd -- SVE broadcast integer immediate (unpredicated) @@ -2133,15 +1811,15 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_SVE_ED_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer min/max immediate (unpredicated) assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidSimm<8>(emitGetInsSC(id)) || isValidUimm<8>(emitGetInsSC(id))); // iiiiiiii - assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isValidSimm8(emitGetInsSC(id)) || isValidUimm8(emitGetInsSC(id))); // iiiiiiii + assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx break; case IF_SVE_EE_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer multiply immediate (unpredicated) assert(insOptsScalableStandard(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); // ddddd - assert(isValidSimm<8>(emitGetInsSC(id))); // iiiiiiii + assert(isValidSimm8(emitGetInsSC(id))); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(id->idInsOpt()))); // xx break; @@ -2166,7 +1844,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg2())); // nnnnn assert(isVectorRegister(id->idReg3())); // mmm assert((REG_V0 <= id->idReg3()) && (id->idReg3() <= REG_V7)); - assert(isValidUimm<4>(emitGetInsSC(id))); // ii rr + assert(isValidUimm4(emitGetInsSC(id))); // ii rr break; case IF_SVE_FA_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) @@ -2177,7 +1855,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg1())); // ddddd assert(isVectorRegister(id->idReg2())); // nnnnn assert(isLowVectorRegister(id->idReg3())); // mmm - assert(isValidUimm<3>(emitGetInsSC(id))); // i rr + assert(isValidUimm3(emitGetInsSC(id))); // i rr break; case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus @@ -2231,7 +1909,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case INS_sve_st2w: case INS_sve_st2d: case INS_sve_st2q: - assert((isValidSimm_MultipleOf<4, 2>(emitGetInsSC(id)))); // iiii + assert(isValidSimm4_MultipleOf2(emitGetInsSC(id))); // iiii break; case INS_sve_ld3b: @@ -2244,7 +1922,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) case INS_sve_st3w: case INS_sve_st3d: case INS_sve_st3q: - assert((isValidSimm_MultipleOf<4, 3>(emitGetInsSC(id)))); // iiii + assert(isValidSimm4_MultipleOf3(emitGetInsSC(id))); // iiii break; case INS_sve_ld4b: @@ -2257,25 +1935,25 @@ void emitter::emitInsSanityCheck(instrDesc* id) case INS_sve_st4w: case INS_sve_st4d: case INS_sve_st4q: - assert((isValidSimm_MultipleOf<4, 4>(emitGetInsSC(id)))); // iiii + assert(isValidSimm4_MultipleOf4(emitGetInsSC(id))); // iiii break; case INS_sve_ld1rqb: case INS_sve_ld1rqd: case INS_sve_ld1rqh: case INS_sve_ld1rqw: - assert((isValidSimm_MultipleOf<4, 16>(emitGetInsSC(id)))); // iiii + assert(isValidSimm4_MultipleOf16(emitGetInsSC(id))); // iiii break; case INS_sve_ld1rob: case INS_sve_ld1rod: case INS_sve_ld1roh: case INS_sve_ld1row: - assert((isValidSimm_MultipleOf<4, 32>(emitGetInsSC(id)))); // iiii + assert(isValidSimm4_MultipleOf32(emitGetInsSC(id))); // iiii break; default: - assert(isValidSimm<4>(emitGetInsSC(id))); // iiii + assert(isValidSimm4(emitGetInsSC(id))); // iiii break; } break; @@ -2330,7 +2008,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isPredicateRegister(id->idReg2())); // ggg assert(isGeneralRegister(id->idReg3())); // nnnnn assert(isScalableVectorSize(elemsize)); // xx - assert(isValidSimm<4>(imm)); // iiii + assert(isValidSimm4(imm)); // iiii break; case IF_SVE_JN_3B: // ..........x.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) @@ -2341,7 +2019,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isPredicateRegister(id->idReg2())); // ggg assert(isGeneralRegister(id->idReg3())); // nnnnn assert(isScalableVectorSize(elemsize)); // x - assert(isValidSimm<4>(imm)); // iiii + assert(isValidSimm4(imm)); // iiii break; case IF_SVE_HW_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled @@ -2621,7 +2299,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(insOptsScalableAtLeastHalf(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); assert(isVectorRegister(id->idReg2())); - assert(isValidUimm<3>(imm)); + assert(isValidUimm3(imm)); assert(isScalableVectorSize(elemsize)); break; @@ -2662,7 +2340,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isScalableVectorSize(elemsize)); assert(isPredicateRegister(id->idReg1())); // TTTT assert(isGeneralRegister(id->idReg2())); // nnnnn - assert(isValidSimm<9>(emitGetInsSC(id))); // iii + assert(isValidSimm9(emitGetInsSC(id))); // iii // iiiiii break; @@ -2671,97 +2349,10 @@ void emitter::emitInsSanityCheck(instrDesc* id) elemsize = id->idOpSize(); assert(insOptsNone(id->idInsOpt())); assert(isScalableVectorSize(elemsize)); - assert(isVectorRegister(id->idReg1())); // ttttt - assert(isGeneralRegister(id->idReg2())); // nnnnn - assert(isValidSimm<9>(emitGetInsSC(id))); // iii - // iiiiii - break; - - case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isValidUimm<2>(emitGetInsSC(id))); // ii - assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); - break; - - case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isValidUimm<2>(emitGetInsSC(id))); // ii - assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); - break; - - case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isValidUimm<3>(emitGetInsSC(id))); // ii - // i - assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); - break; - - case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - assert(insOptsScalable(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); // ddddd - assert(isVectorRegister(id->idReg2())); // nnnnn - assert(isVectorRegister(id->idReg3())); // mmmmm - assert(isValidUimm<1>(emitGetInsSC(id))); // i - assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); - break; - - case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit - // scaled offsets) - elemsize = id->idOpSize(); - assert(insOptsScalable32bitExtends(id->idInsOpt())); - assert(isLowPredicateRegister(id->idReg1())); - assert(isGeneralRegister(id->idReg2())); - assert(isVectorRegister(id->idReg3())); - assert(isScalableVectorSize(elemsize)); - break; - - case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - elemsize = id->idOpSize(); - assert(id->idInsOpt() == INS_OPTS_SCALABLE_D); - assert(isLowPredicateRegister(id->idReg1())); - assert(isGeneralRegister(id->idReg2())); - assert(isVectorRegister(id->idReg3())); - assert(isScalableVectorSize(elemsize)); - break; - - case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) - elemsize = id->idOpSize(); - assert(insOptsNone(id->idInsOpt())); - assert(isLowPredicateRegister(id->idReg1())); - assert(isGeneralRegister(id->idReg2())); - assert(isGeneralRegister(id->idReg3())); - assert(isScalableVectorSize(elemsize)); - break; - - case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) - elemsize = id->idOpSize(); - assert(insOptsScalableWords(id->idInsOpt())); - assert(isLowPredicateRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - assert(isScalableVectorSize(elemsize)); - break; - - case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) - elemsize = id->idOpSize(); - assert(insOptsNone(id->idInsOpt())); - assert(isLowPredicateRegister(id->idReg1())); - assert(isGeneralRegister(id->idReg2())); - assert(isScalableVectorSize(elemsize)); + assert(isVectorRegister(id->idReg1())); // ttttt + assert(isGeneralRegister(id->idReg2())); // nnnnn + assert(isValidSimm9(emitGetInsSC(id))); // iii + // iiiiii break; case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) @@ -2771,7 +2362,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg1())); assert(isLowPredicateRegister(id->idReg2())); assert(isVectorRegister(id->idReg3())); - assert(isValidUimm<5>(emitGetInsSC(id))); + assert(isValidUimm5(emitGetInsSC(id))); break; case IF_SVE_HX_3A_E: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) @@ -2808,7 +2399,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg1())); assert(isLowPredicateRegister(id->idReg2())); assert(isVectorRegister(id->idReg3())); - assert((isValidUimm_MultipleOf<5, 8>(emitGetInsSC(id)))); + assert(isValidUimm5_MultipleOf8(emitGetInsSC(id))); break; case IF_SVE_IC_3A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element @@ -2847,116 +2438,21 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isGeneralRegister(id->idReg3())); break; - case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) - assert(insOptsNone(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); + default: + printf("unexpected format %s\n", emitIfName(id->idInsFmt())); + assert(!"Unexpected format"); break; + } +} +#endif // DEBUG - case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts - assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - break; +bool emitter::emitInsMayWriteToGCReg(instrDesc* id) +{ + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); - case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isGeneralRegisterOrZR(id->idReg2())); // ZR is SP - break; - - case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - break; - - case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator - case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements - case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) - assert(insOptsScalableAtLeastHalf(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - break; - - case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) - case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert - case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(id->idInsOpt()), - emitInsIsVectorRightShift(id->idIns()))); - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - assert(isScalableVectorSize(elemsize)); - break; - - case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element - imm = emitGetInsSC(id); - assert(insOptsScalable(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - assert(isValidBroadcastImm(imm, optGetSveElemsize(id->idInsOpt()))); - break; - - case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(insOptsScalableStandard(id->idInsOpt())); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - assert(isScalableVectorSize(elemsize)); -#ifdef DEBUG - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - break; - - default: - break; - } -#endif // DEBUG - break; - - case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(id->idReg1())); - assert(isVectorRegister(id->idReg2())); - assert(isScalableVectorSize(elemsize)); - assert(isValidUimm<4>(imm)); - break; - - default: - printf("unexpected format %s\n", emitIfName(id->idInsFmt())); - assert(!"Unexpected format"); - break; - } -} -#endif // DEBUG - -bool emitter::emitInsMayWriteToGCReg(instrDesc* id) -{ - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - - switch (fmt) - { + switch (fmt) + { // These are the formats with "destination" registers: @@ -4671,6 +4167,8 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) IF_SVE_IG_4A_E}; const static insFormat formatEncode4A[4] = {IF_SVE_AA_3A, IF_SVE_AU_3A, IF_SVE_BS_1A, IF_SVE_CZ_4A}; const static insFormat formatEncode4B[4] = {IF_SVE_BU_2A, IF_SVE_BV_2B, IF_SVE_EA_1A, IF_SVE_EB_1B}; + const static insFormat formatEncode4C[4] = {IF_SVE_HS_3A, IF_SVE_HS_3A_H, IF_SVE_HS_3A_I, IF_SVE_HS_3A_J}; + const static insFormat formatEncode4D[4] = {IF_SVE_HP_3B, IF_SVE_HP_3B_H, IF_SVE_HP_3B_I, IF_SVE_HP_3B_J}; const static insFormat formatEncode4E[4] = {IF_SVE_BE_3A, IF_SVE_FI_3A, IF_SVE_FI_3B, IF_SVE_FI_3C}; const static insFormat formatEncode4F[4] = {IF_SVE_EM_3A, IF_SVE_FK_3A, IF_SVE_FK_3B, IF_SVE_FK_3C}; const static insFormat formatEncode4G[4] = {IF_SVE_AR_4A, IF_SVE_FF_3A, IF_SVE_FF_3B, IF_SVE_FF_3C}; @@ -4719,6 +4217,7 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) const static insFormat formatEncode2AP[2] = {IF_SVE_GY_3B, IF_SVE_HA_3A}; const static insFormat formatEncode2AQ[2] = {IF_SVE_GO_3A, IF_SVE_HC_3A}; const static insFormat formatEncode2AR[2] = {IF_SVE_AP_3A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2AS[2] = {IF_SVE_HO_3A, IF_SVE_HO_3A_B}; const static insFormat formatEncode2AT[2] = {IF_SVE_AB_3A, IF_SVE_EC_1A}; const static insFormat formatEncode2AU[2] = {IF_SVE_AH_3A, IF_SVE_BI_2A}; const static insFormat formatEncode2AV[2] = {IF_SVE_BM_1A, IF_SVE_BN_1A}; @@ -5013,6 +4512,26 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) } } break; + case IF_SVE_4C: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4C[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4D: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4D[index]) + { + encoding_found = true; + break; + } + } + break; case IF_SVE_4E: for (index = 0; index < 4; index++) { @@ -5493,6 +5012,16 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) } } break; + case IF_SVE_2AS: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AS[index]) + { + encoding_found = true; + break; + } + } + break; case IF_SVE_2AT: for (index = 0; index < 2; index++) { @@ -6043,7 +5572,7 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) * * A helper method to perform a bit Replicate operation * the source is 'value' with a fixed size 'width' set of bits. - * value is replicated to fill out 8/16/32/64 bits as determined by 'size'. + * value is replicated to fill out 32 or 64 bits as determined by 'size'. * * Example * value is '11000011' (0xE3), width is 8 and size is EA_8BYTE @@ -6053,7 +5582,9 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) /*static*/ UINT64 emitter::Replicate_helper(UINT64 value, unsigned width, emitAttr size) { - unsigned immWidth = getBitWidth(size); + assert(emitter::isValidGeneralDatasize(size)); + + unsigned immWidth = (size == EA_8BYTE) ? 64 : 32; assert(width <= immWidth); UINT64 result = value; @@ -6072,11 +5603,13 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) * * Convert an imm(N,r,s) into a 64-bit immediate * inputs 'bmImm' a bitMaskImm struct - * 'size' specifies the size of the result (8/16/32/64 bits) + * 'size' specifies the size of the result (64 or 32 bits) */ /*static*/ INT64 emitter::emitDecodeBitMaskImm(const emitter::bitMaskImm bmImm, emitAttr size) { + assert(isValidGeneralDatasize(size)); // Only EA_4BYTE or EA_8BYTE forms + unsigned N = bmImm.immN; // read the N,R and S values from the 'bitMaskImm' encoding unsigned R = bmImm.immR; unsigned S = bmImm.immS; @@ -6209,7 +5742,7 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) /************************************************************************ * - * returns true if 'imm' of 'size bits (8/16/32/64) can be encoded + * returns true if 'imm' of 'size bits (32/64) can be encoded * using the ARM64 'bitmask immediate' form. * When a non-null value is passed for 'wbBMI' then this method * writes back the 'N','S' and 'R' values use to encode this immediate @@ -6218,32 +5751,10 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) /*static*/ bool emitter::canEncodeBitMaskImm(INT64 imm, emitAttr size, emitter::bitMaskImm* wbBMI) { - unsigned immWidth = getBitWidth(size); - unsigned maxLen; - - switch (size) - { - case EA_1BYTE: - maxLen = 3; - break; - - case EA_2BYTE: - maxLen = 4; - break; - - case EA_4BYTE: - maxLen = 5; - break; - - case EA_8BYTE: - maxLen = 6; - break; + assert(isValidGeneralDatasize(size)); // Only EA_4BYTE or EA_8BYTE forms - default: - assert(!"Invalid size"); - maxLen = 0; - break; - } + unsigned immWidth = (size == EA_8BYTE) ? 64 : 32; + unsigned maxLen = (size == EA_8BYTE) ? 6 : 5; imm = normalizeImm64(imm, size); @@ -6252,7 +5763,7 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) // len=3, elemWidth is 8 bits // len=4, elemWidth is 16 bits // len=5, elemWidth is 32 bits - // len=6, elemWidth is 64 bits + // (optionally) len=6, elemWidth is 64 bits // for (unsigned len = 1; (len <= maxLen); len++) { @@ -7240,18 +6751,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) dst = INS_OPTS_SCALABLE_H; src = INS_OPTS_SCALABLE_D; break; - case INS_OPTS_SCALABLE_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_SCALABLE_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_SCALABLE_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_D; - break; default: noway_assert(!"unreachable"); break; @@ -7311,34 +6810,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) } } -// For the given 'elemsize' returns the 'arrangement' when used in a SVE vector register arrangement. -// Asserts and returns INS_OPTS_NONE if an invalid 'elemsize' is passed -// -/*static*/ insOpts emitter::optGetSveInsOpt(emitAttr elemsize) -{ - switch (elemsize) - { - case EA_1BYTE: - return INS_OPTS_SCALABLE_B; - - case EA_2BYTE: - return INS_OPTS_SCALABLE_H; - - case EA_4BYTE: - return INS_OPTS_SCALABLE_S; - - case EA_8BYTE: - return INS_OPTS_SCALABLE_D; - - case EA_16BYTE: - return INS_OPTS_SCALABLE_Q; - - default: - assert(!"Invalid emitAttr for sve vector register"); - return INS_OPTS_NONE; - } -} - // For the given 'arrangement' returns the 'elemsize' specified by the SVE vector register arrangement // asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed // @@ -7782,16 +7253,17 @@ void emitter::emitIns_R_I(instruction ins, emitAttr attr, regNumber reg, ssize_t imm, - insOpts opt, /* = INS_OPTS_NONE */ + insOpts opt /* = INS_OPTS_NONE */, insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */ DEBUGARG(size_t targetHandle /* = 0 */) DEBUGARG(GenTreeFlags gtFlags /* = GTF_EMPTY */)) { - emitAttr size = EA_SIZE(attr); - emitAttr elemsize = EA_UNKNOWN; - insFormat fmt = IF_NONE; - bool canEncode = false; - bool signedImm = false; - bool hasShift = false; + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; + bool canEncode = false; + bool signedImm = false; + bool optionalShift = false; + bool hasShift = true; /* Figure out the encoding format of the instruction */ switch (ins) @@ -7820,7 +7292,7 @@ void emitter::emitIns_R_I(instruction ins, assert(isValidGeneralDatasize(size)); assert(insOptsNone(opt)); // No LSL here (you must use emitIns_R_I_I if a shift is needed) assert(isGeneralRegister(reg)); - assert(isValidUimm<16>(imm)); + assert(isValidUimm16(imm)); hwi.immHW = 0; hwi.immVal = imm; @@ -7916,7 +7388,7 @@ void emitter::emitIns_R_I(instruction ins, } } imm = imm8; - assert(isValidUimm<8>(imm)); + assert(isValidUimm8(imm)); fmt = IF_DV_1B; break; } @@ -7995,7 +7467,7 @@ void emitter::emitIns_R_I(instruction ins, ins = insReverse(ins); imm = -imm; } - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); canEncode = true; fmt = IF_DI_1A; } @@ -8010,7 +7482,7 @@ void emitter::emitIns_R_I(instruction ins, } assert((imm & 0xfff) == 0); imm >>= 12; - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); canEncode = true; fmt = IF_DI_1A; } @@ -8020,15 +7492,6 @@ void emitter::emitIns_R_I(instruction ins, } break; - case INS_sve_rdvl: - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); - assert(isGeneralRegister(reg)); // ddddd - assert(isValidSimm<6>(imm)); // iiiiii - fmt = IF_SVE_BC_1A; - canEncode = true; - break; - case INS_sve_smax: case INS_sve_smin: signedImm = true; @@ -8042,11 +7505,11 @@ void emitter::emitIns_R_I(instruction ins, if (signedImm) { - assert(isValidSimm<8>(imm)); // iiiiiiii + assert(isValidSimm8(imm)); // iiiiiiii } else { - assert(isValidUimm<8>(imm)); // iiiiiiii + assert(isValidUimm8(imm)); // iiiiiiii } fmt = IF_SVE_ED_1A; @@ -8056,63 +7519,24 @@ void emitter::emitIns_R_I(instruction ins, case INS_sve_mul: assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg)); // ddddd - assert(isValidSimm<8>(imm)); // iiiiiiii + assert(isValidSimm8(imm)); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx fmt = IF_SVE_EE_1A; canEncode = true; break; case INS_sve_mov: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - if (sopt == INS_SCALABLE_OPTS_IMM_BITMASK) - { - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - - if (!useMovDisasmForBitMask(imm)) - { - ins = INS_sve_dupm; - } - - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BT_1A; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (!isValidSimm<8>(imm)) - { - // Size specifier must be able to fit a left-shifted immediate - assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; - } - - fmt = IF_SVE_EB_1A; - canEncode = true; - } - break; - case INS_sve_dup: + optionalShift = true; + hasShift = (sopt == INS_SCALABLE_OPTS_SHIFT); + assert(insOptsScalableStandard(opt)); + // Size specifier must be able to fit left-shifted immediate + assert(!hasShift || insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt) || hasShift); // h assert(isVectorRegister(reg)); // ddddd + assert(isValidSimm8(imm)); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (!isValidSimm<8>(imm)) - { - // Size specifier must be able to fit a left-shifted immediate - assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; - } - fmt = IF_SVE_EB_1A; canEncode = true; @@ -8127,97 +7551,20 @@ void emitter::emitIns_R_I(instruction ins, case INS_sve_uqadd: case INS_sve_uqsub: case INS_sve_subr: + optionalShift = true; + hasShift = (sopt == INS_SCALABLE_OPTS_SHIFT); + assert(insOptsScalableStandard(opt)); + // Size specifier must be able to fit left-shifted immediate + assert(!hasShift || insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt) || hasShift); // h assert(isVectorRegister(reg)); // ddddd + assert(isValidUimm8(imm)); // iiiiiiii assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - if (!isValidUimm<8>(imm)) - { - // Size specifier must be able to fit left-shifted immediate - assert((isValidUimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; - } - fmt = IF_SVE_EC_1A; canEncode = true; break; - case INS_sve_and: - case INS_sve_orr: - case INS_sve_eor: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; - break; - - case INS_sve_bic: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - // AND is an alias for BIC, and is always the preferred disassembly. - ins = INS_sve_and; - imm = -imm - 1; - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; - break; - - case INS_sve_eon: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - // EOR is an alias for EON, and is always the preferred disassembly. - ins = INS_sve_eor; - imm = -imm - 1; - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; - break; - - case INS_sve_orn: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - // ORR is an alias for ORN, and is always the preferred disassembly. - ins = INS_sve_orr; - imm = -imm - 1; - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BS_1A; - break; - - case INS_sve_dupm: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - - bmi.immNRS = 0; - canEncode = canEncodeBitMaskImm(imm, optGetSveElemsize(opt), &bmi); - fmt = IF_SVE_BT_1A; - - if (useMovDisasmForBitMask(imm)) - { - ins = INS_sve_mov; - } - - imm = bmi.immNRS; // iiiiiiiiiiiii - assert(isValidImmNRS(imm, optGetSveElemsize(opt))); - break; - default: unreached(); break; @@ -8227,16 +7574,18 @@ void emitter::emitIns_R_I(instruction ins, assert(canEncode); assert(fmt != IF_NONE); - // For encodings with shifted immediates, we need a way to determine if the immediate has been shifted or not. - // We could just leave the immediate in its unshifted form, and call emitNewInstrSC, - // but that would allocate unnecessarily large descriptors. Therefore: - // - For encodings without any shifting, just call emitNewInstrSC. - // - For unshifted immediates, call emitNewInstrSC. - // If it allocates a small descriptor, idHasShift() will always return false. - // Else, idHasShift still returns false, as we set the dedicated bit in large descriptors to false. - // - For immediates that need a shift, call emitNewInstrCns so a normal or large descriptor is used. - // idHasShift will always check the dedicated bit, as it is always available. We set this bit to true below. - instrDesc* id = !hasShift ? emitNewInstrSC(attr, imm) : emitNewInstrCns(attr, imm); + instrDesc* id; + + if (!optionalShift) + { + id = emitNewInstrSC(attr, imm); + } + else + { + // Instructions with optional shifts (MOV, DUP, etc.) need larger instrDesc to store state + id = emitNewInstrCns(attr, imm); + id->idOptionalShift(hasShift); + } id->idIns(ins); id->idInsFmt(fmt); @@ -8244,8 +7593,6 @@ void emitter::emitIns_R_I(instruction ins, id->idReg1(reg); - id->idHasShift(hasShift); - #ifdef DEBUG id->idDebugOnlyInfo()->idMemCookie = targetHandle; id->idDebugOnlyInfo()->idFlags = gtFlags; @@ -9181,30 +8528,6 @@ void emitter::emitIns_R_R(instruction ins, } break; - case INS_sve_pmov: - if (opt != INS_OPTS_SCALABLE_B) - { - assert(insOptsScalableStandard(opt)); - return emitIns_R_R_I(INS_sve_pmov, attr, reg1, reg2, 0, opt, sopt); - } - if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) - { - assert(isPredicateRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_SVE_CE_2A; - } - else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) - { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - fmt = IF_SVE_CF_2A; - } - else - { - assert(!"invalid instruction"); - } - break; - case INS_sve_movs: { assert(opt == INS_OPTS_SCALABLE_B); @@ -9216,51 +8539,13 @@ void emitter::emitIns_R_R(instruction ins, case INS_sve_mov: { - if (isGeneralRegisterOrSP(reg2)) - { - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); -#ifdef DEBUG - if (opt == INS_OPTS_SCALABLE_D) - { - assert(size == EA_8BYTE); - } - else - { - assert(size == EA_4BYTE); - } -#endif // DEBUG - reg2 = encodingSPtoZR(reg2); - fmt = IF_SVE_CB_2A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // nnnn - fmt = IF_SVE_CZ_4A_L; - } + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // nnnn + fmt = IF_SVE_CZ_4A_L; break; } - case INS_sve_insr: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - if (isVectorRegister(reg2)) - { - fmt = IF_SVE_CC_2A; - } - else if (isGeneralRegisterOrZR(reg2)) - { - fmt = IF_SVE_CD_2A; - } - else - { - unreached(); - } - break; - case INS_sve_pfirst: assert(opt == INS_OPTS_SCALABLE_B); assert(isPredicateRegister(reg1)); // DDDD @@ -9292,22 +8577,10 @@ void emitter::emitIns_R_R(instruction ins, break; case INS_sve_rev: - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_CG_2A; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // NNNN - fmt = IF_SVE_CJ_2A; - } + assert(insOptsScalableStandard(opt)); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // NNNN + fmt = IF_SVE_CJ_2A; break; case INS_sve_ptest: @@ -9429,98 +8702,6 @@ void emitter::emitIns_R_R(instruction ins, fmt = IF_SVE_GK_2A; break; - case INS_sve_frecpe: - case INS_sve_frsqrte: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HF_2A; - break; - - case INS_sve_sunpkhi: - case INS_sve_sunpklo: - case INS_sve_uunpkhi: - case INS_sve_uunpklo: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_CH_2A; - break; - - case INS_sve_fexpa: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_BJ_2A; - break; - - case INS_sve_dup: - assert(insScalableOptsNone(sopt)); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrSP(reg2)); -#ifdef DEBUG - if (opt == INS_OPTS_SCALABLE_D) - { - assert(size == EA_8BYTE); - } - else - { - assert(size == EA_4BYTE); - } -#endif // DEBUG - reg2 = encodingSPtoZR(reg2); - fmt = IF_SVE_CB_2A; - - // DUP is an alias for MOV; - // MOV is the preferred disassembly - ins = INS_sve_mov; - break; - - case INS_sve_bf1cvt: - case INS_sve_bf1cvtlt: - case INS_sve_bf2cvt: - case INS_sve_bf2cvtlt: - case INS_sve_f1cvt: - case INS_sve_f1cvtlt: - case INS_sve_f2cvt: - case INS_sve_f2cvtlt: - assert(insScalableOptsNone(sopt)); - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_HH_2A; - unreached(); // not supported yet - break; - - case INS_sve_movprfx: - assert(insScalableOptsNone(sopt)); - assert(insOptsNone(opt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_BI_2A; - break; - - case INS_sve_fmov: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // gggg - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_BV_2B; - - // CPY is an alias for FMOV, and MOV is an alias for CPY. - // Thus, MOV is the preferred disassembly. - ins = INS_sve_mov; - break; - default: unreached(); break; @@ -9584,7 +8765,7 @@ void emitter::emitIns_R_I_I(instruction ins, case INS_movz: assert(isValidGeneralDatasize(size)); assert(isGeneralRegister(reg)); - assert(isValidUimm<16>(imm1)); + assert(isValidUimm16(imm1)); assert(insOptsLSL(opt)); // Must be INS_OPTS_LSL if (size == EA_8BYTE) @@ -9635,17 +8816,6 @@ void emitter::emitIns_R_I_I(instruction ins, } break; - case INS_sve_index: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg)); // ddddd - assert(isValidSimm<5>(imm1)); // iiiii - assert(isValidSimm<5>(imm2)); // iiiii - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - immOut = insEncodeTwoSimm5(imm1, imm2); - canEncode = true; - fmt = IF_SVE_AX_1A; - break; - default: unreached(); break; @@ -9658,7 +8828,6 @@ void emitter::emitIns_R_I_I(instruction ins, id->idIns(ins); id->idInsFmt(fmt); - id->idInsOpt(opt); id->idReg1(reg); @@ -9694,7 +8863,6 @@ void emitter::emitIns_R_R_I(instruction ins, bool setFlags = false; unsigned scale = 0; bool unscaledOp = false; - bool hasShift = false; /* Figure out the encoding format of the instruction */ switch (ins) @@ -10257,180 +9425,11 @@ void emitter::emitIns_R_R_I(instruction ins, case INS_sve_uqshl: case INS_sve_asrd: isRightShift = emitInsIsVectorRightShift(ins); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - assert(insOptsScalableStandard(opt)); - assert(isScalableVectorSize(size)); - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert((ins == INS_sve_asr) || (ins == INS_sve_lsl) || (ins == INS_sve_lsr)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - fmt = IF_SVE_BF_2A; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_AM_2A; - } - break; - - case INS_sve_xar: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx xx - - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); // iii - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); // xiii - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); // xxiii - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimmFrom1<6>(imm)); // x xxiii - break; - - default: - unreached(); - break; - } - - fmt = IF_SVE_AW_2A; - break; - - case INS_sve_index: - assert(insOptsScalable(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isValidSimm<5>(imm)); // iiiii - assert(isIntegerRegister(reg2)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (sopt == INS_SCALABLE_OPTS_IMM_FIRST) - { - fmt = IF_SVE_AY_2A; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AZ_2A; - } - break; - - case INS_sve_addvl: - case INS_sve_addpl: - assert(insOptsNone(opt)); - assert(size == EA_8BYTE); - assert(isGeneralRegisterOrSP(reg1)); // ddddd - assert(isGeneralRegisterOrSP(reg2)); // nnnnn - assert(isValidSimm<6>(imm)); // iiiiii - reg1 = encodingSPtoZR(reg1); - reg2 = encodingSPtoZR(reg2); - fmt = IF_SVE_BB_2A; - break; - - case INS_sve_mov: - if (sopt == INS_SCALABLE_OPTS_BROADCAST) - { - return emitIns_R_R_I(INS_sve_dup, attr, reg1, reg2, imm, opt, sopt); - } - FALLTHROUGH; - case INS_sve_cpy: assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // DDDDD - assert(isPredicateRegister(reg2)); // GGGG - - if (!isValidSimm<8>(imm)) - { - // Size specifier must be able to fit a left-shifted immediate - assert((isValidSimm_MultipleOf<8, 256>(imm))); // iiiiiiii - assert(insOptsScalableAtLeastHalf(opt)); - hasShift = true; - imm >>= 8; - } - - if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) - { - fmt = IF_SVE_BV_2A_J; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BV_2A; - } - - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; - break; - - case INS_sve_dup: - assert(insOptsScalable(opt)); - assert(isVectorRegister(reg1)); // DDDDD - assert(isVectorRegister(reg2)); // GGGG - assert(isValidBroadcastImm(imm, optGetSveElemsize(opt))); - fmt = IF_SVE_BW_2A; - ins = INS_sve_mov; // Set preferred alias for disassembly - break; - - case INS_sve_pmov: - if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) - { - assert(isPredicateRegister(reg1)); - assert(isVectorRegister(reg2)); - switch (opt) - { - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<3>(imm)); - fmt = IF_SVE_CE_2B; - break; - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - fmt = IF_SVE_CE_2D; - break; - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<1>(imm)); - fmt = IF_SVE_CE_2C; - break; - default: - unreached(); - } - } - else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) - { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - switch (opt) - { - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<3>(imm)); - fmt = IF_SVE_CF_2B; - break; - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - fmt = IF_SVE_CF_2D; - break; - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<1>(imm)); - fmt = IF_SVE_CF_2C; - break; - default: - unreached(); - } - } - else - { - unreached(); - } + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); + fmt = IF_SVE_AM_2A; break; case INS_sve_sqrshrn: @@ -10453,13 +9452,13 @@ void emitter::emitIns_R_R_I(instruction ins, if (sopt == INS_SCALABLE_OPTS_WITH_PREDICATE_PAIR) { - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_DW_2B; } else { assert(insScalableOptsNone(sopt)); - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_DW_2A; } break; @@ -10476,15 +9475,15 @@ void emitter::emitIns_R_R_I(instruction ins, switch (opt) { case INS_OPTS_SCALABLE_B: - assert(isValidUimm<3>(imm)); // iii + assert(isValidUimm3(imm)); // iii break; case INS_OPTS_SCALABLE_H: - assert(isValidUimm<4>(imm)); // x iii + assert(isValidUimm4(imm)); // x iii break; case INS_OPTS_SCALABLE_S: - assert(isValidUimm<5>(imm)); // xx iii + assert(isValidUimm5(imm)); // xx iii break; default: @@ -10519,15 +9518,15 @@ void emitter::emitIns_R_R_I(instruction ins, switch (opt) { case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); // iii + assert(isValidUimm3From1(imm)); // iii break; case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); // x iii + assert(isValidUimm4From1(imm)); // x iii break; case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); // xx iii + assert(isValidUimm5From1(imm)); // xx iii break; default: @@ -10555,7 +9554,7 @@ void emitter::emitIns_R_R_I(instruction ins, assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); assert(isVectorRegister(reg2)); - assert(isValidUimm<3>(imm)); + assert(isValidUimm3(imm)); assert(isScalableVectorSize(size)); fmt = IF_SVE_HN_2A; break; @@ -10564,7 +9563,7 @@ void emitter::emitIns_R_R_I(instruction ins, assert(insOptsNone(opt)); assert(isScalableVectorSize(size)); assert(isGeneralRegister(reg2)); // nnnnn - assert(isValidSimm<9>(imm)); // iii + assert(isValidSimm9(imm)); // iii // iiiiii if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) @@ -10584,7 +9583,7 @@ void emitter::emitIns_R_R_I(instruction ins, assert(insOptsNone(opt)); assert(isScalableVectorSize(size)); assert(isGeneralRegister(reg2)); // nnnnn - assert(isValidSimm<9>(imm)); // iii + assert(isValidSimm9(imm)); // iii // iiiiii if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) @@ -10600,93 +9599,8 @@ void emitter::emitIns_R_R_I(instruction ins, } break; - case INS_sve_sli: - case INS_sve_sri: - isRightShift = emitInsIsVectorRightShift(ins); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_FT_2A; - break; - - case INS_sve_srsra: - case INS_sve_ssra: - case INS_sve_ursra: - case INS_sve_usra: - isRightShift = emitInsIsVectorRightShift(ins); - assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - fmt = IF_SVE_FU_2A; - break; - - case INS_sve_ext: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isValidUimm<8>(imm)); // iiiii iii - - if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) - { - fmt = IF_SVE_BQ_2A; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BQ_2B; - } - break; - - case INS_sve_dupq: - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); -#ifdef DEBUG - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - break; - - default: - break; - } -#endif // DEBUG - fmt = IF_SVE_BX_2A; - break; - - case INS_sve_extq: - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isScalableVectorSize(size)); - assert(isValidUimm<4>(imm)); - fmt = IF_SVE_BY_2A; - break; - - default: - unreached(); + default: + unreached(); break; } // end switch (ins) @@ -10793,7 +9707,7 @@ void emitter::emitIns_R_R_I(instruction ins, ins = insReverse(ins); imm = -imm; } - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); fmt = IF_DI_2A; } else if (canEncodeWithShiftImmBy12(imm)) // Try the shifted by 12 encoding @@ -10807,7 +9721,7 @@ void emitter::emitIns_R_R_I(instruction ins, } assert((imm & 0xfff) == 0); imm >>= 12; - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); fmt = IF_DI_2A; } else @@ -10818,16 +9732,7 @@ void emitter::emitIns_R_R_I(instruction ins, assert(fmt != IF_NONE); - // For encodings with shifted immediates, we need a way to determine if the immediate has been shifted or not. - // We could just leave the immediate in its unshifted form, and call emitNewInstrSC, - // but that would allocate unnecessarily large descriptors. Therefore: - // - For encodings without any shifting, just call emitNewInstrSC. - // - For unshifted immediates, call emitNewInstrSC. - // If it allocates a small descriptor, idHasShift() will always return false. - // Else, idHasShift still returns false, as we set the dedicated bit in large descriptors to false. - // - For immediates that need a shift, call emitNewInstrCns so a normal or large descriptor is used. - // idHasShift will always check the dedicated bit, as it is always available. We set this bit to true below. - instrDesc* id = !hasShift ? emitNewInstrSC(attr, imm) : emitNewInstrCns(attr, imm); + instrDesc* id = emitNewInstrSC(attr, imm); id->idIns(ins); id->idInsFmt(fmt); @@ -10835,9 +9740,6 @@ void emitter::emitIns_R_R_I(instruction ins, id->idReg1(reg1); id->idReg2(reg2); - - id->idHasShift(hasShift); - if (EA_IS_CNS_TLSGD_RELOC(attr)) { assert(imm != 0); @@ -10878,22 +9780,6 @@ void emitter::emitIns_R_R_F( fmt = IF_SVE_HM_2A; break; - case INS_sve_fmov: - case INS_sve_fcpy: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // gggg - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - floatImm8 fpi; - fpi.immFPIVal = 0; - canEncodeFloatImm8(immDbl, &fpi); - imm = fpi.immFPIVal; - fmt = IF_SVE_BU_2A; - - // FMOV is an alias for FCPY, and is always the preferred disassembly. - ins = INS_sve_fmov; - break; - default: unreached(); break; @@ -11635,22 +10521,12 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_sve_bic: case INS_sve_eor: case INS_sve_orr: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // mmmmm - assert(isVectorRegister(reg3)); // ddddd - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_AU_3A; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_AA_3A; - } + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AA_3A; break; case INS_sve_add: @@ -11673,26 +10549,6 @@ void emitter::emitIns_R_R_R(instruction ins, } break; - case INS_sve_addpt: - case INS_sve_subpt: - unreached(); // TODO-SVE: Not yet supported. - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg3)); // mmmmm - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_AT_3B; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_AB_3B; - } - break; - case INS_sve_sdiv: case INS_sve_sdivr: case INS_sve_udiv: @@ -11738,14 +10594,6 @@ void emitter::emitIns_R_R_R(instruction ins, } break; - case INS_sve_pmul: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_BD_3B; - break; - case INS_sve_andv: case INS_sve_eorv: case INS_sve_orv: @@ -11869,544 +10717,107 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_sve_uzp2: case INS_sve_trn2: case INS_sve_zip2: - assert(insOptsScalable(opt)); - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_Q) - { - fmt = IF_SVE_BR_3B; - } - else - { - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_BR_3A; - } - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // NNNN - assert(isPredicateRegister(reg3)); // MMMM - fmt = IF_SVE_CI_3A; - } - break; - - case INS_sve_tbl: assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // NNNN + assert(isPredicateRegister(reg3)); // MMMM + fmt = IF_SVE_CI_3A; + break; - if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + case INS_sve_clz: + case INS_sve_cls: + case INS_sve_cnt: + case INS_sve_cnot: + case INS_sve_not: + case INS_sve_nots: + if (isPredicateRegister(reg1) && sopt != INS_SCALABLE_OPTS_UNPREDICATED) { - fmt = IF_SVE_BZ_3A_A; + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // DDDD + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // NNNN + fmt = IF_SVE_CZ_4A; } else { + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BZ_3A; + fmt = IF_SVE_AP_3A; } break; - case INS_sve_tbx: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_BZ_3A; + case INS_sve_fabs: + case INS_sve_fneg: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableFloat(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AP_3A; break; - case INS_sve_tbxq: + case INS_sve_abs: + case INS_sve_neg: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_CA_3A; + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; break; - case INS_sve_sdot: - case INS_sve_udot: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_H) - { - fmt = IF_SVE_EF_3A; - } - else - { - fmt = IF_SVE_EH_3A; - assert(insOptsScalableWords(opt)); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - } + case INS_sve_sxtb: + case INS_sve_uxtb: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableAtLeastHalf(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; break; - case INS_sve_usdot: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_EI_3A; + case INS_sve_sxth: + case INS_sve_uxth: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableWords(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; break; - case INS_sve_smlalb: - case INS_sve_smlalt: - case INS_sve_umlalb: - case INS_sve_umlalt: - case INS_sve_smlslb: - case INS_sve_smlslt: - case INS_sve_umlslb: - case INS_sve_umlslt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EL_3A; + case INS_sve_sxtw: + case INS_sve_uxtw: + assert(isVectorRegister(reg1)); + assert(isLowPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(opt == INS_OPTS_SCALABLE_D); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_AQ_3A; break; - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: + case INS_sve_index: + assert(isValidScalarDatasize(size)); + assert(isVectorRegister(reg1)); + assert(isGeneralRegisterOrZR(reg2)); + assert(isGeneralRegisterOrZR(reg3)); assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EM_3A; + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BA_3A; break; - case INS_sve_sqdmlalbt: - case INS_sve_sqdmlslbt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EN_3A; - break; - - case INS_sve_sqdmlalb: - case INS_sve_sqdmlalt: - case INS_sve_sqdmlslb: - case INS_sve_sqdmlslt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EO_3A; - break; - - case INS_sve_sclamp: - case INS_sve_uclamp: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EV_3A; - break; - - case INS_sve_zipq1: - case INS_sve_zipq2: - case INS_sve_uzpq1: - case INS_sve_uzpq2: - case INS_sve_tblq: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EX_3A; - break; - - case INS_sve_saddlb: - case INS_sve_saddlt: - case INS_sve_uaddlb: - case INS_sve_uaddlt: - case INS_sve_ssublb: - case INS_sve_ssublt: - case INS_sve_usublb: - case INS_sve_usublt: - case INS_sve_sabdlb: - case INS_sve_sabdlt: - case INS_sve_uabdlb: - case INS_sve_uabdlt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FL_3A; - break; - - case INS_sve_saddwb: - case INS_sve_saddwt: - case INS_sve_uaddwb: - case INS_sve_uaddwt: - case INS_sve_ssubwb: - case INS_sve_ssubwt: - case INS_sve_usubwb: - case INS_sve_usubwt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FM_3A; - break; - - case INS_sve_smullb: - case INS_sve_smullt: - case INS_sve_umullb: - case INS_sve_umullt: - case INS_sve_sqdmullb: - case INS_sve_sqdmullt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FN_3A; - break; - - case INS_sve_pmullb: - case INS_sve_pmullt: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_Q) - { - fmt = IF_SVE_FN_3B; - } - else - { - assert((opt == INS_OPTS_SCALABLE_H) || (opt == INS_OPTS_SCALABLE_D)); - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FN_3A; - } - break; - - case INS_sve_smmla: - case INS_sve_usmmla: - case INS_sve_ummla: - assert(opt == INS_OPTS_SCALABLE_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_FO_3A; - break; - - case INS_sve_rax1: - case INS_sve_sm4ekey: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (ins == INS_sve_rax1) - { - assert(opt == INS_OPTS_SCALABLE_D); - } - else - { - assert(opt == INS_OPTS_SCALABLE_S); - } - - fmt = IF_SVE_GJ_3A; - break; - - case INS_sve_fmlalb: - case INS_sve_fmlalt: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_B) - { - unreached(); // TODO-SVE: Not yet supported. - fmt = IF_SVE_GN_3A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_H); - fmt = IF_SVE_HB_3A; - } - break; - - case INS_sve_fmlslb: - case INS_sve_fmlslt: - case INS_sve_bfmlalb: - case INS_sve_bfmlalt: - case INS_sve_bfmlslb: - case INS_sve_bfmlslt: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HB_3A; - break; - - case INS_sve_bfmmla: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HD_3A; - break; - - case INS_sve_fmmla: - unreached(); // TODO-SVE: Not yet supported. - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HD_3A_A; - break; - - case INS_sve_fmlallbb: - case INS_sve_fmlallbt: - case INS_sve_fmlalltb: - case INS_sve_fmlalltt: - unreached(); // TODO-SVE: Not yet supported. - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_GO_3A; - break; - - case INS_sve_bfclamp: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_GW_3B; - break; - - case INS_sve_bfdot: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_HA_3A; - break; - - case INS_sve_fdot: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_H) - { - fmt = IF_SVE_HA_3A; - } - else if (opt == INS_OPTS_SCALABLE_B) - { - unreached(); // TODO-SVE: Not yet supported. - fmt = IF_SVE_HA_3A_E; - } - else - { - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsNone(opt)); - fmt = IF_SVE_HA_3A_F; - } - break; - - case INS_sve_eorbt: - case INS_sve_eortb: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FP_3A; - break; - - case INS_sve_bext: - case INS_sve_bdep: - case INS_sve_bgrp: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FQ_3A; - break; - - case INS_sve_saddlbt: - case INS_sve_ssublbt: - case INS_sve_ssubltb: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FS_3A; - break; - - case INS_sve_saba: - case INS_sve_uaba: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FW_3A; - break; - - case INS_sve_sabalb: - case INS_sve_sabalt: - case INS_sve_uabalb: - case INS_sve_uabalt: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_FX_3A; - break; - - case INS_sve_addhnb: - case INS_sve_addhnt: - case INS_sve_raddhnb: - case INS_sve_raddhnt: - case INS_sve_subhnb: - case INS_sve_subhnt: - case INS_sve_rsubhnb: - case INS_sve_rsubhnt: - unreached(); // TODO-SVE: Not yet supported. - assert(insOptsScalableWide(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GC_3A; - break; - - case INS_sve_histseg: - assert(opt == INS_OPTS_SCALABLE_B); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GF_3A; - break; - - case INS_sve_fclamp: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GW_3A; - break; - - case INS_sve_clz: - case INS_sve_cls: - case INS_sve_cnt: - case INS_sve_cnot: - case INS_sve_not: - case INS_sve_nots: - if (isPredicateRegister(reg1) && sopt != INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // DDDD - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // NNNN - fmt = IF_SVE_CZ_4A; - } - else - { - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AP_3A; - } - break; - - case INS_sve_fabs: - case INS_sve_fneg: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AP_3A; - break; - - case INS_sve_abs: - case INS_sve_neg: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; - - case INS_sve_sxtb: - case INS_sve_uxtb: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableAtLeastHalf(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; - - case INS_sve_sxth: - case INS_sve_uxth: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableWords(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; - - case INS_sve_sxtw: - case INS_sve_uxtw: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(opt == INS_OPTS_SCALABLE_D); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AQ_3A; - break; - - case INS_sve_index: - assert(isValidScalarDatasize(size)); - assert(isVectorRegister(reg1)); - assert(isGeneralRegisterOrZR(reg2)); - assert(isGeneralRegisterOrZR(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BA_3A; - break; - - case INS_sve_sqdmulh: - case INS_sve_sqrdmulh: - assert(isScalableVectorSize(size)); - assert(isVectorRegister(reg1)); - assert(isVectorRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_BE_3A; + case INS_sve_sqdmulh: + case INS_sve_sqrdmulh: + assert(isScalableVectorSize(size)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_BE_3A; break; case INS_sve_ftssel: @@ -12459,35 +10870,14 @@ void emitter::emitIns_R_R_R(instruction ins, // TODO-SVE: Following checks can be simplified to check reg1 as predicate register only after adding // definitions for predicate registers. Currently, predicate registers P0 to P15 are aliased to simd // registers V0 to V15. - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(ins == INS_sve_mov); - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - fmt = IF_SVE_AU_3A; - // ORR is an alias for MOV, and is always the preferred disassembly. - ins = INS_sve_orr; - } - else if (isPredicateRegister(reg3) && - (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) + if (isPredicateRegister(reg3) && + (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) { assert(opt == INS_OPTS_SCALABLE_B); assert(isPredicateRegister(reg1)); // DDDD assert(isPredicateRegister(reg2)); // gggg assert(isPredicateRegister(reg3)); // NNNN fmt = sopt == INS_SCALABLE_OPTS_NONE ? IF_SVE_CZ_4A : IF_SVE_CZ_4A_K; - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; - } - else if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) - { - assert(isVectorRegister(reg1)); - assert(isPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_CW_4A; } else { @@ -12505,10 +10895,10 @@ void emitter::emitIns_R_R_R(instruction ins, assert(isVectorRegister(reg3)); fmt = IF_SVE_CP_3A; } - - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; } + + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; break; case INS_sve_lasta: @@ -12796,52 +11186,23 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_HJ_3A; break; - case INS_sve_frecps: - case INS_sve_frsqrts: - case INS_sve_ftsmul: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_HK_3A; - break; - - case INS_sve_fadd: - case INS_sve_fsub: - case INS_sve_fmul: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_HK_3A; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_HL_3A; - } - break; - case INS_sve_fabd: + case INS_sve_fadd: case INS_sve_fdiv: case INS_sve_fdivr: case INS_sve_fmax: case INS_sve_fmaxnm: case INS_sve_fmin: case INS_sve_fminnm: + case INS_sve_fmul: case INS_sve_fmulx: case INS_sve_fscale: + case INS_sve_fsub: case INS_sve_fsubr: assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(insOptsScalableAtLeastHalf(opt)); + assert(insOptsScalableFloat(opt)); assert(insScalableOptsNone(sopt)); fmt = IF_SVE_HL_3A; break; @@ -12857,43 +11218,6 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_HL_3A; break; - case INS_sve_bfmul: - case INS_sve_bfadd: - case INS_sve_bfsub: - case INS_sve_bfmaxnm: - case INS_sve_bfminnm: - case INS_sve_bfmax: - case INS_sve_bfmin: - assert(opt == INS_OPTS_SCALABLE_H); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg3)); // mmmmm - - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - assert(isVectorRegister(reg2)); // nnnnn - fmt = IF_SVE_HK_3B; - } - else - { - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg2)); // ggg - fmt = IF_SVE_HL_3B; - } - break; - - case INS_sve_bsl: - case INS_sve_eor3: - case INS_sve_bcax: - case INS_sve_bsl1n: - case INS_sve_bsl2n: - case INS_sve_nbsl: - assert(opt == INS_OPTS_SCALABLE_D); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // mmmmm - assert(isVectorRegister(reg3)); // kkkkk - fmt = IF_SVE_AV_3A; - break; - case INS_sve_frintn: case INS_sve_frintm: case INS_sve_frintp: @@ -12909,47 +11233,6 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_HQ_3A; break; - case INS_sve_bfcvt: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HO_3A; - break; - - case INS_sve_fcvt: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HO_3B; - break; - - case INS_sve_fcvtx: - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HO_3C; - break; - - case INS_sve_fcvtzs: - case INS_sve_fcvtzu: - assert(insOptsScalableFloat(opt) || opt == INS_OPTS_H_TO_S || opt == INS_OPTS_H_TO_D || - opt == INS_OPTS_S_TO_D || opt == INS_OPTS_D_TO_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HP_3B; - break; - - case INS_sve_scvtf: - case INS_sve_ucvtf: - assert(insOptsScalableAtLeastHalf(opt) || opt == INS_OPTS_S_TO_H || opt == INS_OPTS_S_TO_D || - opt == INS_OPTS_D_TO_H || opt == INS_OPTS_D_TO_S); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - fmt = IF_SVE_HS_3A; - break; - case INS_sve_frecpx: case INS_sve_fsqrt: assert(isVectorRegister(reg1)); @@ -13034,6 +11317,9 @@ void emitter::emitIns_R_R_R(instruction ins, assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmmmm fmt = IF_SVE_EW_3A; + + // opt is set only for convenience in emitDispInsHelp + opt = INS_OPTS_SCALABLE_D; break; case INS_sve_madpt: @@ -13043,6 +11329,9 @@ void emitter::emitIns_R_R_R(instruction ins, assert(isVectorRegister(reg2)); // mmmmm assert(isVectorRegister(reg3)); // aaaaa fmt = IF_SVE_EW_3B; + + // opt is set only for convenience in emitDispInsHelp + opt = INS_OPTS_SCALABLE_D; break; case INS_sve_fcmeq: @@ -13070,12 +11359,6 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_HP_3A; break; - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ld1w: - case INS_sve_ld1d: - return emitIns_R_R_R_I(ins, size, reg1, reg2, reg3, 0, opt); - default: unreached(); break; @@ -13242,15 +11525,14 @@ void emitter::emitIns_R_R_R_I_LdStPair(instruction ins, * Add an instruction referencing three registers and a constant. */ -void emitter::emitIns_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - emitAttr attrReg2 /* = EA_UNKNOWN */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +void emitter::emitIns_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + emitAttr attrReg2 /* = EA_UNKNOWN */) { emitAttr size = EA_SIZE(attr); emitAttr elemsize = EA_UNKNOWN; @@ -13540,12 +11822,10 @@ void emitter::emitIns_R_R_R_I(instruction ins, default: // fallback to emit SVE instructions. - return emitInsSve_R_R_R_I(ins, attr, reg1, reg2, reg3, imm, opt, sopt); + return emitInsSve_R_R_R_I(ins, attr, reg1, reg2, reg3, imm, opt, attrReg2); } // end switch (ins) - assert(insScalableOptsNone(sopt)); - if (isLdSt) { assert(!isAddSub); @@ -13704,14 +11984,14 @@ void emitter::emitIns_R_R_R_I(instruction ins, * Add a SVE instruction referencing three registers and a constant. */ -void emitter::emitInsSve_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) +void emitter::emitInsSve_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + emitAttr attrReg2 /* = EA_UNKNOWN */) { emitAttr size = EA_SIZE(attr); emitAttr elemsize = EA_UNKNOWN; @@ -13720,42 +12000,17 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_sve_adr: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidUimm<2>(imm)); - switch (opt) - { - case INS_OPTS_SCALABLE_S: - case INS_OPTS_SCALABLE_D: - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - fmt = IF_SVE_BH_3A; - break; - case INS_OPTS_SCALABLE_D_SXTW: - fmt = IF_SVE_BH_3B; - break; - case INS_OPTS_SCALABLE_D_UXTW: - fmt = IF_SVE_BH_3B_A; - break; - default: - assert(!"invalid instruction"); - break; - } - break; - case INS_sve_cmpeq: case INS_sve_cmpgt: case INS_sve_cmpge: case INS_sve_cmpne: case INS_sve_cmple: case INS_sve_cmplt: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isPredicateRegister(reg1)); // DDDD assert(isLowPredicateRegister(reg2)); // ggg assert(isVectorRegister(reg3)); // nnnnn - assert(isValidSimm<5>(imm)); // iiiii + assert(isValidSimm5(imm)); // iiiii fmt = IF_SVE_CY_3A; break; @@ -13763,18 +12018,16 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_cmphs: case INS_sve_cmplo: case INS_sve_cmpls: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isPredicateRegister(reg1)); // DDDD assert(isLowPredicateRegister(reg2)); // ggg assert(isVectorRegister(reg3)); // nnnnn - assert(isValidUimm<7>(imm)); // iiiii + assert(isValidUimm7(imm)); // iiiii fmt = IF_SVE_CY_3B; break; case INS_sve_sdot: case INS_sve_udot: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13782,19 +12035,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_B) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_EY_3A; } else if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_EG_3A; } else { assert(insOptsNone(opt)); - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i opt = INS_OPTS_SCALABLE_H; fmt = IF_SVE_EY_3B; } @@ -13802,18 +12055,16 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_usdot: case INS_sve_sudot: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_B); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_EZ_3A; break; case INS_sve_mul: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -13822,19 +12073,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, switch (opt) { case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); // iii + assert(isValidUimm3(imm)); // iii assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm fmt = IF_SVE_FD_3A; break; case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm fmt = IF_SVE_FD_3B; break; case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_FD_3C; break; @@ -13845,7 +12096,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_cdot: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -13860,7 +12110,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_cmla: case INS_sve_sqrdcmlah: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -13874,14 +12123,13 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1d: - assert(insScalableOptsNone(sopt)); assert(insOptsScalable(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); if (opt == INS_OPTS_SCALABLE_Q) { fmt = IF_SVE_IH_3A_A; @@ -13896,72 +12144,67 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, { assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 8>(imm))); + assert(isValidUimm5_MultipleOf8(imm)); fmt = IF_SVE_IV_3A; } break; case INS_sve_ldff1d: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 8>(imm))); + assert(isValidUimm5_MultipleOf8(imm)); fmt = IF_SVE_IV_3A; break; case INS_sve_ld1w: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWordsOrQuadwords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IH_3A_F; } else { assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); + assert(isValidUimm5_MultipleOf4(imm)); fmt = IF_SVE_HX_3A_E; } break; case INS_sve_ld1sw: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IJ_3A; } else { assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); + assert(isValidUimm5_MultipleOf4(imm)); fmt = IF_SVE_IV_3A; } break; case INS_sve_ldff1sw: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); + assert(isValidUimm5_MultipleOf4(imm)); fmt = IF_SVE_IV_3A; break; case INS_sve_ld1sb: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13969,148 +12212,138 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (isGeneralRegister(reg3)) { assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IJ_3A_D; } else { assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg3)); - assert(isValidUimm<5>(imm)); + assert(isValidUimm5(imm)); fmt = IF_SVE_HX_3A_B; } break; case INS_sve_ld1b: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IJ_3A_E; } else { assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg3)); - assert(isValidUimm<5>(imm)); + assert(isValidUimm5(imm)); fmt = IF_SVE_HX_3A_B; } break; case INS_sve_ldff1b: case INS_sve_ldff1sb: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(isValidUimm<5>(imm)); + assert(isValidUimm5(imm)); fmt = IF_SVE_HX_3A_B; break; case INS_sve_ld1sh: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IJ_3A_F; } else { assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 2>(imm))); + assert(isValidUimm5_MultipleOf2(imm)); fmt = IF_SVE_HX_3A_E; } break; case INS_sve_ld1h: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IJ_3A_G; } else { assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 2>(imm))); + assert(isValidUimm5_MultipleOf2(imm)); fmt = IF_SVE_HX_3A_E; } break; case INS_sve_ldff1h: case INS_sve_ldff1sh: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 2>(imm))); + assert(isValidUimm5_MultipleOf2(imm)); fmt = IF_SVE_HX_3A_E; break; case INS_sve_ldff1w: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert((isValidUimm_MultipleOf<5, 4>(imm))); + assert(isValidUimm5_MultipleOf4(imm)); fmt = IF_SVE_HX_3A_E; break; case INS_sve_ldnf1sw: case INS_sve_ldnf1d: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IL_3A; break; case INS_sve_ldnf1sh: case INS_sve_ldnf1w: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IL_3A_A; break; case INS_sve_ldnf1h: case INS_sve_ldnf1sb: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IL_3A_B; break; case INS_sve_ldnf1b: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); fmt = IF_SVE_IL_3A_C; break; @@ -14118,12 +12351,11 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldnt1h: case INS_sve_ldnt1w: case INS_sve_ldnt1d: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); #ifdef DEBUG switch (ins) @@ -14161,7 +12393,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld1row: case INS_sve_ld1rqd: case INS_sve_ld1rod: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -14174,14 +12405,14 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld1rqd: case INS_sve_ld1rqh: case INS_sve_ld1rqw: - assert((isValidSimm_MultipleOf<4, 16>(imm))); + assert(isValidSimm4_MultipleOf16(imm)); break; case INS_sve_ld1rob: case INS_sve_ld1rod: case INS_sve_ld1roh: case INS_sve_ld1row: - assert((isValidSimm_MultipleOf<4, 32>(imm))); + assert(isValidSimm4_MultipleOf32(imm)); break; default: @@ -14223,7 +12454,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld2q: case INS_sve_ld3q: case INS_sve_ld4q: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_Q); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -14233,15 +12463,15 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, switch (ins) { case INS_sve_ld2q: - assert((isValidSimm_MultipleOf<4, 2>(imm))); + assert(isValidSimm4_MultipleOf2(imm)); break; case INS_sve_ld3q: - assert((isValidSimm_MultipleOf<4, 3>(imm))); + assert(isValidSimm4_MultipleOf3(imm)); break; case INS_sve_ld4q: - assert((isValidSimm_MultipleOf<4, 4>(imm))); + assert(isValidSimm4_MultipleOf4(imm)); break; default: @@ -14265,7 +12495,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld2d: case INS_sve_ld3d: case INS_sve_ld4d: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -14278,21 +12507,21 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld2h: case INS_sve_ld2w: case INS_sve_ld2d: - assert((isValidSimm_MultipleOf<4, 2>(imm))); + assert(isValidSimm4_MultipleOf2(imm)); break; case INS_sve_ld3b: case INS_sve_ld3h: case INS_sve_ld3w: case INS_sve_ld3d: - assert((isValidSimm_MultipleOf<4, 3>(imm))); + assert(isValidSimm4_MultipleOf3(imm)); break; case INS_sve_ld4b: case INS_sve_ld4h: case INS_sve_ld4w: case INS_sve_ld4d: - assert((isValidSimm_MultipleOf<4, 4>(imm))); + assert(isValidSimm4_MultipleOf4(imm)); break; default: @@ -14338,7 +12567,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st2q: case INS_sve_st3q: case INS_sve_st4q: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_Q); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -14348,15 +12576,15 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, switch (ins) { case INS_sve_st2q: - assert((isValidSimm_MultipleOf<4, 2>(imm))); + assert(isValidSimm4_MultipleOf2(imm)); break; case INS_sve_st3q: - assert((isValidSimm_MultipleOf<4, 3>(imm))); + assert(isValidSimm4_MultipleOf3(imm)); break; case INS_sve_st4q: - assert((isValidSimm_MultipleOf<4, 4>(imm))); + assert(isValidSimm4_MultipleOf4(imm)); break; default: @@ -14372,12 +12600,11 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_stnt1h: case INS_sve_stnt1w: case INS_sve_stnt1d: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); #ifdef DEBUG switch (ins) @@ -14409,13 +12636,12 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st1w: case INS_sve_st1d: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); if (opt == INS_OPTS_SCALABLE_Q && (ins == INS_sve_st1d)) { @@ -14448,13 +12674,13 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, assert(isVectorRegister(reg3)); if ((ins == INS_sve_st1w) && insOptsScalableWords(opt)) { - assert((isValidUimm_MultipleOf<5, 4>(imm))); + assert(isValidUimm5_MultipleOf4(imm)); fmt = IF_SVE_JI_3A_A; } else { assert(ins == INS_sve_st1d); - assert((isValidUimm_MultipleOf<5, 8>(imm))); + assert(isValidUimm5_MultipleOf8(imm)); fmt = IF_SVE_JL_3A; } } @@ -14472,7 +12698,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st2d: case INS_sve_st3d: case INS_sve_st4d: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -14485,21 +12710,21 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st2h: case INS_sve_st2w: case INS_sve_st2d: - assert((isValidSimm_MultipleOf<4, 2>(imm))); + assert(isValidSimm4_MultipleOf2(imm)); break; case INS_sve_st3b: case INS_sve_st3h: case INS_sve_st3w: case INS_sve_st3d: - assert((isValidSimm_MultipleOf<4, 3>(imm))); + assert(isValidSimm4_MultipleOf3(imm)); break; case INS_sve_st4b: case INS_sve_st4h: case INS_sve_st4w: case INS_sve_st4d: - assert((isValidSimm_MultipleOf<4, 4>(imm))); + assert(isValidSimm4_MultipleOf4(imm)); break; default: @@ -14544,13 +12769,12 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st1b: case INS_sve_st1h: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); if (isGeneralRegister(reg3)) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); // st1h is reserved for scalable B assert((ins == INS_sve_st1h) ? insOptsScalableAtLeastHalf(opt) : insOptsScalableStandard(opt)); fmt = IF_SVE_JN_3A; @@ -14564,11 +12788,11 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, switch (ins) { case INS_sve_st1b: - assert(isValidUimm<5>(imm)); + assert(isValidUimm5(imm)); break; case INS_sve_st1h: - assert((isValidUimm_MultipleOf<5, 2>(imm))); + assert(isValidUimm5_MultipleOf2(imm)); break; default: @@ -14583,7 +12807,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_fmla: case INS_sve_fmls: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14591,31 +12814,29 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_S) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_GU_3A; } else { assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_GU_3B; } break; case INS_sve_bfmla: case INS_sve_bfmls: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<3>(imm)); // i ii + assert(isValidUimm3(imm)); // i ii fmt = IF_SVE_GU_3C; break; case INS_sve_fmul: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14623,52 +12844,50 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_S) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_GX_3A; } else { assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_GX_3B; } break; case INS_sve_bfmul: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<3>(imm)); // i ii + assert(isValidUimm3(imm)); // i ii fmt = IF_SVE_GX_3C; break; case INS_sve_fdot: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii if (opt == INS_OPTS_SCALABLE_B) { - unreached(); // TODO-SVE: Not yet supported. - assert(isValidUimm<2>(imm)); // ii + unreached(); // TODO-SVE: Not yet supported. + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_GY_3B_D; } else if (opt == INS_OPTS_SCALABLE_H) { - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_GY_3B; } else { unreached(); // TODO-SVE: Not yet supported. assert(insOptsNone(opt)); - assert(isValidUimm<3>(imm)); // i ii + assert(isValidUimm3(imm)); // i ii // Simplify emitDispInsHelp logic by setting insOpt opt = INS_OPTS_SCALABLE_B; @@ -14677,19 +12896,17 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_bfdot: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_GY_3B; break; case INS_sve_mla: case INS_sve_mls: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14697,19 +12914,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // i ii + assert(isValidUimm3(imm)); // i ii fmt = IF_SVE_FF_3A; } else if (opt == INS_OPTS_SCALABLE_S) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_FF_3B; } else { assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_FF_3C; } break; @@ -14718,7 +12935,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_smullt: case INS_sve_umullb: case INS_sve_umullt: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14726,13 +12942,13 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i + assert(isValidUimm3(imm)); // ii i fmt = IF_SVE_FE_3A; } else { assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // i i + assert(isValidUimm2(imm)); // i i fmt = IF_SVE_FE_3B; } break; @@ -14745,7 +12961,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_smlslt: case INS_sve_umlslb: case INS_sve_umlslt: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14753,20 +12968,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i + assert(isValidUimm3(imm)); // ii i fmt = IF_SVE_FG_3A; } else { assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // i i + assert(isValidUimm2(imm)); // i i fmt = IF_SVE_FG_3B; } break; case INS_sve_sqdmullb: case INS_sve_sqdmullt: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14774,20 +12988,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i + assert(isValidUimm3(imm)); // ii i fmt = IF_SVE_FH_3A; } else { assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // i i + assert(isValidUimm2(imm)); // i i fmt = IF_SVE_FH_3B; } break; case INS_sve_sqdmulh: case INS_sve_sqrdmulh: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14795,19 +13008,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i + assert(isValidUimm3(imm)); // ii i fmt = IF_SVE_FI_3A; } else if (opt == INS_OPTS_SCALABLE_S) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_FI_3B; } else { assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_FI_3C; } break; @@ -14816,7 +13029,6 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_sqdmlalt: case INS_sve_sqdmlslb: case INS_sve_sqdmlslt: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14824,20 +13036,19 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // ii i + assert(isValidUimm3(imm)); // ii i fmt = IF_SVE_FJ_3A; } else { assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_FJ_3B; } break; case INS_sve_sqrdmlah: case INS_sve_sqrdmlsh: - assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -14845,25 +13056,24 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<3>(imm)); // i ii + assert(isValidUimm3(imm)); // i ii fmt = IF_SVE_FK_3A; } else if (opt == INS_OPTS_SCALABLE_S) { assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm - assert(isValidUimm<2>(imm)); // ii + assert(isValidUimm2(imm)); // ii fmt = IF_SVE_FK_3B; } else { assert(opt == INS_OPTS_SCALABLE_D); - assert(isValidUimm<1>(imm)); // i + assert(isValidImm1(imm)); // i fmt = IF_SVE_FK_3C; } break; case INS_sve_fcadd: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -14874,72 +13084,65 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rd: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 8>(imm))); + assert(isValidUimm6_MultipleOf8(imm)); fmt = IF_SVE_IC_3A; break; case INS_sve_ld1rsw: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 4>(imm))); + assert(isValidUimm6_MultipleOf4(imm)); fmt = IF_SVE_IC_3A; break; case INS_sve_ld1rsh: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 2>(imm))); + assert(isValidUimm6_MultipleOf2(imm)); fmt = IF_SVE_IC_3A_A; break; case INS_sve_ld1rw: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 4>(imm))); + assert(isValidUimm6_MultipleOf4(imm)); fmt = IF_SVE_IC_3A_A; break; case INS_sve_ld1rh: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert((isValidUimm_MultipleOf<6, 2>(imm))); + assert(isValidUimm6_MultipleOf2(imm)); fmt = IF_SVE_IC_3A_B; break; case INS_sve_ld1rsb: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidUimm<6>(imm)); + assert(isValidUimm6(imm)); fmt = IF_SVE_IC_3A_B; break; case INS_sve_ld1rb: - assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isGeneralRegister(reg3)); - assert(isValidUimm<6>(imm)); + assert(isValidUimm6(imm)); fmt = IF_SVE_IC_3A_C; break; @@ -14951,65 +13154,15 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_bfmlalt: case INS_sve_bfmlslb: case INS_sve_bfmlslt: - assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm<3>(imm)); // ii i + assert(isValidUimm3(imm)); // ii i fmt = IF_SVE_GZ_3A; break; - case INS_sve_luti2: - assert(insScalableOptsNone(sopt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_H) - { - assert(isValidUimm<3>(imm)); // iii - fmt = IF_SVE_GG_3B; - } - else - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isValidUimm<2>(imm)); // ii i - fmt = IF_SVE_GG_3A; - } - unreached(); - break; - - case INS_sve_luti4: - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - - if (opt == INS_OPTS_SCALABLE_H) - { - assert(isValidUimm<2>(imm)); - - if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) - { - fmt = IF_SVE_GH_3B; - } - else - { - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_GH_3B_B; - } - } - else - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(insScalableOptsNone(sopt)); - assert(isValidUimm<1>(imm)); // i - fmt = IF_SVE_GH_3A; - } - unreached(); - break; - default: unreached(); break; @@ -15060,14 +13213,14 @@ void emitter::emitIns_R_R_R_I_I(instruction ins, if (opt == INS_OPTS_SCALABLE_B) { - assert(isValidUimm<2>(imm1)); // ii + assert(isValidUimm2(imm1)); // ii assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm fmt = IF_SVE_FA_3A; } else { assert(opt == INS_OPTS_SCALABLE_H); - assert(isValidUimm<1>(imm1)); // i + assert(isValidImm1(imm1)); // i fmt = IF_SVE_FA_3B; } break; @@ -15082,14 +13235,14 @@ void emitter::emitIns_R_R_R_I_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { - assert(isValidUimm<2>(imm1)); // ii + assert(isValidUimm2(imm1)); // ii assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm fmt = IF_SVE_FB_3A; } else { assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<1>(imm1)); // i + assert(isValidImm1(imm1)); // i fmt = IF_SVE_FB_3B; } break; @@ -15104,14 +13257,14 @@ void emitter::emitIns_R_R_R_I_I(instruction ins, if (opt == INS_OPTS_SCALABLE_H) { - assert(isValidUimm<2>(imm1)); // ii + assert(isValidUimm2(imm1)); // ii assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm fmt = IF_SVE_FC_3A; } else { assert(opt == INS_OPTS_SCALABLE_S); - assert(isValidUimm<1>(imm1)); // i + assert(isValidImm1(imm1)); // i fmt = IF_SVE_FC_3B; } break; @@ -15121,7 +13274,7 @@ void emitter::emitIns_R_R_R_I_I(instruction ins, assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm - assert(isValidUimm<1>(imm1)); // i + assert(isValidImm1(imm1)); // i assert(isValidRot(imm2)); // rr // Convert imm2 from rotation value (0-270) to bitwise representation (0-3) @@ -15482,33 +13635,6 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_sve_sel: - if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) - { - if (reg1 == reg4) - { - // mov is a preferred alias for sel - return emitIns_R_R_R(INS_sve_mov, attr, reg1, reg2, reg3, opt, INS_SCALABLE_OPTS_PREDICATE_MERGE); - } - - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isPredicateRegister(reg2)); // VVVV - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - fmt = IF_SVE_CW_4A; - } - else - { - assert(opt == INS_OPTS_SCALABLE_B); - assert(isPredicateRegister(reg1)); // dddd - assert(isPredicateRegister(reg2)); // gggg - assert(isPredicateRegister(reg3)); // nnnn - assert(isPredicateRegister(reg4)); // mmmm - fmt = IF_SVE_CZ_4A; - } - break; - case INS_sve_cmpeq: case INS_sve_cmpgt: case INS_sve_cmpge: @@ -15544,6 +13670,7 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, case INS_sve_bic: case INS_sve_orn: case INS_sve_bics: + case INS_sve_sel: case INS_sve_eors: case INS_sve_nor: case INS_sve_nand: @@ -15613,29 +13740,6 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, fmt = IF_SVE_AR_4A; break; - case INS_sve_histcnt: - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_GI_4A; - break; - - case INS_sve_fmla: - case INS_sve_fmls: - case INS_sve_fnmla: - case INS_sve_fnmls: - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg - assert(isVectorRegister(reg3)); // nnnnn - assert(isVectorRegister(reg4)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_HU_4A; - break; - case INS_sve_mad: case INS_sve_msb: assert(insOptsScalableStandard(opt)); @@ -16611,19 +14715,19 @@ void emitter::emitIns_R_R_R_R_I(instruction ins, switch (opt) { case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); + assert(isValidUimm4(imm)); break; case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); + assert(isValidUimm3(imm)); break; case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); + assert(isValidUimm2(imm)); break; case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); + assert(isValidImm1(imm)); break; default: @@ -16841,7 +14945,7 @@ void emitter::emitIns_R_R_FLAGS_COND( */ void emitter::emitIns_R_I_FLAGS_COND( - instruction ins, emitAttr attr, regNumber reg, ssize_t imm, insCflags flags, insCond cond) + instruction ins, emitAttr attr, regNumber reg, int imm, insCflags flags, insCond cond) { insFormat fmt = IF_NONE; condFlagsImm cfi; @@ -16858,7 +14962,7 @@ void emitter::emitIns_R_I_FLAGS_COND( ins = insReverse(ins); imm = -imm; } - if (isValidUimm<5>(imm)) + if (isValidUimm5(imm)) { cfi.imm5 = imm; cfi.flags = flags; @@ -16898,7 +15002,8 @@ void emitter::emitIns_R_I_FLAGS_COND( void emitter::emitIns_R_PATTERN( instruction ins, emitAttr attr, regNumber reg1, insOpts opt, insSvePattern pattern /* = SVE_PATTERN_ALL*/) { - insFormat fmt = IF_NONE; + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; /* Figure out the encoding format of the instruction */ switch (ins) @@ -16907,237 +15012,8 @@ void emitter::emitIns_R_PATTERN( case INS_sve_ptrues: assert(isPredicateRegister(reg1)); assert(isScalableVectorSize(attr)); - assert(insOptsScalableStandard(opt)); - fmt = IF_SVE_DE_1A; - break; - - default: - unreached(); - break; - - } // end switch (ins) - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstr(attr); - - id->idIns(ins); - id->idInsFmt(fmt); - - id->idReg1(reg1); - id->idInsOpt(opt); - id->idSvePattern(pattern); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing a register, a SVE Pattern and an immediate. - */ - -void emitter::emitIns_R_PATTERN_I(instruction ins, - emitAttr attr, - regNumber reg1, - insSvePattern pattern, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_sve_cntb: - case INS_sve_cntd: - case INS_sve_cnth: - case INS_sve_cntw: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidUimmFrom1<4>(imm)); // iiii - assert(size == EA_8BYTE); - fmt = IF_SVE_BL_1A; - break; - - case INS_sve_incd: - case INS_sve_inch: - case INS_sve_incw: - case INS_sve_decd: - case INS_sve_dech: - case INS_sve_decw: - assert(isValidUimmFrom1<4>(imm)); // iiii - - if (insOptsNone(opt)) - { - assert(isGeneralRegister(reg1)); // ddddd - assert(size == EA_8BYTE); - fmt = IF_SVE_BM_1A; - } - else - { - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - fmt = IF_SVE_BN_1A; - } - break; - - case INS_sve_incb: - case INS_sve_decb: - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidUimmFrom1<4>(imm)); // iiii - assert(size == EA_8BYTE); - fmt = IF_SVE_BM_1A; - break; - - case INS_sve_sqincb: - case INS_sve_uqincb: - case INS_sve_sqdecb: - case INS_sve_uqdecb: - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidUimmFrom1<4>(imm)); // iiii - assert(isValidGeneralDatasize(size)); // X - fmt = IF_SVE_BO_1A; - break; - - case INS_sve_sqinch: - case INS_sve_uqinch: - case INS_sve_sqdech: - case INS_sve_uqdech: - case INS_sve_sqincw: - case INS_sve_uqincw: - case INS_sve_sqdecw: - case INS_sve_uqdecw: - case INS_sve_sqincd: - case INS_sve_uqincd: - case INS_sve_sqdecd: - case INS_sve_uqdecd: - assert(isValidUimmFrom1<4>(imm)); // iiii - - if (insOptsNone(opt)) - { - assert(isGeneralRegister(reg1)); // ddddd - assert(isValidGeneralDatasize(size)); // X - fmt = IF_SVE_BO_1A; - } - else - { - assert(insOptsScalableAtLeastHalf(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isScalableVectorSize(size)); - fmt = IF_SVE_BP_1A; - } - break; - - default: - unreached(); - break; - - } // end switch (ins) - assert(fmt != IF_NONE); - - instrDesc* id = emitNewInstrCns(attr, imm); - - id->idIns(ins); - id->idInsFmt(fmt); - id->idInsOpt(opt); - id->idOpSize(size); - - id->idReg1(reg1); - id->idSvePattern(pattern); - - dispIns(id); - appendToCurIG(id); -} - -/***************************************************************************** - * - * Add an instruction referencing three registers and a SVE 'prfop'. - */ - -void emitter::emitIns_PRFOP_R_R_R(instruction ins, - emitAttr attr, - insSvePrfop prfop, - regNumber reg1, - regNumber reg2, - regNumber reg3, - insOpts opt /* = INS_OPTS_NONE */, - insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) -{ - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; - - /* Figure out the encoding format of the instruction */ - switch (ins) - { - case INS_sve_prfb: - assert(insScalableOptsNone(sopt)); - assert(isLowPredicateRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isScalableVectorSize(size)); - - if (insOptsScalable32bitExtends(opt)) - { - assert(isVectorRegister(reg3)); - - if (insOptsScalableSingleWord32bitExtends(opt)) - { - fmt = IF_SVE_HY_3A; - } - else - { - assert(insOptsScalableDoubleWord32bitExtends(opt)); - fmt = IF_SVE_HY_3A_A; - } - } - else if (isVectorRegister(reg3)) - { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_HY_3B; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg3)); - fmt = IF_SVE_IB_3A; - } - break; - - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - assert(isLowPredicateRegister(reg1)); - assert(isGeneralRegister(reg2)); - assert(isScalableVectorSize(size)); - - if (sopt == INS_SCALABLE_OPTS_MOD_N) - { - if (insOptsScalableSingleWord32bitExtends(opt)) - { - fmt = IF_SVE_HY_3A; - } - else - { - assert(insOptsScalableDoubleWord32bitExtends(opt)); - fmt = IF_SVE_HY_3A_A; - } - } - else - { - assert(sopt == INS_SCALABLE_OPTS_LSL_N); - if (isVectorRegister(reg3)) - { - assert(opt == INS_OPTS_SCALABLE_D); - fmt = IF_SVE_HY_3B; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg3)); - fmt = IF_SVE_IB_3A; - } - } + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_DE_1A; break; default: @@ -17150,13 +15026,11 @@ void emitter::emitIns_PRFOP_R_R_R(instruction ins, instrDesc* id = emitNewInstr(attr); id->idIns(ins); - id->idInsOpt(opt); id->idInsFmt(fmt); id->idReg1(reg1); - id->idReg2(reg2); - id->idReg3(reg3); - id->idSvePrfop(prfop); + id->idInsOpt(opt); + id->idSvePattern(pattern); dispIns(id); appendToCurIG(id); @@ -17164,67 +15038,26 @@ void emitter::emitIns_PRFOP_R_R_R(instruction ins, /***************************************************************************** * - * Add an instruction referencing two registers, a SVE 'prfop' and an immediate. + * Add an instruction referencing a register, a SVE Pattern and an immediate. */ -void emitter::emitIns_PRFOP_R_R_I(instruction ins, - emitAttr attr, - insSvePrfop prfop, - regNumber reg1, - regNumber reg2, - int imm, - insOpts opt /* = INS_OPTS_NONE */) +void emitter::emitIns_R_PATTERN_I(instruction ins, emitAttr attr, regNumber reg1, insSvePattern pattern, int imm) { - emitAttr size = EA_SIZE(attr); - insFormat fmt = IF_NONE; + emitAttr size = EA_SIZE(attr); + emitAttr elemsize = EA_UNKNOWN; + insFormat fmt = IF_NONE; /* Figure out the encoding format of the instruction */ switch (ins) { - case INS_sve_prfb: - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - assert(isLowPredicateRegister(reg1)); - assert(isScalableVectorSize(size)); - - if (isVectorRegister(reg2)) - { - assert(insOptsScalableWords(opt)); - -#ifdef DEBUG - switch (ins) - { - case INS_sve_prfb: - assert(isValidUimm<5>(imm)); - break; - - case INS_sve_prfh: - assert((isValidUimm_MultipleOf<5, 2>(imm))); - break; - - case INS_sve_prfw: - assert((isValidUimm_MultipleOf<5, 4>(imm))); - break; - - case INS_sve_prfd: - assert((isValidUimm_MultipleOf<5, 8>(imm))); - break; - - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG - fmt = IF_SVE_HZ_2A_B; - } - else - { - assert(insOptsNone(opt)); - assert(isGeneralRegister(reg2)); - assert(isValidSimm<6>(imm)); - fmt = IF_SVE_IA_2A; - } + case INS_sve_cntb: + case INS_sve_cntd: + case INS_sve_cnth: + case INS_sve_cntw: + assert(isGeneralRegister(reg1)); + assert(size == EA_8BYTE); + assert(isValidUimm4From1(imm)); + fmt = IF_SVE_BL_1A; break; default: @@ -17237,12 +15070,10 @@ void emitter::emitIns_PRFOP_R_R_I(instruction ins, instrDesc* id = emitNewInstrCns(attr, imm); id->idIns(ins); - id->idInsOpt(opt); id->idInsFmt(fmt); id->idReg1(reg1); - id->idReg2(reg2); - id->idSvePrfop(prfop); + id->idSvePattern(pattern); dispIns(id); appendToCurIG(id); @@ -18786,7 +16617,7 @@ void emitter::emitIns_Call(EmitCallType callType, { assert(isLowPredicateRegister(reg)); emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 7)); + assert((ureg >= 0) && (ureg <= 15)); return ureg << 10; } @@ -19348,18 +17179,6 @@ void emitter::emitIns_Call(EmitCallType callType, return (bits << 11); // bits at locations [14,13,12,11] } -/***************************************************************************** - * - * Returns the encoding for an immediate in the SVE variant of dup (indexed) - */ -/*static*/ emitter::code_t emitter::insEncodeSveBroadcastIndex(emitAttr elemsize, ssize_t index) -{ - unsigned lane_bytes = genLog2(elemsize) + 1; - code_t tsz = (1 << (lane_bytes - 1)); - code_t imm = (code_t)index << lane_bytes | tsz; - return insEncodeSplitUimm<23, 22, 20, 16>(imm); -} - /***************************************************************************** * * Returns the encoding to select the 'index' for an Arm64 'mul' by element instruction @@ -19927,28 +17746,6 @@ void emitter::emitIns_Call(EmitCallType callType, return 0; } -/***************************************************************************** - * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'sz' at bit location '20'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_20(emitAttr size) -{ - switch (size) - { - case EA_4BYTE: - return 0; - - case EA_8BYTE: - return (1 << 20); - - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; -} - /***************************************************************************** * * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction @@ -19974,10 +17771,10 @@ void emitter::emitIns_Call(EmitCallType callType, /***************************************************************************** * * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:20-19'. + * This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_23_tszl_20_to_19(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_22_tszl_20_to_19(emitAttr size) { switch (size) { @@ -19990,9 +17787,6 @@ void emitter::emitIns_Call(EmitCallType callType, case EA_4BYTE: return 0x400000; // set the bit at location 22 - case EA_8BYTE: - return 0x800000; // set the bit at location 23 - default: assert(!"Invalid size for vector register"); } @@ -20065,7 +17859,7 @@ void emitter::emitIns_Call(EmitCallType callType, switch (opt) { case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); + assert(isValidUimm4(imm)); encoding = 0x040000; // set the bit at location 18 // encode immediate at location 23-22:20-19 encoding |= ((imm & 0b1100) << 22); @@ -20073,7 +17867,7 @@ void emitter::emitIns_Call(EmitCallType callType, break; case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); + assert(isValidUimm3(imm)); encoding = 0x080000; // set the bit at location 19 // encode immediate at location 23-22:20 encoding |= ((imm & 0b110) << 22); @@ -20081,13 +17875,13 @@ void emitter::emitIns_Call(EmitCallType callType, break; case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); + assert(isValidUimm2(imm)); encoding = 0x100000; // set the bit at location 20 encoding |= (imm << 22); // encode immediate at location 23:22 break; case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); + assert(isValidImm1(imm)); encoding = 0x400000; // set the bit at location 22 encoding |= (imm << 23); // encode immediate at location 23 break; @@ -20100,94 +17894,6 @@ void emitter::emitIns_Call(EmitCallType callType, return encoding; } -/***************************************************************************** - * - * Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, - ssize_t imm, - bool isRightShift) -{ - code_t encoding = 0; - - imm = insEncodeShiftImmediate(optGetSveElemsize(opt), isRightShift, imm); - - switch (opt) - { - case INS_OPTS_SCALABLE_B: - imm = imm & 0b111; // bits 18-16 - encoding |= (1 << 19); // bit 19 - break; - - case INS_OPTS_SCALABLE_H: - imm = imm & 0b1111; // bits 19-16 - encoding |= (1 << 20); // bit 20 - break; - - case INS_OPTS_SCALABLE_S: - imm = imm & 0b11111; // bits 20-16 - encoding |= (1 << 22); // bit 22 - break; - - case INS_OPTS_SCALABLE_D: - // this gets the last bit of 'imm' and tries to set bit 22 - encoding |= ((imm >> 5) << 22); - imm = imm & 0b11111; // bits 20-16 - encoding |= (1 << 23); // bit 23 - break; - - default: - assert(!"Invalid size for vector register"); - break; - } - - return (encoding | (code_t)(imm << 16)); -} - -/***************************************************************************** - * - * Returns the encoding for the field 'i1:tsz' at bit locations '20:19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithImmediate_i1_tsz(const insOpts opt, ssize_t imm) -{ - code_t encoding = 0; - - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - encoding |= (1 << 16); // bit 16 - encoding |= (imm << 17); // bits 20-17 - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - encoding |= (1 << 17); // bit 17 - encoding |= (imm << 18); // bits 20-18 - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - encoding |= (1 << 18); // bit 18 - encoding |= (imm << 19); // bits 20-19 - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - encoding |= (1 << 19); // bit 19 - encoding |= (imm << 20); // bit 20 - break; - - default: - assert(!"Invalid size for vector register"); - break; - } - - return encoding; -} - /***************************************************************************** * * Returns the encoding to select the elemsize for an Arm64 SVE vector instruction plus an immediate. @@ -20458,8 +18164,13 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_BU_2A: case IF_SVE_BV_2B: case IF_SVE_HS_3A: - case IF_SVE_HP_3A: + case IF_SVE_HS_3A_H: + case IF_SVE_HS_3A_I: + case IF_SVE_HS_3A_J: case IF_SVE_HP_3B: + case IF_SVE_HP_3B_H: + case IF_SVE_HP_3B_I: + case IF_SVE_HP_3B_J: case IF_SVE_AR_4A: case IF_SVE_BV_2A_A: case IF_SVE_AB_3A: @@ -20473,8 +18184,7 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_GT_4A: case IF_SVE_AP_3A: case IF_SVE_HO_3A: - case IF_SVE_HO_3B: - case IF_SVE_HO_3C: + case IF_SVE_HO_3A_B: case IF_SVE_GQ_3A: case IF_SVE_HU_4B: case IF_SVE_AQ_3A: @@ -20490,15 +18200,12 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_HQ_3A: case IF_SVE_AS_4A: case IF_SVE_CT_3A: + case IF_SVE_HP_3A: case IF_SVE_HV_4A: return PREDICATE_MERGE; case IF_SVE_CZ_4A_A: case IF_SVE_CZ_4A_L: - case IF_SVE_CE_2A: - case IF_SVE_CE_2B: - case IF_SVE_CE_2C: - case IF_SVE_CE_2D: case IF_SVE_CF_2A: case IF_SVE_CF_2B: case IF_SVE_CF_2C: @@ -20942,20 +18649,6 @@ void emitter::emitIns_Call(EmitCallType callType, } break; - case IF_SVE_HY_3B: - case IF_SVE_IB_3A: - switch (ins) - { - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - return true; - - default: - break; - } - break; - default: break; } @@ -21087,21 +18780,6 @@ void emitter::emitIns_Call(EmitCallType callType, } break; - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - switch (ins) - { - case INS_sve_prfb: - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - return true; - - default: - break; - } - break; - default: break; } @@ -21576,49 +19254,6 @@ void emitter::emitIns_Call(EmitCallType callType, } break; - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - assert(!insSveIsLslN(ins, fmt)); - assert(insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_prfb: - return 0; - - case INS_sve_prfh: - return 1; - - case INS_sve_prfw: - return 2; - - case INS_sve_prfd: - return 3; - - default: - break; - } - break; - - case IF_SVE_HY_3B: - case IF_SVE_IB_3A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_prfh: - return 1; - - case INS_sve_prfw: - return 2; - - case INS_sve_prfd: - return 3; - - default: - break; - } - break; - default: break; } @@ -21978,7 +19613,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm4_19_to_16(ssize_t imm) { - assert(isValidSimm<4>(imm)); + assert(isValidSimm4(imm)); if (imm < 0) { imm = (imm & 0xF); @@ -21993,7 +19628,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm9h9l_21_to_16_and_12_to_10(ssize_t imm) { - assert(isValidSimm<9>(imm)); + assert(isValidSimm9(imm)); if (imm < 0) { @@ -22013,7 +19648,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf2_19_to_16(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 2>(imm))); + assert(isValidSimm4_MultipleOf2(imm)); return insEncodeSimm4_19_to_16(imm / 2); } @@ -22024,7 +19659,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf3_19_to_16(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 3>(imm))); + assert(isValidSimm4_MultipleOf3(imm)); return insEncodeSimm4_19_to_16(imm / 3); } @@ -22035,7 +19670,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf4_19_to_16(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 4>(imm))); + assert(isValidSimm4_MultipleOf4(imm)); return insEncodeSimm4_19_to_16(imm / 4); } @@ -22046,7 +19681,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf16_19_to_16(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 16>(imm))); + assert(isValidSimm4_MultipleOf16(imm)); return insEncodeSimm4_19_to_16(imm / 16); } @@ -22057,91 +19692,76 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf32_19_to_16(ssize_t imm) { - assert((isValidSimm_MultipleOf<4, 32>(imm))); + assert(isValidSimm4_MultipleOf32(imm)); return insEncodeSimm4_19_to_16(imm / 32); } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 2 as 5-bits at bit locations '20-16'. + * // Returns the encoding for the immediate value that is a multiple of 2 as 5-bits at bit locations '20-16'. */ /*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf2_20_to_16(ssize_t imm) { - assert((isValidUimm_MultipleOf<5, 2>(imm))); + assert(isValidUimm5_MultipleOf2(imm)); return insEncodeUimm5_20_to_16(imm / 2); } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 4 as 5-bits at bit locations '20-16'. + * // Returns the encoding for the immediate value that is a multiple of 4 as 5-bits at bit locations '20-16'. */ /*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm) { - assert((isValidUimm_MultipleOf<5, 4>(imm))); + assert(isValidUimm5_MultipleOf4(imm)); return insEncodeUimm5_20_to_16(imm / 4); } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. + * // Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. */ /*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm) { - assert((isValidUimm_MultipleOf<5, 8>(imm))); + assert(isValidUimm5_MultipleOf8(imm)); return insEncodeUimm5_20_to_16(imm / 8); } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. + * // Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. */ /*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf2_21_to_16(ssize_t imm) { - assert((isValidUimm_MultipleOf<6, 2>(imm))); + assert(isValidUimm6_MultipleOf2(imm)); return insEncodeUimm6_21_to_16(imm / 2); } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 4 as 6-bits at bit locations '21-16'. + * // Returns the encoding for the immediate value that is a multiple of 4 as 6-bits at bit locations '21-16'. */ /*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf4_21_to_16(ssize_t imm) { - assert((isValidUimm_MultipleOf<6, 4>(imm))); + assert(isValidUimm6_MultipleOf4(imm)); return insEncodeUimm6_21_to_16(imm / 4); } /***************************************************************************** * - * Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. + * // Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. */ /*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf8_21_to_16(ssize_t imm) { - assert((isValidUimm_MultipleOf<6, 8>(imm))); + assert(isValidUimm6_MultipleOf8(imm)); return insEncodeUimm6_21_to_16(imm / 8); } -/***************************************************************************** - * - * Returns the encoding for the immediate value as 5-bits at bit locations '9-5'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm5_9_to_5(ssize_t imm) -{ - assert(isValidSimm<5>(imm)); - if (imm < 0) - { - imm = (imm & 0x1F); - } - return (code_t)imm << 5; -} - /***************************************************************************** * * Returns the encoding for the immediate value as 5-bits at bit locations '20-16'. @@ -22149,7 +19769,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeSimm5_20_to_16(ssize_t imm) { - assert(isValidSimm<5>(imm)); + assert(isValidSimm5(imm)); if (imm < 0) { imm = (imm & 0x1F); @@ -22157,36 +19777,6 @@ void emitter::emitIns_Call(EmitCallType callType, return (code_t)imm << 16; } -/***************************************************************************** - * - * Returns the encoding for the immediate value as 6-bits at bit locations '10-5'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm6_10_to_5(ssize_t imm) -{ - assert(isValidSimm<6>(imm)); - if (imm < 0) - { - imm = (imm & 0x3F); - } - return (code_t)imm << 5; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 6-bits at bit locations '20-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm6_21_to_16(ssize_t imm) -{ - assert(isValidSimm<6>(imm)); - if (imm < 0) - { - imm = (imm & 0x3F); - } - return (code_t)imm << 16; -} - /***************************************************************************** * * Returns the encoding for the immediate value as 2-bits at bit locations '9-8'. @@ -22194,7 +19784,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm2_9_to_8(ssize_t imm) { - assert(isValidUimm<2>(imm)); + assert(isValidUimm2(imm)); return (code_t)imm << 8; } @@ -22205,7 +19795,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm2_11_to_10(ssize_t imm) { - assert(isValidUimm<2>(imm)); + assert(isValidUimm2(imm)); return (code_t)imm << 10; } @@ -22214,47 +19804,10 @@ void emitter::emitIns_Call(EmitCallType callType, * Returns the encoding for the immediate value as 2-bits at bit locations '20-19'. */ -/*static*/ emitter::code_t emitter::insEncodeUimm2_20_to_19(ssize_t imm) -{ - assert(isValidUimm<2>(imm)); - return (code_t)imm << 19; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 2-bits at bit locations '23-22'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm2_23_to_22(ssize_t imm) -{ - assert(isValidUimm<2>(imm)); - return (code_t)imm << 22; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 1-bit at bit locations '23'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm1_23(ssize_t imm) -{ - assert(isValidUimm<1>(imm)); - return (code_t)imm << 23; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm) -{ - assert(isValidUimm<3>(imm)); - - code_t h = (code_t)(imm & 0x6) << 21; // encode high 2-bits at locations '23-22' - code_t l = (code_t)(imm & 0x1) << 12; // encode low 1-bit at locations '12' - - return (h | l); +/*static*/ emitter::code_t emitter::insEncodeUimm2_20_to_19(ssize_t imm) +{ + assert(isValidUimm2(imm)); + return (code_t)imm << 19; } /***************************************************************************** @@ -22264,7 +19817,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeImm1_10(ssize_t imm) { - assert(isValidUimm<1>(imm)); + assert(isValidImm1(imm)); return (code_t)imm << 10; } @@ -22275,7 +19828,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeImm1_11(ssize_t imm) { - assert(isValidUimm<1>(imm)); + assert(isValidImm1(imm)); return (code_t)imm << 11; } @@ -22286,7 +19839,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeImm1_22(ssize_t imm) { - assert(isValidUimm<1>(imm)); + assert(isValidImm1(imm)); return (code_t)imm << 22; } @@ -22297,21 +19850,10 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm7_20_to_14(ssize_t imm) { - assert(isValidUimm<7>(imm)); + assert(isValidUimm7(imm)); return (code_t)imm << 14; } -/***************************************************************************** - * - * Returns the encoding for the immediate value as 4-bits at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm4_19_to_16(ssize_t imm) -{ - assert(isValidUimm<4>(imm)); - return (code_t)imm << 16; -} - /***************************************************************************** * * Returns the encoding for the immediate value as 4-bits starting from 1, at bit locations '19-16'. @@ -22319,7 +19861,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm4From1_19_to_16(ssize_t imm) { - assert(isValidUimmFrom1<4>(imm)); + assert(isValidUimm4From1(imm)); return (code_t)(imm - 1) << 16; } @@ -22330,7 +19872,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm5_20_to_16(ssize_t imm) { - assert(isValidUimm<5>(imm)); + assert(isValidUimm5(imm)); return (code_t)imm << 16; } @@ -22341,7 +19883,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm6_21_to_16(ssize_t imm) { - assert(isValidUimm<6>(imm)); + assert(isValidUimm6(imm)); return (code_t)imm << 16; } @@ -22352,21 +19894,10 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeImm8_12_to_5(ssize_t imm) { - assert(isValidSimm<8>(imm) || isValidUimm<8>(imm)); + assert(isValidSimm8(imm) || isValidUimm8(imm)); return (code_t)((imm & 0xFF) << 5); } -/***************************************************************************** - * - * Returns the encoding for the unsigned immediate value as 3-bits at bit locations '12-10'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm3_12_to_10(ssize_t imm) -{ - assert(isValidUimm<3>(imm)); - return (code_t)imm << 10; -} - /***************************************************************************** * * Returns the encoding for the unsigned immediate value as 3-bits at bit locations '18-16'. @@ -22374,7 +19905,7 @@ void emitter::emitIns_Call(EmitCallType callType, /*static*/ emitter::code_t emitter::insEncodeUimm3_18_to_16(ssize_t imm) { - assert(isValidUimm<3>(imm)); + assert(isValidUimm3(imm)); return (code_t)imm << 16; } @@ -22404,21 +19935,17 @@ void emitter::emitIns_Call(EmitCallType callType, switch (opt) { case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); + assert(isValidUimm3From1(imm)); return (8 - imm); case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); + assert(isValidUimm4From1(imm)); return (16 - imm); case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); + assert(isValidUimm5From1(imm)); return (32 - imm); - case INS_OPTS_SCALABLE_D: - assert(isValidUimmFrom1<6>(imm)); - return (64 - imm); - default: unreached(); break; @@ -22427,69 +19954,6 @@ void emitter::emitIns_Call(EmitCallType callType, return 0; } -/***************************************************************************** - * - * Returns the two 5-bit signed immediates encoded in the following format: - * njjj jjmi iiii - * - iiiii: the absolute value of imm1 - * - m: 1 if imm1 is negative, 0 otherwise - * - jjjjj: the absolute value of imm2 - * - n: 1 if imm2 is negative, 0 otherwise - */ -/*static*/ ssize_t emitter::insEncodeTwoSimm5(ssize_t imm1, ssize_t imm2) -{ - assert(isValidSimm<5>(imm1)); - assert(isValidSimm<5>(imm2)); - ssize_t immOut = 0; - - if (imm1 < 0) - { - // Set bit location 5 to indicate imm1 is negative - immOut |= 0x20; - imm1 *= -1; - } - - if (imm2 < 0) - { - // Set bit location 11 to indicate imm2 is negative - immOut |= 0x800; - imm2 *= -1; - } - - immOut |= imm1; - immOut |= (imm2 << 6); - return immOut; -} - -/***************************************************************************** - * - * Decodes imm into two 5-bit signed immediates, - * using the encoding format from insEncodeTwoSimm5. - */ -/*static*/ void emitter::insDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2) -{ - assert(imm1 != nullptr); - assert(imm2 != nullptr); - - *imm1 = (imm & 0x1F); - - if ((imm & 0x20) != 0) - { - *imm1 *= -1; - } - - imm >>= 6; - *imm2 = (imm & 0x1F); - - if ((imm & 0x20) != 0) - { - *imm2 *= -1; - } - - assert(isValidSimm<5>(*imm1)); - assert(isValidSimm<5>(*imm2)); -} - /***************************************************************************** * * Returns the encoding to select an insSvePattern @@ -22521,7 +19985,7 @@ BYTE* emitter::emitOutputLoadLabel(BYTE* dst, BYTE* srcAddr, BYTE* dstAddr, inst // add x, x, page offs -- compute address = page addr + page offs ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits - assert(isValidUimm<12>(imm12)); + assert(isValidUimm12(imm12)); code_t code = emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) code |= insEncodeDatasize(EA_8BYTE); // X @@ -22642,7 +20106,7 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) dst = emitOutputShortAddress(dst, ins, fmt, relPageAddr, addrReg); ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits - assert(isValidUimm<12>(imm12)); + assert(isValidUimm12(imm12)); // Special case: emit add + ld1 instructions for loading 16-byte data into vector register. if (isVectorRegister(dstReg) && (opSize == EA_16BYTE)) @@ -22883,7 +20347,7 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) // Branch offset encodings are scaled by 4. noway_assert((distVal & 3) == 0); distVal >>= 2; - noway_assert(isValidSimm<26>(distVal)); + noway_assert(isValidSimm26(distVal)); // Insert offset into unconditional branch instruction distVal &= 0x3FFFFFFLL; @@ -22928,14 +20392,14 @@ BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, if (fmt == IF_BI_0A) { // INS_b or INS_bl_local - noway_assert(isValidSimm<26>(distVal)); + noway_assert(isValidSimm26(distVal)); distVal &= 0x3FFFFFFLL; code |= distVal; } else if (fmt == IF_BI_0B) // BI_0B 01010100iiiiiiii iiiiiiiiiiiXXXXX simm19:00 { // INS_beq, INS_bne, etc... - noway_assert(isValidSimm<19>(distVal)); + noway_assert(isValidSimm19(distVal)); distVal &= 0x7FFFFLL; code |= distVal << 5; } @@ -22946,7 +20410,7 @@ BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, code |= insEncodeDatasize(id->idOpSize()); // X code |= insEncodeReg_Rt(id->idReg1()); // ttttt - noway_assert(isValidSimm<19>(distVal)); + noway_assert(isValidSimm19(distVal)); distVal &= 0x7FFFFLL; // 19 bits code |= distVal << 5; } @@ -22964,7 +20428,7 @@ BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, code |= ((imm & 0x1F) << 19); // bbbbb code |= insEncodeReg_Rt(id->idReg1()); // ttttt - noway_assert(isValidSimm<14>(distVal)); + noway_assert(isValidSimm14(distVal)); distVal &= 0x3FFFLL; // 14 bits code |= distVal << 5; } @@ -22993,7 +20457,7 @@ BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, // INS_adr or INS_adrp code |= insEncodeReg_Rd(reg); // ddddd - noway_assert(isValidSimm<19>(distVal)); + noway_assert(isValidSimm19(distVal)); distVal &= 0x7FFFFLL; // 19 bits code |= distVal << 5; code |= loBits << 29; // 2 bits @@ -23026,7 +20490,7 @@ BYTE* emitter::emitOutputShortConstant( noway_assert(loBits == 0); ssize_t distVal = imm >> 2; // load offset encodings are scaled by 4. - noway_assert(isValidSimm<19>(distVal)); + noway_assert(isValidSimm19(distVal)); // Is the target a vector register? if (isVectorRegister(reg)) @@ -23055,7 +20519,7 @@ BYTE* emitter::emitOutputShortConstant( { // ldr Rt,[Xn+pimm12] LS_2B 1X11100101iiiiii iiiiiinnnnnttttt B940 0000 imm(0-4095<<{2,3}) // INS_ldr or INS_ldrsw (PC-Relative) - noway_assert(isValidUimm<12>(imm)); + noway_assert(isValidUimm12(imm)); assert(isGeneralRegister(reg)); if (opSize == EA_8BYTE) @@ -23380,7 +20844,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) assert(insOptsNone(id->idInsOpt())); imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); code = emitInsCode(ins, fmt); // Is the target a vector register? if (isVectorRegister(id->idReg1())) @@ -23593,7 +21057,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_DI_1A: // DI_1A X.......shiiiiii iiiiiinnnnn..... Rn imm(i12,sh) assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); code = emitInsCode(ins, fmt); code |= insEncodeDatasize(id->idOpSize()); // X code |= insEncodeShiftImm12(id->idInsOpt()); // sh @@ -23671,7 +21135,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_DI_2A: // DI_2A X.......shiiiiii iiiiiinnnnnddddd Rd Rn imm(i12,sh) assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); + assert(isValidUimm12(imm)); code = emitInsCode(ins, fmt); code |= insEncodeDatasize(id->idOpSize()); // X code |= insEncodeShiftImm12(id->idInsOpt()); // sh @@ -24440,7 +21904,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_SI_0A: // SI_0A ...........iiiii iiiiiiiiiii..... imm16 imm = emitGetInsSC(id); - assert(isValidUimm<16>(imm)); + assert(isValidUimm16(imm)); code = emitInsCode(ins, fmt); code |= ((code_t)imm << 5); // iiiii iiiiiiiiiii dst += emitOutput_Instr(dst, code); @@ -24709,15 +22173,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - // Scalable with Merge or Zero predicate case IF_SVE_AH_3A: // ........xx.....M ...gggnnnnnddddd -- SVE constructive prefix (predicated) code = emitInsCodeSve(ins, fmt); @@ -24746,8 +22201,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) // Scalable, 4 regs. Reg4 in mmmmm. case IF_SVE_AR_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE integer multiply-accumulate writing addend // (predicated) - case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) - case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg @@ -24770,35 +22223,12 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) break; // Scalable, 3 regs, no predicates - case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high - // (unpredicated) - case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) - case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient - case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) - case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long - case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high - case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long - case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) - case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long - case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide - case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved - case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute - case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long - case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate - case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long - case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part - case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) - case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high + // (unpredicated) + case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) + case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn @@ -24818,29 +22248,8 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm2_11_to_10(emitGetInsSC(id)); // hh - code |= insEncodeImm1_22(id->idInsOpt() == INS_OPTS_SCALABLE_D ? 1 : 0); - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm2_11_to_10(emitGetInsSC(id)); // hh - dst += emitOutput_Instr(dst, code); - break; - - // Immediate and pattern to general purpose. + // Immediate and patterm to general purpose. case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count - case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count imm = emitGetInsSC(id); code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_Rd(id->idReg1()); // ddddd @@ -24849,167 +22258,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - code |= insEncodeSveElemsize_sz_20(id->idOpSize()); // X - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) - case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn/mmmmm - code |= insEncodeUimm3_12_to_10(imm & 0b111); // iii - code |= insEncodeUimm5_20_to_16(imm >> 3); // iiiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count - case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) - case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= (imm << 5); - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= (id->idHasShift() ? 0x2000 : 0); // h - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) - // In emitIns, we set this format's instruction to MOV, as that is the preferred disassembly. - // However, passing (MOV, IF_SVE_BV_2B) to emitInsCodeSve will assert with "encoding_found", - // as FMOV is the only instruction associated with this encoding format. - // Thus, always pass FMOV here, and use MOV elsewhere for simplicity. - code = emitInsCodeSve(INS_sve_fmov, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveBroadcastIndex(optGetSveElemsize(id->idInsOpt()), imm); - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD @@ -25082,156 +22330,55 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) break; case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) - case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // VVV - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn/mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) - { - regNumber reg4 = (ins == INS_sve_mov ? id->idReg1() : id->idReg4()); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_13_to_10(id->idReg2()); // VVVV - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(reg4); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } - - case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors - case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors - case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match - case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeReg_V_20_to_16(id->idReg4()); // nnnnn - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm5_20_to_16(imm); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm7_20_to_14(imm); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate - case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) - case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product - case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product - case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations - case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long - case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long - case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long - case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeReg_V_9_to_5(id->idReg3()); // kkkkk - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm - imm = insGetImmDiff(emitGetInsSC(id), id->idInsOpt()); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm5_20_to_16(imm & 0b11111); // xxiii - code |= insEncodeImm1_22(imm >> 5); // x - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate - // increment) - { - ssize_t imm1; - ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm5_9_to_5(imm1); // iiiii - code |= insEncodeSimm5_20_to_16(imm2); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } - - case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register - // increment) + case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm5_9_to_5(emitGetInsSC(id)); // iiiii - code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeReg_P_12_to_10(id->idReg2()); // VVV + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn/mmmmm code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx dst += emitOutput_Instr(dst, code); break; - case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate - // increment) + case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors + case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors + case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match + case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSimm5_20_to_16(emitGetInsSC(id)); // iiiii + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeReg_V_20_to_16(id->idReg4()); // nnnnn + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm5_20_to_16(imm); // iiiii code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx dst += emitOutput_Instr(dst, code); break; - case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate + imm = emitGetInsSC(id); code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm6_10_to_5(emitGetInsSC(id)); // iiiiii - code |= insEncodeReg_R_20_to_16(id->idReg2()); // nnnnn + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm7_20_to_14(imm); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx dst += emitOutput_Instr(dst, code); break; - case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm6_10_to_5(emitGetInsSC(id)); // iiiiii + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; @@ -25516,36 +22663,29 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) case IF_SVE_GD_2A: // .........x.xx... ......nnnnnddddd -- SVE2 saturating extract narrow code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - // Bit 23 should not be set by below call - assert(insOptsScalableWide(id->idInsOpt())); - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsize_tszh_22_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx // x dst += emitOutput_Instr(dst, code); break; case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm5_20_to_16(emitGetInsSC(id)); // iii - // Bit 23 should not be set by below call - assert(insOptsScalableWide(id->idInsOpt())); - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm5_20_to_16(emitGetInsSC(id)); // iii + code |= insEncodeSveElemsize_tszh_22_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx // x dst += emitOutput_Instr(dst, code); break; case IF_SVE_GB_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift right narrow - // Bit 23 should not be set by call to insEncodeSveElemsize_tszh_23_tszl_20_to_19, - // nor should we pass INS_OPTS_SCALABLE_D to insGetImmDiff. - assert(insOptsScalableWide(id->idInsOpt())); code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn code |= insEncodeUimm5_20_to_16(insGetImmDiff(emitGetInsSC(id), id->idInsOpt())); // iii - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + code |= insEncodeSveElemsize_tszh_22_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx // x dst += emitOutput_Instr(dst, code); break; @@ -25697,14 +22837,16 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) + { imm = emitGetInsSC(id); code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= (id->idHasShift() ? 0x2000 : 0); // h + code |= (id->idOptionalShift() ? 0x2000 : 0); // h + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx dst += emitOutput_Instr(dst, code); break; + } case IF_SVE_EB_1B: // ........xx...... ...........ddddd -- SVE broadcast integer immediate (unpredicated) // ins is MOV for this encoding, as it is the preferred disassembly, so pass FMOV to emitInsCodeSve @@ -25733,126 +22875,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HO_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - switch (id->idInsOpt()) - { - case INS_OPTS_H_TO_S: - code |= (1 << 16); - break; - case INS_OPTS_H_TO_D: - code |= (1 << 22) | (1 << 16); - break; - case INS_OPTS_S_TO_H: - break; - case INS_OPTS_S_TO_D: - code |= (1 << 22) | (3 << 16); - break; - case INS_OPTS_D_TO_H: - code |= (1 << 22); - break; - case INS_OPTS_D_TO_S: - code |= (1 << 22) | (1 << 17); - break; - default: - unreached(); - } - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HO_3C: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_H: - code |= (1 << 22) | (1 << 17); - break; - case INS_OPTS_H_TO_S: - code |= (1 << 22) | (1 << 18); - break; - case INS_OPTS_H_TO_D: - code |= (1 << 22) | (3 << 17); - break; - case INS_OPTS_SCALABLE_S: - code |= (1 << 23) | (1 << 18); - break; - case INS_OPTS_S_TO_D: - code |= (3 << 22) | (1 << 18); - break; - case INS_OPTS_D_TO_S: - code |= (3 << 22); - break; - case INS_OPTS_SCALABLE_D: - code |= (3 << 22) | (3 << 17); - break; - default: - unreached(); - break; - } - - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_H: - code |= (1 << 22) | (1 << 17); - break; - case INS_OPTS_S_TO_H: - code |= (1 << 22) | (1 << 18); - break; - case INS_OPTS_SCALABLE_S: - code |= (1 << 23) | (1 << 18); - break; - case INS_OPTS_S_TO_D: - code |= (1 << 23) | (1 << 22); - break; - case INS_OPTS_D_TO_H: - code |= (1 << 22) | (3 << 17); - break; - case INS_OPTS_D_TO_S: - code |= (3 << 22) | (1 << 18); - break; - case INS_OPTS_SCALABLE_D: - code |= (3 << 22) | (3 << 17); - break; - default: - unreached(); - break; - } - - dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus // immediate) case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus @@ -26294,119 +23316,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm2_23_to_22(imm); // ii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm3h3l_23_to_22_and_12(imm); // ii - // i - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm1_23(imm); // i - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit - // scaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 22); // h - break; - - default: - break; - } - - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= id->idSvePrfop(); // oooo - - if (id->idInsOpt() == INS_OPTS_SCALABLE_D) - { - code |= (1 << 30); // set bit '30' to make it a double-word - } - - switch (ins) - { - case INS_sve_prfh: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; - - case INS_sve_prfw: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; - - case INS_sve_prfd: - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii - break; - - default: - assert(ins == INS_sve_prfb); - } - dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) imm = emitGetInsSC(id); code = emitInsCodeSve(ins, fmt); @@ -26486,16 +23395,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= id->idSvePrfop(); // oooo - code |= insEncodeSimm6_21_to_16(imm); // iiiiii - dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_IC_3A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element imm = emitGetInsSC(id); code = emitInsCodeSve(ins, fmt); @@ -26547,63 +23446,6 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) - case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator - case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements - case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements - case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) - case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert - case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsizeWithShift_tszh_tszl_imm3(id->idInsOpt(), imm, - emitInsIsVectorRightShift(ins)); // xx xxiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsizeWithImmediate_i1_tsz(id->idInsOpt(), imm); // ixxxx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm4_19_to_16(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; - default: assert(!"Unexpected format"); break; @@ -26883,17 +23725,11 @@ void emitter::emitDispSveExtendOpts(insOpts opt) { switch (opt) { - case INS_OPTS_LSL: - printf("lsl"); - break; - - case INS_OPTS_UXTW: case INS_OPTS_SCALABLE_S_UXTW: case INS_OPTS_SCALABLE_D_UXTW: printf("uxtw"); break; - case INS_OPTS_SXTW: case INS_OPTS_SCALABLE_S_SXTW: case INS_OPTS_SCALABLE_D_SXTW: printf("sxtw"); @@ -26910,18 +23746,27 @@ void emitter::emitDispSveExtendOpts(insOpts opt) * Prints the encoding for the Extend Type encoding along with the N value */ -void emitter::emitDispSveExtendOptsModN(insOpts opt, ssize_t imm) -{ - assert(imm >= 0 && imm <= 3); +void emitter::emitDispSveExtendOptsModN(insOpts opt, int n) +{ + assert(n >= 0 && n <= 3); + + emitDispSveExtendOpts(opt); + switch (n) + { + case 3: + printf(" #3"); + break; + + case 2: + printf(" #2"); + break; + + case 1: + printf(" #1"); + break; - if (imm == 0 && opt != INS_OPTS_LSL) - { - emitDispSveExtendOpts(opt); - } - else if (imm > 0) - { - emitDispSveExtendOpts(opt); - printf(" #%d", (int)imm); + default: + break; } } @@ -26999,24 +23844,6 @@ void emitter::emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2 printf("]"); } -/***************************************************************************** - * - * Prints the encoding for format [.S{, #}] - */ -void emitter::emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt) -{ - printf("["); - emitDispSveReg(reg1, opt, imm != 0); - if (imm != 0) - { - // This does not have to be printed as hex. - // We only do it because the capstone disassembly displays this immediate as hex. - // We could not modify capstone without affecting other cases. - emitDispImm(imm, false, /* alwaysHex */ true); - } - printf("]"); -} - /***************************************************************************** * * Prints the encoding for format [{, #, MUL VL}] @@ -27091,46 +23918,20 @@ void emitter::emitDispReg(regNumber reg, emitAttr attr, bool addComma) emitDispComma(); } -//------------------------------------------------------------------------ -// emitDispSveReg: Display a scalable vector register name -// -void emitter::emitDispSveReg(regNumber reg, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - - if (addComma) - emitDispComma(); -} - //------------------------------------------------------------------------ // emitDispSveReg: Display a scalable vector register name with an arrangement suffix // void emitter::emitDispSveReg(regNumber reg, insOpts opt, bool addComma) { + assert(insOptsScalable(opt) || insOptsScalable32bitExtends(opt)); assert(isVectorRegister(reg)); printf(emitSveRegName(reg)); - - if (opt != INS_OPTS_NONE) - { - assert(insOptsScalable(opt) || insOptsScalable32bitExtends(opt)); - emitDispArrangement(opt); - } + emitDispArrangement(opt); if (addComma) emitDispComma(); } -//------------------------------------------------------------------------ -// emitDispSveRegIndex: Display a scalable vector register with indexed element -// -void emitter::emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - emitDispElementIndex(index, addComma); -} - //------------------------------------------------------------------------ // emitDispVectorReg: Display a SIMD vector register name with an arrangement suffix // @@ -27612,89 +24413,6 @@ void emitter::emitDispSvePattern(insSvePattern pattern, bool addComma) } } -/***************************************************************************** - * - * Display an insSvePrfop - */ -void emitter::emitDispSvePrfop(insSvePrfop prfop, bool addComma) -{ - switch (prfop) - { - case SVE_PRFOP_PLDL1KEEP: - printf("pldl1keep"); - break; - - case SVE_PRFOP_PLDL1STRM: - printf("pldl1strm"); - break; - - case SVE_PRFOP_PLDL2KEEP: - printf("pldl2keep"); - break; - - case SVE_PRFOP_PLDL2STRM: - printf("pldl2strm"); - break; - - case SVE_PRFOP_PLDL3KEEP: - printf("pldl3keep"); - break; - - case SVE_PRFOP_PLDL3STRM: - printf("pldl3strm"); - break; - - case SVE_PRFOP_PSTL1KEEP: - printf("pstl1keep"); - break; - - case SVE_PRFOP_PSTL1STRM: - printf("pstl1strm"); - break; - - case SVE_PRFOP_PSTL2KEEP: - printf("pstl2keep"); - break; - - case SVE_PRFOP_PSTL2STRM: - printf("pstl2strm"); - break; - - case SVE_PRFOP_PSTL3KEEP: - printf("pstl3keep"); - break; - - case SVE_PRFOP_PSTL3STRM: - printf("pstl3strm"); - break; - - case SVE_PRFOP_CONST6: - printf("#6"); - break; - - case SVE_PRFOP_CONST7: - printf("#7"); - break; - - case SVE_PRFOP_CONST14: - printf("#0xE"); - break; - - case SVE_PRFOP_CONST15: - printf("#0xF"); - break; - - default: - assert(!"Invalid prfop"); - break; - } - - if (addComma) - { - emitDispComma(); - } -} - /***************************************************************************** * * Display (optionally) the instruction encoding in hex @@ -29054,10 +25772,6 @@ void emitter::emitDispInsHelp( // (predicated) case IF_SVE_GR_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 floating-point pairwise operations case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - // .D, /M, .D, .D - case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - // .H, /M, .H, .H - case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispLowPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // ggg emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd @@ -29096,9 +25810,6 @@ void emitter::emitDispInsHelp( // (predicated) case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand // (predicated) - case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend - // ., /Z, ., . - case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispLowPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // ggg emitDispSveReg(id->idReg3(), id->idInsOpt(), true); @@ -29110,134 +25821,14 @@ void emitter::emitDispInsHelp( case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high // (unpredicated) - case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved - case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient - case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - // ., ., . - case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high - case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate - // .Q, .Q, .Q - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - // .D, .D, .D - case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - // .D, .D, .D - case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) - // .B, .B, .B - case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) - case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - // .D, .D, .D - // .S, .S, .S - case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations - // .H, .H, .H - case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn/mmmmm - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm/aaaaa - break; - // .D, .D, .D case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) // .D, .D, .D case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_D, true); // ddddd - emitDispSveReg(id->idReg2(), INS_OPTS_SCALABLE_D, true); // nnnnn - emitDispSveReg(id->idReg3(), INS_OPTS_SCALABLE_D, false); // mmmmm - break; - - // .D, .D, .D, .D - case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // mmmmm - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // kkkkk - break; - - // ., #, # - case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate - // increment) - { - ssize_t imm1; - ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispImm(imm1, true); // iiiii - emitDispImm(imm2, false); // iiiii - break; - } - - // ., #, - case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register - // increment) - { - const emitAttr intRegSize = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? EA_8BYTE : EA_4BYTE; - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispImm(emitGetInsSC(id), true); // iiiii - emitDispReg(id->idReg2(), intRegSize, false); // mmmmm - break; - } - - // ., , # - case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate - // increment) - { - const emitAttr intRegSize = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? EA_8BYTE : EA_4BYTE; - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispReg(id->idReg2(), intRegSize, true); // mmmmm - emitDispImm(emitGetInsSC(id), false); // iiiii - break; - } - - // .H, .B, .B - case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long - case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_H, true); // ddddd - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm - break; - - // ., ., . - // ., {.}, . - case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - if (id->idIns() == INS_sve_tbl) - { - emitDispSveConsecutiveRegList(id->idReg2(), 1, id->idInsOpt(), true); // nnnnn - } - else - { - assert(id->idIns() == INS_sve_tbx); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn - } - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm - break; - - // ., ., . - // ., {.}, . - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - if (id->idIns() == INS_sve_tblq) - { - emitDispSveConsecutiveRegList(id->idReg2(), 1, id->idInsOpt(), true); // nnnnn - } - else - { - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn - } - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm - break; - - // ., {., .}, . - case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveConsecutiveRegList(id->idReg2(), 2, id->idInsOpt(), true); // nnnnn - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn/mmmmm + emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm/aaaaa break; // ., , @@ -29248,108 +25839,17 @@ void emitter::emitDispInsHelp( emitDispReg(id->idReg3(), size, false); // mmmmm break; - // {, {, MUL #}} case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count - // {, {, MUL #}} - case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count imm = emitGetInsSC(id); emitDispReg(id->idReg1(), size, true); // ddddd emitDispSvePattern(id->idSvePattern(), (imm > 1)); // ppppp if (imm > 1) { printf("mul "); - emitDispImm(imm, false, false); // iiii - } - break; - - // .D{, {, MUL #}} - // .H{, {, MUL #}} - // .S{, {, MUL #}} - case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count - case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSvePattern(id->idSvePattern(), (imm > 1)); // ppppp - if (imm > 1) - { - printf("mul "); - emitDispImm(imm, false, false); // iiii - } - break; - - // ., ., # - case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - - FALLTHROUGH; - // ., # - case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - bmi.immNRS = (unsigned)emitGetInsSC(id); - imm = emitDecodeBitMaskImm(bmi, optGetSveElemsize(id->idInsOpt())); - emitDispImm(imm, false); // iiiiiiiiiiiii - break; - - // , {, {, MUL #}} - // {, {, MUL #}} - // {, {, MUL #}} - case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count - switch (id->idIns()) - { - case INS_sve_sqincb: - case INS_sve_sqdecb: - case INS_sve_sqinch: - case INS_sve_sqdech: - case INS_sve_sqincw: - case INS_sve_sqdecw: - case INS_sve_sqincd: - case INS_sve_sqdecd: - emitDispReg(id->idReg1(), EA_8BYTE, true); // ddddd - - if (size == EA_4BYTE) - { - emitDispReg(id->idReg1(), EA_4BYTE, true); - } - break; - - default: - emitDispReg(id->idReg1(), size, true); // ddddd - break; - } - - imm = emitGetInsSC(id); - emitDispSvePattern(id->idSvePattern(), (imm > 1)); // ppppp - if (imm > 1) - { - printf("mul "); - emitDispImm(imm, false, false); // iiii + emitDispImm(emitGetInsSC(id), false, false); // iiii } break; - // .B, {.B, .B }, # - case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispVectorRegList(id->idReg2(), 2, id->idInsOpt(), true); // nnnnn - emitDispImm(imm, false); // iiiii iii - break; - - // .B, .B, .B, # - case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // mmmmm - emitDispImm(imm, false); // iiiii iii - break; - - // ., /M, # - case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(id->idInsFmt()), INS_OPTS_NONE, true); // gggg - emitDispFloatImm(emitGetInsSC(id)); // iiiiiiii - break; - // ., ., .D case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd @@ -29357,48 +25857,6 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg3(), INS_OPTS_SCALABLE_D, false); // mmmmm break; - // ., [., .{, }] - case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - printf("["); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); - emitDispSveReg(id->idReg3(), id->idInsOpt(), emitGetInsSC(id) > 0); - emitDispSveExtendOptsModN(INS_OPTS_LSL, emitGetInsSC(id)); - printf("]"); - break; - - // .D, [.D, .D, SXTW{ }] - case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - printf("["); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); - emitDispSveReg(id->idReg3(), id->idInsOpt(), true); - emitDispSveExtendOptsModN(INS_OPTS_SXTW, emitGetInsSC(id)); - printf("]"); - break; - - // .D, [.D, .D, UXTW{ }] - case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - printf("["); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); - emitDispSveReg(id->idReg3(), id->idInsOpt(), true); - emitDispSveExtendOptsModN(INS_OPTS_UXTW, emitGetInsSC(id)); - printf("]"); - break; - - // ., - case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispReg(id->idReg2(), optGetSveElemsize(id->idInsOpt()), false); // mmmmm - break; - - // ., - case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispReg(id->idReg2(), id->idInsOpt() == INS_OPTS_SCALABLE_D ? EA_8BYTE : EA_4BYTE, false); // mmmmm - break; - // .H, .B case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_H, true); // DDDD @@ -29465,39 +25923,6 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm break; - case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_B, true); // DDDD - emitDispSveReg(id->idReg2(), false); // nnnnn - break; - case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_D, true); // DDDD - emitDispSveRegIndex(id->idReg2(), emitGetInsSC(id), false); // nnnnn - break; - case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_H, true); // DDDD - emitDispSveRegIndex(id->idReg2(), emitGetInsSC(id), false); // nnnnn - break; - case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_S, true); // DDDD - emitDispSveRegIndex(id->idReg2(), emitGetInsSC(id), false); // nnnnn - break; - case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector - emitDispSveReg(id->idReg1(), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_B, false); // NNNN - break; - case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector - emitDispSveRegIndex(id->idReg1(), emitGetInsSC(id), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_D, false); // NNNN - break; - case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector - emitDispSveRegIndex(id->idReg1(), emitGetInsSC(id), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_H, false); // NNNN - break; - case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector - emitDispSveRegIndex(id->idReg1(), emitGetInsSC(id), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_S, false); // NNNN - break; - // ., ., . case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt, 1), id->idInsOpt(), true); // DDDD @@ -29549,25 +25974,6 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm break; - // MOV ., /M, . or SEL ., , ., . - case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) - { - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - - if (id->idIns() == INS_sve_mov) - { - emitDispPredicateReg(id->idReg2(), PREDICATE_MERGE, id->idInsOpt(), true); // VVVV - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // nnnnn - } - else - { - emitDispPredicateReg(id->idReg2(), PREDICATE_NONE, id->idInsOpt(), true); // VVVV - emitDispSveReg(id->idReg3(), id->idInsOpt(), true); // nnnnn - emitDispSveReg(id->idReg4(), id->idInsOpt(), false); // mmmmm - } - break; - } - // ., /Z, ., . case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match @@ -29619,25 +26025,6 @@ void emitter::emitDispInsHelp( emitDispElementIndex(emitGetInsSC(id), false); // ii/iii break; - // .S, .H, .H - case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product - case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long - case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product - case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long - emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_S, true); // ddddd - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm - break; - - // .S, .B, .B - case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_S, true); // ddddd - emitDispSveReg(id->idReg2(), INS_OPTS_SCALABLE_B, true); // nnnnn - emitDispSveReg(id->idReg3(), INS_OPTS_SCALABLE_B, false); // mmmmm - break; - // .D, .S, .S[] case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) @@ -29903,23 +26290,6 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg2(), optWidenSveElemsizeArrangement(id->idInsOpt()), false); // nnnnn break; - // , , # - case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment - { - const regNumber reg1 = (id->idReg1() == REG_ZR) ? REG_SP : id->idReg1(); - const regNumber reg2 = (id->idReg2() == REG_ZR) ? REG_SP : id->idReg2(); - emitDispReg(reg1, id->idOpSize(), true); // ddddd - emitDispReg(reg2, id->idOpSize(), true); // nnnnn - emitDispImm(emitGetInsSC(id), false); // iiiiii - break; - } - - // , # - case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size - emitDispReg(id->idReg1(), id->idOpSize(), true); // ddddd - emitDispImm(emitGetInsSC(id), false); // iiiiii - break; - // ., ., # case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long { @@ -30033,18 +26403,8 @@ void emitter::emitDispInsHelp( // FMOV ., # case IF_SVE_EA_1A: // ........xx...... ...iiiiiiiiddddd -- SVE broadcast floating-point immediate (unpredicated) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispFloatImm(emitGetInsSC(id)); // iiiiiiii - break; - - // DUP ., #{, } - // MOV ., #{, } - case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) - { - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispImmOptsLSL(imm, id->idHasShift(), 8); // h iiiiiiii + emitDispFloatImm(emitGetInsSC(id)); // iiiiiiii break; - } // ADD ., ., #{, } // SQADD ., ., #{, } @@ -30054,13 +26414,15 @@ void emitter::emitDispInsHelp( // SQSUB ., ., #{, } // UQSUB ., ., #{, } case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) - { - imm = emitGetInsSC(id); emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispImmOptsLSL(imm, id->idHasShift(), 8); // h iiiiiiii + + FALLTHROUGH; + // DUP ., #{, } + // MOV ., #{, } + case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispImmOptsLSL(emitGetInsSC(id), id->idOptionalShift(), 8); // iiiiiiii, h break; - } // FMOV ., #0.0 // (Preferred disassembly: FMOV ., #0) @@ -30081,57 +26443,6 @@ void emitter::emitDispInsHelp( emitDispImm(emitGetInsSC(id), false); // iiiiiiii break; - // ., ., . - case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) - // .S, .B, .B - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate - { - const insOpts smallSizeSpecifier = (insOpts)(id->idInsOpt() - 2); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), smallSizeSpecifier, true); // nnnnn - emitDispSveReg(id->idReg3(), smallSizeSpecifier, false); // mmmmm - break; - } - - // ., ., . - case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long - case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long - case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long - case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long - // ., ., . - case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long - case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long - // .Q, .D, .D - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - { - const insOpts smallSizeSpecifier = (insOpts)(id->idInsOpt() - 1); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), smallSizeSpecifier, true); // nnnnn - emitDispSveReg(id->idReg3(), smallSizeSpecifier, false); // mmmmm - break; - } - - // ., ., . - case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part - { - const insOpts largeSizeSpecifier = (insOpts)(id->idInsOpt() + 1); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), largeSizeSpecifier, true); // nnnnn - emitDispSveReg(id->idReg3(), largeSizeSpecifier, false); // mmmmm - break; - } - - // ., ., . - case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide - { - const insOpts smallSizeSpecifier = (insOpts)(id->idInsOpt() - 1); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn - emitDispSveReg(id->idReg3(), smallSizeSpecifier, false); // mmmmm - break; - } - // CDOT ., ., ., case IF_SVE_EJ_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer dot product { @@ -30216,18 +26527,7 @@ void emitter::emitDispInsHelp( // .S, /M, .D // .D, /M, .S // .S, /M, .H - // .D, /M, .D - // .S, /M, .S - // .D, /M, .H - // .H, /M, .H - // .H, /M, .D - // .H, /M, .S case IF_SVE_GQ_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision odd elements - case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - case IF_SVE_HO_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - case IF_SVE_HO_3C: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer - case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point { insOpts opt = id->idInsOpt(); @@ -30240,12 +26540,6 @@ void emitter::emitDispInsHelp( case INS_sve_bfcvtnt: opt = INS_OPTS_S_TO_H; break; - case INS_sve_fcvtx: - opt = INS_OPTS_D_TO_S; - break; - case INS_sve_bfcvt: - opt = INS_OPTS_S_TO_H; - break; default: break; } @@ -30663,7 +26957,6 @@ void emitter::emitDispInsHelp( // ., ., ., # case IF_SVE_HN_2A: // ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient - case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm emitDispSveReg(id->idReg1(), id->idInsOpt(), true); emitDispSveReg(id->idReg1(), id->idInsOpt(), true); emitDispSveReg(id->idReg2(), id->idInsOpt(), true); @@ -30688,73 +26981,6 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg4(), id->idInsOpt(), false); break; - // .B, { .B }, [] - case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - // .B, { .B }, [] - case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - // .H, { .H }, [] - case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - // .H, { .H, .H }, [] - case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - // .H, {.H }, [] - case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveConsecutiveRegList(id->idReg1(), 1, id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), false); - emitDispElementIndex(imm, false); - break; - - // , , [, .S, ] - // , , [, .S, #1] - // , , [, .S, #2] - // , , [, .S, #3] - case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - // , , [, .D, ] - // , , [, .D, #1] - // , , [, .D, #2] - // , , [, .D, #3] - case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit - // scaled offsets) - // , , [, .D] - // , , [, .D, LSL #1] - // , , [, .D, LSL #2] - // , , [, .D, LSL #3] - case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - // , , [, ] - // , , [, , LSL #1] - // , , [, , LSL #2] - // , , [, , LSL #3] - case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) - emitDispSvePrfop(id->idSvePrfop(), true); - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), id->idInsOpt(), true); - emitDispSveModAddr(ins, id->idReg2(), id->idReg3(), id->idInsOpt(), fmt); - break; - - // , , [.S{, #}] - // , , [.D{, #}] - case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) - imm = emitGetInsSC(id); - emitDispSvePrfop(id->idSvePrfop(), true); - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), id->idInsOpt(), true); - emitDispSveImm(id->idReg2(), imm, id->idInsOpt()); - break; - - // , , [{, #, MUL VL}] - case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) - imm = emitGetInsSC(id); - emitDispSvePrfop(id->idSvePrfop(), true); - emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), id->idInsOpt(), true); - emitDispSveImmMulVl(id->idReg2(), imm); - break; - // {.S }, /Z, [.S{, #}] // {.D }, /Z, [.D{, #}] case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) @@ -30782,101 +27008,6 @@ void emitter::emitDispInsHelp( emitDispSveImmIndex(id->idReg3(), id->idInsOpt(), imm); break; - // , - case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), false); - break; - - // ., - case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispReg(encodingZRtoSP(id->idReg2()), size, false); - break; - - // .H, .B - case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts - // ., . - case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), (insOpts)((unsigned)id->idInsOpt() - 1), false); - break; - - // ., . - case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator - // ., . - case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements - // ., . - case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), false); - break; - - // ., ., # - case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) - // ., ., # - case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert - // ., ., # - case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); - emitDispImm(imm, false); - break; - - // ., /Z, #{, } - // ., /M, #{, } - case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - { - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // gggg - emitDispImmOptsLSL(imm, id->idHasShift(), 8); // iiiiiiii, h - break; - } - - // ., /M, # - case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // gggg - emitDispImm(0, false); - break; - - // ., .[] - // ., - case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - if (imm > 0) - { - emitDispSveReg(id->idReg2(), id->idInsOpt(), false); // nnnnn - emitDispElementIndex(imm, false); - } - else - { - assert(imm == 0); - emitDispReg(id->idReg2(), optGetSveElemsize(id->idInsOpt()), false); - } - break; - - // ., .[] - case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), false); - emitDispElementIndex(imm, false); - break; - - // .B, .B, .B, # - case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq - imm = emitGetInsSC(id); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg1(), id->idInsOpt(), true); - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); - emitDispImm(imm, false); - break; - default: printf("unexpected format %s", emitIfName(id->idInsFmt())); assert(!"unexpectedFormat"); @@ -33193,14 +29324,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_FK_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) case IF_SVE_FK_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) case IF_SVE_FK_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high result.insThroughput = PERFSCORE_THROUGHPUT_2X; result.insLatency = PERFSCORE_LATENCY_5C; break; case IF_SVE_GU_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) case IF_SVE_GU_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_4C; break; @@ -33229,216 +29358,40 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } break; - case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - switch (ins) - { - case INS_sve_fdot: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_bfdot: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_4C; - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long - switch (ins) - { - case INS_sve_fmlalb: - case INS_sve_fmlalt: - case INS_sve_fmlslb: - case INS_sve_fmlslt: - case INS_sve_bfmlalb: - case INS_sve_bfmlalt: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_4C; - break; - case INS_sve_bfmlslb: - case INS_sve_bfmlslt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations - switch (ins) - { - case INS_sve_eor3: - case INS_sve_bcax: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - case INS_sve_bsl: - case INS_sve_bsl1n: - case INS_sve_bsl2n: - case INS_sve_nbsl: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) - case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long - case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix result.insLatency = PERFSCORE_LATENCY_1C; // need to fix break; - case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long - case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide - case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved - case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long - case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part - case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) - case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) - case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) - case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment - case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_2C; break; - case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute - result.insThroughput = PERFSCORE_THROUGHPUT_2X; - result.insLatency = PERFSCORE_LATENCY_6C; - break; - - case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long - switch (ins) - { - case INS_sve_smullb: - case INS_sve_smullt: - case INS_sve_umullb: - case INS_sve_umullt: - case INS_sve_sqdmullb: - case INS_sve_sqdmullt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_4C; - break; - case INS_sve_pmullb: - case INS_sve_pmullt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - case IF_SVE_BA_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE index generation (register start, register // increment) - case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate - // increment) - case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register - // increment) - case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate - // increment) result.insThroughput = PERFSCORE_THROUGHPUT_2X; result.insLatency = PERFSCORE_LATENCY_8C; break; - case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation - case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count - case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count - case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count - case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count - case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count - case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) - case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) - case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) - case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) - case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_2C; break; case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_3C; break; case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm result.insThroughput = PERFSCORE_THROUGHPUT_1C; result.insLatency = PERFSCORE_LATENCY_2C; break; - case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - - case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_3C; - break; - - case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector - case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector - case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector - case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector - case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector - case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector - case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector - case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector - result.insThroughput = PERFSCORE_THROUGHPUT_140C; // @ToDo currently undocumented - result.insLatency = PERFSCORE_LATENCY_140C; - break; - - case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register - case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_5C; - break; - case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements case IF_SVE_CJ_2A: // ........xx...... .......NNNN.DDDD -- SVE reverse predicate elements case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements @@ -33484,11 +29437,6 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insThroughput = PERFSCORE_THROUGHPUT_1C; break; - case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) - result.insLatency = PERFSCORE_LATENCY_2C; - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - break; - case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate @@ -33504,34 +29452,10 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) case IF_SVE_FJ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) case IF_SVE_FJ_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) - case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) - case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long - case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long - case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long - case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate - case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long result.insLatency = PERFSCORE_LATENCY_4C; result.insThroughput = PERFSCORE_THROUGHPUT_1C; break; - case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations - switch (ins) - { - case INS_sve_rax1: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - case INS_sve_sm4ekey: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_4C; - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - case IF_SVE_GZ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE floating-point multiply-add long (indexed) switch (ins) { @@ -33780,9 +29704,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins // Arithmetic, shift complex case IF_SVE_EU_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating/rounding bitwise shift left // (predicated) + result.insLatency = PERFSCORE_LATENCY_4C; + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + break; + // Arithmetic, pairwise add and accum long case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long - case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product result.insLatency = PERFSCORE_LATENCY_4C; result.insThroughput = PERFSCORE_THROUGHPUT_1C; break; @@ -33802,41 +29729,13 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins // Floating point reduction, F64. (Note: Worse for F32 and F16) case IF_SVE_HE_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction result.insLatency = PERFSCORE_LATENCY_2C; - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - break; - - // Floating point associative add, F64. (Note: Worse for F32 and F16) - case IF_SVE_HJ_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point serial reduction (predicated) - result.insLatency = PERFSCORE_LATENCY_4C; - result.insThroughput = PERFSCORE_THROUGHPUT_2X; - break; - - case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - switch (ins) - { - case INS_sve_frecps: - case INS_sve_frsqrts: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_4C; - break; - - case INS_sve_fmul: - case INS_sve_ftsmul: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_3C; - break; - - case INS_sve_fadd: - case INS_sve_fsub: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + break; + + // Floating point associative add, F64. (Note: Worse for F32 and F16) + case IF_SVE_HJ_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point serial reduction (predicated) + result.insLatency = PERFSCORE_LATENCY_4C; + result.insThroughput = PERFSCORE_THROUGHPUT_2X; break; case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) @@ -33885,14 +29784,6 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } break; - case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - case IF_SVE_HO_3B: - case IF_SVE_HO_3C: - case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_3C; - break; - // Floating point round to integral, F64. (Note: Worse for F32 and F16) case IF_SVE_HQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point round to integral value result.insLatency = PERFSCORE_LATENCY_3C; @@ -33921,11 +29812,6 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } break; - case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point - result.insThroughput = PERFSCORE_THROUGHPUT_4X; - result.insLatency = PERFSCORE_LATENCY_6C; - break; - case IF_SVE_DL_2A: // ........xx...... .....l.NNNNddddd -- SVE predicate count (predicate-as-counter) result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_2C; @@ -34725,14 +30611,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins break; case IF_SVE_GP_3A: // ........xx.....r ...gggmmmmmddddd -- SVE floating-point complex add (predicated) - case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_3C; break; case IF_SVE_GV_3A: // ...........immmm ....rrnnnnnddddd -- SVE floating-point complex multiply-add (indexed) case IF_SVE_GT_4A: // ........xx.mmmmm .rrgggnnnnnddddd -- SVE floating-point complex multiply-add (predicated) - case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_5C; break; @@ -34790,7 +30674,6 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_HV_4A: // ........xx.aaaaa ...gggmmmmmddddd -- SVE floating-point multiply-accumulate writing // multiplicand - case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_4C; break; @@ -34807,195 +30690,6 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_2C; break; - case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - - case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - - case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - - case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - - case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - - case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - switch (ins) - { - case INS_sve_prfb: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfh: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfw: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfd: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit - // scaled offsets) - switch (ins) - { - case INS_sve_prfb: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfh: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfw: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfd: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - switch (ins) - { - case INS_sve_prfb: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfh: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfw: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfd: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) - switch (ins) - { - case INS_sve_prfb: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfh: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfw: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfd: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) - switch (ins) - { - case INS_sve_prfb: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfh: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfw: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfd: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) - switch (ins) - { - case INS_sve_prfb: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfh: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfw: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_prfd: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) case IF_SVE_HX_3A_E: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) case IF_SVE_IV_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit gather load (vector plus immediate) @@ -35017,121 +30711,6 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_6C; break; - case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - - case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts - switch (ins) - { - case INS_sve_f1cvt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_f2cvt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_bf1cvt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_bf2cvt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_f1cvtlt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_f2cvtlt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_bf1cvtlt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - case INS_sve_bf2cvtlt: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_3C; - break; - - case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register - switch (ins) - { - case INS_sve_mov: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - case INS_sve_dup: - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_3C; - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements - switch (ins) - { - case INS_sve_rev: - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - default: - // all other instructions - perfScoreUnhandledInstruction(id, &result); - break; - } - break; - - case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements - result.insThroughput = PERFSCORE_THROUGHPUT_2C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - - case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_3C; - break; - - case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) - case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_2C; - break; - - case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - result.insLatency = PERFSCORE_LATENCY_4C; - break; - - case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - - case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq - result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix - result.insLatency = PERFSCORE_LATENCY_1C; // need to fix - break; - default: // all other instructions perfScoreUnhandledInstruction(id, &result); diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index bf603155b36fcc..cd4b3bc9738198 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -51,16 +51,13 @@ void emitDispBarrier(insBarrier barrier); void emitDispShiftOpts(insOpts opt); void emitDispExtendOpts(insOpts opt); void emitDispSveExtendOpts(insOpts opt); -void emitDispSveExtendOptsModN(insOpts opt, ssize_t imm); +void emitDispSveExtendOptsModN(insOpts opt, int n); void emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts opt, insFormat fmt); -void emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt); void emitDispSveImmMulVl(regNumber reg1, ssize_t imm); void emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm); void emitDispLSExtendOpts(insOpts opt); void emitDispReg(regNumber reg, emitAttr attr, bool addComma); -void emitDispSveReg(regNumber reg, bool addComma); void emitDispSveReg(regNumber reg, insOpts opt, bool addComma); -void emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma); void emitDispVectorReg(regNumber reg, insOpts opt, bool addComma); void emitDispVectorRegIndex(regNumber reg, emitAttr elemsize, ssize_t index, bool addComma); void emitDispVectorRegList(regNumber firstReg, unsigned listSize, insOpts opt, bool addComma); @@ -78,7 +75,6 @@ void emitDispExtendReg(regNumber reg, insOpts opt, ssize_t imm); void emitDispAddrRI(regNumber reg, insOpts opt, ssize_t imm); void emitDispAddrRRExt(regNumber reg1, regNumber reg2, insOpts opt, bool isScaled, emitAttr size); void emitDispSvePattern(insSvePattern pattern, bool addComma); -void emitDispSvePrfop(insSvePrfop prfop, bool addComma); /************************************************************************/ /* Private members that deal with target-dependent instr. descriptors */ @@ -468,9 +464,6 @@ static code_t insEncodeVectorIndex(emitAttr elemsize, ssize_t index); // Returns the encoding to select 'index2' for an Arm64 'ins' elem instruction static code_t insEncodeVectorIndex2(emitAttr elemsize, ssize_t index2); -// Returns the encoding for an immediate in the SVE variant of dup (indexed) -static code_t insEncodeSveBroadcastIndex(emitAttr elemsize, ssize_t index); - // Returns the encoding to select 'index' for an Arm64 'mul' elem instruction static code_t insEncodeVectorIndexLMH(emitAttr elemsize, ssize_t index); @@ -533,17 +526,13 @@ static code_t insEncodeSveElemsize_22_to_21(emitAttr size); // This specifically encodes the size at bit locations '18-17'. static code_t insEncodeSveElemsize_18_to_17(emitAttr size); -// Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction -// This specifically encodes the field 'sz' at bit location '20'. -static code_t insEncodeSveElemsize_sz_20(emitAttr size); - // Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction // This specifically encodes the field 'sz' at bit location '21'. static code_t insEncodeSveElemsize_sz_21(emitAttr size); // Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 SVE vector instruction -// This specifically encodes the field 'tszh:tszl' at bit locations '23-22:20-19'. -static code_t insEncodeSveElemsize_tszh_23_tszl_20_to_19(emitAttr size); +// This specifically encodes the field 'tszh:tszl' at bit locations '22:20-19'. +static code_t insEncodeSveElemsize_tszh_22_tszl_20_to_19(emitAttr size); // Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction at bit location '30' or // '21'. @@ -553,12 +542,6 @@ static code_t insEncodeSveElemsize_30_or_21(insFormat fmt, emitAttr size); // Returns the encoding for the field 'i1:tszh:tszl' at bit locations '23-22:20-18'. static code_t insEncodeSveElemsize_tszh_tszl_and_imm(const insOpts opt, const ssize_t imm); -// Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. -static code_t insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, ssize_t imm, bool isRightShift); - -// Returns the encoding for the field 'i1:tsz' at bit locations '20:19-16'. -static code_t insEncodeSveElemsizeWithImmediate_i1_tsz(const insOpts opt, ssize_t imm); - // Returns the encoding to select the constant values 90 or 270 for an Arm64 SVE vector instruction // This specifically encode the field 'rot' at bit location '16'. static code_t insEncodeSveImm90_or_270_rot(ssize_t imm); @@ -606,58 +589,6 @@ static code_t insEncodeSveElemsize_dtype_ld1w(instruction ins, insFormat fmt, em // for the 'dtypeh' and 'dtypel' fields. static code_t insEncodeSveElemsize_dtypeh_dtypel(instruction ins, insFormat fmt, emitAttr size, code_t code); -// Encodes an immediate value in consecutive bits from most significant position 'hi' to least significant -// position 'lo'. -template -static code_t insEncodeUimm(size_t imm) -{ - // lo <= hi < 32 - static_assert((hi >= lo) && (hi < sizeof(code_t) * BITS_PER_BYTE)); - - const size_t imm_bits = hi - lo + 1; - static_assert(imm_bits < sizeof(code_t) * BITS_PER_BYTE); - - const size_t imm_max = 1 << imm_bits; - assert(imm < imm_max); - - code_t result = static_cast(imm << lo); - assert((result >> lo) == imm); - return result; -} - -// Encodes an immediate value across two ranges of consecutive bits, splitting the bits of the immediate -// value between them. The bit ranges are from hi1-lo1, and hi2-lo2 where the second range is at a less -// significant position relative to the first. -template -static code_t insEncodeSplitUimm(size_t imm) -{ - static_assert((hi1 >= lo1) && (lo1 > hi2) && (hi2 >= lo2)); - static_assert(hi1 < sizeof(code_t) * BITS_PER_BYTE); - - const size_t hi_bits = hi1 - lo1 + 1; - const size_t lo_bits = hi2 - lo2 + 1; - - const size_t imm_max = 1 << (hi_bits + lo_bits); - assert(imm < imm_max); - - const size_t hi_max = 1 << hi_bits; - const size_t lo_max = 1 << lo_bits; - - size_t immhi = (imm >> lo_bits) & (hi_max - 1); - size_t immlo = imm & (lo_max - 1); - - code_t result = insEncodeUimm(immhi) | insEncodeUimm(immlo); - - // Calculate and generate a mask for the number of bits between hi2-lo1, and assert that these bits - // are not set in the result. Note if between_bits == 0 then the mask will always be 0 and this will - // pass. - size_t between_bits = lo1 - hi2 - 1; - code_t between_mask = ((1 << between_bits) - 1) << (hi2 + 1); - assert((result & between_mask) == 0); - - return result; -} - // Returns the encoding for the immediate value as 4-bits at bit locations '19-16'. static code_t insEncodeSimm4_19_to_16(ssize_t imm); @@ -688,12 +619,6 @@ static code_t insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm); // Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. static code_t insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm); -// Returns the encoding for the immediate value as 6-bits at bit locations '10-5'. -static code_t insEncodeSimm6_10_to_5(ssize_t imm); - -// Returns the encoding for the immediate value as 6-bits at bit locations '21-16'. -static code_t insEncodeSimm6_21_to_16(ssize_t imm); - // Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. static code_t insEncodeUimm6_MultipleOf2_21_to_16(ssize_t imm); @@ -703,9 +628,6 @@ static code_t insEncodeUimm6_MultipleOf4_21_to_16(ssize_t imm); // Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. static code_t insEncodeUimm6_MultipleOf8_21_to_16(ssize_t imm); -// Returns the encoding for the immediate value as 5-bits at bit locations '9-5'. -static code_t insEncodeSimm5_9_to_5(ssize_t imm); - // Returns the encoding for the immediate value as 5-bits at bit locations '20-16'. static code_t insEncodeSimm5_20_to_16(ssize_t imm); @@ -718,15 +640,6 @@ static code_t insEncodeUimm2_11_to_10(ssize_t imm); // Returns the encoding for the immediate value as 2-bits at bit locations '20-19'. static code_t insEncodeUimm2_20_to_19(ssize_t imm); -// Returns the encoding for the immediate value as 2-bits at bit locations '23-22'. -static code_t insEncodeUimm2_23_to_22(ssize_t imm); - -// Returns the encoding for the immediate value as 1-bit at bit locations '23'. -static code_t insEncodeUimm1_23(ssize_t imm); - -// Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. -static code_t insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm); - // Returns the encoding for the immediate value as 1 bit at bit location '10'. static code_t insEncodeImm1_10(ssize_t imm); @@ -739,9 +652,6 @@ static code_t insEncodeImm1_22(ssize_t imm); // Returns the encoding for the immediate value as 7-bits at bit locations '20-14'. static code_t insEncodeUimm7_20_to_14(ssize_t imm); -// Returns the encoding for the immediate value as 4-bits at bit locations '19-16'. -static code_t insEncodeUimm4_19_to_16(ssize_t imm); - // Returns the encoding for the immediate value as 4-bits starting from 1, at bit locations '19-16'. static code_t insEncodeUimm4From1_19_to_16(ssize_t imm); @@ -754,9 +664,6 @@ static code_t insEncodeUimm6_21_to_16(ssize_t imm); // Returns the encoding for the immediate value as 8-bits at bit locations '12-5'. static code_t insEncodeImm8_12_to_5(ssize_t imm); -// Returns the encoding for the unsigned immediate value as 3-bits at bit locations '12-10'. -static code_t insEncodeUimm3_12_to_10(ssize_t imm); - // Returns the encoding for the unsigned immediate value as 3-bits at bit locations '18-16'. static code_t insEncodeUimm3_18_to_16(ssize_t imm); @@ -771,12 +678,6 @@ static code_t insEncodeSveElemsize_R_22(emitAttr size); // Returns the immediate value for instructions that encode it as a difference from tszh:tszl:imm3. static ssize_t insGetImmDiff(const ssize_t imm, const insOpts opt); -// Returns the two 5-bit signed immediates encoded as one ssize_t. -static ssize_t insEncodeTwoSimm5(ssize_t imm1, ssize_t imm2); - -// Decodes imm into two 5-bit signed immediates, using the encoding format from insEncodeTwoSimm5. -static void insDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2); - // Returns the encoding to select an insSvePattern static code_t insEncodeSvePattern(insSvePattern pattern); @@ -792,173 +693,204 @@ static bool isStackRegister(regNumber reg) return (reg == REG_ZR) || (reg == REG_FP); } // ZR (R31) encodes the SP register -// Returns true if 'value' is a legal unsigned immediate with 'bits' number of bits. -template -static bool isValidUimm(ssize_t value) +// Returns true if 'value' is a legal signed immediate 4 bit encoding (such as for LDNF1SW). +static bool isValidSimm4(ssize_t value) { - constexpr size_t max = 1 << bits; - return (0 <= value) && (value < max); -} + return (-8 <= value) && (value <= 7); +}; -// Returns true if 'value' is a legal unsigned immediate with 'bits' number of bits, starting from 1. -template -static bool isValidUimmFrom1(ssize_t value) +// Returns true if 'value' is a legal signed immediate 9 bit encoding (such as for LDR). +static bool isValidSimm9(ssize_t value) { - return isValidUimm(value - 1); -} + return (-0x100 <= value) && (value <= 0xFF); +}; -// Returns true if 'value' is a legal unsigned multiple of 'mod' immediate with 'bits' number of bits. -template -static bool isValidUimm_MultipleOf(ssize_t value) +// Returns true if 'value' is a legal signed multiple of 2 immediate 4 bit encoding (such as for LD2Q). +static bool isValidSimm4_MultipleOf2(ssize_t value) { - static_assert(mod != 0); - return isValidUimm(value / mod) && (value % mod == 0); -} + return (-16 <= value) && (value <= 14) && (value % 2 == 0); +}; -// Returns true if 'value' is a legal signed immediate with 'bits' number of bits. -template -static bool isValidSimm(ssize_t value) +// Returns true if 'value' is a legal signed multiple of 3 immediate 4 bit encoding (such as for LD3Q). +static bool isValidSimm4_MultipleOf3(ssize_t value) { - constexpr ssize_t max = 1 << (bits - 1); - return (-max <= value) && (value < max); -} + return (-24 <= value) && (value <= 21) && (value % 3 == 0); +}; -// Returns true if 'value' is a legal signed multiple of 'mod' immediate with 'bits' number of bits. -template -static bool isValidSimm_MultipleOf(ssize_t value) +// Returns true if 'value' is a legal signed multiple of 4 immediate 4 bit encoding (such as for LD4Q). +static bool isValidSimm4_MultipleOf4(ssize_t value) { - static_assert(mod != 0); - return isValidSimm(value / mod) && (value % mod == 0); -} + return (-32 <= value) && (value <= 28) && (value % 4 == 0); +}; -// Returns true if 'imm' is a valid broadcast immediate for some SVE DUP variants -static bool isValidBroadcastImm(ssize_t imm, emitAttr laneSize) +// Returns true if 'value' is a legal signed multiple of 16 immediate 4 bit encoding (such as for LD1RQB). +static bool isValidSimm4_MultipleOf16(ssize_t value) { - // imm fits within 0 <= imm < 2**(7 - (log2(bytes_in_lane) + 1)) - // e.g. for B => imm < 2**6, H => imm < 2**5, ... - ssize_t max = 0; - switch (laneSize) - { - case EA_16BYTE: - max = 4; - break; - case EA_8BYTE: - max = 8; - break; - case EA_4BYTE: - max = 16; - break; - case EA_2BYTE: - max = 32; - break; - case EA_1BYTE: - max = 64; - break; - default: - unreached(); - }; + return (-128 <= value) && (value <= 112) && (value % 16 == 0); +}; - return (imm >= 0) && (imm < max); -} +// Returns true if 'value' is a legal signed multiple of 32 immediate 4 bit encoding (such as for LD1ROB). +static bool isValidSimm4_MultipleOf32(ssize_t value) +{ + return (-256 <= value) && (value <= 224) && (value % 32 == 0); +}; -// Returns true if 'value' is a legal rotation value (such as for CDOT, CMLA). -static bool isValidRot(ssize_t value) +// Returns true if 'value' is a legal signed multiple of 2 immediate 5 bit encoding (such as for LD1H). +static bool isValidUimm5_MultipleOf2(ssize_t value) { - return (value == 0) || (value == 90) || (value == 180) || (value == 270); -} + return (0 <= value) && (value <= 62) && (value % 2 == 0); +}; -// Returns true if 'value' represents a valid 'bitmask immediate' encoding. -static bool isValidImmNRS(size_t value, emitAttr size) +// Returns true if 'value' is a legal signed multiple of 4 immediate 5 bit encoding (such as for LD1W). +static bool isValidUimm5_MultipleOf4(ssize_t value) { - return (value >= 0) && (value < 0x2000); -} // any unsigned 13-bit immediate + return (0 <= value) && (value <= 124) && (value % 4 == 0); +}; -// Returns one of the following patterns, depending on width, where `mn` is imm: -// 0xFFFFFFFFFFFFFFmn, 0xFFFFFFmnFFFFFFmn, 0xFFmnFFmnFFmnFFmn, -// 0xFFFFFFFFFFFFmnFF, 0xFFFFmnFFFFFFmnFF, 0xmnFFmnFFmnFFmnFF, -// 0xmnmnmnmnmnmnmnmn -static ssize_t getBitMaskOnes(const ssize_t imm, const unsigned width) +// Returns true if 'value' is a legal signed multiple of 8 immediate 5 bit encoding (such as for LD1D). +static bool isValidUimm5_MultipleOf8(ssize_t value) { - assert(isValidUimm<16>(imm)); - assert((width % 8) == 0); - assert(isValidGeneralLSDatasize((emitAttr)(width / 8))); - const unsigned immWidth = isValidUimm<8>(imm) ? 8 : 16; + return (0 <= value) && (value <= 248) && (value % 8 == 0); +}; - const unsigned numIterations = 64 / width; - const ssize_t ones = ((UINT64)-1) >> (64 - width + immWidth); - ssize_t mask = 0; +// Returns true if 'value' is a legal signed multiple of 2 immediate 6 bit encoding (such as for LD1RH). +static bool isValidUimm6_MultipleOf2(ssize_t value) +{ + return (0 <= value) && (value <= 126) && (value % 2 == 0); +}; - for (unsigned i = 0; i < numIterations; i++) - { - mask <<= width; - mask |= (ones << immWidth) | imm; - } +// Returns true if 'value' is a legal signed multiple of 4 immediate 6 bit encoding (such as for LD1RSW). +static bool isValidUimm6_MultipleOf4(ssize_t value) +{ + return (0 <= value) && (value <= 252) && (value % 4 == 0); +}; - return mask; -} +// Returns true if 'value' is a legal signed multiple of 8 immediate 6 bit encoding (such as for LD1RD). +static bool isValidUimm6_MultipleOf8(ssize_t value) +{ + return (0 <= value) && (value <= 504) && (value % 8 == 0); +}; -// Returns one of the following patterns, depending on width, where `mn` is imm: -// 0x00000000000000mn, 0x000000mn000000mn, 0x00mn00mn00mn00mn, -// 0x000000000000mn00, 0x0000mn000000mn00, 0xmn00mn00mn00mn00, -// 0xmnmnmnmnmnmnmnmn -static ssize_t getBitMaskZeroes(const ssize_t imm, const unsigned width) +// Returns true if 'value' is a legal immediate 1 bit encoding (such as for PEXT). +static bool isValidImm1(ssize_t value) { - assert(isValidUimm<16>(imm)); - assert((width % 8) == 0); - assert(isValidGeneralLSDatasize((emitAttr)(width / 8))); - const unsigned numIterations = 64 / width; - ssize_t mask = 0; + return (value == 0) || (value == 1); +}; - for (unsigned i = 0; i < numIterations; i++) - { - mask <<= width; - mask |= imm; - } +// Returns true if 'value' is a legal unsigned immediate 2 bit encoding (such as for PEXT). +static bool isValidUimm2(ssize_t value) +{ + return (0 <= value) || (value <= 3); +}; + +// Returns true if 'value' is a legal unsigned immediate 3 bit encoding. +static bool isValidUimm3(ssize_t value) +{ + return (0 <= value) && (value <= 7); +}; + +// Returns true if 'value' is a legal unsigned immediate 3 bit encoding, starting from 1 (such as for SHRNB). +static bool isValidUimm3From1(ssize_t value) +{ + return (1 <= value) && (value <= 8); +}; + +// Returns true if 'value' is a legal unsigned immediate 4 bit encoding. +static bool isValidUimm4(ssize_t value) +{ + return (0 <= value) && (value <= 0xF); +}; + +// Returns true if 'value' is a legal unsigned immediate 4 bit encoding, starting from 1 (such as for CNTB). +static bool isValidUimm4From1(ssize_t value) +{ + return (1 <= value) && (value <= 0x10); +}; - return mask; +// Returns true if 'value' is a legal unsigned immediate 5 bit encoding (such as for CCMP). +static bool isValidUimm5(ssize_t value) +{ + return (0 <= value) && (value <= 0x1FLL); +}; + +// Returns true if 'value' is a legal unsigned immediate 6 bit encoding (such as for LD1RD). +static bool isValidUimm6(ssize_t value) +{ + return (0 <= value) && (value <= 63); } -// For the IF_SVE_BT_1A encoding, we prefer the DUPM disasm for the following immediate patterns, -// where 'mn' is some nonzero value: -// 0xFFFFFFFFFFFFFFmn, 0x00000000000000mn, 0xFFFFFFFFFFFFmn00, 0x000000000000mn00 -// 0xFFFFFFmnFFFFFFmn, 0x000000mn000000mn, 0xFFFFmn00FFFFmn00, 0x0000mn000000mn00 -// 0xFFmnFFmnFFmnFFmn, 0x00mn00mn00mn00mn, 0xmn00mn00mn00mn00 -// 0xmnmnmnmnmnmnmnmn -// Else, we prefer the MOV disasm. -static bool useMovDisasmForBitMask(const ssize_t value) +// Returns true if 'value' is a legal unsigned immediate 5 bit encoding, starting from 1 (such as for SHRNB). +static bool isValidUimm5From1(ssize_t value) { - ssize_t imm = value & 0xFF; - unsigned minFieldSize; + return (1 <= value) && (value <= 0x20); +}; - if (imm == 0) - { - imm = value & 0xFF00; - minFieldSize = 16; - } - else - { - minFieldSize = 8; - } +// Returns true if 'value' is a legal unsigned immediate 7 bit encoding (such as for CMPLT, CMPNE). +static bool isValidUimm7(ssize_t value) +{ + return (0 <= value) && (value <= 0x7FLL); +}; + +// Returns true if 'value' is a legal unsigned immediate 8 bit encoding (such as for FMOV). +static bool isValidUimm8(ssize_t value) +{ + return (0 <= value) && (value <= 0xFFLL); +}; - assert(isValidUimm<16>(imm)); +// Returns true if 'value' is a legal signed immediate 8 bit encoding (such as for SMAX, SMIN). +static bool isValidSimm8(ssize_t value) +{ + return (-0x80 <= value) && (value <= 0x7F); +}; - // Check for all possible bit field sizes - for (unsigned width = minFieldSize; width <= 64; width <<= 1) - { - if (value == getBitMaskZeroes(imm, width)) - { - return false; - } - - if (value == getBitMaskOnes(imm, width)) - { - return false; - } - } +// Returns true if 'value' is a legal unsigned immediate 12 bit encoding (such as for CMP, CMN). +static bool isValidUimm12(ssize_t value) +{ + return (0 <= value) && (value <= 0xFFFLL); +}; + +// Returns true if 'value' is a legal unsigned immediate 16 bit encoding (such as for MOVZ, MOVN, MOVK). +static bool isValidUimm16(ssize_t value) +{ + return (0 <= value) && (value <= 0xFFFFLL); +}; + +// Returns true if 'value' is a legal signed immediate 26 bit encoding (such as for B or BL). +static bool isValidSimm26(ssize_t value) +{ + return (-0x2000000LL <= value) && (value <= 0x1FFFFFFLL); +}; + +// Returns true if 'value' is a legal signed immediate 19 bit encoding (such as for B.cond, CBNZ, CBZ). +static bool isValidSimm19(ssize_t value) +{ + return (-0x40000LL <= value) && (value <= 0x3FFFFLL); +}; + +// Returns true if 'value' is a legal signed immediate 14 bit encoding (such as for TBNZ, TBZ). +static bool isValidSimm14(ssize_t value) +{ + return (-0x2000LL <= value) && (value <= 0x1FFFLL); +}; + +// Returns true if 'value' is a legal signed immediate 5 bit encoding (such as for CMPLO, CMPHI). +static bool isValidSimm5(ssize_t value) +{ + return (-0x10LL <= value) && (value <= 0xFLL); +}; - return true; +// Returns true if 'value' is a legal rotation value (such as for CDOT, CMLA). +static bool isValidRot(ssize_t value) +{ + return (value == 0) || (value == 90) || (value == 180) || (value == 270); } +// Returns true if 'value' represents a valid 'bitmask immediate' encoding. +static bool isValidImmNRS(size_t value, emitAttr size) +{ + return (value >= 0) && (value < 0x2000); +} // any unsigned 13-bit immediate + // Returns true if 'value' represents a valid 'halfword immediate' encoding. static bool isValidImmHWVal(size_t value, emitAttr size) { @@ -1002,9 +934,6 @@ static emitAttr optGetDatasize(insOpts arrangement); // For the given 'arrangement' returns the 'elemsize' specified by the vector register arrangement static emitAttr optGetElemsize(insOpts arrangement); -// For the given 'elemsize' returns the 'arrangement' when used in a SVE vector register arrangement. -static insOpts optGetSveInsOpt(emitAttr elemsize); - // For the given 'arrangement' returns the 'elemsize' specified by the SVE vector register arrangement static emitAttr optGetSveElemsize(insOpts arrangement); @@ -1498,24 +1427,23 @@ void emitIns_R_R_R(instruction ins, insOpts opt = INS_OPTS_NONE, insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); -void emitIns_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt = INS_OPTS_NONE, - emitAttr attrReg2 = EA_UNKNOWN, - insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); - -void emitInsSve_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt = INS_OPTS_NONE, - insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); +void emitIns_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt = INS_OPTS_NONE, + emitAttr attrReg2 = EA_UNKNOWN); + +void emitInsSve_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt = INS_OPTS_NONE, + emitAttr attrReg2 = EA_UNKNOWN); void emitIns_R_R_R_I_I(instruction ins, emitAttr attr, @@ -1573,30 +1501,12 @@ void emitIns_R_R_R_COND(instruction ins, emitAttr attr, regNumber reg1, regNumbe void emitIns_R_R_FLAGS_COND( instruction ins, emitAttr attr, regNumber reg1, regNumber reg2, insCflags flags, insCond cond); -void emitIns_R_I_FLAGS_COND(instruction ins, emitAttr attr, regNumber reg1, ssize_t imm, insCflags flags, insCond cond); +void emitIns_R_I_FLAGS_COND(instruction ins, emitAttr attr, regNumber reg1, int imm, insCflags flags, insCond cond); void emitIns_R_PATTERN( instruction ins, emitAttr attr, regNumber reg1, insOpts opt, insSvePattern pattern = SVE_PATTERN_ALL); -void emitIns_R_PATTERN_I( - instruction ins, emitAttr attr, regNumber reg1, insSvePattern pattern, ssize_t imm, insOpts opt = INS_OPTS_NONE); - -void emitIns_PRFOP_R_R_R(instruction ins, - emitAttr attr, - insSvePrfop prfop, - regNumber reg1, - regNumber reg2, - regNumber reg3, - insOpts opt = INS_OPTS_NONE, - insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); - -void emitIns_PRFOP_R_R_I(instruction ins, - emitAttr attr, - insSvePrfop prfop, - regNumber reg1, - regNumber reg2, - int imm, - insOpts opt = INS_OPTS_NONE); +void emitIns_R_PATTERN_I(instruction ins, emitAttr attr, regNumber reg1, insSvePattern pattern, int imm); void emitIns_BARR(instruction ins, insBarrier barrier); diff --git a/src/coreclr/jit/emitfmtsarm64sve.h b/src/coreclr/jit/emitfmtsarm64sve.h index cd27f567478cf0..13137166b7d72a 100644 --- a/src/coreclr/jit/emitfmtsarm64sve.h +++ b/src/coreclr/jit/emitfmtsarm64sve.h @@ -395,13 +395,18 @@ IF_DEF(SVE_HL_3B, IS_NONE, NONE) // SVE_HL_3B ................ ...gggmmmmmddd IF_DEF(SVE_HM_2A, IS_NONE, NONE) // SVE_HM_2A ........xx...... ...ggg....iddddd -- SVE floating-point arithmetic with immediate (predicated) IF_DEF(SVE_HN_2A, IS_NONE, NONE) // SVE_HN_2A ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient IF_DEF(SVE_HO_3A, IS_NONE, NONE) // SVE_HO_3A ................ ...gggnnnnnddddd -- SVE floating-point convert precision -IF_DEF(SVE_HO_3B, IS_NONE, NONE) // SVE_HO_3B ................ ...gggnnnnnddddd -- -IF_DEF(SVE_HO_3C, IS_NONE, NONE) // SVE_HO_3C ................ ...gggnnnnnddddd -- +IF_DEF(SVE_HO_3A_B, IS_NONE, NONE) // SVE_HO_3A_B ................ ...gggnnnnnddddd -- IF_DEF(SVE_HP_3A, IS_NONE, NONE) // SVE_HP_3A .............xx. ...gggnnnnnddddd -- SVE floating-point convert to integer -IF_DEF(SVE_HP_3B, IS_NONE, NONE) // SVE_HP_3B .............xx. ...gggnnnnnddddd -- SVE floating-point convert to integer +IF_DEF(SVE_HP_3B, IS_NONE, NONE) // SVE_HP_3B ................ ...gggnnnnnddddd -- SVE floating-point convert to integer +IF_DEF(SVE_HP_3B_H, IS_NONE, NONE) // SVE_HP_3B_H ................ ...gggnnnnnddddd -- +IF_DEF(SVE_HP_3B_I, IS_NONE, NONE) // SVE_HP_3B_I ................ ...gggnnnnnddddd -- +IF_DEF(SVE_HP_3B_J, IS_NONE, NONE) // SVE_HP_3B_J ................ ...gggnnnnnddddd -- IF_DEF(SVE_HQ_3A, IS_NONE, NONE) // SVE_HQ_3A ........xx...... ...gggnnnnnddddd -- SVE floating-point round to integral value IF_DEF(SVE_HR_3A, IS_NONE, NONE) // SVE_HR_3A ........xx...... ...gggnnnnnddddd -- SVE floating-point unary operations IF_DEF(SVE_HS_3A, IS_NONE, NONE) // SVE_HS_3A ................ ...gggnnnnnddddd -- SVE integer convert to floating-point +IF_DEF(SVE_HS_3A_H, IS_NONE, NONE) // SVE_HS_3A_H ................ ...gggnnnnnddddd -- +IF_DEF(SVE_HS_3A_I, IS_NONE, NONE) // SVE_HS_3A_I ................ ...gggnnnnnddddd -- +IF_DEF(SVE_HS_3A_J, IS_NONE, NONE) // SVE_HS_3A_J ................ ...gggnnnnnddddd -- IF_DEF(SVE_HT_4A, IS_NONE, NONE) // SVE_HT_4A ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors IF_DEF(SVE_HU_4A, IS_NONE, NONE) // SVE_HU_4A ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend IF_DEF(SVE_HU_4B, IS_NONE, NONE) // SVE_HU_4B ...........mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp index 6aaf00a973c8a7..0ea2546213a7aa 100644 --- a/src/coreclr/jit/emitloongarch64.cpp +++ b/src/coreclr/jit/emitloongarch64.cpp @@ -4,7 +4,7 @@ /*XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XX XX -XX emitloongarch64.cpp XX +XX emitloongarch64.cpp XX XX XX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX @@ -4598,7 +4598,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR int offset = 0; DWORD lsl = 0; - if (addr->OperIs(GT_LEA)) + if (addr->OperGet() == GT_LEA) { offset = addr->AsAddrMode()->Offset(); if (addr->AsAddrMode()->gtScale > 0) @@ -4980,7 +4980,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, { emitIns_R_R_R(ins, attr, dst->GetRegNum(), src1->GetRegNum(), src2->GetRegNum()); } - else if (dst->OperIs(GT_MUL)) + else if (dst->OperGet() == GT_MUL) { if (!needCheckOv) { @@ -5048,14 +5048,10 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, // TODO-LOONGARCH64-CQ: here sign-extend dst when deal with 32bit data is too conservative. if (EA_SIZE(attr) == EA_4BYTE) - { emitIns_R_R_I(INS_slli_w, attr, dst->GetRegNum(), dst->GetRegNum(), 0); - } } else { - assert(dst->OperIs(GT_ADD, GT_SUB)); - regNumber regOp1 = src1->GetRegNum(); regNumber regOp2 = src2->GetRegNum(); regNumber saveOperReg1 = REG_NA; @@ -5068,38 +5064,26 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, assert(REG_R21 != dst->GetRegNum()); assert(REG_RA != dst->GetRegNum()); - if (dst->OperIs(GT_ADD)) + if (dst->GetRegNum() == regOp1) { - saveOperReg1 = (dst->GetRegNum() == regOp1) ? regOp2 : regOp1; + assert(REG_R21 != regOp1); + assert(REG_RA != regOp1); + saveOperReg1 = REG_R21; + saveOperReg2 = regOp2; + emitIns_R_R_R(INS_or, attr, REG_R21, regOp1, REG_R0); } - else + else if (dst->GetRegNum() == regOp2) { - if (dst->GetRegNum() == regOp1) - { - assert(REG_R21 != regOp1); - assert(REG_RA != regOp1); - saveOperReg1 = REG_R21; - emitIns_R_R_R(INS_or, attr, REG_R21, regOp1, REG_R0); - } - else - { - saveOperReg1 = regOp1; - } + assert(REG_R21 != regOp2); + assert(REG_RA != regOp2); + saveOperReg1 = regOp1; + saveOperReg2 = REG_R21; + emitIns_R_R_R(INS_or, attr, REG_R21, regOp2, REG_R0); } - - if ((dst->gtFlags & GTF_UNSIGNED) == 0) + else { - saveOperReg2 = dst->GetSingleTempReg(); - assert((saveOperReg2 != REG_RA) && (saveOperReg2 != REG_R21)); - assert(REG_RA != regOp1); - assert(saveOperReg2 != regOp2); - - ssize_t ui6 = (attr == EA_4BYTE) ? 31 : 63; - if (dst->OperIs(GT_ADD)) - { - emitIns_R_R_I(INS_srli_d, attr, REG_RA, regOp1, ui6); - } - emitIns_R_R_I(INS_srli_d, attr, saveOperReg2, regOp2, ui6); + saveOperReg1 = regOp1; + saveOperReg2 = regOp2; } } @@ -5107,56 +5091,86 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, if (needCheckOv) { - // ADD : A = B + C - // SUB : A = B - C <=> B = A + C - if ((dst->gtFlags & GTF_UNSIGNED) != 0) + if (dst->OperGet() == GT_ADD || dst->OperGet() == GT_SUB) { - // ADD: if A < B, goto overflow - // SUB: if B < A, goto overflow - codeGen->genJumpToThrowHlpBlk_la(SCK_OVERFLOW, INS_bltu, - dst->OperIs(GT_ADD) ? dst->GetRegNum() : saveOperReg1, nullptr, - dst->OperIs(GT_ADD) ? saveOperReg1 : dst->GetRegNum()); - } - else - { - if (dst->OperIs(GT_SUB)) + ssize_t imm; + regNumber tempReg1; + regNumber tempReg2; + // ADD : A = B + C + // SUB : C = A - B + if ((dst->gtFlags & GTF_UNSIGNED) != 0) { - emitIns_R_R_I(INS_srli_d, attr, REG_RA, dst->GetRegNum(), (attr == EA_4BYTE) ? 31 : 63); + // if A < B, goto overflow + if (dst->OperGet() == GT_ADD) + { + tempReg1 = dst->GetRegNum(); + tempReg2 = saveOperReg1; + } + else + { + tempReg1 = saveOperReg1; + tempReg2 = saveOperReg2; + } + codeGen->genJumpToThrowHlpBlk_la(SCK_OVERFLOW, INS_bltu, tempReg1, nullptr, tempReg2); } - - emitIns_R_R_R(INS_xor, attr, REG_RA, REG_RA, saveOperReg2); - if (attr == EA_4BYTE) + else { - emitIns_R_R_I(INS_andi, attr, REG_RA, REG_RA, 1); - emitIns_R_R_I(INS_andi, attr, saveOperReg2, saveOperReg2, 1); - } - // ADD: if (B > 0 && C < 0) || (B < 0 && C > 0), skip overflow - // SUB: if (A > 0 && C < 0) || (A < 0 && C > 0), skip overflow - BasicBlock* tmpLabel1 = codeGen->genCreateTempLabel(); - BasicBlock* tmpLabel2 = codeGen->genCreateTempLabel(); - BasicBlock* tmpLabel3 = codeGen->genCreateTempLabel(); + tempReg1 = REG_RA; + tempReg2 = dst->GetSingleTempReg(); + assert(tempReg1 != tempReg2); + assert(tempReg1 != saveOperReg1); + assert(tempReg2 != saveOperReg2); + + ssize_t ui6 = (attr == EA_4BYTE) ? 31 : 63; + if (dst->OperGet() == GT_ADD) + { + emitIns_R_R_I(INS_srli_d, attr, tempReg1, saveOperReg1, ui6); + } + else + { + emitIns_R_R_I(INS_srli_d, attr, tempReg1, dst->GetRegNum(), ui6); + } + emitIns_R_R_I(INS_srli_d, attr, tempReg2, saveOperReg2, ui6); - emitIns_J_cond_la(INS_bne, tmpLabel1, REG_RA, REG_R0); + emitIns_R_R_R(INS_xor, attr, tempReg1, tempReg1, tempReg2); + if (attr == EA_4BYTE) + { + imm = 1; + emitIns_R_R_I(INS_andi, attr, tempReg1, tempReg1, imm); + emitIns_R_R_I(INS_andi, attr, tempReg2, tempReg2, imm); + } + // if (B > 0 && C < 0) || (B < 0 && C > 0), skip overflow + BasicBlock* tmpLabel = codeGen->genCreateTempLabel(); + BasicBlock* tmpLabel2 = codeGen->genCreateTempLabel(); + BasicBlock* tmpLabel3 = codeGen->genCreateTempLabel(); + + emitIns_J_cond_la(INS_bne, tmpLabel, tempReg1, REG_R0); - emitIns_J_cond_la(INS_bne, tmpLabel3, saveOperReg2, REG_R0); + emitIns_J_cond_la(INS_bne, tmpLabel3, tempReg2, REG_R0); - // ADD: B > 0 and C > 0, if A < B, goto overflow - // SUB: A > 0 and C > 0, if B < A, goto overflow - emitIns_J_cond_la(INS_bge, tmpLabel1, dst->OperIs(GT_ADD) ? dst->GetRegNum() : saveOperReg1, - dst->OperIs(GT_ADD) ? saveOperReg1 : dst->GetRegNum()); + // B > 0 and C > 0, if A < B, goto overflow + emitIns_J_cond_la(INS_bge, tmpLabel, dst->OperGet() == GT_ADD ? dst->GetRegNum() : saveOperReg1, + dst->OperGet() == GT_ADD ? saveOperReg1 : saveOperReg2); - codeGen->genDefineTempLabel(tmpLabel2); + codeGen->genDefineTempLabel(tmpLabel2); - codeGen->genJumpToThrowHlpBlk(EJ_jmp, SCK_OVERFLOW); + codeGen->genJumpToThrowHlpBlk(EJ_jmp, SCK_OVERFLOW); - codeGen->genDefineTempLabel(tmpLabel3); + codeGen->genDefineTempLabel(tmpLabel3); - // ADD: B < 0 and C < 0, if A > B, goto overflow - // SUB: A < 0 and C < 0, if B > A, goto overflow - emitIns_J_cond_la(INS_blt, tmpLabel2, dst->OperIs(GT_ADD) ? saveOperReg1 : dst->GetRegNum(), - dst->OperIs(GT_ADD) ? dst->GetRegNum() : saveOperReg1); + // B < 0 and C < 0, if A > B, goto overflow + emitIns_J_cond_la(INS_blt, tmpLabel2, dst->OperGet() == GT_ADD ? saveOperReg1 : saveOperReg2, + dst->OperGet() == GT_ADD ? dst->GetRegNum() : saveOperReg1); - codeGen->genDefineTempLabel(tmpLabel1); + codeGen->genDefineTempLabel(tmpLabel); + } + } + else + { +#ifdef DEBUG + printf("---------[LOONGARCH64]-NOTE: UnsignedOverflow instruction %d\n", ins); +#endif + assert(!"unimplemented on LOONGARCH yet"); } } } diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp index 9e9f9004170964..6ac9e13293aa85 100644 --- a/src/coreclr/jit/emitriscv64.cpp +++ b/src/coreclr/jit/emitriscv64.cpp @@ -625,8 +625,7 @@ void emitter::emitIns_R_R( assert(isGeneralRegisterOrR0(reg2)); code |= (reg1 & 0x1f) << 7; code |= reg2 << 15; - if (INS_fcvt_d_w != ins && INS_fcvt_d_wu != ins) // fcvt.d.w[u] always produces an exact result - code |= 0x7 << 12; // round according to frm status register + code |= 0x7 << 12; } else if (INS_fcvt_s_d == ins || INS_fcvt_d_s == ins) { @@ -634,8 +633,7 @@ void emitter::emitIns_R_R( assert(isFloatReg(reg2)); code |= (reg1 & 0x1f) << 7; code |= (reg2 & 0x1f) << 15; - if (INS_fcvt_d_s != ins) // fcvt.d.s never rounds - code |= 0x7 << 12; // round according to frm status register + code |= 0x7 << 12; } else { @@ -2120,8 +2118,8 @@ void emitter::emitJumpDistBind() unsigned emitter::emitOutput_Instr(BYTE* dst, code_t code) const { assert(dst != nullptr); - static_assert(sizeof(code_t) == 4, "code_t must be 4 bytes"); - memcpy(dst + writeableOffset, &code, sizeof(code)); + assert(sizeof(code_t) == 4); + memcpy(dst + writeableOffset, &code, sizeof(code_t)); return sizeof(code_t); } @@ -3159,7 +3157,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) instruction ins; size_t sz = 0; - static_assert(REG_NA == static_cast(REG_NA), "REG_NA must fit in an int"); + assert(REG_NA == static_cast(REG_NA)); + assert(writeableOffset == 0); insOpts insOp = id->idInsOpt(); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 1bafb6796d8075..9e80da3bfaf9ab 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -4849,7 +4849,7 @@ void emitter::emitInsLoadInd(instruction ins, emitAttr attr, regNumber dstReg, G GenTree* addr = mem->Addr(); - if (addr->isContained() && addr->OperIs(GT_LCL_ADDR)) + if (addr->OperIs(GT_LCL_ADDR)) { GenTreeLclVarCommon* varNode = addr->AsLclVarCommon(); unsigned offset = varNode->GetLclOffs(); @@ -4899,7 +4899,7 @@ void emitter::emitInsStoreInd(instruction ins, emitAttr attr, GenTreeStoreInd* m data = data->gtGetOp1(); } - if (addr->isContained() && addr->OperIs(GT_LCL_ADDR)) + if (addr->OperIs(GT_LCL_ADDR)) { GenTreeLclVarCommon* varNode = addr->AsLclVarCommon(); unsigned offset = varNode->GetLclOffs(); @@ -5140,18 +5140,18 @@ regNumber emitter::emitInsBinary(instruction ins, emitAttr attr, GenTree* dst, G switch (memBase->OperGet()) { case GT_LCL_ADDR: - if (memBase->isContained()) - { - varNum = memBase->AsLclFld()->GetLclNum(); - offset = memBase->AsLclFld()->GetLclOffs(); + { + assert(memBase->isContained()); + varNum = memBase->AsLclFld()->GetLclNum(); + offset = memBase->AsLclFld()->GetLclOffs(); - // Ensure that all the GenTreeIndir values are set to their defaults. - assert(!memIndir->HasIndex()); - assert(memIndir->Scale() == 1); - assert(memIndir->Offset() == 0); - break; - } - FALLTHROUGH; + // Ensure that all the GenTreeIndir values are set to their defaults. + assert(!memIndir->HasIndex()); + assert(memIndir->Scale() == 1); + assert(memIndir->Offset() == 0); + + break; + } default: // Addressing mode [base + index * scale + offset] { @@ -12475,9 +12475,9 @@ BYTE* emitter::emitOutputAlign(insGroup* ig, instrDesc* id, BYTE* dst) assert(paddingToAdd == paddingNeeded); } } -#endif - emitComp->Metrics.LoopsAligned++; + emitComp->loopsAligned++; +#endif #ifdef DEBUG // Under STRESS_EMITTER, if this is the 'align' before the 'jmp' instruction, @@ -15330,10 +15330,13 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) if (id->idIsCnsReloc()) { - if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && id->idAddr()->iiaSecRel) + if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI)) { - // For section relative, the immediate offset is relocatable and hence need IMAGE_REL_SECREL - emitRecordRelocation((void*)(dst - (unsigned)EA_SIZE(size)), (void*)(size_t)val, IMAGE_REL_SECREL); + if (id->idAddr()->iiaSecRel) + { + // For section relative, the immediate offset is relocatable and hence need IMAGE_REL_SECREL + emitRecordRelocation((void*)(dst - (unsigned)EA_SIZE(size)), (void*)(size_t)val, IMAGE_REL_SECREL); + } } else { diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index 92a6a58f21347f..3800e2ffe89d94 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -30,6 +30,8 @@ void Compiler::fgInit() fgRangeUsedInEdgeWeights = true; fgCalledCount = BB_ZERO_WEIGHT; + fgReturnBlocksComputed = false; + /* Initialize the basic block list */ fgFirstBB = nullptr; @@ -47,7 +49,8 @@ void Compiler::fgInit() fgBBcount = 0; #ifdef DEBUG - fgBBOrder = nullptr; + fgBBcountAtCodegen = 0; + fgBBOrder = nullptr; #endif // DEBUG fgMightHaveNaturalLoops = false; @@ -204,10 +207,17 @@ bool Compiler::fgEnsureFirstBBisScratch() assert(fgFirstBBScratch == nullptr); - BasicBlock* block; + BasicBlock* block = BasicBlock::New(this, BBJ_ALWAYS, fgFirstBB); + block->SetFlags(BBF_NONE_QUIRK); if (fgFirstBB != nullptr) { + // If we have profile data the new block will inherit fgFirstBlock's weight + if (fgFirstBB->hasProfileWeight()) + { + block->inheritWeight(fgFirstBB); + } + // The first block has an implicit ref count which we must // remove. Note the ref count could be greater than one, if // the first block is not scratch and is targeted by a @@ -215,23 +225,14 @@ bool Compiler::fgEnsureFirstBBisScratch() assert(fgFirstBB->bbRefs >= 1); fgFirstBB->bbRefs--; - block = BasicBlock::New(this); - - // If we have profile data the new block will inherit fgFirstBlock's weight - if (fgFirstBB->hasProfileWeight()) - { - block->inheritWeight(fgFirstBB); - } - // The new scratch bb will fall through to the old first bb FlowEdge* const edge = fgAddRefPred(fgFirstBB, block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, edge); + edge->setLikelihood(1.0); fgInsertBBbefore(fgFirstBB, block); } else { noway_assert(fgLastBB == nullptr); - block = BasicBlock::New(this, BBJ_ALWAYS); fgFirstBB = block; fgLastBB = block; } @@ -239,7 +240,7 @@ bool Compiler::fgEnsureFirstBBisScratch() noway_assert(fgLastBB != nullptr); // Set the expected flags - block->SetFlags(BBF_INTERNAL | BBF_IMPORTED | BBF_NONE_QUIRK); + block->SetFlags(BBF_INTERNAL | BBF_IMPORTED); // This new first BB has an implicit ref, and no others. // @@ -357,7 +358,7 @@ void Compiler::fgConvertBBToThrowBB(BasicBlock* block) fgRemoveBlockAsPred(block); // Update jump kind after the scrub. - block->SetKindAndTargetEdge(BBJ_THROW); + block->SetKindAndTarget(BBJ_THROW); block->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY // Any block with a throw is rare @@ -381,26 +382,36 @@ void Compiler::fgChangeSwitchBlock(BasicBlock* oldSwitchBlock, BasicBlock* newSw assert(fgPredsComputed); // Walk the switch's jump table, updating the predecessor for each branch. - BBswtDesc* swtDesc = oldSwitchBlock->GetSwitchTargets(); - - for (unsigned i = 0; i < swtDesc->bbsCount; i++) + for (BasicBlock* const bJump : oldSwitchBlock->SwitchTargets()) { - FlowEdge* succEdge = swtDesc->bbsDstTab[i]; - assert(succEdge != nullptr); + noway_assert(bJump != nullptr); - if (succEdge->getSourceBlock() != oldSwitchBlock) - { - // swtDesc can have duplicate targets, so we may have updated this edge already - // - assert(succEdge->getSourceBlock() == newSwitchBlock); - assert(succEdge->getDupCount() > 1); - } - else + // Note that if there are duplicate branch targets in the switch jump table, + // fgRemoveRefPred()/fgAddRefPred() will do the right thing: the second and + // subsequent duplicates will simply subtract from and add to the duplicate + // count (respectively). + // + // However this does the "wrong" thing with respect to edge profile + // data; the old edge is not returned by fgRemoveRefPred until it has + // a dup count of 0, and the fgAddRefPred only uses the optional + // old edge arg when the new edge is first created. + // + // Remove the old edge [oldSwitchBlock => bJump] + // + assert(bJump->countOfInEdges() > 0); + FlowEdge* const oldEdge = fgRemoveRefPred(bJump, oldSwitchBlock); + + // + // Create the new edge [newSwitchBlock => bJump] + // + FlowEdge* const newEdge = fgAddRefPred(bJump, newSwitchBlock); + + // Handle the profile update, once we get our hands on the old edge. + // + if (oldEdge != nullptr) { - // Redirect edge's source block from oldSwitchBlock to newSwitchBlock, - // and keep successor block's pred list in order - // - fgReplacePred(succEdge, newSwitchBlock); + assert(!newEdge->hasLikelihood()); + newEdge->setLikelihood(oldEdge->getLikelihood()); } } @@ -643,101 +654,91 @@ void Compiler::fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, Bas case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: // This function can be called before import, so we still have BBJ_LEAVE + { assert(block->TargetIs(oldTarget)); - fgRedirectTargetEdge(block, newTarget); + block->SetTarget(newTarget); + FlowEdge* const oldEdge = fgRemoveRefPred(oldTarget, block); + fgAddRefPred(newTarget, block, oldEdge); break; + } case BBJ_COND: + if (block->TrueTargetIs(oldTarget)) { - if (block->FalseEdgeIs(block->GetTrueEdge())) + if (block->FalseTargetIs(oldTarget)) { - // Branch was degenerate, simplify it first - // + // fgRemoveRefPred returns nullptr for BBJ_COND blocks with two flow edges to target fgRemoveConditionalJump(block); assert(block->KindIs(BBJ_ALWAYS)); assert(block->TargetIs(oldTarget)); - fgRedirectTargetEdge(block, newTarget); + block->SetTarget(newTarget); } else { - fgRedirectTrueEdge(block, newTarget); + block->SetTrueTarget(newTarget); + } + + // fgRemoveRefPred should have removed the flow edge + FlowEdge* oldEdge = fgRemoveRefPred(oldTarget, block); + assert(oldEdge != nullptr); + + // TODO-NoFallThrough: Proliferate weight from oldEdge + // (as a quirk, we avoid doing so for the true target to reduce diffs for now) + FlowEdge* const newEdge = fgAddRefPred(newTarget, block); + if (block->KindIs(BBJ_ALWAYS)) + { + newEdge->setLikelihood(1.0); + } + else if (oldEdge->hasLikelihood()) + { + newEdge->setLikelihood(oldEdge->getLikelihood()); } } else { - // Already degenerate cases should have taken the true path above - // assert(block->FalseTargetIs(oldTarget)); - assert(!block->TrueEdgeIs(block->GetFalseEdge())); - fgRedirectFalseEdge(block, newTarget); - } - if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) - { - // Block became degenerate, simplify - // - fgRemoveConditionalJump(block); - assert(block->KindIs(BBJ_ALWAYS)); - assert(block->TargetIs(newTarget)); + // fgRemoveRefPred should have removed the flow edge + FlowEdge* oldEdge = fgRemoveRefPred(oldTarget, block); + assert(oldEdge != nullptr); + block->SetFalseTarget(newTarget); + fgAddRefPred(newTarget, block, oldEdge); } - break; case BBJ_SWITCH: { - unsigned const jumpCnt = block->GetSwitchTargets()->bbsCount; - FlowEdge** const jumpTab = block->GetSwitchTargets()->bbsDstTab; - bool existingEdge = false; - FlowEdge* oldEdge = nullptr; - FlowEdge* newEdge = nullptr; - bool changed = false; + unsigned const jumpCnt = block->GetSwitchTargets()->bbsCount; + BasicBlock** const jumpTab = block->GetSwitchTargets()->bbsDstTab; + bool changed = false; for (unsigned i = 0; i < jumpCnt; i++) { - if (jumpTab[i]->getDestinationBlock() == newTarget) + if (jumpTab[i] == oldTarget) { - // The new target already has an edge from this switch statement. - // We'll need to add the likelihood from the edge we're redirecting - // to the existing edge. Note that if there is no existing edge, - // then we'll copy the likelihood from the existing edge we pass to - // `fgAddRefPred`. Note also that we can visit the same edge multiple - // times if there are multiple switch cases with the same target. The - // edge has a dup count and a single likelihood for all the possible - // paths to the target, so we only want to add the likelihood once - // despite visiting the duplicated edges in the `jumpTab` array - // multiple times. - existingEdge = true; - } + jumpTab[i] = newTarget; + changed = true; + FlowEdge* const oldEdge = fgRemoveRefPred(oldTarget, block); + FlowEdge* const newEdge = fgAddRefPred(newTarget, block, oldEdge); - if (jumpTab[i]->getDestinationBlock() == oldTarget) - { - assert((oldEdge == nullptr) || (oldEdge == jumpTab[i])); - oldEdge = jumpTab[i]; - fgRemoveRefPred(oldEdge); - newEdge = fgAddRefPred(newTarget, block, oldEdge); - jumpTab[i] = newEdge; - changed = true; + // Handle the profile update, once we get our hands on the old edge. + // (see notes in fgChangeSwitchBlock for why this extra step is necessary) + // + // We do it slightly differently here so we don't lose the old + // edge weight propagation that would sometimes happen + // + if ((oldEdge != nullptr) && !newEdge->hasLikelihood()) + { + newEdge->setLikelihood(oldEdge->getLikelihood()); + } } } - if (existingEdge) + if (changed) { - assert(oldEdge != nullptr); - assert(oldEdge->getSourceBlock() == block); - assert(oldEdge->getDestinationBlock() == oldTarget); - assert(newEdge != nullptr); - assert(newEdge->getSourceBlock() == block); - assert(newEdge->getDestinationBlock() == newTarget); - - if (newEdge->hasLikelihood() && oldEdge->hasLikelihood()) - { - newEdge->addLikelihood(oldEdge->getLikelihood()); - } + InvalidateUniqueSwitchSuccMap(); } - - assert(changed); - InvalidateUniqueSwitchSuccMap(); break; } @@ -752,6 +753,54 @@ void Compiler::fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, Bas } } +//------------------------------------------------------------------------ +// fgReplacePred: update the predecessor list, swapping one pred for another +// +// Arguments: +// block - block with the pred list we want to update +// oldPred - pred currently appearing in block's pred list +// newPred - pred that will take oldPred's place. +// +// Notes: +// +// A block can only appear once in the preds list. If a predecessor has multiple +// ways to get to this block, then the pred edge DupCount will be >1. +// +// This function assumes that all branches from the predecessor (practically, that all +// switch cases that target this block) are changed to branch from the new predecessor, +// with the same dup count. +// +// Note that the block bbRefs is not changed, since 'block' has the same number of +// references as before, just from a different predecessor block. +// +// Also note this may cause sorting of the pred list. +// +void Compiler::fgReplacePred(BasicBlock* block, BasicBlock* oldPred, BasicBlock* newPred) +{ + noway_assert(block != nullptr); + noway_assert(oldPred != nullptr); + noway_assert(newPred != nullptr); + + bool modified = false; + + for (FlowEdge* const pred : block->PredEdges()) + { + if (oldPred == pred->getSourceBlock()) + { + pred->setSourceBlock(newPred); + modified = true; + break; + } + } + + // We may now need to reorder the pred list. + // + if (modified) + { + block->ensurePredListOrder(this); + } +} + //------------------------------------------------------------------------ // fgReplacePred: redirects the given edge to a new predecessor block // @@ -1303,10 +1352,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed break; } - case NI_System_SpanHelpers_ClearWithoutReferences: - case NI_System_SpanHelpers_Fill: case NI_System_SpanHelpers_SequenceEqual: - case NI_System_SpanHelpers_Memmove: + case NI_System_Buffer_Memmove: { if (FgStack::IsConstArgument(pushedStack.Top(), impInlineInfo)) { @@ -2939,21 +2986,10 @@ void Compiler::fgLinkBasicBlocks() { BasicBlock* const trueTarget = fgLookupBB(curBBdesc->GetTargetOffs()); BasicBlock* const falseTarget = curBBdesc->Next(); - FlowEdge* const trueEdge = fgAddRefPred(trueTarget, curBBdesc); - FlowEdge* const falseEdge = fgAddRefPred(falseTarget, curBBdesc); - curBBdesc->SetTrueEdge(trueEdge); - curBBdesc->SetFalseEdge(falseEdge); - - if (trueEdge == falseEdge) - { - assert(trueEdge->getDupCount() == 2); - trueEdge->setLikelihood(1.0); - } - else - { - trueEdge->setLikelihood(0.5); - falseEdge->setLikelihood(0.5); - } + curBBdesc->SetTrueTarget(trueTarget); + curBBdesc->SetFalseTarget(falseTarget); + fgAddRefPred(trueTarget, curBBdesc); + fgAddRefPred(falseTarget, curBBdesc); if (trueTarget->bbNum <= curBBdesc->bbNum) { @@ -2976,10 +3012,10 @@ void Compiler::fgLinkBasicBlocks() assert(!(curBBdesc->IsLast() && jumpsToNext)); BasicBlock* const jumpDest = jumpsToNext ? curBBdesc->Next() : fgLookupBB(curBBdesc->GetTargetOffs()); - // Redundantly use SetKindAndTargetEdge() instead of SetTargetEdge() just this once, - // so we don't break the HasInitializedTarget() invariant of SetTargetEdge(). - FlowEdge* const newEdge = fgAddRefPred(jumpDest, curBBdesc); - curBBdesc->SetKindAndTargetEdge(curBBdesc->GetKind(), newEdge); + // Redundantly use SetKindAndTarget() instead of SetTarget() just this once, + // so we don't break the HasInitializedTarget() invariant of SetTarget(). + curBBdesc->SetKindAndTarget(curBBdesc->GetKind(), jumpDest); + fgAddRefPred(jumpDest, curBBdesc); if (curBBdesc->GetTarget()->bbNum <= curBBdesc->bbNum) { @@ -3005,26 +3041,23 @@ void Compiler::fgLinkBasicBlocks() case BBJ_SWITCH: { - const unsigned numSucc = curBBdesc->GetSwitchTargets()->bbsCount; - unsigned jumpCnt = numSucc; - FlowEdge** jumpPtr = curBBdesc->GetSwitchTargets()->bbsDstTab; + unsigned jumpCnt = curBBdesc->GetSwitchTargets()->bbsCount; + BasicBlock** jumpPtr = curBBdesc->GetSwitchTargets()->bbsDstTab; do { - BasicBlock* jumpDest = fgLookupBB((unsigned)*(size_t*)jumpPtr); - FlowEdge* const newEdge = fgAddRefPred(jumpDest, curBBdesc); - - newEdge->setLikelihood((1.0 / numSucc) * newEdge->getDupCount()); - *jumpPtr = newEdge; - if (jumpDest->bbNum <= curBBdesc->bbNum) + BasicBlock* jumpDest = fgLookupBB((unsigned)*(size_t*)jumpPtr); + *jumpPtr = jumpDest; + fgAddRefPred(jumpDest, curBBdesc); + if ((*jumpPtr)->bbNum <= curBBdesc->bbNum) { - fgMarkBackwardJump(jumpDest, curBBdesc); + fgMarkBackwardJump(*jumpPtr, curBBdesc); } } while (++jumpPtr, --jumpCnt); /* Default case of CEE_SWITCH (next block), is at end of jumpTab[] */ - noway_assert(curBBdesc->NextIs((*(jumpPtr - 1))->getDestinationBlock())); + noway_assert(curBBdesc->NextIs(*(jumpPtr - 1))); break; } @@ -3187,8 +3220,8 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F unsigned jmpBase; unsigned jmpCnt; // # of switch cases (excluding default) - FlowEdge** jmpTab; - FlowEdge** jmpPtr; + BasicBlock** jmpTab; + BasicBlock** jmpPtr; /* Allocate the switch descriptor */ @@ -3205,7 +3238,7 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F /* Allocate the jump table */ - jmpPtr = jmpTab = new (this, CMK_FlowEdge) FlowEdge*[jmpCnt + 1]; + jmpPtr = jmpTab = new (this, CMK_BasicBlock) BasicBlock*[jmpCnt + 1]; /* Fill in the jump table */ @@ -3215,12 +3248,12 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F codeAddr += 4; // store the offset in the pointer. We change these in fgLinkBasicBlocks(). - *jmpPtr++ = (FlowEdge*)(size_t)(jmpBase + jmpDist); + *jmpPtr++ = (BasicBlock*)(size_t)(jmpBase + jmpDist); } /* Append the default label to the target table */ - *jmpPtr++ = (FlowEdge*)(size_t)jmpBase; + *jmpPtr++ = (BasicBlock*)(size_t)jmpBase; /* Make sure we found the right number of labels */ @@ -3557,7 +3590,7 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F noway_assert(codeAddr == codeEndp); - /* Finally link up the targets of the blocks together */ + /* Finally link up the bbTarget of the blocks together */ fgLinkBasicBlocks(); @@ -3875,9 +3908,10 @@ void Compiler::fgFindBasicBlocks() if (block->KindIs(BBJ_EHFILTERRET)) { // Mark catch handler as successor. + block->SetTarget(hndBegBB); FlowEdge* const newEdge = fgAddRefPred(hndBegBB, block); - block->SetTargetEdge(newEdge); - assert(hndBegBB->bbCatchTyp == BBCT_FILTER_HANDLER); + newEdge->setLikelihood(1.0); + assert(block->GetTarget()->bbCatchTyp == BBCT_FILTER_HANDLER); break; } } @@ -4209,7 +4243,10 @@ void Compiler::fgFixEntryFlowForOSR() // fgEnsureFirstBBisScratch(); assert(fgFirstBB->KindIs(BBJ_ALWAYS) && fgFirstBB->JumpsToNext()); - fgRedirectTargetEdge(fgFirstBB, fgOSREntryBB); + fgRemoveRefPred(fgFirstBB->GetTarget(), fgFirstBB); + fgFirstBB->SetKindAndTarget(BBJ_ALWAYS, fgOSREntryBB); + FlowEdge* const edge = fgAddRefPred(fgOSREntryBB, fgFirstBB); + edge->setLikelihood(1.0); // We don't know the right weight for this block, since // execution of the method was interrupted within the @@ -4758,15 +4795,14 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr) { // For each successor of the original block, set the new block as their predecessor. - for (FlowEdge* const succEdge : curr->SuccEdges()) + for (BasicBlock* const succ : curr->Succs(this)) { - // For non-switch blocks, successor iterator should not iterate duplicates. - assert(succEdge->getSourceBlock() != newBlock); - - BasicBlock* const succBlock = succEdge->getDestinationBlock(); - JITDUMP(FMT_BB " previous predecessor was " FMT_BB ", now is " FMT_BB "\n", succBlock->bbNum, curr->bbNum, - newBlock->bbNum); - fgReplacePred(succEdge, newBlock); + if (succ != newBlock) + { + JITDUMP(FMT_BB " previous predecessor was " FMT_BB ", now is " FMT_BB "\n", succ->bbNum, curr->bbNum, + newBlock->bbNum); + fgReplacePred(succ, curr, newBlock); + } } } @@ -4798,17 +4834,19 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr) // Remove flags from the old block that are no longer possible. curr->RemoveFlags(BBF_HAS_JMP | BBF_RETLESS_CALL); - // Default to fallthrough, and add the arc for that. - FlowEdge* const newEdge = fgAddRefPred(newBlock, curr); - // Transfer the kind and target. Do this after the code above, to avoid null-ing out the old targets used by the - // above code. + // above code (and so newBlock->bbNext is valid, so SetCond() can initialize bbFalseTarget if newBlock is a + // BBJ_COND). newBlock->TransferTarget(curr); - curr->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + // Default to fallthrough, and add the arc for that. + curr->SetKindAndTarget(BBJ_ALWAYS, newBlock); curr->SetFlags(BBF_NONE_QUIRK); assert(curr->JumpsToNext()); + FlowEdge* const newEdge = fgAddRefPred(newBlock, curr); + newEdge->setLikelihood(1.0); + return newBlock; } @@ -5031,14 +5069,15 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) // an immediately following block of a BBJ_SWITCH (which has // no fall-through path). For this case, simply insert a new // fall-through block after 'curr'. - // TODO-NoFallThrough: Once false target can diverge from bbNext, this will be unnecessary for BBJ_COND - newBlock = fgNewBBafter(BBJ_ALWAYS, curr, true /* extendRegion */); + // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, this will be unnecessary for BBJ_COND + newBlock = fgNewBBafter(BBJ_ALWAYS, curr, true /* extendRegion */, /* jumpDest */ succ); newBlock->SetFlags(BBF_NONE_QUIRK); + assert(newBlock->JumpsToNext()); } else { // The new block always jumps to 'succ' - newBlock = fgNewBBinRegion(BBJ_ALWAYS, curr, /* isRunRarely */ curr->isRunRarely()); + newBlock = fgNewBBinRegion(BBJ_ALWAYS, curr, /* jumpDest */ succ, /* isRunRarely */ curr->isRunRarely()); } newBlock->CopyFlags(curr, succ->GetFlagsRaw() & BBF_BACKWARD_JUMP); @@ -5050,7 +5089,7 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) // And 'succ' has 'newBlock' as a new predecessor. FlowEdge* const newEdge = fgAddRefPred(succ, newBlock); - newBlock->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); // This isn't accurate, but it is complex to compute a reasonable number so just assume that we take the // branch 50% of the time. @@ -5311,9 +5350,9 @@ BasicBlock* Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) * First, remove 'block' from the predecessor list of succBlock. */ - fgRemoveRefPred(block->GetTargetEdge()); + fgRemoveRefPred(succBlock, block); - for (BasicBlock* const predBlock : block->PredBlocksEditing()) + for (BasicBlock* const predBlock : block->PredBlocks()) { /* change all jumps/refs to the removed block */ switch (predBlock->GetKind()) @@ -5353,7 +5392,7 @@ BasicBlock* Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) assert(!bPrev->FalseTargetIs(block)); /* Check if both sides of the BBJ_COND now jump to the same block */ - if (bPrev->TrueEdgeIs(bPrev->GetFalseEdge())) + if (bPrev->TrueTargetIs(bPrev->GetFalseTarget())) { fgRemoveConditionalJump(bPrev); } @@ -5429,21 +5468,16 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) if (bSrc->KindIs(BBJ_COND) && bSrc->FalseTargetIs(bDst) && !bSrc->NextIs(bDst)) { // Add a new block after bSrc which jumps to 'bDst' - jmpBlk = fgNewBBafter(BBJ_ALWAYS, bSrc, true); - FlowEdge* const oldEdge = bSrc->GetFalseEdge(); - // Access the likelihood of oldEdge before - // it gets reset by SetTargetEdge below. - // - FlowEdge* const newEdge = fgAddRefPred(jmpBlk, bSrc, oldEdge); - fgReplacePred(oldEdge, jmpBlk); - jmpBlk->SetTargetEdge(oldEdge); - assert(jmpBlk->TargetIs(bDst)); - bSrc->SetFalseEdge(newEdge); + jmpBlk = fgNewBBafter(BBJ_ALWAYS, bSrc, true, bDst); + bSrc->SetFalseTarget(jmpBlk); + fgAddRefPred(jmpBlk, bSrc, fgGetPredForBlock(bDst, bSrc)); // When adding a new jmpBlk we will set the bbWeight and bbFlags // if (fgHaveValidEdgeWeights && fgHaveProfileWeights()) { + FlowEdge* const newEdge = fgGetPredForBlock(jmpBlk, bSrc); + jmpBlk->bbWeight = (newEdge->edgeWeightMin() + newEdge->edgeWeightMax()) / 2; if (bSrc->bbWeight == BB_ZERO_WEIGHT) { @@ -5481,6 +5515,8 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) } } + fgReplacePred(bDst, bSrc, jmpBlk); + JITDUMP("Added an unconditional jump to " FMT_BB " after block " FMT_BB "\n", jmpBlk->GetTarget()->bbNum, bSrc->bbNum); } @@ -5933,6 +5969,12 @@ BasicBlock* Compiler::fgRelocateEHRange(unsigned regionIndex, FG_RELOCATE_TYPE r insertAfterBlk->Next()); // We insert at the end, not at the beginning, of the funclet region. } + // These asserts assume we aren't moving try regions (which we might need to do). Only + // try regions can have fall through into or out of the region. + + noway_assert(!bPrev->bbFallsThrough()); // There can be no fall through into a filter or handler region + noway_assert(!bLast->bbFallsThrough()); // There can be no fall through out of a handler region + #ifdef DEBUG if (verbose) { @@ -6031,17 +6073,25 @@ BasicBlock* Compiler::fgRelocateEHRange(unsigned regionIndex, FG_RELOCATE_TYPE r * Insert a BasicBlock before the given block. */ -BasicBlock* Compiler::fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool extendRegion) +BasicBlock* Compiler::fgNewBBbefore(BBKinds jumpKind, + BasicBlock* block, + bool extendRegion, + BasicBlock* jumpDest /* = nullptr */) { // Create a new BasicBlock and chain it in - BasicBlock* newBlk = BasicBlock::New(this, jumpKind); + BasicBlock* newBlk = BasicBlock::New(this, jumpKind, jumpDest); newBlk->SetFlags(BBF_INTERNAL); fgInsertBBbefore(block, newBlk); newBlk->bbRefs = 0; + if (newBlk->bbFallsThrough() && block->isRunRarely()) + { + newBlk->bbSetRunRarely(); + } + if (extendRegion) { fgExtendEHRegionBefore(block); @@ -6065,17 +6115,25 @@ BasicBlock* Compiler::fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool ex * Insert a BasicBlock after the given block. */ -BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool extendRegion) +BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, + BasicBlock* block, + bool extendRegion, + BasicBlock* jumpDest /* = nullptr */) { // Create a new BasicBlock and chain it in - BasicBlock* newBlk = BasicBlock::New(this, jumpKind); + BasicBlock* newBlk = BasicBlock::New(this, jumpKind, jumpDest); newBlk->SetFlags(BBF_INTERNAL); fgInsertBBafter(block, newBlk); newBlk->bbRefs = 0; + if (block->bbFallsThrough() && block->isRunRarely()) + { + newBlk->bbSetRunRarely(); + } + if (extendRegion) { fgExtendEHRegionAfter(block); @@ -6104,6 +6162,7 @@ BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool ext // tree - tree that will be wrapped into a statement and // inserted in the new block. // debugInfo - debug info to propagate into the new statement. +// jumpDest - the jump target of the new block. Defaults to nullptr. // updateSideEffects - update side effects for the whole statement. // // Return Value: @@ -6112,10 +6171,14 @@ BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool ext // Notes: // The new block will have BBF_INTERNAL flag and EH region will be extended // -BasicBlock* Compiler::fgNewBBFromTreeAfter( - BBKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, bool updateSideEffects /* = false */) +BasicBlock* Compiler::fgNewBBFromTreeAfter(BBKinds jumpKind, + BasicBlock* block, + GenTree* tree, + DebugInfo& debugInfo, + BasicBlock* jumpDest /* = nullptr */, + bool updateSideEffects /* = false */) { - BasicBlock* newBlock = fgNewBBafter(jumpKind, block, true); + BasicBlock* newBlock = fgNewBBafter(jumpKind, block, true, jumpDest); newBlock->SetFlags(BBF_INTERNAL); Statement* stmt = fgNewStmtFromTree(tree, debugInfo); fgInsertStmtAtEnd(newBlock, stmt); @@ -6537,6 +6600,7 @@ BasicBlock* Compiler::fgFindInsertPoint(unsigned regionIndex, // [0..compHndBBtabCount]. // nearBlk - insert the new block closely after this block, if possible. If nullptr, put the new block anywhere // in the requested region. +// jumpDest - the jump target of the new block. Defaults to nullptr. // putInFilter - put the new block in the filter region given by hndIndex, as described above. // runRarely - 'true' if the new block is run rarely. // insertAtEnd - 'true' if the block should be inserted at the end of the region. Note: this is currently only @@ -6549,6 +6613,7 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, unsigned tryIndex, unsigned hndIndex, BasicBlock* nearBlk, + BasicBlock* jumpDest /* = nullptr */, bool putInFilter /* = false */, bool runRarely /* = false */, bool insertAtEnd /* = false */) @@ -6674,7 +6739,7 @@ _FoundAfterBlk:; bbKindNames[jumpKind], tryIndex, hndIndex, dspBool(putInFilter), dspBool(runRarely), dspBool(insertAtEnd), afterBlk->bbNum); - return fgNewBBinRegionWorker(jumpKind, afterBlk, regionIndex, putInTryRegion); + return fgNewBBinRegionWorker(jumpKind, afterBlk, regionIndex, putInTryRegion, jumpDest); } //------------------------------------------------------------------------ @@ -6685,6 +6750,7 @@ _FoundAfterBlk:; // Arguments: // jumpKind - the jump kind of the new block to create. // srcBlk - insert the new block in the same EH region as this block, and closely after it if possible. +// jumpDest - the jump target of the new block. Defaults to nullptr. // runRarely - 'true' if the new block is run rarely. // insertAtEnd - 'true' if the block should be inserted at the end of the region. Note: this is currently only // implemented when inserting into the main function (not into any EH region). @@ -6694,6 +6760,7 @@ _FoundAfterBlk:; BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, BasicBlock* srcBlk, + BasicBlock* jumpDest /* = nullptr */, bool runRarely /* = false */, bool insertAtEnd /* = false */) { @@ -6712,7 +6779,7 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, putInFilter = ehGetDsc(hndIndex - 1)->InFilterRegionBBRange(srcBlk); } - return fgNewBBinRegion(jumpKind, tryIndex, hndIndex, srcBlk, putInFilter, runRarely, insertAtEnd); + return fgNewBBinRegion(jumpKind, tryIndex, hndIndex, srcBlk, jumpDest, putInFilter, runRarely, insertAtEnd); } //------------------------------------------------------------------------ @@ -6722,13 +6789,14 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, // // Arguments: // jumpKind - the jump kind of the new block to create. +// jumpDest - the jump target of the new block. Defaults to nullptr. // // Return Value: // The new block. -BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind) +BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, BasicBlock* jumpDest /* = nullptr */) { - return fgNewBBinRegion(jumpKind, 0, 0, nullptr, /* putInFilter */ false, /* runRarely */ false, + return fgNewBBinRegion(jumpKind, 0, 0, nullptr, jumpDest, /* putInFilter */ false, /* runRarely */ false, /* insertAtEnd */ true); } @@ -6747,6 +6815,7 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind) // set its handler index to the most nested handler region enclosing that 'try' region. // Otherwise, put the block in the handler region specified by 'regionIndex', and set its 'try' // index to the most nested 'try' region enclosing that handler region. +// jumpDest - the jump target of the new block. Defaults to nullptr. // // Return Value: // The new block. @@ -6754,12 +6823,13 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind) BasicBlock* Compiler::fgNewBBinRegionWorker(BBKinds jumpKind, BasicBlock* afterBlk, unsigned regionIndex, - bool putInTryRegion) + bool putInTryRegion, + BasicBlock* jumpDest /* = nullptr */) { /* Insert the new block */ BasicBlock* afterBlkNext = afterBlk->Next(); (void)afterBlkNext; // prevent "unused variable" error from GCC - BasicBlock* newBlk = fgNewBBafter(jumpKind, afterBlk, false); + BasicBlock* newBlk = fgNewBBafter(jumpKind, afterBlk, false, jumpDest); if (putInTryRegion) { diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index f49f863b16dc57..900a58fb404033 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -134,7 +134,7 @@ void Compiler::fgDebugCheckUpdate() // Check for an unnecessary jumps to the next block. // A conditional branch should never jump to the next block as it can be folded into a BBJ_ALWAYS. - if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) + if (block->KindIs(BBJ_COND) && block->TrueTargetIs(block->GetFalseTarget())) { noway_assert(!"Unnecessary jump to the next block!"); } @@ -1101,7 +1101,7 @@ bool Compiler::fgDumpFlowGraph(Phases phase, PhasePosition pos) { fprintf(fgxFile, "\n switchCases=\"%d\"", edge->getDupCount()); } - if (bSource->GetSwitchTargets()->getDefault()->getDestinationBlock() == bTarget) + if (bSource->GetSwitchTargets()->getDefault() == bTarget) { fprintf(fgxFile, "\n switchDefault=\"true\""); } @@ -1806,7 +1806,6 @@ void Compiler::fgDumpFlowGraphLoops(FILE* file) void Compiler::fgTableDispBasicBlock(const BasicBlock* block, const BasicBlock* nextBlock /* = nullptr */, - bool printEdgeLikelihoods /* = true */, int blockTargetFieldWidth /* = 21 */, int ibcColWidth /* = 0 */) { @@ -1932,41 +1931,27 @@ void Compiler::fgTableDispBasicBlock(const BasicBlock* block, // Call `dspBlockNum()` to get the block number to print, and update `printedBlockWidth` with the width // of the generated string. Note that any computation using `printedBlockWidth` must be done after all // calls to this function. - auto dspBlockNum = [printEdgeLikelihoods, terseNext, nextBlock, - &printedBlockWidth](const FlowEdge* e) -> const char* { + auto dspBlockNum = [terseNext, nextBlock, &printedBlockWidth](const BasicBlock* b) -> const char* { static char buffers[3][64]; // static array of 3 to allow 3 concurrent calls in one printf() static int nextBufferIndex = 0; - auto& buffer = buffers[nextBufferIndex]; - nextBufferIndex = (nextBufferIndex + 1) % ArrLen(buffers); - const size_t sizeOfBuffer = ArrLen(buffer); - int written; + auto& buffer = buffers[nextBufferIndex]; + nextBufferIndex = (nextBufferIndex + 1) % ArrLen(buffers); - const BasicBlock* b = e->getDestinationBlock(); if (b == nullptr) { - written = _snprintf_s(buffer, sizeOfBuffer, sizeOfBuffer, "NULL"); - printedBlockWidth += written; + _snprintf_s(buffer, ArrLen(buffer), ArrLen(buffer), "NULL"); + printedBlockWidth += 4; } else if (terseNext && (b == nextBlock)) { - written = _snprintf_s(buffer, sizeOfBuffer, sizeOfBuffer, "*"); - printedBlockWidth += written; + _snprintf_s(buffer, ArrLen(buffer), ArrLen(buffer), "*"); + printedBlockWidth += 1; } else { - written = _snprintf_s(buffer, sizeOfBuffer, sizeOfBuffer, FMT_BB, b->bbNum); - printedBlockWidth += written; - } - - if (printEdgeLikelihoods) - { - if (e->hasLikelihood()) - { - written = _snprintf_s(buffer + written, sizeOfBuffer - written, sizeOfBuffer - written, - "(" FMT_WT_NARROW ")", e->getLikelihood()); - printedBlockWidth += written; - } + _snprintf_s(buffer, ArrLen(buffer), ArrLen(buffer), FMT_BB, b->bbNum); + printedBlockWidth += 2 /* BB */ + max(CountDigits(b->bbNum), 2); } return buffer; @@ -1983,19 +1968,19 @@ void Compiler::fgTableDispBasicBlock(const BasicBlock* block, { case BBJ_COND: printedBlockWidth = 3 /* "-> " */ + 1 /* comma */ + 9 /* kind */; - printf("-> %s,%s", dspBlockNum(block->GetTrueEdgeRaw()), dspBlockNum(block->GetFalseEdgeRaw())); + printf("-> %s,%s", dspBlockNum(block->GetTrueTargetRaw()), dspBlockNum(block->GetFalseTargetRaw())); printf("%*s ( cond )", blockTargetFieldWidth - printedBlockWidth, ""); break; case BBJ_CALLFINALLY: printedBlockWidth = 3 /* "-> " */ + 9 /* kind */; - printf("-> %s", dspBlockNum(block->GetTargetEdgeRaw())); + printf("-> %s", dspBlockNum(block->GetTargetRaw())); printf("%*s (callf )", blockTargetFieldWidth - printedBlockWidth, ""); break; case BBJ_CALLFINALLYRET: printedBlockWidth = 3 /* "-> " */ + 9 /* kind */; - printf("-> %s", dspBlockNum(block->GetTargetEdgeRaw())); + printf("-> %s", dspBlockNum(block->GetFinallyContinuation())); printf("%*s (callfr)", blockTargetFieldWidth - printedBlockWidth, ""); break; @@ -2003,13 +1988,13 @@ void Compiler::fgTableDispBasicBlock(const BasicBlock* block, const char* label; label = (flags & BBF_KEEP_BBJ_ALWAYS) ? "ALWAYS" : "always"; printedBlockWidth = 3 /* "-> " */ + 9 /* kind */; - printf("-> %s", dspBlockNum(block->GetTargetEdgeRaw())); + printf("-> %s", dspBlockNum(block->GetTargetRaw())); printf("%*s (%s)", blockTargetFieldWidth - printedBlockWidth, "", label); break; case BBJ_LEAVE: printedBlockWidth = 3 /* "-> " */ + 9 /* kind */; - printf("-> %s", dspBlockNum(block->GetTargetEdgeRaw())); + printf("-> %s", dspBlockNum(block->GetTargetRaw())); printf("%*s (leave )", blockTargetFieldWidth - printedBlockWidth, ""); break; @@ -2034,7 +2019,7 @@ void Compiler::fgTableDispBasicBlock(const BasicBlock* block, for (unsigned i = 0; i < jumpCnt; i++) { printedBlockWidth += 1 /* space/comma */; - printf("%c%s", (i == 0) ? ' ' : ',', dspBlockNum(jumpTab[i])); + printf("%c%s", (i == 0) ? ' ' : ',', dspBlockNum(jumpTab[i]->getDestinationBlock())); } } @@ -2054,13 +2039,13 @@ void Compiler::fgTableDispBasicBlock(const BasicBlock* block, case BBJ_EHFILTERRET: printedBlockWidth = 3 /* "-> " */ + 9 /* kind */; - printf("-> %s", dspBlockNum(block->GetTargetEdgeRaw())); + printf("-> %s", dspBlockNum(block->GetTargetRaw())); printf("%*s (fltret)", blockTargetFieldWidth - printedBlockWidth, ""); break; case BBJ_EHCATCHRET: printedBlockWidth = 3 /* "-> " */ + 9 /* kind */; - printf("-> %s", dspBlockNum(block->GetTargetEdgeRaw())); + printf("-> %s", dspBlockNum(block->GetTargetRaw())); printf("%*s ( cret )", blockTargetFieldWidth - printedBlockWidth, ""); break; @@ -2081,7 +2066,7 @@ void Compiler::fgTableDispBasicBlock(const BasicBlock* block, const BBswtDesc* const jumpSwt = block->GetSwitchTargets(); const unsigned jumpCnt = jumpSwt->bbsCount; - FlowEdge** const jumpTab = jumpSwt->bbsDstTab; + BasicBlock** const jumpTab = jumpSwt->bbsDstTab; for (unsigned i = 0; i < jumpCnt; i++) { @@ -2336,16 +2321,10 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, maxBlockNumWidth = max(maxBlockNumWidth, 2); int padWidth = maxBlockNumWidth - 2; // Account for functions with a large number of blocks. - const bool printEdgeLikelihoods = true; // TODO: parameterize? - - // Edge likelihoods are printed as "(0.123)", so take 7 characters maxmimum. - int edgeLikelihoodsWidth = printEdgeLikelihoods ? 7 : 0; - // Calculate the field width allocated for the block target. The field width is allocated to allow for two blocks // for BBJ_COND. It does not include any extra space for variable-sized BBJ_EHFINALLYRET and BBJ_SWITCH. - int blockTargetFieldWidth = 3 /* "-> " */ + 2 /* BB */ + maxBlockNumWidth + edgeLikelihoodsWidth + 1 /* comma */ + - 2 /* BB */ + maxBlockNumWidth + edgeLikelihoodsWidth + 1 /* space */ + - 8 /* kind: "(xxxxxx)" */; + int blockTargetFieldWidth = 3 /* "-> " */ + 2 /* BB */ + maxBlockNumWidth + 1 /* comma */ + 2 /* BB */ + + maxBlockNumWidth + 1 /* space */ + 8 /* kind: "(xxxxxx)" */; // clang-format off @@ -2353,7 +2332,7 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, printf("------%*s-------------------------------------%*s--------------------------%*s--------------------------\n", padWidth, "------------", // ibcColWidth, "------------", // - blockTargetFieldWidth, "----------------------------------------------"); // + blockTargetFieldWidth, "-----------------------"); // printf("BBnum %*sBBid ref try hnd %s weight %*s%s [IL range] [jump]%*s [EH region] [flags]\n", padWidth, "", (fgPredsComputed ? "preds " @@ -2366,7 +2345,7 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, printf("------%*s-------------------------------------%*s--------------------------%*s--------------------------\n", padWidth, "------------", // ibcColWidth, "------------", // - blockTargetFieldWidth, "----------------------------------------------"); // + blockTargetFieldWidth, "-----------------------"); // // clang-format on @@ -2398,9 +2377,9 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, { printf("~~~~~~%*s~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~%*s~~~~~~~~~~~~~~~~~~~~~~~~~~%*s~~~~~~~~~~" "~~~~~~~~~~~~~~~~\n", - padWidth, "~~~~~~~~~~~~", // - ibcColWidth, "~~~~~~~~~~~~", // - blockTargetFieldWidth, "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"); // + padWidth, "~~~~~~~~~~~~", // + ibcColWidth, "~~~~~~~~~~~~", // + blockTargetFieldWidth, "~~~~~~~~~~~~~~~~~~~~~~~"); // } #if defined(FEATURE_EH_FUNCLETS) @@ -2408,13 +2387,13 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, { printf("++++++%*s+++++++++++++++++++++++++++++++++++++%*s++++++++++++++++++++++++++%*s++++++++++" "++++++++++++++++ funclets follow\n", - padWidth, "++++++++++++", // - ibcColWidth, "++++++++++++", // - blockTargetFieldWidth, "++++++++++++++++++++++++++++++++++++++++++++++"); // + padWidth, "++++++++++++", // + ibcColWidth, "++++++++++++", // + blockTargetFieldWidth, "+++++++++++++++++++++++"); // } #endif // FEATURE_EH_FUNCLETS - fgTableDispBasicBlock(block, nextBlock, printEdgeLikelihoods, blockTargetFieldWidth, ibcColWidth); + fgTableDispBasicBlock(block, nextBlock, blockTargetFieldWidth, ibcColWidth); if (block == lastBlock) { @@ -2424,9 +2403,9 @@ void Compiler::fgDispBasicBlocks(BasicBlock* firstBlock, BasicBlock* lastBlock, printf("------%*s-------------------------------------%*s--------------------------%*s------------------" "--------\n", - padWidth, "------------", // - ibcColWidth, "------------", // - blockTargetFieldWidth, "----------------------------------------------"); // + padWidth, "------------", // + ibcColWidth, "------------", // + blockTargetFieldWidth, "-----------------------"); // if (dumpTrees) { @@ -2807,7 +2786,6 @@ bool BBPredsChecker::CheckJump(BasicBlock* blockPred, BasicBlock* block) case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: assert(blockPred->TargetIs(block)); - assert(blockPred->GetTargetEdge()->getLikelihood() == 1.0); return true; case BBJ_EHFINALLYRET: @@ -3025,7 +3003,7 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, bool checkBBRef { for (unsigned i = 0; i < sd.numDistinctSuccs; i++) { - const BasicBlock* const nonDuplicateSucc = sd.nonDuplicates[i]->getDestinationBlock(); + const BasicBlock* const nonDuplicateSucc = sd.nonDuplicates[i]; assert(nonDuplicateSucc != nullptr); assert(nonDuplicateSucc->bbTraversalStamp == curTraversalStamp); } @@ -3411,17 +3389,20 @@ void Compiler::fgDebugCheckFlags(GenTree* tree, BasicBlock* block) GenTreeFlags handleKind = op1->GetIconHandleFlag(); // Some of these aren't handles to invariant data... - if (GenTree::HandleKindDataIsInvariant(handleKind) && (handleKind != GTF_ICON_FTN_ADDR)) - { - expectedFlags |= GTF_IND_INVARIANT; - } - else + if ((handleKind == GTF_ICON_STATIC_HDL) || // Pointer to a mutable class Static variable + (handleKind == GTF_ICON_BBC_PTR) || // Pointer to a mutable basic block count value + (handleKind == GTF_ICON_FTN_ADDR) || // Pointer to a potentially mutable VM slot + (handleKind == GTF_ICON_GLOBAL_PTR)) // Pointer to mutable data from the VM state { // For statics, we expect the GTF_GLOB_REF to be set. However, we currently // fail to set it in a number of situations, and so this check is disabled. // TODO: enable checking of GTF_GLOB_REF. // expectedFlags |= GTF_GLOB_REF; } + else // All the other handle indirections are considered invariant + { + expectedFlags |= GTF_IND_INVARIANT; + } // Currently we expect all indirections with constant addresses to be nonfaulting. expectedFlags |= GTF_IND_NONFAULTING; @@ -4000,8 +3981,7 @@ void Compiler::fgDebugCheckBlockLinks() assert(uniqueSuccSet.numDistinctSuccs == count); for (unsigned i = 0; i < uniqueSuccSet.numDistinctSuccs; i++) { - assert(BitVecOps::IsMember(&bitVecTraits, succBlocks, - uniqueSuccSet.nonDuplicates[i]->getDestinationBlock()->bbNum)); + assert(BitVecOps::IsMember(&bitVecTraits, succBlocks, uniqueSuccSet.nonDuplicates[i]->bbNum)); } } } @@ -4762,17 +4742,10 @@ void Compiler::fgDebugCheckLoops() } //------------------------------------------------------------------------------ -// fgDebugCheckFlowGraphAnnotations: Checks that all flow graph annotations -// that are currently non-null are valid. +// fgDebugCheckDfsTree: Checks that the DFS tree matches the current flow graph. // -void Compiler::fgDebugCheckFlowGraphAnnotations() +void Compiler::fgDebugCheckDfsTree() { - if (m_dfsTree == nullptr) - { - assert((m_loops == nullptr) && (m_domTree == nullptr) && (m_reachabilitySets == nullptr)); - return; - } - unsigned count = fgRunDfs([](BasicBlock* block, unsigned preorderNum) { assert(block->bbPreorderNum == preorderNum); }, [=](BasicBlock* block, unsigned postorderNum) { @@ -4782,10 +4755,6 @@ void Compiler::fgDebugCheckFlowGraphAnnotations() [](BasicBlock* block, BasicBlock* succ) {}); assert(m_dfsTree->GetPostOrderCount() == count); - - assert((m_loops == nullptr) || (m_loops->GetDfsTree() == m_dfsTree)); - assert((m_domTree == nullptr) || (m_domTree->GetDfsTree() == m_dfsTree)); - assert((m_reachabilitySets == nullptr) || (m_reachabilitySets->GetDfsTree() == m_dfsTree)); } /*****************************************************************************/ diff --git a/src/coreclr/jit/fgehopt.cpp b/src/coreclr/jit/fgehopt.cpp index a437a3da128d4c..4c9a6e3b1dab10 100644 --- a/src/coreclr/jit/fgehopt.cpp +++ b/src/coreclr/jit/fgehopt.cpp @@ -167,11 +167,13 @@ PhaseStatus Compiler::fgRemoveEmptyFinally() fgPrepareCallFinallyRetForRemoval(leaveBlock); fgRemoveBlock(leaveBlock, /* unreachable */ true); - // Ref count updates. - fgRedirectTargetEdge(currentBlock, postTryFinallyBlock); - currentBlock->SetKind(BBJ_ALWAYS); + currentBlock->SetKindAndTarget(BBJ_ALWAYS, postTryFinallyBlock); currentBlock->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY + // Ref count updates. + fgAddRefPred(postTryFinallyBlock, currentBlock); + fgRemoveRefPred(firstBlock, currentBlock); + // Cleanup the postTryFinallyBlock fgCleanupContinuation(postTryFinallyBlock); @@ -522,8 +524,8 @@ PhaseStatus Compiler::fgRemoveEmptyTry() GenTree* finallyRetExpr = finallyRet->GetRootNode(); assert(finallyRetExpr->gtOper == GT_RETFILT); fgRemoveStmt(block, finallyRet); - FlowEdge* const newEdge = fgAddRefPred(continuation, block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + block->SetKindAndTarget(BBJ_ALWAYS, continuation); + fgAddRefPred(continuation, block); } } @@ -1091,13 +1093,13 @@ PhaseStatus Compiler::fgCloneFinally() GenTree* finallyRetExpr = finallyRet->GetRootNode(); assert(finallyRetExpr->gtOper == GT_RETFILT); fgRemoveStmt(newBlock, finallyRet); + newBlock->SetKindAndTarget(BBJ_ALWAYS, normalCallFinallyReturn); - FlowEdge* const newEdge = fgAddRefPred(normalCallFinallyReturn, newBlock); - newBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + fgAddRefPred(normalCallFinallyReturn, newBlock); } else { - optSetMappedBlockTargets(block, newBlock, &blockMap); + optRedirectBlock(block, newBlock, &blockMap); } } @@ -1133,12 +1135,13 @@ PhaseStatus Compiler::fgCloneFinally() fgPrepareCallFinallyRetForRemoval(leaveBlock); fgRemoveBlock(leaveBlock, /* unreachable */ true); - // Ref count updates. - fgRedirectTargetEdge(currentBlock, firstCloneBlock); - // This call returns to the expected spot, so retarget it to branch to the clone. + currentBlock->SetKindAndTarget(BBJ_ALWAYS, firstCloneBlock); currentBlock->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY - currentBlock->SetKind(BBJ_ALWAYS); + + // Ref count updates. + fgAddRefPred(firstCloneBlock, currentBlock); + fgRemoveRefPred(firstBlock, currentBlock); // Make sure iteration isn't going off the deep end. assert(leaveBlock != endCallFinallyRangeBlock); @@ -1754,8 +1757,10 @@ bool Compiler::fgRetargetBranchesToCanonicalCallFinally(BasicBlock* block, JITDUMP("Redirecting branch in " FMT_BB " from " FMT_BB " to " FMT_BB ".\n", block->bbNum, callFinally->bbNum, canonicalCallFinally->bbNum); + block->SetTarget(canonicalCallFinally); + fgAddRefPred(canonicalCallFinally, block); assert(callFinally->bbRefs > 0); - fgRedirectTargetEdge(block, canonicalCallFinally); + fgRemoveRefPred(callFinally, block); // Update profile counts // @@ -2002,12 +2007,36 @@ PhaseStatus Compiler::fgTailMergeThrows() // Walk pred list of the non canonical block, updating flow to target // the canonical block instead. - for (BasicBlock* const predBlock : nonCanonicalBlock->PredBlocksEditing()) + for (FlowEdge* predEdge = nonCanonicalBlock->bbPreds; predEdge != nullptr; predEdge = nextPredEdge) { + BasicBlock* const predBlock = predEdge->getSourceBlock(); + nextPredEdge = predEdge->getNextPredEdge(); + switch (predBlock->GetKind()) { case BBJ_ALWAYS: + { + fgTailMergeThrowsJumpToHelper(predBlock, nonCanonicalBlock, canonicalBlock, predEdge); + updated = true; + } + break; + case BBJ_COND: + { + // Flow to non canonical block could be via fall through or jump or both. + if (predBlock->FalseTargetIs(nonCanonicalBlock)) + { + fgTailMergeThrowsFallThroughHelper(predBlock, nonCanonicalBlock, canonicalBlock, predEdge); + } + + if (predBlock->TrueTargetIs(nonCanonicalBlock)) + { + fgTailMergeThrowsJumpToHelper(predBlock, nonCanonicalBlock, canonicalBlock, predEdge); + } + updated = true; + } + break; + case BBJ_SWITCH: { JITDUMP("*** " FMT_BB " now branching to " FMT_BB "\n", predBlock->bbNum, canonicalBlock->bbNum); @@ -2050,3 +2079,87 @@ PhaseStatus Compiler::fgTailMergeThrows() fgModified = false; return PhaseStatus::MODIFIED_EVERYTHING; } + +//------------------------------------------------------------------------ +// fgTailMergeThrowsFallThroughHelper: fixup flow for fall throughs to mergeable throws +// +// Arguments: +// predBlock - block falling through to the throw helper +// nonCanonicalBlock - original fall through target +// canonicalBlock - new (jump) target +// predEdge - original flow edge +// +// Notes: +// Alters fall through flow of predBlock so it jumps to the +// canonicalBlock via a new basic block. Does not try and fix +// jump-around flow; we leave that to optOptimizeFlow which runs +// just afterwards. +// +void Compiler::fgTailMergeThrowsFallThroughHelper(BasicBlock* predBlock, + BasicBlock* nonCanonicalBlock, + BasicBlock* canonicalBlock, + FlowEdge* predEdge) +{ + assert(predBlock->KindIs(BBJ_COND)); + assert(predBlock->FalseTargetIs(nonCanonicalBlock)); + + BasicBlock* const newBlock = fgNewBBafter(BBJ_ALWAYS, predBlock, true, canonicalBlock); + predBlock->SetFalseTarget(newBlock); + + JITDUMP("*** " FMT_BB " now falling through to empty " FMT_BB " and then to " FMT_BB "\n", predBlock->bbNum, + newBlock->bbNum, canonicalBlock->bbNum); + + // Remove the old flow + fgRemoveRefPred(nonCanonicalBlock, predBlock); + + // Wire up the new flow + fgAddRefPred(newBlock, predBlock, predEdge); + + fgAddRefPred(canonicalBlock, newBlock, predEdge); + + // If nonCanonicalBlock has only one pred, all its flow transfers. + // If it has multiple preds, then we need edge counts or likelihoods + // to figure things out. + // + // For now just do a minimal update. + // + newBlock->inheritWeight(nonCanonicalBlock); +} + +//------------------------------------------------------------------------ +// fgTailMergeThrowsJumpToHelper: fixup flow for jumps to mergeable throws +// +// Arguments: +// predBlock - block jumping to the throw helper +// nonCanonicalBlock - original jump target +// canonicalBlock - new jump target +// predEdge - original flow edge +// +// Notes: +// Alters jumpDest of predBlock so it jumps to the canonicalBlock. +// +void Compiler::fgTailMergeThrowsJumpToHelper(BasicBlock* predBlock, + BasicBlock* nonCanonicalBlock, + BasicBlock* canonicalBlock, + FlowEdge* predEdge) +{ + JITDUMP("*** " FMT_BB " now branching to " FMT_BB "\n", predBlock->bbNum, canonicalBlock->bbNum); + + // Remove the old flow + fgRemoveRefPred(nonCanonicalBlock, predBlock); + + // Wire up the new flow + if (predBlock->KindIs(BBJ_ALWAYS)) + { + assert(predBlock->TargetIs(nonCanonicalBlock)); + predBlock->SetTarget(canonicalBlock); + } + else + { + assert(predBlock->KindIs(BBJ_COND)); + assert(predBlock->TrueTargetIs(nonCanonicalBlock)); + predBlock->SetTrueTarget(canonicalBlock); + } + + fgAddRefPred(canonicalBlock, predBlock, predEdge); +} diff --git a/src/coreclr/jit/fgflow.cpp b/src/coreclr/jit/fgflow.cpp index f9a1721218683b..a7766b64cae6bb 100644 --- a/src/coreclr/jit/fgflow.cpp +++ b/src/coreclr/jit/fgflow.cpp @@ -117,8 +117,8 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE // dependency on this order. Note also that we don't allow duplicates in the list; we maintain a DupCount // count of duplication. This also necessitates walking the flow list for every edge we add. // - FlowEdge* flow = nullptr; - FlowEdge** listp; + FlowEdge* flow = nullptr; + FlowEdge** listp = &block->bbPreds; if (initializingPreds) { @@ -126,7 +126,6 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE // increasing blockPred->bbNum order. The only possible // dup list entry is the last one. // - listp = &block->bbPreds; FlowEdge* flowLast = block->bbLastPred; if (flowLast != nullptr) { @@ -137,6 +136,32 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (flowLast->getSourceBlock() == blockPred) { flow = flowLast; + + // This edge should have been given a likelihood when it was created. + // Since we're increasing its duplicate count, update the likelihood. + // + assert(flow->hasLikelihood()); + const unsigned numSucc = blockPred->NumSucc(); + assert(numSucc > 0); + + if (numSucc == 1) + { + // BasicBlock::NumSucc() returns 1 for BBJ_CONDs with the same true/false target. + // For blocks that only ever have one successor (BBJ_ALWAYS, BBJ_LEAVE, etc.), + // their successor edge should never have a duplicate count over 1. + // + assert(blockPred->KindIs(BBJ_COND)); + assert(blockPred->TrueTargetIs(blockPred->GetFalseTarget())); + flow->setLikelihood(1.0); + } + else + { + // Duplicate count isn't updated until later, so add 1 for now. + // + const unsigned dupCount = flow->getDupCount() + 1; + assert(dupCount > 1); + flow->setLikelihood((1.0 / numSucc) * dupCount); + } } } } @@ -144,7 +169,10 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE { // References are added randomly, so we have to search. // - listp = fgGetPredInsertPoint(blockPred, block); + while ((*listp != nullptr) && ((*listp)->getSourceBlock()->bbNum < blockPred->bbNum)) + { + listp = (*listp)->getNextPredEdgeRef(); + } if ((*listp != nullptr) && ((*listp)->getSourceBlock() == blockPred)) { @@ -155,7 +183,6 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (flow != nullptr) { // The predecessor block already exists in the flow list; simply add to its duplicate count. - // noway_assert(flow->getDupCount()); flow->incrementDupCount(); } @@ -184,6 +211,14 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (initializingPreds) { block->bbLastPred = flow; + + // When initializing preds, ensure edge likelihood is set, + // such that this edge is as likely as any other successor edge + // + const unsigned numSucc = blockPred->NumSucc(); + assert(numSucc > 0); + assert(flow->getDupCount() == 1); + flow->setLikelihood(1.0 / numSucc); } else if ((oldEdge != nullptr) && oldEdge->hasLikelihood()) { @@ -233,6 +268,10 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE // assert(block->checkPredListOrder()); + // When initializing preds, edge likelihood should always be set. + // + assert(!initializingPreds || flow->hasLikelihood()); + return flow; } @@ -244,6 +283,62 @@ template FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowEdge* oldEdge /* = nullptr */); +//------------------------------------------------------------------------ +// fgRemoveRefPred: Decrements the reference count of a predecessor edge from "blockPred" to "block", +// removing the edge if it is no longer necessary. +// +// Arguments: +// block -- A block to operate on. +// blockPred -- The predecessor block to remove from the predecessor list. It must be a predecessor of "block". +// +// Return Value: +// If the flow edge was removed (the predecessor has a "dup count" of 1), +// returns the flow graph edge that was removed. This means "blockPred" is no longer a predecessor of "block". +// Otherwise, returns nullptr. This means that "blockPred" is still a predecessor of "block" (because "blockPred" +// is a switch with multiple cases jumping to "block", or a BBJ_COND with both conditional and fall-through +// paths leading to "block"). +// +// Assumptions: +// -- "blockPred" must be a predecessor block of "block". +// +// Notes: +// -- block->bbRefs is decremented by one to account for the reduction in incoming edges. +// -- block->bbRefs is adjusted even if preds haven't been computed. If preds haven't been computed, +// the preds themselves aren't touched. +// -- fgModified is set if a flow edge is removed (but not if an existing flow edge dup count is decremented), +// indicating that the flow graph shape has changed. +// +FlowEdge* Compiler::fgRemoveRefPred(BasicBlock* block, BasicBlock* blockPred) +{ + noway_assert(block != nullptr); + noway_assert(blockPred != nullptr); + noway_assert(block->countOfInEdges() > 0); + assert(fgPredsComputed); + block->bbRefs--; + + FlowEdge** ptrToPred; + FlowEdge* pred = fgGetPredForBlock(block, blockPred, &ptrToPred); + noway_assert(pred != nullptr); + noway_assert(pred->getDupCount() > 0); + + pred->decrementDupCount(); + + if (pred->getDupCount() == 0) + { + // Splice out the predecessor edge since it's no longer necessary. + *ptrToPred = pred->getNextPredEdge(); + + // Any changes to the flow graph invalidate the dominator sets. + fgModified = true; + + return pred; + } + else + { + return nullptr; + } +} + //------------------------------------------------------------------------ // fgRemoveRefPred: Decrements the reference count of `edge`, removing it from its successor block's pred list // if the reference count is zero. @@ -340,12 +435,12 @@ void Compiler::fgRemoveBlockAsPred(BasicBlock* block) case BBJ_ALWAYS: case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: - fgRemoveRefPred(block->GetTargetEdge()); + fgRemoveRefPred(block->GetTarget(), block); break; case BBJ_COND: - fgRemoveRefPred(block->GetTrueEdge()); - fgRemoveRefPred(block->GetFalseEdge()); + fgRemoveRefPred(block->GetTrueTarget(), block); + fgRemoveRefPred(block->GetFalseTarget(), block); break; case BBJ_EHFINALLYRET: @@ -364,14 +459,11 @@ void Compiler::fgRemoveBlockAsPred(BasicBlock* block) break; case BBJ_SWITCH: - { - BBswtDesc* const swtDesc = block->GetSwitchTargets(); - for (unsigned i = 0; i < swtDesc->bbsCount; i++) + for (BasicBlock* const bTarget : block->SwitchTargets()) { - fgRemoveRefPred(swtDesc->bbsDstTab[i]); + fgRemoveRefPred(bTarget, block); } break; - } default: noway_assert(!"Block doesn't have a valid bbKind!!!!"); @@ -379,188 +471,6 @@ void Compiler::fgRemoveBlockAsPred(BasicBlock* block) } } -//------------------------------------------------------------------------ -// fgGetPredInsertPoint: Searches newTarget->bbPreds for where to insert an edge from blockPred. -// -// Arguments: -// blockPred -- The block we want to make a predecessor of newTarget (it could already be one). -// newTarget -- The block whose pred list we will search. -// -// Return Value: -// Returns a pointer to the next pointer of an edge in newTarget's pred list. -// A new edge from blockPred to newTarget can be inserted here -// without disrupting bbPreds' sorting invariant. -// -FlowEdge** Compiler::fgGetPredInsertPoint(BasicBlock* blockPred, BasicBlock* newTarget) -{ - assert(blockPred != nullptr); - assert(newTarget != nullptr); - assert(fgPredsComputed); - - FlowEdge** listp = &newTarget->bbPreds; - - // Search pred list for insertion point - // - while ((*listp != nullptr) && ((*listp)->getSourceBlock()->bbNum < blockPred->bbNum)) - { - listp = (*listp)->getNextPredEdgeRef(); - } - - return listp; -} - -//------------------------------------------------------------------------ -// fgRedirectTargetEdge: Sets block->bbTargetEdge's target block to newTarget, -// updating pred lists as necessary. -// -// Arguments: -// block -- The block we want to make a predecessor of newTarget. -// It could be one already, in which case nothing changes. -// newTarget -- The new successor of block. -// -void Compiler::fgRedirectTargetEdge(BasicBlock* block, BasicBlock* newTarget) -{ - assert(block != nullptr); - assert(newTarget != nullptr); - - FlowEdge* edge = block->GetTargetEdge(); - assert(edge->getDupCount() == 1); - - // Update oldTarget's pred list. - // We could call fgRemoveRefPred, but since we're removing the one and only ref from block to oldTarget, - // fgRemoveAllRefPreds is slightly more efficient (one fewer branch, doesn't update edge's dup count, etc). - // - BasicBlock* oldTarget = edge->getDestinationBlock(); - fgRemoveAllRefPreds(oldTarget, block); - - // Splice edge into new target block's pred list - // - FlowEdge** predListPtr = fgGetPredInsertPoint(block, newTarget); - edge->setNextPredEdge(*predListPtr); - edge->setDestinationBlock(newTarget); - *predListPtr = edge; - - // Pred list of target should (still) be ordered - // - assert(newTarget->checkPredListOrder()); - - // Edge should still have only one ref - assert(edge->getDupCount() == 1); - newTarget->bbRefs++; -} - -//------------------------------------------------------------------------ -// fgRedirectTrueEdge: Sets block->bbTrueEdge's target block to newTarget, -// updating pred lists as necessary. -// -// Arguments: -// block -- The block we want to make a predecessor of newTarget. -// It could be one already, in which case nothing changes. -// newTarget -- The new successor of block. -// -// Notes: -// This assumes block's true and false targets are different. -// If setting block's true target to its false target, -// fgRedirectTrueEdge increments the false edge's dup count, -// and ensures block->bbTrueEdge == block->bbFalseEdge. -// We don't update newTarget->bbPreds in this case, -// as we don't want to have more than one edge from the same predecessor. -// -void Compiler::fgRedirectTrueEdge(BasicBlock* block, BasicBlock* newTarget) -{ - assert(block != nullptr); - assert(newTarget != nullptr); - assert(block->KindIs(BBJ_COND)); - assert(!block->TrueEdgeIs(block->GetFalseEdge())); - - // Update oldTarget's pred list. - // We could call fgRemoveRefPred, but since we're removing the one and only ref from block to oldTarget, - // fgRemoveAllRefPreds is slightly more efficient (one fewer branch, doesn't update edge's dup count, etc). - // - FlowEdge* trueEdge = block->GetTrueEdge(); - BasicBlock* oldTarget = trueEdge->getDestinationBlock(); - fgRemoveAllRefPreds(oldTarget, block); - - // Splice edge into new target block's pred list - // - FlowEdge** predListPtr = fgGetPredInsertPoint(block, newTarget); - FlowEdge* predEdge = *predListPtr; - - if (block->FalseEdgeIs(predEdge)) - { - block->SetTrueEdge(predEdge); - predEdge->incrementDupCount(); - } - else - { - trueEdge->setNextPredEdge(predEdge); - trueEdge->setDestinationBlock(newTarget); - *predListPtr = trueEdge; - } - - newTarget->bbRefs++; - - // Pred list of target should (still) be ordered - // - assert(newTarget->checkPredListOrder()); -} - -//------------------------------------------------------------------------ -// fgRedirectFalseEdge: Sets block->bbFalseEdge's target block to newTarget, -// updating pred lists as necessary. -// -// Arguments: -// block -- The block we want to make a predecessor of newTarget. -// It could be one already, in which case nothing changes. -// newTarget -- The new successor of block. -// -// Notes: -// This assumes block's true and false targets are different. -// If setting block's false target to its true target, -// fgRedirectFalseEdge increments the true edge's dup count, -// and ensures block->bbTrueEdge == block->bbFalseEdge. -// We don't update newTarget->bbPreds in this case, -// as we don't want to have more than one edge from the same predecessor. -// -void Compiler::fgRedirectFalseEdge(BasicBlock* block, BasicBlock* newTarget) -{ - assert(block != nullptr); - assert(newTarget != nullptr); - assert(block->KindIs(BBJ_COND)); - assert(!block->TrueEdgeIs(block->GetFalseEdge())); - - // Update oldTarget's pred list. - // We could call fgRemoveRefPred, but since we're removing the one and only ref from block to oldTarget, - // fgRemoveAllRefPreds is slightly more efficient (one fewer branch, doesn't update edge's dup count, etc). - // - FlowEdge* falseEdge = block->GetFalseEdge(); - BasicBlock* oldTarget = falseEdge->getDestinationBlock(); - fgRemoveAllRefPreds(oldTarget, block); - - // Splice edge into new target block's pred list - // - FlowEdge** predListPtr = fgGetPredInsertPoint(block, newTarget); - FlowEdge* predEdge = *predListPtr; - - if (block->TrueEdgeIs(predEdge)) - { - block->SetFalseEdge(predEdge); - predEdge->incrementDupCount(); - } - else - { - falseEdge->setNextPredEdge(predEdge); - falseEdge->setDestinationBlock(newTarget); - *predListPtr = falseEdge; - } - - newTarget->bbRefs++; - - // Pred list of target should (still) be ordered - // - assert(newTarget->checkPredListOrder()); -} - Compiler::SwitchUniqueSuccSet Compiler::GetDescriptorForSwitch(BasicBlock* switchBlk) { assert(switchBlk->KindIs(BBJ_SWITCH)); @@ -589,20 +499,16 @@ Compiler::SwitchUniqueSuccSet Compiler::GetDescriptorForSwitch(BasicBlock* switc // Now we have a set of unique successors. unsigned numNonDups = BitVecOps::Count(&blockVecTraits, uniqueSuccBlocks); - FlowEdge** nonDups = new (getAllocator()) FlowEdge*[numNonDups]; + BasicBlock** nonDups = new (getAllocator()) BasicBlock*[numNonDups]; unsigned nonDupInd = 0; - // At this point, all unique targets are in "uniqueSuccBlocks". As we encounter each, // add to nonDups, remove from "uniqueSuccBlocks". - BBswtDesc* const swtDesc = switchBlk->GetSwitchTargets(); - for (unsigned i = 0; i < swtDesc->bbsCount; i++) + for (BasicBlock* const targ : switchBlk->SwitchTargets()) { - FlowEdge* const succEdge = swtDesc->bbsDstTab[i]; - BasicBlock* const targ = succEdge->getDestinationBlock(); if (BitVecOps::IsMember(&blockVecTraits, uniqueSuccBlocks, targ->bbNum)) { - nonDups[nonDupInd] = succEdge; + nonDups[nonDupInd] = targ; nonDupInd++; BitVecOps::RemoveElemD(&blockVecTraits, uniqueSuccBlocks, targ->bbNum); } @@ -617,6 +523,87 @@ Compiler::SwitchUniqueSuccSet Compiler::GetDescriptorForSwitch(BasicBlock* switc } } +void Compiler::SwitchUniqueSuccSet::UpdateTarget(CompAllocator alloc, + BasicBlock* switchBlk, + BasicBlock* from, + BasicBlock* to) +{ + assert(switchBlk->KindIs(BBJ_SWITCH)); // Precondition. + + // Is "from" still in the switch table (because it had more than one entry before?) + bool fromStillPresent = false; + for (BasicBlock* const bTarget : switchBlk->SwitchTargets()) + { + if (bTarget == from) + { + fromStillPresent = true; + break; + } + } + + // Is "to" already in "this"? + bool toAlreadyPresent = false; + for (unsigned i = 0; i < numDistinctSuccs; i++) + { + if (nonDuplicates[i] == to) + { + toAlreadyPresent = true; + break; + } + } + + // Four cases: + // If "from" is still present, and "to" is already present, do nothing + // If "from" is still present, and "to" is not, must reallocate to add an entry. + // If "from" is not still present, and "to" is not present, write "to" where "from" was. + // If "from" is not still present, but "to" is present, remove "from". + if (fromStillPresent && toAlreadyPresent) + { + return; + } + else if (fromStillPresent && !toAlreadyPresent) + { + // reallocate to add an entry + BasicBlock** newNonDups = new (alloc) BasicBlock*[numDistinctSuccs + 1]; + memcpy(newNonDups, nonDuplicates, numDistinctSuccs * sizeof(BasicBlock*)); + newNonDups[numDistinctSuccs] = to; + numDistinctSuccs++; + nonDuplicates = newNonDups; + } + else if (!fromStillPresent && !toAlreadyPresent) + { + // write "to" where "from" was + INDEBUG(bool foundFrom = false); + for (unsigned i = 0; i < numDistinctSuccs; i++) + { + if (nonDuplicates[i] == from) + { + nonDuplicates[i] = to; + INDEBUG(foundFrom = true); + break; + } + } + assert(foundFrom); + } + else + { + assert(!fromStillPresent && toAlreadyPresent); + // remove "from". + INDEBUG(bool foundFrom = false); + for (unsigned i = 0; i < numDistinctSuccs; i++) + { + if (nonDuplicates[i] == from) + { + nonDuplicates[i] = nonDuplicates[numDistinctSuccs - 1]; + numDistinctSuccs--; + INDEBUG(foundFrom = true); + break; + } + } + assert(foundFrom); + } +} + /***************************************************************************** * * Simple utility function to remove an entry for a block in the switch desc @@ -631,3 +618,20 @@ void Compiler::fgInvalidateSwitchDescMapEntry(BasicBlock* block) m_switchDescMap->Remove(block); } } + +void Compiler::UpdateSwitchTableTarget(BasicBlock* switchBlk, BasicBlock* from, BasicBlock* to) +{ + if (m_switchDescMap == nullptr) + { + return; // No mappings, nothing to do. + } + + // Otherwise... + BlockToSwitchDescMap* switchMap = GetSwitchDescMap(); + SwitchUniqueSuccSet* res = switchMap->LookupPointer(switchBlk); + if (res != nullptr) + { + // If no result, nothing to do. Otherwise, update it. + res->UpdateTarget(getAllocator(), switchBlk, from, to); + } +} diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index b72b99228fbc70..54003637e7e36e 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -675,14 +675,14 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsIntegralConst(0)) { - m_compiler->fgRemoveRefPred(block->GetTrueEdge()); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetFalseEdge()); + m_compiler->fgRemoveRefPred(block->GetTrueTarget(), block); + block->SetKindAndTarget(BBJ_ALWAYS, block->Next()); block->SetFlags(BBF_NONE_QUIRK); } else { - m_compiler->fgRemoveRefPred(block->GetFalseEdge()); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetTrueEdge()); + m_compiler->fgRemoveRefPred(block->GetFalseTarget(), block); + block->SetKind(BBJ_ALWAYS); } } } @@ -1177,7 +1177,6 @@ void Compiler::fgInvokeInlineeCompiler(GenTreeCall* call, InlineResult* inlineRe inlineInfo.retExprClassHnd = nullptr; inlineInfo.retExprClassHndIsExact = false; inlineInfo.inlineResult = inlineResult; - inlineInfo.inlInstParamArgInfo = nullptr; #ifdef FEATURE_SIMD inlineInfo.hasSIMDTypeArgLocalOrReturn = false; #endif // FEATURE_SIMD @@ -1534,8 +1533,9 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo) JITDUMP("\nConvert bbKind of " FMT_BB " to BBJ_ALWAYS to bottomBlock " FMT_BB "\n", block->bbNum, bottomBlock->bbNum); + block->SetKindAndTarget(BBJ_ALWAYS, bottomBlock); FlowEdge* const newEdge = fgAddRefPred(bottomBlock, block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); if (block == InlineeCompiler->fgLastBB) { @@ -1551,10 +1551,11 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo) // Insert inlinee's blocks into inliner's block list. assert(topBlock->KindIs(BBJ_ALWAYS)); assert(topBlock->TargetIs(bottomBlock)); - fgRedirectTargetEdge(topBlock, InlineeCompiler->fgFirstBB); - topBlock->SetNext(InlineeCompiler->fgFirstBB); + topBlock->SetTarget(topBlock->Next()); topBlock->SetFlags(BBF_NONE_QUIRK); + FlowEdge* const oldEdge = fgRemoveRefPred(bottomBlock, topBlock); + fgAddRefPred(InlineeCompiler->fgFirstBB, topBlock, oldEdge); InlineeCompiler->fgLastBB->SetNext(bottomBlock); // @@ -1665,175 +1666,6 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo) iciStmt->SetRootNode(gtNewNothingNode()); } -//------------------------------------------------------------------------ -// fgInsertInlineeArgument: wire up the given argument from the callsite with the inlinee -// -// Arguments: -// argInfo - information about the argument -// block - block to insert the argument into -// afterStmt - statement to insert the argument after -// newStmt - updated with the new statement -// callDI - debug info for the call -// -void Compiler::fgInsertInlineeArgument( - const InlArgInfo& argInfo, BasicBlock* block, Statement** afterStmt, Statement** newStmt, const DebugInfo& callDI) -{ - const bool argIsSingleDef = !argInfo.argHasLdargaOp && !argInfo.argHasStargOp; - CallArg* arg = argInfo.arg; - GenTree* argNode = arg->GetNode(); - - assert(!argNode->OperIs(GT_RET_EXPR)); - - if (argInfo.argHasTmp) - { - noway_assert(argInfo.argIsUsed); - - /* argBashTmpNode is non-NULL iff the argument's value was - referenced exactly once by the original IL. This offers an - opportunity to avoid an intermediate temp and just insert - the original argument tree. - - However, if the temp node has been cloned somewhere while - importing (e.g. when handling isinst or dup), or if the IL - took the address of the argument, then argBashTmpNode will - be set (because the value was only explicitly retrieved - once) but the optimization cannot be applied. - */ - - GenTree* argSingleUseNode = argInfo.argBashTmpNode; - - if ((argSingleUseNode != nullptr) && !(argSingleUseNode->gtFlags & GTF_VAR_MOREUSES) && argIsSingleDef) - { - // Change the temp in-place to the actual argument. - // We currently do not support this for struct arguments, so it must not be a GT_BLK. - assert(argNode->gtOper != GT_BLK); - argSingleUseNode->ReplaceWith(argNode, this); - return; - } - else - { - // We're going to assign the argument value to the temp we use for it in the inline body. - GenTree* store = gtNewTempStore(argInfo.argTmpNum, argNode); - - *newStmt = gtNewStmt(store, callDI); - fgInsertStmtAfter(block, *afterStmt, *newStmt); - *afterStmt = *newStmt; - DISPSTMT(*afterStmt); - } - } - else if (argInfo.argIsByRefToStructLocal) - { - // Do nothing. Arg was directly substituted as we read - // the inlinee. - } - else - { - // The argument is either not used or a const or lcl var - noway_assert(!argInfo.argIsUsed || argInfo.argIsInvariant || argInfo.argIsLclVar); - noway_assert((argInfo.argIsLclVar == 0) == - (argNode->gtOper != GT_LCL_VAR || (argNode->gtFlags & GTF_GLOB_REF))); - - // If the argument has side effects, append it - if (argInfo.argHasSideEff) - { - noway_assert(argInfo.argIsUsed == false); - *newStmt = nullptr; - bool append = true; - - if (argNode->gtOper == GT_BLK || argNode->gtOper == GT_MKREFANY) - { - // Don't put GT_BLK node under a GT_COMMA. - // Codegen can't deal with it. - // Just hang the address here in case there are side-effect. - *newStmt = gtNewStmt(gtUnusedValNode(argNode->AsOp()->gtOp1), callDI); - } - else - { - // In some special cases, unused args with side effects can - // trigger further changes. - // - // (1) If the arg is a static field access and the field access - // was produced by a call to EqualityComparer.get_Default, the - // helper call to ensure the field has a value can be suppressed. - // This helper call is marked as a "Special DCE" helper during - // importation, over in fgGetStaticsCCtorHelper. - // - // (2) NYI. If we find that the actual arg expression - // has no side effects, we can skip appending all - // together. This will help jit TP a bit. - // - assert(!argNode->OperIs(GT_RET_EXPR)); - - // For case (1) - // - // Look for the following tree shapes - // prejit: (IND (ADD (CONST, CALL(special dce helper...)))) - // jit : (COMMA (CALL(special dce helper...), (FIELD ...))) - if (argNode->gtOper == GT_COMMA) - { - // Look for (COMMA (CALL(special dce helper...), (FIELD ...))) - GenTree* op1 = argNode->AsOp()->gtOp1; - GenTree* op2 = argNode->AsOp()->gtOp2; - if (op1->IsCall() && ((op1->AsCall()->gtCallMoreFlags & GTF_CALL_M_HELPER_SPECIAL_DCE) != 0) && - op2->OperIs(GT_IND) && op2->gtGetOp1()->IsIconHandle() && ((op2->gtFlags & GTF_EXCEPT) == 0)) - { - JITDUMP("\nPerforming special dce on unused arg [%06u]:" - " actual arg [%06u] helper call [%06u]\n", - argNode->gtTreeID, argNode->gtTreeID, op1->gtTreeID); - // Drop the whole tree - append = false; - } - } - else if (argNode->gtOper == GT_IND) - { - // Look for (IND (ADD (CONST, CALL(special dce helper...)))) - GenTree* addr = argNode->AsOp()->gtOp1; - - if (addr->gtOper == GT_ADD) - { - GenTree* op1 = addr->AsOp()->gtOp1; - GenTree* op2 = addr->AsOp()->gtOp2; - if (op1->IsCall() && ((op1->AsCall()->gtCallMoreFlags & GTF_CALL_M_HELPER_SPECIAL_DCE) != 0) && - op2->IsCnsIntOrI()) - { - // Drop the whole tree - JITDUMP("\nPerforming special dce on unused arg [%06u]:" - " actual arg [%06u] helper call [%06u]\n", - argNode->gtTreeID, argNode->gtTreeID, op1->gtTreeID); - append = false; - } - } - } - } - - if (!append) - { - assert(*newStmt == nullptr); - JITDUMP("Arg tree side effects were discardable, not appending anything for arg\n"); - } - else - { - // If we don't have something custom to append, - // just append the arg node as an unused value. - if (*newStmt == nullptr) - { - *newStmt = gtNewStmt(gtUnusedValNode(argNode), callDI); - } - - fgInsertStmtAfter(block, *afterStmt, *newStmt); - *afterStmt = *newStmt; - DISPSTMT(*afterStmt); - } - } - else if (argNode->IsBoxedValue()) - { - // Try to clean up any unnecessary boxing side effects - // since the box itself will be ignored. - gtTryRemoveBoxUpstreamEffects(argNode); - } - } -} - //------------------------------------------------------------------------ // fgInlinePrependStatements: prepend statements needed to match up // caller and inlined callee @@ -1855,18 +1687,28 @@ void Compiler::fgInsertInlineeArgument( // Newly added statements are placed just after the original call // and are are given the same inline context as the call any calls // added here will appear to have been part of the immediate caller. -// + Statement* Compiler::fgInlinePrependStatements(InlineInfo* inlineInfo) { BasicBlock* block = inlineInfo->iciBlock; Statement* callStmt = inlineInfo->iciStmt; const DebugInfo& callDI = callStmt->GetDebugInfo(); + Statement* postStmt = callStmt->GetNextStmt(); Statement* afterStmt = callStmt; // afterStmt is the place where the new statements should be inserted after. Statement* newStmt = nullptr; GenTreeCall* call = inlineInfo->iciCall->AsCall(); noway_assert(call->gtOper == GT_CALL); +#ifdef DEBUG + if (0 && verbose) + { + printf("\nfgInlinePrependStatements for iciCall= "); + printTreeID(call); + printf(":\n"); + } +#endif + // Prepend statements for any initialization / side effects InlArgInfo* inlArgInfo = inlineInfo->inlArgInfo; @@ -1887,7 +1729,7 @@ Statement* Compiler::fgInlinePrependStatements(InlineInfo* inlineInfo) if (call->gtFlags & GTF_CALL_NULLCHECK && !inlineInfo->thisDereferencedFirst) { // Call impInlineFetchArg to "reserve" a temp for the "this" pointer. - GenTree* thisOp = impInlineFetchArg(inlArgInfo[0], lclVarInfo[0]); + GenTree* thisOp = impInlineFetchArg(0, inlArgInfo, lclVarInfo); if (fgAddrCouldBeNull(thisOp)) { nullcheck = gtNewNullCheck(thisOp, block); @@ -1896,19 +1738,183 @@ Statement* Compiler::fgInlinePrependStatements(InlineInfo* inlineInfo) } } - // Append the InstParam - if (inlineInfo->inlInstParamArgInfo != nullptr) - { - fgInsertInlineeArgument(*inlineInfo->inlInstParamArgInfo, block, &afterStmt, &newStmt, callDI); - } - - // Treat arguments that had to be assigned to temps + /* Treat arguments that had to be assigned to temps */ if (inlineInfo->argCnt) { - JITDUMP("\nArguments setup:\n"); + +#ifdef DEBUG + if (verbose) + { + printf("\nArguments setup:\n"); + } +#endif // DEBUG + for (unsigned argNum = 0; argNum < inlineInfo->argCnt; argNum++) { - fgInsertInlineeArgument(inlArgInfo[argNum], block, &afterStmt, &newStmt, callDI); + const InlArgInfo& argInfo = inlArgInfo[argNum]; + const bool argIsSingleDef = !argInfo.argHasLdargaOp && !argInfo.argHasStargOp; + CallArg* arg = argInfo.arg; + GenTree* argNode = arg->GetNode(); + + assert(!argNode->OperIs(GT_RET_EXPR)); + + if (argInfo.argHasTmp) + { + noway_assert(argInfo.argIsUsed); + + /* argBashTmpNode is non-NULL iff the argument's value was + referenced exactly once by the original IL. This offers an + opportunity to avoid an intermediate temp and just insert + the original argument tree. + + However, if the temp node has been cloned somewhere while + importing (e.g. when handling isinst or dup), or if the IL + took the address of the argument, then argBashTmpNode will + be set (because the value was only explicitly retrieved + once) but the optimization cannot be applied. + */ + + GenTree* argSingleUseNode = argInfo.argBashTmpNode; + + if ((argSingleUseNode != nullptr) && !(argSingleUseNode->gtFlags & GTF_VAR_MOREUSES) && argIsSingleDef) + { + // Change the temp in-place to the actual argument. + // We currently do not support this for struct arguments, so it must not be a GT_BLK. + assert(argNode->gtOper != GT_BLK); + argSingleUseNode->ReplaceWith(argNode, this); + continue; + } + else + { + // We're going to assign the argument value to the temp we use for it in the inline body. + GenTree* store = gtNewTempStore(argInfo.argTmpNum, argNode); + + newStmt = gtNewStmt(store, callDI); + fgInsertStmtAfter(block, afterStmt, newStmt); + afterStmt = newStmt; + + DISPSTMT(afterStmt); + } + } + else if (argInfo.argIsByRefToStructLocal) + { + // Do nothing. Arg was directly substituted as we read + // the inlinee. + } + else + { + // The argument is either not used or a const or lcl var + noway_assert(!argInfo.argIsUsed || argInfo.argIsInvariant || argInfo.argIsLclVar); + noway_assert((argInfo.argIsLclVar == 0) == + (argNode->gtOper != GT_LCL_VAR || (argNode->gtFlags & GTF_GLOB_REF))); + + // If the argument has side effects, append it + if (argInfo.argHasSideEff) + { + noway_assert(argInfo.argIsUsed == false); + newStmt = nullptr; + bool append = true; + + if (argNode->gtOper == GT_BLK || argNode->gtOper == GT_MKREFANY) + { + // Don't put GT_BLK node under a GT_COMMA. + // Codegen can't deal with it. + // Just hang the address here in case there are side-effect. + newStmt = gtNewStmt(gtUnusedValNode(argNode->AsOp()->gtOp1), callDI); + } + else + { + // In some special cases, unused args with side effects can + // trigger further changes. + // + // (1) If the arg is a static field access and the field access + // was produced by a call to EqualityComparer.get_Default, the + // helper call to ensure the field has a value can be suppressed. + // This helper call is marked as a "Special DCE" helper during + // importation, over in fgGetStaticsCCtorHelper. + // + // (2) NYI. If we find that the actual arg expression + // has no side effects, we can skip appending all + // together. This will help jit TP a bit. + // + assert(!argNode->OperIs(GT_RET_EXPR)); + + // For case (1) + // + // Look for the following tree shapes + // prejit: (IND (ADD (CONST, CALL(special dce helper...)))) + // jit : (COMMA (CALL(special dce helper...), (FIELD ...))) + if (argNode->gtOper == GT_COMMA) + { + // Look for (COMMA (CALL(special dce helper...), (FIELD ...))) + GenTree* op1 = argNode->AsOp()->gtOp1; + GenTree* op2 = argNode->AsOp()->gtOp2; + if (op1->IsCall() && + ((op1->AsCall()->gtCallMoreFlags & GTF_CALL_M_HELPER_SPECIAL_DCE) != 0) && + op2->OperIs(GT_IND) && op2->gtGetOp1()->IsIconHandle() && + ((op2->gtFlags & GTF_EXCEPT) == 0)) + { + JITDUMP("\nPerforming special dce on unused arg [%06u]:" + " actual arg [%06u] helper call [%06u]\n", + argNode->gtTreeID, argNode->gtTreeID, op1->gtTreeID); + // Drop the whole tree + append = false; + } + } + else if (argNode->gtOper == GT_IND) + { + // Look for (IND (ADD (CONST, CALL(special dce helper...)))) + GenTree* addr = argNode->AsOp()->gtOp1; + + if (addr->gtOper == GT_ADD) + { + GenTree* op1 = addr->AsOp()->gtOp1; + GenTree* op2 = addr->AsOp()->gtOp2; + if (op1->IsCall() && + ((op1->AsCall()->gtCallMoreFlags & GTF_CALL_M_HELPER_SPECIAL_DCE) != 0) && + op2->IsCnsIntOrI()) + { + // Drop the whole tree + JITDUMP("\nPerforming special dce on unused arg [%06u]:" + " actual arg [%06u] helper call [%06u]\n", + argNode->gtTreeID, argNode->gtTreeID, op1->gtTreeID); + append = false; + } + } + } + } + + if (!append) + { + assert(newStmt == nullptr); + JITDUMP("Arg tree side effects were discardable, not appending anything for arg\n"); + } + else + { + // If we don't have something custom to append, + // just append the arg node as an unused value. + if (newStmt == nullptr) + { + newStmt = gtNewStmt(gtUnusedValNode(argNode), callDI); + } + + fgInsertStmtAfter(block, afterStmt, newStmt); + afterStmt = newStmt; +#ifdef DEBUG + if (verbose) + { + gtDispStmt(afterStmt); + } +#endif // DEBUG + } + } + else if (argNode->IsBoxedValue()) + { + // Try to clean up any unnecessary boxing side effects + // since the box itself will be ignored. + gtTryRemoveBoxUpstreamEffects(argNode); + } + } } } diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 59e795f874959f..acce2402044762 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -30,6 +30,8 @@ void Compiler::fgComputeReturnBlocks() } } + fgReturnBlocksComputed = true; + #ifdef DEBUG if (verbose) { @@ -130,7 +132,7 @@ bool Compiler::fgRemoveUnreachableBlocks(CanRemoveBlockBody canRemoveBlock) block->RemoveFlags(BBF_REMOVED | BBF_INTERNAL); block->SetFlags(BBF_IMPORTED); - block->SetKindAndTargetEdge(BBJ_THROW); + block->SetKindAndTarget(BBJ_THROW); block->bbSetRunRarely(); } else @@ -162,6 +164,35 @@ bool Compiler::fgRemoveUnreachableBlocks(CanRemoveBlockBody canRemoveBlock) return changed; } +//------------------------------------------------------------------------ +// fgComputeReachability: Compute the dominator and reachable sets. +// +// Returns: +// Suitable phase status +// +// Notes: +// Also computes the list of return blocks `fgReturnBlocks` +// and set of enter blocks `fgEnterBlks`. +// +// Delete unreachable blocks. +// +// Assumes the predecessor lists are computed and correct. +// +PhaseStatus Compiler::fgComputeReachability() +{ + assert(fgPredsComputed); + + bool madeChanges = fgDfsBlocksAndRemove() != PhaseStatus::MODIFIED_NOTHING; + + madeChanges |= fgRenumberBlocks(); + + fgComputeReturnBlocks(); + m_reachabilitySets = BlockReachabilitySets::Build(m_dfsTree); + m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); + + return madeChanges ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; +} + //------------------------------------------------------------------------ // fgRemoveDeadBlocks: Identify all the unreachable blocks and remove them. // @@ -276,10 +307,7 @@ bool Compiler::fgRemoveDeadBlocks() PhaseStatus Compiler::fgComputeDominators() { assert(m_dfsTree != nullptr); - if (m_domTree == nullptr) - { - m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); - } + m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); bool anyHandlers = false; for (EHblkDsc* const HBtab : EHClauses(this)) @@ -595,8 +623,8 @@ PhaseStatus Compiler::fgPostImportationCleanup() // What follows is similar to fgNewBBInRegion, but we can't call that // here as the oldTryEntry is no longer in the main bb list. - newTryEntry = BasicBlock::New(this); - newTryEntry->SetFlags(BBF_IMPORTED | BBF_INTERNAL); + newTryEntry = BasicBlock::New(this, BBJ_ALWAYS, tryEntryPrev->Next()); + newTryEntry->SetFlags(BBF_IMPORTED | BBF_INTERNAL | BBF_NONE_QUIRK); newTryEntry->bbRefs = 0; // Set the right EH region indices on this new block. @@ -615,13 +643,12 @@ PhaseStatus Compiler::fgPostImportationCleanup() // plausible flow target. Simplest is to just mark it as a throw. if (bbIsHandlerBeg(newTryEntry->Next())) { - newTryEntry->SetKindAndTargetEdge(BBJ_THROW); + newTryEntry->SetKindAndTarget(BBJ_THROW); } else { FlowEdge* const newEdge = fgAddRefPred(newTryEntry->Next(), newTryEntry); - newTryEntry->SetFlags(BBF_NONE_QUIRK); - newTryEntry->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); } JITDUMP("OSR: changing start of try region #%u from " FMT_BB " to new " FMT_BB "\n", @@ -747,7 +774,7 @@ PhaseStatus Compiler::fgPostImportationCleanup() fromBlock->SetFlags(BBF_INTERNAL); newBlock->RemoveFlags(BBF_DONT_REMOVE); addedBlocks++; - FlowEdge* const normalTryEntryEdge = fromBlock->GetTargetEdge(); + FlowEdge* const normalTryEntryEdge = fgGetPredForBlock(newBlock, fromBlock); GenTree* const entryStateLcl = gtNewLclvNode(entryStateVar, TYP_INT); GenTree* const compareEntryStateToZero = @@ -755,9 +782,9 @@ PhaseStatus Compiler::fgPostImportationCleanup() GenTree* const jumpIfEntryStateZero = gtNewOperNode(GT_JTRUE, TYP_VOID, compareEntryStateToZero); fgNewStmtAtBeg(fromBlock, jumpIfEntryStateZero); + fromBlock->SetCond(toBlock, newBlock); FlowEdge* const osrTryEntryEdge = fgAddRefPred(toBlock, fromBlock); newBlock->inheritWeight(fromBlock); - fromBlock->SetCond(osrTryEntryEdge, normalTryEntryEdge); // Not sure what the correct edge likelihoods are just yet; // for now we'll say the OSR path is the likely one. @@ -806,7 +833,9 @@ PhaseStatus Compiler::fgPostImportationCleanup() if (entryJumpTarget != osrEntry) { - fgRedirectTargetEdge(fgFirstBB, entryJumpTarget); + fgFirstBB->SetTarget(entryJumpTarget); + FlowEdge* const oldEdge = fgRemoveRefPred(osrEntry, fgFirstBB); + fgAddRefPred(entryJumpTarget, fgFirstBB, oldEdge); JITDUMP("OSR: redirecting flow from method entry " FMT_BB " to OSR entry " FMT_BB " via step blocks.\n", @@ -977,7 +1006,7 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) noway_assert(block->hasTryIndex() == bNext->hasTryIndex()); JITDUMP("\nCompacting " FMT_BB " into " FMT_BB ":\n", bNext->bbNum, block->bbNum); - fgRemoveRefPred(block->GetTargetEdge()); + fgRemoveRefPred(bNext, block); if (bNext->countOfInEdges() > 0) { @@ -1257,31 +1286,24 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) case BBJ_ALWAYS: case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: - { - /* Update the predecessor list for bNext's target */ - FlowEdge* const targetEdge = bNext->GetTargetEdge(); - fgReplacePred(targetEdge, block); + block->SetKindAndTarget(bNext->GetKind(), bNext->GetTarget()); - block->SetKindAndTargetEdge(bNext->GetKind(), targetEdge); + /* Update the predecessor list for 'bNext->bbTarget' */ + fgReplacePred(bNext->GetTarget(), bNext, block); break; - } case BBJ_COND: - { - /* Update the predecessor list for bNext's true target */ - FlowEdge* const trueEdge = bNext->GetTrueEdge(); - FlowEdge* const falseEdge = bNext->GetFalseEdge(); - fgReplacePred(trueEdge, block); + block->SetCond(bNext->GetTrueTarget(), bNext->GetFalseTarget()); + + /* Update the predecessor list for 'bNext->bbTrueTarget' */ + fgReplacePred(bNext->GetTrueTarget(), bNext, block); - /* Update the predecessor list for bNext's false target if it is different from the true target */ - if (trueEdge != falseEdge) + /* Update the predecessor list for 'bNext->bbFalseTarget' if it is different than 'bNext->bbTrueTarget' */ + if (!bNext->TrueTargetIs(bNext->GetFalseTarget())) { - fgReplacePred(falseEdge, block); + fgReplacePred(bNext->GetFalseTarget(), bNext, block); } - - block->SetCond(trueEdge, falseEdge); break; - } case BBJ_EHFINALLYRET: block->SetEhf(bNext->GetEhfTargets()); @@ -1503,7 +1525,7 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc bDest->SetFlags(BBF_RUN_RARELY); // Set the RarelyRun flag } - FlowEdge* edge2 = bDest->GetTargetEdge(); + FlowEdge* edge2 = fgGetPredForBlock(bDest->GetTarget(), bDest); if (edge2 != nullptr) { @@ -1539,21 +1561,19 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc { case BBJ_ALWAYS: case BBJ_CALLFINALLYRET: - { - fgRedirectTargetEdge(block, bDest->GetTarget()); + block->SetTarget(bDest->GetTarget()); break; - } case BBJ_COND: if (block->TrueTargetIs(bDest)) { assert(!block->FalseTargetIs(bDest)); - fgRedirectTrueEdge(block, bDest->GetTarget()); + block->SetTrueTarget(bDest->GetTarget()); } else { assert(block->FalseTargetIs(bDest)); - fgRedirectFalseEdge(block, bDest->GetTarget()); + block->SetFalseTarget(bDest->GetTarget()); } break; @@ -1561,6 +1581,8 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc unreached(); } + fgAddRefPred(bDest->GetTarget(), block, fgRemoveRefPred(bDest, block)); + return true; } return false; @@ -1617,6 +1639,15 @@ bool Compiler::fgOptimizeEmptyBlock(BasicBlock* block) break; } } + else + { + // TODO-NoFallThrough: Once BBJ_COND blocks have pointers to their false branches, + // allow removing empty BBJ_ALWAYS and pointing bPrev's false branch to block->bbTarget. + if (bPrev->bbFallsThrough() && !block->JumpsToNext()) + { + break; + } + } /* Do not remove a block that jumps to itself - used for while (true){} */ if (block->TargetIs(block)) @@ -1776,16 +1807,16 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) { assert(block->KindIs(BBJ_SWITCH)); - unsigned jmpCnt = block->GetSwitchTargets()->bbsCount; - FlowEdge** jmpTab = block->GetSwitchTargets()->bbsDstTab; - BasicBlock* bNewDest; // the new jump target for the current switch case - BasicBlock* bDest; - bool modified = false; + unsigned jmpCnt = block->GetSwitchTargets()->bbsCount; + BasicBlock** jmpTab = block->GetSwitchTargets()->bbsDstTab; + BasicBlock* bNewDest; // the new jump target for the current switch case + BasicBlock* bDest; + bool returnvalue = false; do { REPEAT_SWITCH:; - bDest = (*jmpTab)->getDestinationBlock(); + bDest = *jmpTab; bNewDest = bDest; // Do we have a JUMP to an empty unconditional JUMP block? @@ -1825,7 +1856,7 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) { if (fgHaveValidEdgeWeights) { - FlowEdge* edge = *jmpTab; + FlowEdge* edge = fgGetPredForBlock(bDest, block); weight_t branchThroughWeight = edge->edgeWeightMin(); if (bDest->bbWeight > branchThroughWeight) @@ -1841,45 +1872,20 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) } // Update the switch jump table - FlowEdge* const oldEdge = *jmpTab; - fgRemoveRefPred(oldEdge); - FlowEdge* const newEdge = fgAddRefPred(bNewDest, block, oldEdge); - *jmpTab = newEdge; - - // Update edge likelihoods - // Note old edge may still be "in use" so we decrease its likelihood. - // - if (oldEdge->hasLikelihood()) - { - // We want to move this much likelihood from old->new - // - const weight_t likelihoodFraction = oldEdge->getLikelihood() / (oldEdge->getDupCount() + 1); + *jmpTab = bNewDest; - if (newEdge->getDupCount() == 1) - { - newEdge->setLikelihood(likelihoodFraction); - } - else - { - newEdge->addLikelihood(likelihoodFraction); - } + // Maintain, if necessary, the set of unique targets of "block." + UpdateSwitchTableTarget(block, bDest, bNewDest); - oldEdge->addLikelihood(-likelihoodFraction); - } + fgAddRefPred(bNewDest, block, fgRemoveRefPred(bDest, block)); // we optimized a Switch label - goto REPEAT_SWITCH to follow this new jump - modified = true; + returnvalue = true; goto REPEAT_SWITCH; } } while (++jmpTab, --jmpCnt); - if (modified) - { - // Invalidate the set of unique targets for block, since we modified the targets - fgInvalidateSwitchDescMapEntry(block); - } - Statement* switchStmt = nullptr; LIR::Range* blockRange = nullptr; @@ -1919,7 +1925,6 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) { printf("\nRemoving a switch jump with a single target (" FMT_BB ")\n", block->bbNum); printf("BEFORE:\n"); - fgDispBasicBlocks(); } #endif // DEBUG @@ -1991,16 +1996,18 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) } // Change the switch jump into a BBJ_ALWAYS - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetSwitchTargets()->bbsDstTab[0]); - for (unsigned i = 1; i < jmpCnt; ++i) + block->SetKindAndTarget(BBJ_ALWAYS, block->GetSwitchTargets()->bbsDstTab[0]); + if (jmpCnt > 1) { - fgRemoveRefPred(jmpTab[i]); + for (unsigned i = 1; i < jmpCnt; ++i) + { + (void)fgRemoveRefPred(jmpTab[i], block); + } } return true; } - else if ((block->GetSwitchTargets()->bbsCount == 2) && - block->NextIs(block->GetSwitchTargets()->bbsDstTab[1]->getDestinationBlock())) + else if ((block->GetSwitchTargets()->bbsCount == 2) && block->NextIs(block->GetSwitchTargets()->bbsDstTab[1])) { /* Use a BBJ_COND(switchVal==0) for a switch with only one significant clause besides the default clause, if the @@ -2053,16 +2060,14 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) fgSetStmtSeq(switchStmt); } - FlowEdge* const trueEdge = block->GetSwitchTargets()->bbsDstTab[0]; - FlowEdge* const falseEdge = block->GetSwitchTargets()->bbsDstTab[1]; - block->SetCond(trueEdge, falseEdge); + block->SetCond(block->GetSwitchTargets()->bbsDstTab[0], block->GetSwitchTargets()->bbsDstTab[1]); JITDUMP("After:\n"); DISPNODE(switchTree); return true; } - return modified; + return returnvalue; } //------------------------------------------------------------- @@ -2463,29 +2468,26 @@ bool Compiler::fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* fgInsertStmtAtEnd(block, cloneStmt); } - // Fix up block's flow. - // Assume edge likelihoods transfer over. + // add an unconditional block after this block to jump to the target block's fallthrough block // - fgRedirectTargetEdge(block, target->GetTrueTarget()); - block->GetTargetEdge()->setLikelihood(target->GetTrueEdge()->getLikelihood()); + assert(!target->IsLast()); + BasicBlock* next = fgNewBBafter(BBJ_ALWAYS, block, true, target->GetFalseTarget()); - FlowEdge* const falseEdge = fgAddRefPred(target->GetFalseTarget(), block, target->GetFalseEdge()); - block->SetCond(block->GetTargetEdge(), falseEdge); + // Fix up block's flow + // + block->SetCond(target->GetTrueTarget(), next); + fgAddRefPred(block->GetTrueTarget(), block); + fgRemoveRefPred(target, block); - JITDUMP("fgOptimizeUncondBranchToSimpleCond(from " FMT_BB " to cond " FMT_BB "), modified " FMT_BB "\n", - block->bbNum, target->bbNum, block->bbNum); - JITDUMP(" expecting opts to key off V%02u in " FMT_BB "\n", lclNum, block->bbNum); + // The new block 'next' will inherit its weight from 'block' + // + next->inheritWeight(block); + fgAddRefPred(next, block); + fgAddRefPred(next->GetTarget(), next); - if (target->hasProfileWeight() && block->hasProfileWeight()) - { - // Remove weight from target since block now bypasses it... - // - weight_t targetWeight = target->bbWeight; - weight_t blockWeight = block->bbWeight; - target->setBBProfileWeight(max(0, targetWeight - blockWeight)); - JITDUMP("Decreased " FMT_BB " profile weight from " FMT_WT " to " FMT_WT "\n", target->bbNum, targetWeight, - target->bbWeight); - } + JITDUMP("fgOptimizeUncondBranchToSimpleCond(from " FMT_BB " to cond " FMT_BB "), created new uncond " FMT_BB "\n", + block->bbNum, target->bbNum, next->bbNum); + JITDUMP(" expecting opts to key off V%02u in " FMT_BB "\n", lclNum, block->bbNum); return true; } @@ -2500,7 +2502,7 @@ bool Compiler::fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* void Compiler::fgRemoveConditionalJump(BasicBlock* block) { assert(block->KindIs(BBJ_COND)); - assert(block->TrueEdgeIs(block->GetFalseEdge())); + assert(block->TrueTargetIs(block->GetFalseTarget())); BasicBlock* target = block->GetTrueTarget(); @@ -2610,7 +2612,7 @@ void Compiler::fgRemoveConditionalJump(BasicBlock* block) /* Conditional is gone - always jump to target */ - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetTrueEdge()); + block->SetKind(BBJ_ALWAYS); assert(block->TargetIs(target)); // TODO-NoFallThrough: Set BBF_NONE_QUIRK only when false target is the next block @@ -2620,7 +2622,7 @@ void Compiler::fgRemoveConditionalJump(BasicBlock* block) * block are counted twice so we have to remove one of them */ noway_assert(target->countOfInEdges() > 1); - fgRemoveRefPred(block->GetTargetEdge()); + fgRemoveRefPred(target, block); } //------------------------------------------------------------- @@ -2880,31 +2882,21 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) // We need to update the following flags of the bJump block if they were set in the bDest block bJump->CopyFlags(bDest, BBF_COPY_PROPAGATE); - // Update bbRefs and bbPreds - // - // For now we set the likelihood of the new branch to match - // the likelihood of the old branch. - // - // This may or may not match the block weight adjustments we're - // making. All this becomes easier to reconcile once we rely on - // edge likelihoods more and have synthesis running. - // - // Until then we won't worry that edges and blocks are potentially - // out of sync. - // - FlowEdge* const destFalseEdge = bDest->GetFalseEdge(); - FlowEdge* const destTrueEdge = bDest->GetTrueEdge(); + bJump->SetCond(bDestNormalTarget, bJump->Next()); + + /* Update bbRefs and bbPreds */ // bJump now falls through into the next block // - FlowEdge* const falseEdge = fgAddRefPred(bJump->Next(), bJump, destFalseEdge); + fgAddRefPred(bJump->GetFalseTarget(), bJump); - // bJump now jumps to bDest's normal jump target + // bJump no longer jumps to bDest // - fgRedirectTargetEdge(bJump, bDestNormalTarget); - bJump->GetTargetEdge()->setLikelihood(destTrueEdge->getLikelihood()); + fgRemoveRefPred(bDest, bJump); - bJump->SetCond(bJump->GetTargetEdge(), falseEdge); + // bJump now jumps to bDest's normal jump target + // + fgAddRefPred(bDestNormalTarget, bJump); if (weightJump > 0) { @@ -3007,7 +2999,7 @@ bool Compiler::fgOptimizeSwitchJumps() // The dominant case should not be the default case, as we already peel that one. // assert(dominantCase < (block->GetSwitchTargets()->bbsCount - 1)); - BasicBlock* const dominantTarget = block->GetSwitchTargets()->bbsDstTab[dominantCase]->getDestinationBlock(); + BasicBlock* const dominantTarget = block->GetSwitchTargets()->bbsDstTab[dominantCase]; Statement* const switchStmt = block->lastStmt(); GenTree* const switchTree = switchStmt->GetRootNode(); assert(switchTree->OperIs(GT_SWITCH)); @@ -3050,9 +3042,11 @@ bool Compiler::fgOptimizeSwitchJumps() // Wire up the new control flow. // + block->SetCond(dominantTarget, newBlock); FlowEdge* const blockToTargetEdge = fgAddRefPred(dominantTarget, block); FlowEdge* const blockToNewBlockEdge = newBlock->bbPreds; - block->SetCond(blockToTargetEdge, blockToNewBlockEdge); + assert(blockToNewBlockEdge->getSourceBlock() == block); + assert(blockToTargetEdge->getSourceBlock() == block); // Update profile data // @@ -3063,9 +3057,7 @@ bool Compiler::fgOptimizeSwitchJumps() newBlock->setBBProfileWeight(blockToNewBlockWeight); blockToTargetEdge->setEdgeWeights(blockToTargetWeight, blockToTargetWeight, dominantTarget); - blockToTargetEdge->setLikelihood(fraction); blockToNewBlockEdge->setEdgeWeights(blockToNewBlockWeight, blockToNewBlockWeight, block); - blockToNewBlockEdge->setLikelihood(max(0, 1.0 - fraction)); // There may be other switch cases that lead to this same block, but there's just // one edge in the flowgraph. So we need to subtract off the profile data that now flows @@ -3523,11 +3515,11 @@ bool Compiler::fgReorderBlocks(bool useProfile) assert(test->OperIsConditionalJump()); test->AsOp()->gtOp1 = gtReverseCond(test->AsOp()->gtOp1); - FlowEdge* const newFalseEdge = block->GetTrueEdge(); - FlowEdge* const newTrueEdge = block->GetFalseEdge(); - block->SetTrueEdge(newTrueEdge); - block->SetFalseEdge(newFalseEdge); - assert(block->CanRemoveJumpToTarget(block->GetFalseTarget(), this)); + BasicBlock* newFalseTarget = block->GetTrueTarget(); + BasicBlock* newTrueTarget = block->GetFalseTarget(); + block->SetTrueTarget(newTrueTarget); + block->SetFalseTarget(newFalseTarget); + assert(block->CanRemoveJumpToTarget(newFalseTarget, this)); } else { @@ -3659,7 +3651,7 @@ bool Compiler::fgReorderBlocks(bool useProfile) // The edge bPrev -> bDest must have a higher minimum weight // than every other edge into bDest // - FlowEdge* edgeFromPrev = bPrev->GetTargetEdge(); + FlowEdge* edgeFromPrev = fgGetPredForBlock(bDest, bPrev); noway_assert(edgeFromPrev != nullptr); // Examine all of the other edges into bDest @@ -3751,9 +3743,8 @@ bool Compiler::fgReorderBlocks(bool useProfile) // V // bDest ---------------> [BB08, weight 21] // - assert(bPrev->FalseTargetIs(block)); - FlowEdge* edgeToDest = bPrev->GetTrueEdge(); - FlowEdge* edgeToBlock = bPrev->GetFalseEdge(); + FlowEdge* edgeToDest = fgGetPredForBlock(bDest, bPrev); + FlowEdge* edgeToBlock = fgGetPredForBlock(block, bPrev); noway_assert(edgeToDest != nullptr); noway_assert(edgeToBlock != nullptr); // @@ -4585,10 +4576,10 @@ bool Compiler::fgReorderBlocks(bool useProfile) noway_assert(condTest->gtOper == GT_JTRUE); condTest->AsOp()->gtOp1 = gtReverseCond(condTest->AsOp()->gtOp1); - FlowEdge* const trueEdge = bPrev->GetTrueEdge(); - FlowEdge* const falseEdge = bPrev->GetFalseEdge(); - bPrev->SetTrueEdge(falseEdge); - bPrev->SetFalseEdge(trueEdge); + BasicBlock* trueTarget = bPrev->GetTrueTarget(); + BasicBlock* falseTarget = bPrev->GetFalseTarget(); + bPrev->SetTrueTarget(falseTarget); + bPrev->SetFalseTarget(trueTarget); // may need to rethread // @@ -4824,11 +4815,13 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh if (doTailDuplication && fgOptimizeUncondBranchToSimpleCond(block, bDest)) { assert(block->KindIs(BBJ_COND)); - assert(bNext == block->Next()); - change = true; - modified = true; - bDest = block->GetTrueTarget(); - bFalseDest = block->GetFalseTarget(); + change = true; + modified = true; + bDest = block->GetTrueTarget(); + bNext = block->GetFalseTarget(); + + // TODO-NoFallThrough: Adjust the above logic once bbFalseTarget can diverge from bbNext + assert(block->NextIs(bNext)); } } @@ -4983,6 +4976,20 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh { ehUpdateLastBlocks(bNext, bDest); } + + // Add fall through fixup block, if needed. + // + if (bDest->KindIs(BBJ_COND) && !bDest->NextIs(bDest->GetFalseTarget())) + { + BasicBlock* const bDestFalseTarget = bDest->GetFalseTarget(); + BasicBlock* const bFixup = fgNewBBafter(BBJ_ALWAYS, bDest, true, bDestFalseTarget); + bDest->SetFalseTarget(bFixup); + bFixup->inheritWeight(bDestFalseTarget); + + fgRemoveRefPred(bDestFalseTarget, bDest); + fgAddRefPred(bFixup, bDest); + fgAddRefPred(bDestFalseTarget, bFixup); + } } } @@ -5009,20 +5016,10 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh } // Optimize the Conditional JUMP to go to the new target - // - FlowEdge* const oldFalseEdge = block->GetFalseEdge(); - FlowEdge* const oldTrueEdge = block->GetTrueEdge(); - FlowEdge* const oldNextEdge = bNext->GetTargetEdge(); + block->SetTrueTarget(bNext->GetTarget()); + block->SetFalseTarget(bNext->Next()); - // bNext no longer flows to target - // - fgRemoveRefPred(oldNextEdge); - - // Rewire flow from block - // - block->SetFalseEdge(oldTrueEdge); - block->SetTrueEdge(oldFalseEdge); - fgRedirectTrueEdge(block, bNext->GetTarget()); + fgAddRefPred(bNext->GetTarget(), block, fgRemoveRefPred(bNext->GetTarget(), bNext)); /* Unlink bNext from the BasicBlock list; note that we can @@ -5034,6 +5031,7 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh to the final target by the time we're done here. */ + fgRemoveRefPred(bNext, block); fgUnlinkBlockForRemoval(bNext); /* Mark the block as removed */ @@ -5263,7 +5261,7 @@ PhaseStatus Compiler::fgDfsBlocksAndRemove() #ifdef DEBUG if (verbose) { - printf("%u/%u blocks are unreachable and will be removed:\n", fgBBcount - m_dfsTree->GetPostOrderCount(), + printf("%u/%u blocks are unreachable and will be removed\n", fgBBcount - m_dfsTree->GetPostOrderCount(), fgBBcount); for (BasicBlock* block : Blocks()) { @@ -5273,7 +5271,7 @@ PhaseStatus Compiler::fgDfsBlocksAndRemove() } } } -#endif // DEBUG +#endif // The DFS we run is not precise around call-finally, so // `fgRemoveUnreachableBlocks` can expose newly unreachable blocks @@ -5301,24 +5299,6 @@ PhaseStatus Compiler::fgDfsBlocksAndRemove() m_dfsTree = fgComputeDfs(); } -#ifdef DEBUG - // Did we actually remove all the blocks we said we were going to? - if (verbose) - { - if (m_dfsTree->GetPostOrderCount() != fgBBcount) - { - printf("%u unreachable blocks were not removed:\n", fgBBcount - m_dfsTree->GetPostOrderCount()); - for (BasicBlock* block : Blocks()) - { - if (!m_dfsTree->Contains(block)) - { - printf(" " FMT_BB "\n", block->bbNum); - } - } - } - } -#endif // DEBUG - status = PhaseStatus::MODIFIED_EVERYTHING; } @@ -5684,16 +5664,13 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early) // Fix up the flow. // + predBlock->SetKindAndTarget(BBJ_ALWAYS, crossJumpTarget); + if (commSucc != nullptr) { - assert(predBlock->KindIs(BBJ_ALWAYS)); - fgRedirectTargetEdge(predBlock, crossJumpTarget); - } - else - { - FlowEdge* const newEdge = fgAddRefPred(crossJumpTarget, predBlock); - predBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + fgRemoveRefPred(commSucc, predBlock); } + fgAddRefPred(crossJumpTarget, predBlock); } // We changed things @@ -5862,7 +5839,7 @@ bool Compiler::fgTryOneHeadMerge(BasicBlock* block, bool early) // ternaries in C#). // The logic below could be generalized to BBJ_SWITCH, but this currently // has almost no CQ benefit but does have a TP impact. - if (!block->KindIs(BBJ_COND) || block->TrueEdgeIs(block->GetFalseEdge())) + if (!block->KindIs(BBJ_COND) || block->TrueTargetIs(block->GetFalseTarget())) { return false; } diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index 4693107168ba61..9e29cc25792734 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -507,11 +507,12 @@ void BlockCountInstrumentor::RelocateProbes() // if (criticalPreds.Height() > 0) { - BasicBlock* const intermediary = m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true); + BasicBlock* const intermediary = + m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true, /* jumpDest */ block); intermediary->SetFlags(BBF_IMPORTED | BBF_MARKED | BBF_NONE_QUIRK); intermediary->inheritWeight(block); FlowEdge* const newEdge = m_comp->fgAddRefPred(block, intermediary); - intermediary->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); SetModifiedFlow(); while (criticalPreds.Height() > 0) @@ -1678,11 +1679,12 @@ void EfficientEdgeCountInstrumentor::RelocateProbes() // if (criticalPreds.Height() > 0) { - BasicBlock* intermediary = m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true); + BasicBlock* intermediary = + m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true, /* jumpDest */ block); intermediary->SetFlags(BBF_IMPORTED | BBF_NONE_QUIRK); intermediary->inheritWeight(block); FlowEdge* const newEdge = m_comp->fgAddRefPred(block, intermediary); - intermediary->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); NewRelocatedProbe(intermediary, probe->source, probe->target, &leader); SetModifiedFlow(); @@ -1945,7 +1947,7 @@ class ValueHistogramProbeVisitor final : public GenTreeVisitorIsCall() && node->AsCall()->IsSpecialIntrinsic()) { const NamedIntrinsic ni = m_compiler->lookupNamedIntrinsic(node->AsCall()->gtCallMethHnd); - if ((ni == NI_System_SpanHelpers_Memmove) || (ni == NI_System_SpanHelpers_SequenceEqual)) + if ((ni == NI_System_Buffer_Memmove) || (ni == NI_System_SpanHelpers_SequenceEqual)) { m_functor(m_compiler, node); } @@ -2272,7 +2274,7 @@ class ValueHistogramProbeInserter return; } - assert(node->AsCall()->IsSpecialIntrinsic(compiler, NI_System_SpanHelpers_Memmove) || + assert(node->AsCall()->IsSpecialIntrinsic(compiler, NI_System_Buffer_Memmove) || node->AsCall()->IsSpecialIntrinsic(compiler, NI_System_SpanHelpers_SequenceEqual)); const ICorJitInfo::PgoInstrumentationSchema& countEntry = m_schema[*m_currentSchemaIndex]; @@ -2538,13 +2540,10 @@ PhaseStatus Compiler::fgPrepareToInstrumentMethod() // These are marked as [Intrinsic] only to be handled (unrolled) for constant inputs. // In other cases they have large managed implementations we want to profile. case NI_System_String_Equals: - case NI_System_SpanHelpers_Memmove: + case NI_System_Buffer_Memmove: case NI_System_MemoryExtensions_Equals: case NI_System_MemoryExtensions_SequenceEqual: case NI_System_MemoryExtensions_StartsWith: - case NI_System_SpanHelpers_Fill: - case NI_System_SpanHelpers_SequenceEqual: - case NI_System_SpanHelpers_ClearWithoutReferences: // Same here, these are only folded when JIT knows the exact types case NI_System_Type_IsAssignableFrom: @@ -3860,22 +3859,18 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, { // We expect one pseudo-edge and at least one normal edge. // - Edge* pseudoEdge = nullptr; - weight_t pseudoEdgeWeight = 0; - unsigned nEdges = 0; - weight_t successorWeight = BB_ZERO_WEIGHT; + Edge* pseudoEdge = nullptr; + unsigned nEdges = 0; for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { if (edge->m_isPseudoEdge) { assert(pseudoEdge == nullptr); - pseudoEdge = edge; - pseudoEdgeWeight = edge->m_weight; + pseudoEdge = edge; continue; } - successorWeight += edge->m_weight; nEdges++; } @@ -3892,25 +3887,28 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, assert(nEdges == nSucc); - if ((info->m_weight == BB_ZERO_WEIGHT) || (successorWeight == BB_ZERO_WEIGHT)) + if (info->m_weight == BB_ZERO_WEIGHT) { - JITDUMP("\nPropagate: OSR entry block or successor weight is zero\n"); + JITDUMP("\nPropagate: OSR entry block weight is zero\n"); EntryWeightZero(); return; } // Transfer model edge weight onto the FlowEdges as likelihoods. // - JITDUMP("Normalizing OSR successor likelihoods with factor 1/" FMT_WT "\n", successorWeight); + assert(nEdges == nSucc); + weight_t totalLikelihood = 0; for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { assert(block == edge->m_sourceBlock); - // The pseudo edge doesn't correspond to a flow edge. + // The pseudo edge doesn't correspond to a flow edge, + // but it carries away some flow. // if (edge == pseudoEdge) { + totalLikelihood += edge->m_weight / info->m_weight; continue; } @@ -3919,26 +3917,58 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, assert(flowEdge != nullptr); - // Naive likelihood should have been set during pred initialization in fgLinkBasicBlocks + // Naive likelihood should have been set during pred initialization in fgAddRefPred // assert(flowEdge->hasLikelihood()); weight_t likelihood = 0; if (nEdges == 1) { + // Conceptually we could assert(edge->m_weight == info->m_weight); + // but we can have inconsistencies. + // // Go with what we know for sure, edge should be 100% likely. // likelihood = 1.0; JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (uniq)\n", block->bbNum, edge->m_targetBlock->bbNum, likelihood); flowEdge->setLikelihood(likelihood); + totalLikelihood += likelihood; break; } - likelihood = edge->m_weight / successorWeight; - JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (pgo)\n", block->bbNum, - edge->m_targetBlock->bbNum, likelihood); + assert(info->m_weight != BB_ZERO_WEIGHT); + + // We may see nonsensical weights here, cap likelihood. + // + bool capped = false; + if (edge->m_weight > info->m_weight) + { + capped = true; + likelihood = 1.0; + } + else + { + likelihood = edge->m_weight / info->m_weight; + } + JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (%s)\n", block->bbNum, + edge->m_targetBlock->bbNum, likelihood, capped ? "pgo -- capped" : "pgo"); flowEdge->setLikelihood(likelihood); + totalLikelihood += likelihood; + } + + // Note we expect real flow imbalances here as it's likely there + // was no observed flow from the OSR entry to some of its successors. + // Since we added in the pseudo edge likelihood above, the check below + // probably won't flag this. + // + // Seems like for OSR we will always want to run synthesis/repair. + // + if (totalLikelihood != 1.0) + { + // Consider what to do here... flag this method as needing immediate profile repairs? + // + JITDUMP(FMT_BB " total outgoing likelihood inaccurate: " FMT_WT "\n", block->bbNum, totalLikelihood); } } @@ -3951,6 +3981,10 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, // info - model info for the block // nSucc - number of successors of the block in the flow graph // +// Notes: +// This block requires special handling because original method flow +// was interrupted here. +// void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInfo* info, unsigned nSucc) { // There is at least one FlowEdge. @@ -3958,9 +3992,8 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // Check the reconstruction graph edges. For normal blocks, if we have // any pseudo-edges there should be only one pseudo-edge, and no regular edges. // - Edge* pseudoEdge = nullptr; - unsigned nEdges = 0; - weight_t successorWeight = BB_ZERO_WEIGHT; + Edge* pseudoEdge = nullptr; + unsigned nEdges = 0; for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { @@ -3971,15 +4004,14 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf continue; } - successorWeight += edge->m_weight; nEdges++; } // If there is a pseudo edge, // There should be only one successor for block. The flow // from block to successor will not represent real flow. - // Likelihood should be set to 1.0 already, as we already know - // this block has only one successor. + // We set likelihood anyways so we can assert later + // that all flow edges have known likelihood. // // Note the flowEdge target may not be the same as the pseudo edge target. // @@ -3988,9 +4020,9 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf assert(nSucc == 1); assert(block == pseudoEdge->m_sourceBlock); assert(block->HasInitializedTarget()); - FlowEdge* const flowEdge = block->GetTargetEdge(); + FlowEdge* const flowEdge = m_comp->fgGetPredForBlock(block->GetTarget(), block); assert(flowEdge != nullptr); - assert(flowEdge->getLikelihood() == 1.0); + flowEdge->setLikelihood(1.0); return; } @@ -3998,7 +4030,7 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // // This can happen because bome BBJ_LEAVE blocks may have been missed during // our spanning tree walk since we don't know where all the finallies can return - // to just yet (specially, in WalkSpanningTree, we may not add the target of + // to just yet (specially, in WalkSpanningTree, we may not add the bbTarget of // a BBJ_LEAVE to the worklist). // // Worst case those missed blocks dominate other blocks so we can't limit @@ -4011,19 +4043,19 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // // (TODO: use synthesis here) // - if ((nEdges != nSucc) || (info->m_weight == BB_ZERO_WEIGHT) || (successorWeight == BB_ZERO_WEIGHT)) + if ((nEdges != nSucc) || (info->m_weight == BB_ZERO_WEIGHT)) { JITDUMP(FMT_BB " %s , setting outgoing likelihoods heuristically\n", block->bbNum, (nEdges != nSucc) ? "has inaccurate flow model" : "has zero weight"); weight_t equalLikelihood = 1.0 / nSucc; - for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) + for (BasicBlock* succ : block->Succs(m_comp)) { - BasicBlock* const succBlock = succEdge->getDestinationBlock(); - JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (heur)\n", block->bbNum, - succBlock->bbNum, equalLikelihood); - succEdge->setLikelihood(equalLikelihood); + FlowEdge* const flowEdge = m_comp->fgGetPredForBlock(succ, block); + JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (heur)\n", block->bbNum, succ->bbNum, + equalLikelihood); + flowEdge->setLikelihood(equalLikelihood); } return; @@ -4032,7 +4064,7 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // Transfer model edge weight onto the FlowEdges as likelihoods. // assert(nEdges == nSucc); - JITDUMP("Normalizing successor likelihoods with factor 1/" FMT_WT "\n", successorWeight); + weight_t totalLikelihood = 0; for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { @@ -4044,17 +4076,45 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf if (nEdges == 1) { assert(nSucc == 1); + + // Conceptually we could assert(edge->m_weight == info->m_weight); + // but we can have inconsistencies. + // + // Go with what we know for sure, edge should be 100% likely. + // likelihood = 1.0; JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (uniq)\n", block->bbNum, edge->m_targetBlock->bbNum, likelihood); flowEdge->setLikelihood(likelihood); + totalLikelihood += likelihood; break; } - likelihood = edge->m_weight / successorWeight; - JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (pgo)\n", block->bbNum, - edge->m_targetBlock->bbNum, likelihood); + assert(info->m_weight != BB_ZERO_WEIGHT); + + // We may see nonsensical weights here, cap likelihood. + // + bool capped = false; + if (edge->m_weight > info->m_weight) + { + capped = true; + likelihood = 1.0; + } + else + { + likelihood = edge->m_weight / info->m_weight; + } + JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (%s)\n", block->bbNum, + edge->m_targetBlock->bbNum, likelihood, capped ? "pgo -- capped" : "pgo"); flowEdge->setLikelihood(likelihood); + totalLikelihood += likelihood; + } + + if (totalLikelihood != 1.0) + { + // Consider what to do here... flag this method as needing immediate profile repairs? + // + JITDUMP(FMT_BB " total outgoing likelihood inaccurate: " FMT_WT "\n", block->bbNum, totalLikelihood); } } @@ -4171,19 +4231,18 @@ void EfficientEdgeCountReconstructor::MarkInterestingSwitches(BasicBlock* block, // If it turns out often we fail at this stage, we might consider building a histogram of switch case // values at runtime, similar to what we do for classes at virtual call sites. // - const unsigned caseCount = block->GetSwitchTargets()->bbsCount; - FlowEdge** const jumpTab = block->GetSwitchTargets()->bbsDstTab; - unsigned dominantCase = caseCount; + const unsigned caseCount = block->GetSwitchTargets()->bbsCount; + BasicBlock** const jumpTab = block->GetSwitchTargets()->bbsDstTab; + unsigned dominantCase = caseCount; for (unsigned i = 0; i < caseCount; i++) { - BasicBlock* jumpTarget = jumpTab[i]->getDestinationBlock(); - if (jumpTarget == dominantEdge->m_targetBlock) + if (jumpTab[i] == dominantEdge->m_targetBlock) { if (dominantCase != caseCount) { JITDUMP("Both case %u and %u lead to " FMT_BB "-- can't optimize\n", i, dominantCase, - jumpTarget->bbNum); + jumpTab[i]->bbNum); dominantCase = caseCount; break; } @@ -4915,13 +4974,13 @@ PhaseStatus Compiler::fgComputeEdgeWeights() BasicBlock* otherDst; if (bSrc->FalseTargetIs(bDst)) { - otherEdge = bSrc->GetTrueEdge(); + otherDst = bSrc->GetTrueTarget(); } else { - otherEdge = bSrc->GetFalseEdge(); + otherDst = bSrc->GetFalseTarget(); } - otherDst = otherEdge->getDestinationBlock(); + otherEdge = fgGetPredForBlock(otherDst, bSrc); // If we see min/max violations, just give up on the computations // @@ -5233,8 +5292,7 @@ void Compiler::fgDebugCheckProfileWeights() } else { - ProfileChecks checks = - ProfileChecks::CHECK_HASLIKELIHOOD | ProfileChecks::CHECK_LIKELIHOODSUM | ProfileChecks::RAISE_ASSERT; + ProfileChecks checks = ProfileChecks::CHECK_HASLIKELIHOOD | ProfileChecks::RAISE_ASSERT; fgDebugCheckProfileWeights(checks); } } @@ -5266,7 +5324,6 @@ void Compiler::fgDebugCheckProfileWeights(ProfileChecks checks) const bool verifyClassicWeights = fgEdgeWeightsComputed && hasFlag(checks, ProfileChecks::CHECK_CLASSIC); const bool verifyLikelyWeights = hasFlag(checks, ProfileChecks::CHECK_LIKELY); const bool verifyHasLikelihood = hasFlag(checks, ProfileChecks::CHECK_HASLIKELIHOOD); - const bool verifyLikelihoodSum = hasFlag(checks, ProfileChecks::CHECK_LIKELIHOODSUM); const bool assertOnFailure = hasFlag(checks, ProfileChecks::RAISE_ASSERT); const bool checkAllBlocks = hasFlag(checks, ProfileChecks::CHECK_ALL_BLOCKS); @@ -5417,10 +5474,6 @@ void Compiler::fgDebugCheckProfileWeights(ProfileChecks checks) JITDUMP("Profile is self-consistent (%d profiled blocks, %d unprofiled)\n", profiledBlocks, unprofiledBlocks); } - else if (verifyLikelihoodSum) - { - JITDUMP("All block successor flow edge likelihoods sum to 1.0\n"); - } else if (verifyHasLikelihood) { JITDUMP("All flow edges have likelihoods\n"); @@ -5561,10 +5614,10 @@ bool Compiler::fgDebugCheckIncomingProfileData(BasicBlock* block, ProfileChecks bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks checks) { const bool verifyClassicWeights = fgEdgeWeightsComputed && hasFlag(checks, ProfileChecks::CHECK_CLASSIC); + const bool verifyLikelyWeights = hasFlag(checks, ProfileChecks::CHECK_LIKELY); const bool verifyHasLikelihood = hasFlag(checks, ProfileChecks::CHECK_HASLIKELIHOOD); - const bool verifyLikelihoodSum = hasFlag(checks, ProfileChecks::CHECK_LIKELIHOODSUM); - if (!(verifyClassicWeights || verifyHasLikelihood || verifyLikelihoodSum)) + if (!(verifyClassicWeights || verifyLikelyWeights || verifyHasLikelihood)) { return true; } @@ -5588,10 +5641,17 @@ bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks unsigned missingEdges = 0; unsigned missingLikelihood = 0; - for (FlowEdge* succEdge : block->SuccEdges(this)) + for (unsigned i = 0; i < numSuccs; i++) { - assert(succEdge != nullptr); - BasicBlock* succBlock = succEdge->getDestinationBlock(); + BasicBlock* succBlock = block->GetSucc(i, this); + FlowEdge* succEdge = fgGetPredForBlock(succBlock, block); + + if (succEdge == nullptr) + { + missingEdges++; + JITDUMP(" " FMT_BB " can't find successor edge to " FMT_BB "\n", block->bbNum, succBlock->bbNum); + continue; + } outgoingWeightMin += succEdge->edgeWeightMin(); outgoingWeightMax += succEdge->edgeWeightMax(); @@ -5647,7 +5707,7 @@ bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks } } - if (verifyLikelihoodSum) + if (verifyLikelyWeights) { if (!fgProfileWeightsConsistent(outgoingLikelihood, 1.0)) { @@ -5664,25 +5724,6 @@ bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks else { likelyWeightsValid = false; - -#ifdef DEBUG - if (verbose) - { - for (const FlowEdge* succEdge : block->SuccEdges(this)) - { - const BasicBlock* succBlock = succEdge->getDestinationBlock(); - if (succEdge->hasLikelihood()) - { - printf(" " FMT_BB " -> " FMT_BB ": " FMT_WT "\n", block->bbNum, succBlock->bbNum, - succEdge->getLikelihood()); - } - else - { - printf(" " FMT_BB " -> " FMT_BB ": no likelihood\n", block->bbNum, succBlock->bbNum); - } - } - } -#endif // DEBUG } } } diff --git a/src/coreclr/jit/fgprofilesynthesis.cpp b/src/coreclr/jit/fgprofilesynthesis.cpp index a8d16a5b7d2456..e315e33015e138 100644 --- a/src/coreclr/jit/fgprofilesynthesis.cpp +++ b/src/coreclr/jit/fgprofilesynthesis.cpp @@ -11,6 +11,7 @@ // TODO // +// * faster way of doing fgGetPredForBlock // * vet against some real data // * IR based heuristics (perhaps) // * During Cp, avoid repeatedly propagating through nested loops @@ -141,6 +142,14 @@ void ProfileSynthesis::AssignLikelihoods() break; case BBJ_CALLFINALLY: + // Single successor next cases + // + // Note we handle flow to the finally + // specially; this represents return + // from the finally. + AssignLikelihoodNext(block); + break; + case BBJ_ALWAYS: case BBJ_CALLFINALLYRET: case BBJ_LEAVE: @@ -166,16 +175,29 @@ void ProfileSynthesis::AssignLikelihoods() } } +//------------------------------------------------------------------------ +// AssignLikelihoodNext: update edge likelihood for block that always +// transfers control to bbNext +// +// Arguments; +// block -- block in question +// +void ProfileSynthesis::AssignLikelihoodNext(BasicBlock* block) +{ + FlowEdge* const edge = m_comp->fgGetPredForBlock(block->Next(), block); + edge->setLikelihood(1.0); +} + //------------------------------------------------------------------------ // AssignLikelihoodJump: update edge likelihood for a block that always -// transfers control to its target block +// transfers control to bbTarget // // Arguments; // block -- block in question // void ProfileSynthesis::AssignLikelihoodJump(BasicBlock* block) { - FlowEdge* const edge = block->GetTargetEdge(); + FlowEdge* const edge = m_comp->fgGetPredForBlock(block->GetTarget(), block); edge->setLikelihood(1.0); } @@ -188,37 +210,36 @@ void ProfileSynthesis::AssignLikelihoodJump(BasicBlock* block) // void ProfileSynthesis::AssignLikelihoodCond(BasicBlock* block) { - FlowEdge* const trueEdge = block->GetTrueEdge(); - FlowEdge* const falseEdge = block->GetFalseEdge(); + BasicBlock* const jump = block->GetTrueTarget(); + BasicBlock* const next = block->GetFalseTarget(); // Watch for degenerate case // - if (trueEdge == falseEdge) + if (jump == next) { - assert(trueEdge->getDupCount() == 2); - trueEdge->setLikelihood(1.0); + AssignLikelihoodNext(block); return; } - BasicBlock* trueTarget = trueEdge->getDestinationBlock(); - BasicBlock* falseTarget = falseEdge->getDestinationBlock(); + FlowEdge* const jumpEdge = m_comp->fgGetPredForBlock(jump, block); + FlowEdge* const nextEdge = m_comp->fgGetPredForBlock(next, block); // THROW heuristic // - bool const isTrueThrow = trueTarget->KindIs(BBJ_THROW); - bool const isFalseThrow = falseTarget->KindIs(BBJ_THROW); + bool const isJumpThrow = jump->KindIs(BBJ_THROW); + bool const isNextThrow = next->KindIs(BBJ_THROW); - if (isTrueThrow != isFalseThrow) + if (isJumpThrow != isNextThrow) { - if (isTrueThrow) + if (isJumpThrow) { - trueEdge->setLikelihood(0.0); - falseEdge->setLikelihood(1.0); + jumpEdge->setLikelihood(0.0); + nextEdge->setLikelihood(1.0); } else { - trueEdge->setLikelihood(1.0); - falseEdge->setLikelihood(0.0); + jumpEdge->setLikelihood(1.0); + nextEdge->setLikelihood(0.0); } return; @@ -226,22 +247,22 @@ void ProfileSynthesis::AssignLikelihoodCond(BasicBlock* block) // LOOP BACK EDGE heuristic // - bool const isTrueEdgeBackEdge = m_loops->IsLoopBackEdge(trueEdge); - bool const isFalseEdgeBackEdge = m_loops->IsLoopBackEdge(falseEdge); + bool const isJumpEdgeBackEdge = m_loops->IsLoopBackEdge(jumpEdge); + bool const isNextEdgeBackEdge = m_loops->IsLoopBackEdge(nextEdge); - if (isTrueEdgeBackEdge != isFalseEdgeBackEdge) + if (isJumpEdgeBackEdge != isNextEdgeBackEdge) { - if (isTrueEdgeBackEdge) + if (isJumpEdgeBackEdge) { - JITDUMP(FMT_BB "->" FMT_BB " is loop back edge\n", block->bbNum, trueTarget->bbNum); - trueEdge->setLikelihood(loopBackLikelihood); - falseEdge->setLikelihood(1.0 - loopBackLikelihood); + JITDUMP(FMT_BB "->" FMT_BB " is loop back edge\n", block->bbNum, jump->bbNum); + jumpEdge->setLikelihood(loopBackLikelihood); + nextEdge->setLikelihood(1.0 - loopBackLikelihood); } else { - JITDUMP(FMT_BB "->" FMT_BB " is loop back edge\n", block->bbNum, falseTarget->bbNum); - trueEdge->setLikelihood(1.0 - loopBackLikelihood); - falseEdge->setLikelihood(loopBackLikelihood); + JITDUMP(FMT_BB "->" FMT_BB " is loop back edge\n", block->bbNum, next->bbNum); + jumpEdge->setLikelihood(1.0 - loopBackLikelihood); + nextEdge->setLikelihood(loopBackLikelihood); } return; @@ -252,22 +273,22 @@ void ProfileSynthesis::AssignLikelihoodCond(BasicBlock* block) // Consider: adjust probability if loop has multiple exit edges, so that // overall exit probability is around 0.1. // - bool const isTrueEdgeExitEdge = m_loops->IsLoopExitEdge(trueEdge); - bool const isFalseEdgeExitEdge = m_loops->IsLoopExitEdge(falseEdge); + bool const isJumpEdgeExitEdge = m_loops->IsLoopExitEdge(jumpEdge); + bool const isNextEdgeExitEdge = m_loops->IsLoopExitEdge(nextEdge); - if (isTrueEdgeExitEdge != isFalseEdgeExitEdge) + if (isJumpEdgeExitEdge != isNextEdgeExitEdge) { - if (isTrueEdgeExitEdge) + if (isJumpEdgeExitEdge) { - JITDUMP(FMT_BB "->" FMT_BB " is loop exit edge\n", block->bbNum, trueTarget->bbNum); - trueEdge->setLikelihood(1.0 - loopExitLikelihood); - falseEdge->setLikelihood(loopExitLikelihood); + JITDUMP(FMT_BB "->" FMT_BB " is loop exit edge\n", block->bbNum, jump->bbNum); + jumpEdge->setLikelihood(1.0 - loopExitLikelihood); + nextEdge->setLikelihood(loopExitLikelihood); } else { - JITDUMP(FMT_BB "->" FMT_BB " is loop exit edge\n", block->bbNum, falseTarget->bbNum); - trueEdge->setLikelihood(loopExitLikelihood); - falseEdge->setLikelihood(1.0 - loopExitLikelihood); + JITDUMP(FMT_BB "->" FMT_BB " is loop exit edge\n", block->bbNum, next->bbNum); + jumpEdge->setLikelihood(loopExitLikelihood); + nextEdge->setLikelihood(1.0 - loopExitLikelihood); } return; @@ -275,20 +296,20 @@ void ProfileSynthesis::AssignLikelihoodCond(BasicBlock* block) // RETURN heuristic // - bool const isJumpReturn = trueTarget->KindIs(BBJ_RETURN); - bool const isNextReturn = falseTarget->KindIs(BBJ_RETURN); + bool const isJumpReturn = jump->KindIs(BBJ_RETURN); + bool const isNextReturn = next->KindIs(BBJ_RETURN); if (isJumpReturn != isNextReturn) { if (isJumpReturn) { - trueEdge->setLikelihood(returnLikelihood); - falseEdge->setLikelihood(1.0 - returnLikelihood); + jumpEdge->setLikelihood(returnLikelihood); + nextEdge->setLikelihood(1.0 - returnLikelihood); } else { - trueEdge->setLikelihood(1.0 - returnLikelihood); - falseEdge->setLikelihood(returnLikelihood); + jumpEdge->setLikelihood(1.0 - returnLikelihood); + nextEdge->setLikelihood(returnLikelihood); } return; @@ -298,8 +319,8 @@ void ProfileSynthesis::AssignLikelihoodCond(BasicBlock* block) // // Give slight preference to bbNext // - trueEdge->setLikelihood(1.0 - ilNextLikelihood); - falseEdge->setLikelihood(ilNextLikelihood); + jumpEdge->setLikelihood(1.0 - ilNextLikelihood); + nextEdge->setLikelihood(ilNextLikelihood); } //------------------------------------------------------------------------ @@ -321,9 +342,10 @@ void ProfileSynthesis::AssignLikelihoodSwitch(BasicBlock* block) // Each unique edge gets some multiple of that basic probability // - for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) + for (BasicBlock* const succ : block->Succs(m_comp)) { - succEdge->setLikelihood(p * succEdge->getDupCount()); + FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); + edge->setLikelihood(p * edge->getDupCount()); } } @@ -346,9 +368,10 @@ weight_t ProfileSynthesis::SumOutgoingLikelihoods(BasicBlock* block, WeightVecto likelihoods->clear(); } - for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) + for (BasicBlock* const succ : block->Succs(m_comp)) { - weight_t likelihood = succEdge->getLikelihood(); + FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); + weight_t likelihood = edge->getLikelihood(); if (likelihoods != nullptr) { likelihoods->push_back(likelihood); @@ -383,6 +406,11 @@ void ProfileSynthesis::RepairLikelihoods() break; case BBJ_CALLFINALLY: + // Single successor next cases. + // Just assign 1.0 + AssignLikelihoodNext(block); + break; + case BBJ_ALWAYS: case BBJ_CALLFINALLYRET: case BBJ_LEAVE: @@ -470,6 +498,11 @@ void ProfileSynthesis::BlendLikelihoods() break; case BBJ_CALLFINALLY: + // Single successor next cases. + // Just assign 1.0 + AssignLikelihoodNext(block); + break; + case BBJ_ALWAYS: case BBJ_CALLFINALLYRET: case BBJ_LEAVE: @@ -527,15 +560,15 @@ void ProfileSynthesis::BlendLikelihoods() JITDUMP("Blending likelihoods in " FMT_BB " with blend factor " FMT_WT " \n", block->bbNum, blendFactor); iter = likelihoods.begin(); - for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) + for (BasicBlock* const succ : block->Succs(m_comp)) { - weight_t newLikelihood = succEdge->getLikelihood(); - weight_t oldLikelihood = *iter; + FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); + weight_t newLikelihood = edge->getLikelihood(); + weight_t oldLikelihood = *iter; - succEdge->setLikelihood((blendFactor * oldLikelihood) + ((1.0 - blendFactor) * newLikelihood)); - BasicBlock* const succBlock = succEdge->getDestinationBlock(); - JITDUMP(FMT_BB " -> " FMT_BB " was " FMT_WT " now " FMT_WT "\n", block->bbNum, succBlock->bbNum, - oldLikelihood, succEdge->getLikelihood()); + edge->setLikelihood((blendFactor * oldLikelihood) + ((1.0 - blendFactor) * newLikelihood)); + JITDUMP(FMT_BB " -> " FMT_BB " was " FMT_WT " now " FMT_WT "\n", block->bbNum, succ->bbNum, + oldLikelihood, edge->getLikelihood()); iter++; } @@ -555,9 +588,10 @@ void ProfileSynthesis::ClearLikelihoods() { for (BasicBlock* const block : m_comp->Blocks()) { - for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) + for (BasicBlock* const succ : block->Succs(m_comp)) { - succEdge->clearLikelihood(); + FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); + edge->clearLikelihood(); } } } @@ -630,9 +664,10 @@ void ProfileSynthesis::RandomizeLikelihoods() } i = 0; - for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) + for (BasicBlock* const succ : block->Succs(m_comp)) { - succEdge->setLikelihood(likelihoods[i++] / sum); + FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); + edge->setLikelihood(likelihoods[i++] / sum); } } #endif // DEBUG @@ -824,26 +859,28 @@ void ProfileSynthesis::ComputeCyclicProbabilities(FlowGraphNaturalLoop* loop) " to reflect capping; current likelihood is " FMT_WT "\n", exitBlock->bbNum, exitEdge->getLikelihood()); - FlowEdge* const trueEdge = exitBlock->GetTrueEdge(); - FlowEdge* const falseEdge = exitBlock->GetFalseEdge(); - weight_t const exitLikelihood = (missingExitWeight + currentExitWeight) / exitBlockWeight; - weight_t const continueLikelihood = 1.0 - exitLikelihood; + BasicBlock* const jump = exitBlock->GetTrueTarget(); + BasicBlock* const next = exitBlock->GetFalseTarget(); + FlowEdge* const jumpEdge = m_comp->fgGetPredForBlock(jump, exitBlock); + FlowEdge* const nextEdge = m_comp->fgGetPredForBlock(next, exitBlock); + weight_t const exitLikelihood = (missingExitWeight + currentExitWeight) / exitBlockWeight; + weight_t const continueLikelihood = 1.0 - exitLikelihood; // We are making it more likely that the loop exits, so the new exit likelihood // should be greater than the old. // assert(exitLikelihood > exitEdge->getLikelihood()); - if (trueEdge == exitEdge) + if (jumpEdge == exitEdge) { - trueEdge->setLikelihood(exitLikelihood); - falseEdge->setLikelihood(continueLikelihood); + jumpEdge->setLikelihood(exitLikelihood); + nextEdge->setLikelihood(continueLikelihood); } else { - assert(falseEdge == exitEdge); - trueEdge->setLikelihood(continueLikelihood); - falseEdge->setLikelihood(exitLikelihood); + assert(nextEdge == exitEdge); + jumpEdge->setLikelihood(continueLikelihood); + nextEdge->setLikelihood(exitLikelihood); } adjustedExit = true; diff --git a/src/coreclr/jit/fgprofilesynthesis.h b/src/coreclr/jit/fgprofilesynthesis.h index 304ca58d9da4f9..9297357049e8aa 100644 --- a/src/coreclr/jit/fgprofilesynthesis.h +++ b/src/coreclr/jit/fgprofilesynthesis.h @@ -58,6 +58,7 @@ class ProfileSynthesis weight_t SumOutgoingLikelihoods(BasicBlock* block, WeightVector* likelihoods = nullptr); void AssignLikelihoods(); + void AssignLikelihoodNext(BasicBlock* block); void AssignLikelihoodJump(BasicBlock* block); void AssignLikelihoodCond(BasicBlock* block); void AssignLikelihoodSwitch(BasicBlock* block); diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index 8c0c4daf4acf51..1c40239b35beeb 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -266,11 +266,17 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) // I want to create: // top -> poll -> bottom (lexically) // so that we jump over poll to get to bottom. - BasicBlock* top = block; + BasicBlock* top = block; + BBKinds oldJumpKind = top->GetKind(); BasicBlock* poll = fgNewBBafter(BBJ_ALWAYS, top, true); bottom = fgNewBBafter(top->GetKind(), poll, true); + poll->SetTarget(bottom); + assert(poll->JumpsToNext()); + + bottom->TransferTarget(top); + // Update block flags const BasicBlockFlags originalFlags = top->GetFlagsRaw() | BBF_GC_SAFE_POINT; @@ -294,7 +300,7 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) } // Remove the last statement from Top and add it to Bottom if necessary. - if (top->KindIs(BBJ_COND, BBJ_RETURN, BBJ_THROW)) + if ((oldJumpKind == BBJ_COND) || (oldJumpKind == BBJ_RETURN) || (oldJumpKind == BBJ_THROW)) { Statement* stmt = top->firstStmt(); while (stmt->GetNextStmt() != nullptr) @@ -358,49 +364,38 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) } #endif + top->SetCond(bottom, poll); // Bottom has Top and Poll as its predecessors. Poll has just Top as a predecessor. - FlowEdge* const trueEdge = fgAddRefPred(bottom, top); - FlowEdge* const falseEdge = fgAddRefPred(poll, top); - trueEdge->setLikelihood(1.0); - falseEdge->setLikelihood(0.0); - - FlowEdge* const newEdge = fgAddRefPred(bottom, poll); - poll->SetTargetEdge(newEdge); - assert(poll->JumpsToNext()); + fgAddRefPred(bottom, poll); + fgAddRefPred(bottom, top); + fgAddRefPred(poll, top); // Replace Top with Bottom in the predecessor list of all outgoing edges from Bottom // (1 for unconditional branches, 2 for conditional branches, N for switches). - switch (top->GetKind()) + switch (oldJumpKind) { case BBJ_RETURN: case BBJ_THROW: // no successors break; - case BBJ_COND: // replace predecessor in true/false successors. noway_assert(!bottom->IsLast()); - fgReplacePred(top->GetFalseEdge(), bottom); - fgReplacePred(top->GetTrueEdge(), bottom); + fgReplacePred(bottom->GetFalseTarget(), top, bottom); + fgReplacePred(bottom->GetTrueTarget(), top, bottom); break; case BBJ_ALWAYS: case BBJ_CALLFINALLY: - fgReplacePred(top->GetTargetEdge(), bottom); + fgReplacePred(bottom->GetTarget(), top, bottom); break; - case BBJ_SWITCH: NO_WAY("SWITCH should be a call rather than an inlined poll."); break; - default: NO_WAY("Unknown block type for updating predecessor lists."); - break; } - bottom->TransferTarget(top); - top->SetCond(trueEdge, falseEdge); - if (compCurBB == top) { compCurBB = bottom; @@ -1038,7 +1033,7 @@ GenTree* Compiler::fgOptimizeDelegateConstructor(GenTreeCall* call, GenTree* targetObjPointers = call->gtArgs.GetArgByIndex(1)->GetNode(); CORINFO_LOOKUP pLookup; info.compCompHnd->getReadyToRunDelegateCtorHelper(&ldftnToken->m_token, ldftnToken->m_tokenConstraint, - clsHnd, info.compMethodHnd, &pLookup); + clsHnd, &pLookup); if (!pLookup.lookupKind.needsRuntimeLookup) { call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_DELEGATE_CTOR, TYP_VOID, thisPointer, @@ -1050,8 +1045,7 @@ GenTree* Compiler::fgOptimizeDelegateConstructor(GenTreeCall* call, assert(oper != GT_FTN_ADDR); CORINFO_CONST_LOOKUP genericLookup; info.compCompHnd->getReadyToRunHelper(&ldftnToken->m_token, &pLookup.lookupKind, - CORINFO_HELP_READYTORUN_GENERIC_HANDLE, info.compMethodHnd, - &genericLookup); + CORINFO_HELP_READYTORUN_GENERIC_HANDLE, &genericLookup); GenTree* ctxTree = getRuntimeContextTree(pLookup.lookupKind.runtimeLookupKind); call = gtNewHelperCallNode(CORINFO_HELP_READYTORUN_DELEGATE_CTOR, TYP_VOID, thisPointer, targetObjPointers, ctxTree); @@ -1074,7 +1068,7 @@ GenTree* Compiler::fgOptimizeDelegateConstructor(GenTreeCall* call, CORINFO_LOOKUP entryPoint; info.compCompHnd->getReadyToRunDelegateCtorHelper(&ldftnToken->m_token, ldftnToken->m_tokenConstraint, - clsHnd, info.compMethodHnd, &entryPoint); + clsHnd, &entryPoint); assert(!entryPoint.lookupKind.needsRuntimeLookup); call->setEntryPoint(entryPoint.constLookup); } @@ -1631,8 +1625,9 @@ void Compiler::fgConvertSyncReturnToLeave(BasicBlock* block) assert(ehDsc->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX); // Convert the BBJ_RETURN to BBJ_ALWAYS, jumping to genReturnBB. + block->SetKindAndTarget(BBJ_ALWAYS, genReturnBB); FlowEdge* const newEdge = fgAddRefPred(genReturnBB, block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); #ifdef DEBUG if (verbose) @@ -2102,8 +2097,9 @@ class MergedReturns // Change BBJ_RETURN to BBJ_ALWAYS targeting const return block. assert((comp->info.compFlags & CORINFO_FLG_SYNCH) == 0); + returnBlock->SetKindAndTarget(BBJ_ALWAYS, constReturnBlock); FlowEdge* const newEdge = comp->fgAddRefPred(constReturnBlock, returnBlock); - returnBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); // Remove GT_RETURN since constReturnBlock returns the constant. assert(returnBlock->lastStmt()->GetRootNode()->OperIs(GT_RETURN)); @@ -2762,20 +2758,21 @@ void Compiler::fgInsertFuncletPrologBlock(BasicBlock* block) /* Allocate a new basic block */ - BasicBlock* newHead = BasicBlock::New(this); + BasicBlock* newHead = BasicBlock::New(this, BBJ_ALWAYS, block); newHead->SetFlags(BBF_INTERNAL | BBF_NONE_QUIRK); newHead->inheritWeight(block); newHead->bbRefs = 0; fgInsertBBbefore(block, newHead); // insert the new block in the block list - fgExtendEHRegionBefore(block); // Update the EH table to make the prolog block the first block in the block's EH - // block. + assert(newHead->JumpsToNext()); + fgExtendEHRegionBefore(block); // Update the EH table to make the prolog block the first block in the block's EH + // block. // Distribute the pred list between newHead and block. Incoming edges coming from outside // the handler go to the prolog. Edges coming from with the handler are back-edges, and // go to the existing 'block'. - for (BasicBlock* const predBlock : block->PredBlocksEditing()) + for (BasicBlock* const predBlock : block->PredBlocks()) { if (!fgIsIntraHandlerPred(predBlock, block)) { @@ -2785,11 +2782,11 @@ void Compiler::fgInsertFuncletPrologBlock(BasicBlock* block) switch (predBlock->GetKind()) { case BBJ_CALLFINALLY: - { noway_assert(predBlock->TargetIs(block)); - fgRedirectTargetEdge(predBlock, newHead); + predBlock->SetTarget(newHead); + fgRemoveRefPred(block, predBlock); + fgAddRefPred(newHead, predBlock); break; - } default: // The only way into the handler is via a BBJ_CALLFINALLY (to a finally handler), or @@ -2800,10 +2797,10 @@ void Compiler::fgInsertFuncletPrologBlock(BasicBlock* block) } } - assert(fgGetPredForBlock(block, newHead) == nullptr); - FlowEdge* const newEdge = fgAddRefPred(block, newHead); - newHead->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); - assert(newHead->JumpsToNext()); + assert(nullptr == fgGetPredForBlock(block, newHead)); + fgAddRefPred(block, newHead); + + assert(newHead->HasFlag(BBF_INTERNAL)); } //------------------------------------------------------------------------ @@ -3377,7 +3374,7 @@ PhaseStatus Compiler::fgCreateThrowHelperBlocks() assert((add->acdKind == SCK_FAIL_FAST) || (bbThrowIndex(srcBlk) == add->acdData)); assert(add->acdKind != SCK_NONE); - BasicBlock* const newBlk = fgNewBBinRegion(jumpKinds[add->acdKind], srcBlk, + BasicBlock* const newBlk = fgNewBBinRegion(jumpKinds[add->acdKind], srcBlk, /* jumpDest */ nullptr, /* runRarely */ true, /* insertAtEnd */ true); // Update the descriptor @@ -3441,7 +3438,7 @@ PhaseStatus Compiler::fgCreateThrowHelperBlocks() #endif // DEBUG // Mark the block as added by the compiler and not removable by future flow - // graph optimizations. Note that no target block points to these blocks. + // graph optimizations. Note that no bbTarget points to these blocks. // newBlk->SetFlags(BBF_IMPORTED | BBF_DONT_REMOVE); @@ -3908,26 +3905,6 @@ void Compiler::fgLclFldAssign(unsigned lclNum) } } -#ifdef DEBUG - -//------------------------------------------------------------------------ -// FlowGraphDfsTree::Dump: Dump a textual representation of the DFS tree. -// -void FlowGraphDfsTree::Dump() const -{ - printf("DFS tree. %s.\n", HasCycle() ? "Has cycle" : "No cycle"); - printf("PO RPO -> BB [pre, post]\n"); - for (unsigned i = 0; i < GetPostOrderCount(); i++) - { - unsigned rpoNum = GetPostOrderCount() - i - 1; - BasicBlock* const block = GetPostOrder(i); - printf("%02u %02u -> " FMT_BB "[%u, %u]\n", i, rpoNum, block->bbNum, block->bbPreorderNum, - block->bbPostorderNum); - } -} - -#endif // DEBUG - //------------------------------------------------------------------------ // FlowGraphDfsTree::Contains: Check if a block is contained in the DFS tree; // i.e., if it is reachable. @@ -5626,8 +5603,8 @@ void FlowGraphNaturalLoop::Duplicate(BasicBlock** insertAfter, BlockToBlockMap* assert(!newBlk->HasInitializedTarget()); // Redirect the new block according to "blockMap". - // optSetMappedBlockTargets will set newBlk's successors, and add pred edges for the successors. - comp->optSetMappedBlockTargets(blk, newBlk, map); + // opRedirectBlock will set newBlk's successors, and add pred edges for the successors. + comp->optRedirectBlock(blk, newBlk, map); return BasicBlockVisit::Continue; }); @@ -6152,37 +6129,6 @@ BlockToNaturalLoopMap* BlockToNaturalLoopMap::Build(FlowGraphNaturalLoops* loops return new (comp, CMK_Loops) BlockToNaturalLoopMap(loops, indices); } -#ifdef DEBUG - -//------------------------------------------------------------------------ -// BlockToNaturalLoopMap::Dump: Dump a textual representation of the map. -// -void BlockToNaturalLoopMap::Dump() const -{ - const FlowGraphDfsTree* dfs = m_loops->GetDfsTree(); - unsigned blockCount = dfs->GetPostOrderCount(); - - printf("Block -> natural loop map: %u blocks\n", blockCount); - if (blockCount > 0) - { - printf("block : loop index\n"); - for (unsigned i = 0; i < blockCount; i++) - { - if (m_indices[i] == UINT_MAX) - { - // Just leave the loop space empty if there is no enclosing loop - printf(FMT_BB " : \n", dfs->GetPostOrder(i)->bbNum); - } - else - { - printf(FMT_BB " : " FMT_LP "\n", dfs->GetPostOrder(i)->bbNum, m_indices[i]); - } - } - } -} - -#endif // DEBUG - //------------------------------------------------------------------------ // BlockReachabilitySets::Build: Build the reachability sets. // @@ -6196,7 +6142,7 @@ void BlockToNaturalLoopMap::Dump() const // This algorithm consumes O(n^2) memory because we're using dense // bitsets to represent reachability. // -BlockReachabilitySets* BlockReachabilitySets::Build(const FlowGraphDfsTree* dfsTree) +BlockReachabilitySets* BlockReachabilitySets::Build(FlowGraphDfsTree* dfsTree) { Compiler* comp = dfsTree->GetCompiler(); BitVecTraits postOrderTraits = dfsTree->PostOrderTraits(); diff --git a/src/coreclr/jit/forwardsub.cpp b/src/coreclr/jit/forwardsub.cpp index 85e33a62d82e2c..cfb11357206227 100644 --- a/src/coreclr/jit/forwardsub.cpp +++ b/src/coreclr/jit/forwardsub.cpp @@ -221,7 +221,7 @@ class ForwardSubVisitor final : public GenTreeVisitor // fgGetStubAddrArg cannot handle complex trees (it calls gtClone) // bool isCallTarget = false; - if ((parent != nullptr) && parent->IsCall()) + if (parent->IsCall()) { GenTreeCall* const parentCall = parent->AsCall(); isCallTarget = (parentCall->gtCallType == CT_INDIRECT) && (parentCall->gtCallAddr == node); @@ -319,7 +319,7 @@ class ForwardSubVisitor final : public GenTreeVisitor bool IsCallArg() const { - return (m_parentNode != nullptr) && m_parentNode->IsCall(); + return m_parentNode->IsCall(); } unsigned GetComplexity() const @@ -749,7 +749,7 @@ bool Compiler::fgForwardSubStatement(Statement* stmt) if (fsv.IsCallArg() && fsv.GetNode()->TypeIs(TYP_STRUCT) && !fwdSubNode->OperIs(GT_BLK, GT_LCL_VAR, GT_LCL_FLD, GT_MKREFANY)) { - JITDUMP(" use is a struct arg; fwd sub node is not BLK/LCL_VAR/LCL_FLD/MKREFANY\n"); + JITDUMP(" use is a struct arg; fwd sub node is not OBJ/LCL_VAR/LCL_FLD/MKREFANY\n"); return false; } @@ -772,7 +772,7 @@ bool Compiler::fgForwardSubStatement(Statement* stmt) { GenTree* const parentNode = fsv.GetParentNode(); - if ((parentNode == nullptr) || !parentNode->OperIs(GT_STORE_LCL_VAR)) + if (!parentNode->OperIs(GT_STORE_LCL_VAR)) { JITDUMP(" multi-reg struct node, parent not STORE_LCL_VAR\n"); return false; @@ -794,8 +794,7 @@ bool Compiler::fgForwardSubStatement(Statement* stmt) // for them on all 32 bit targets is a CQ regression due to some bad // interaction between decomposition and RA. // - if (compMethodReturnsMultiRegRetType() && (fsv.GetParentNode() != nullptr) && - fsv.GetParentNode()->OperIs(GT_RETURN)) + if (compMethodReturnsMultiRegRetType() && fsv.GetParentNode()->OperIs(GT_RETURN)) { #if defined(TARGET_X86) if (fwdSubNode->TypeGet() == TYP_LONG) diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index d039cb3169379e..96005e6057662d 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -1564,7 +1564,7 @@ size_t GCInfo::gcInfoBlockHdrSave( header->syncStartOffset = INVALID_SYNC_OFFSET; header->syncEndOffset = INVALID_SYNC_OFFSET; -#if !defined(FEATURE_EH_FUNCLETS) +#ifndef UNIX_X86_ABI // JIT is responsible for synchronization on funclet-based EH model that x86/Linux uses. if (compiler->info.compFlags & CORINFO_FLG_SYNCH) { @@ -4698,7 +4698,7 @@ void GCInfo::gcMakeVarPtrTable(GcInfoEncoder* gcInfoEncoder, MakeRegPtrMode mode // unused by alignment C_ASSERT((OFFSET_MASK + 1) <= sizeof(int)); -#if defined(DEBUG) && defined(JIT32_GCENCODER) && !defined(FEATURE_EH_FUNCLETS) +#ifdef DEBUG if (mode == MAKE_REG_PTR_MODE_ASSIGN_SLOTS) { // Tracked variables can't be pinned, and the encoding takes @@ -4712,7 +4712,7 @@ void GCInfo::gcMakeVarPtrTable(GcInfoEncoder* gcInfoEncoder, MakeRegPtrMode mode assert((flags & this_OFFSET_FLAG) == 0); } } -#endif +#endif // DEBUG // Only need to do this once, and only if we have EH. if ((mode == MAKE_REG_PTR_MODE_ASSIGN_SLOTS) && compiler->ehAnyFunclets()) diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 0db384e5c69a0c..ad68efef307b77 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -252,6 +252,7 @@ void GenTree::InitNodeSize() GenTree::s_gtNodeSizes[GT_FIELD_ADDR] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_CMPXCHG] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_QMARK] = TREE_NODE_SZ_LARGE; + GenTree::s_gtNodeSizes[GT_STORE_DYN_BLK] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_INTRINSIC] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_ALLOCOBJ] = TREE_NODE_SZ_LARGE; #if USE_HELPERS_FOR_INT_DIV @@ -317,6 +318,7 @@ void GenTree::InitNodeSize() static_assert_no_msg(sizeof(GenTreeStoreInd) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeAddrMode) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeBlk) <= TREE_NODE_SZ_SMALL); + static_assert_no_msg(sizeof(GenTreeStoreDynBlk) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeRetExpr) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeILOffset) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreePhiArg) <= TREE_NODE_SZ_SMALL); @@ -1817,15 +1819,6 @@ regNumber CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension c } } -#if defined(TARGET_AMD64) && defined(SWIFT_SUPPORT) - // TODO-Cleanup: Unify this with hasFixedRetBuffReg. That will - // likely be necessary for the reverse pinvoke support regardless. - if (cc == CorInfoCallConvExtension::Swift) - { - return REG_SWIFT_ARG_RET_BUFF; - } -#endif - break; case WellKnownArg::VirtualStubCell: @@ -1852,17 +1845,6 @@ regNumber CallArgs::GetCustomRegister(Compiler* comp, CorInfoCallConvExtension c case WellKnownArg::DispatchIndirectCallTarget: return REG_DISPATCH_INDIRECT_CALL_ADDR; #endif - -#ifdef SWIFT_SUPPORT - case WellKnownArg::SwiftError: - assert(cc == CorInfoCallConvExtension::Swift); - return REG_SWIFT_ERROR; - - case WellKnownArg::SwiftSelf: - assert(cc == CorInfoCallConvExtension::Swift); - return REG_SWIFT_SELF; -#endif // SWIFT_SUPPORT - default: break; } @@ -2919,7 +2901,6 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) case GT_NOP: case GT_LABEL: - case GT_SWIFT_ERROR: return true; default: @@ -3170,6 +3151,11 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) Compare(op1->AsCmpXchg()->Data(), op2->AsCmpXchg()->Data()) && Compare(op1->AsCmpXchg()->Comparand(), op2->AsCmpXchg()->Comparand()); + case GT_STORE_DYN_BLK: + return Compare(op1->AsStoreDynBlk()->Addr(), op2->AsStoreDynBlk()->Addr()) && + Compare(op1->AsStoreDynBlk()->Data(), op2->AsStoreDynBlk()->Data()) && + Compare(op1->AsStoreDynBlk()->gtDynamicSize, op2->AsStoreDynBlk()->gtDynamicSize); + default: assert(!"unexpected operator"); } @@ -3415,13 +3401,6 @@ unsigned Compiler::gtHashValue(GenTree* tree) { #if defined(FEATURE_SIMD) #if defined(TARGET_XARCH) - case TYP_MASK: - { - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[1]); - add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[0]); - break; - } - case TYP_SIMD64: { add = genTreeHashAdd(ulo32(add), vecCon->gtSimdVal.u32[15]); @@ -3725,6 +3704,12 @@ unsigned Compiler::gtHashValue(GenTree* tree) hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->Comparand())); break; + case GT_STORE_DYN_BLK: + hash = genTreeHashAdd(hash, gtHashValue(tree->AsStoreDynBlk()->Data())); + hash = genTreeHashAdd(hash, gtHashValue(tree->AsStoreDynBlk()->Addr())); + hash = genTreeHashAdd(hash, gtHashValue(tree->AsStoreDynBlk()->gtDynamicSize)); + break; + default: #ifdef DEBUG gtDispTree(tree); @@ -4535,6 +4520,12 @@ bool Compiler::gtGetIndNodeCost(GenTreeIndir* node, int* pCostEx, int* pCostSz) { // See if we can form a complex addressing mode. bool doAddrMode = true; + + // TODO-1stClassStructs: delete once IND nodes are no more. + if (node->TypeGet() == TYP_STRUCT) + { + doAddrMode = false; + } #ifdef TARGET_ARM64 if (node->IsVolatile()) { @@ -6393,6 +6384,22 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } break; + case GT_STORE_DYN_BLK: + level = gtSetEvalOrder(tree->AsStoreDynBlk()->Addr()); + costEx = tree->AsStoreDynBlk()->Addr()->GetCostEx(); + costSz = tree->AsStoreDynBlk()->Addr()->GetCostSz(); + + lvl2 = gtSetEvalOrder(tree->AsStoreDynBlk()->Data()); + level = max(level, lvl2); + costEx += tree->AsStoreDynBlk()->Data()->GetCostEx(); + costSz += tree->AsStoreDynBlk()->Data()->GetCostSz(); + + lvl2 = gtSetEvalOrder(tree->AsStoreDynBlk()->gtDynamicSize); + level = max(level, lvl2); + costEx += tree->AsStoreDynBlk()->gtDynamicSize->GetCostEx(); + costSz += tree->AsStoreDynBlk()->gtDynamicSize->GetCostSz(); + break; + case GT_SELECT: level = gtSetEvalOrder(tree->AsConditional()->gtCond); costEx = tree->AsConditional()->gtCond->GetCostEx(); @@ -6706,7 +6713,6 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: - case GT_SWIFT_ERROR: return false; // Standard unary operators @@ -6840,6 +6846,27 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) return false; } + case GT_STORE_DYN_BLK: + { + GenTreeStoreDynBlk* const dynBlock = this->AsStoreDynBlk(); + if (operand == dynBlock->gtOp1) + { + *pUse = &dynBlock->gtOp1; + return true; + } + if (operand == dynBlock->gtOp2) + { + *pUse = &dynBlock->gtOp2; + return true; + } + if (operand == dynBlock->gtDynamicSize) + { + *pUse = &dynBlock->gtDynamicSize; + return true; + } + return false; + } + case GT_CALL: { GenTreeCall* const call = this->AsCall(); @@ -6999,6 +7026,7 @@ bool GenTree::OperRequiresAsgFlag() const case GT_STORE_LCL_FLD: case GT_STOREIND: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_XADD: case GT_XORR: case GT_XAND: @@ -7038,9 +7066,6 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) const case GT_KEEPALIVE: return true; - case GT_SWIFT_ERROR: - return true; - case GT_INTRINSIC: return comp->IsIntrinsicImplementedByUserCall(this->AsIntrinsic()->gtIntrinsicName); @@ -7111,6 +7136,7 @@ bool GenTree::OperIsImplicitIndir() const case GT_CMPXCHG: case GT_BLK: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_BOX: case GT_ARR_ELEM: case GT_ARR_LENGTH: @@ -7207,6 +7233,7 @@ ExceptionSetFlags GenTree::OperExceptions(Compiler* comp) case GT_BLK: case GT_NULLCHECK: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_ARR_LENGTH: case GT_MDARR_LENGTH: case GT_MDARR_LOWER_BOUND: @@ -7326,6 +7353,7 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const case GT_STOREIND: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_XADD: case GT_XORR: case GT_XAND: @@ -7334,7 +7362,6 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const case GT_CMPXCHG: case GT_MEMORYBARRIER: case GT_KEEPALIVE: - case GT_SWIFT_ERROR: return true; case GT_CALL: @@ -7384,6 +7411,7 @@ bool GenTree::OperSupportsOrderingSideEffect() const case GT_STOREIND: case GT_NULLCHECK: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_XADD: case GT_XORR: case GT_XAND: @@ -7392,7 +7420,6 @@ bool GenTree::OperSupportsOrderingSideEffect() const case GT_CMPXCHG: case GT_MEMORYBARRIER: case GT_CATCH_ARG: - case GT_SWIFT_ERROR: return true; default: return false; @@ -7726,8 +7753,8 @@ GenTreeFlags Compiler::gtTokenToIconFlags(unsigned token) // Returns a GT_IND node representing value at the address provided by 'addr' // // Notes: -// The GT_IND node is marked as non-faulting. -// If the indirection is not invariant, we also mark the indNode as GTF_GLOB_REF. +// The GT_IND node is marked as non-faulting +// If the indType is GT_REF we also mark the indNode as GTF_GLOB_REF // GenTree* Compiler::gtNewIndOfIconHandleNode(var_types indType, size_t addr, GenTreeFlags iconFlags, bool isInvariant) { @@ -7736,7 +7763,9 @@ GenTree* Compiler::gtNewIndOfIconHandleNode(var_types indType, size_t addr, GenT if (isInvariant) { - assert(GenTree::HandleKindDataIsInvariant(iconFlags)); + assert(iconFlags != GTF_ICON_STATIC_HDL); // Pointer to a mutable class Static variable + assert(iconFlags != GTF_ICON_BBC_PTR); // Pointer to a mutable basic block count value + assert(iconFlags != GTF_ICON_GLOBAL_PTR); // Pointer to mutable data from the VM state // This indirection also is invariant. indirFlags |= GTF_IND_INVARIANT; @@ -8622,26 +8651,6 @@ GenTreeBlk* Compiler::gtNewBlkIndir(ClassLayout* layout, GenTree* addr, GenTreeF return blkNode; } -//------------------------------------------------------------------------ -// gtNewMemoryBarrier: Create a memory barrier node -// -// Arguments: -// loadOnly - relaxes the full memory barrier to be load-only -// -// Return Value: -// The created GT_MEMORYBARRIER node. -// -GenTree* Compiler::gtNewMemoryBarrier(bool loadOnly) -{ - GenTree* tree = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); - tree->gtFlags |= GTF_GLOB_REF | GTF_ASG; - if (loadOnly) - { - tree->gtFlags |= GTF_MEMORYBARRIER_LOAD; - } - return tree; -} - //------------------------------------------------------------------------------ // gtNewIndir : Create an indirection node. // @@ -8716,12 +8725,40 @@ GenTreeBlk* Compiler::gtNewStoreBlkNode(ClassLayout* layout, GenTree* addr, GenT return store; } +//------------------------------------------------------------------------------ +// gtNewStoreDynBlkNode : Create a dynamic block store node. +// +// Arguments: +// addr - Destination address +// data - Value to store (init val or indirection representing a location) +// dynamicSize - Node that computes number of bytes to store +// indirFlags - Indirection flags +// +// Return Value: +// The created GT_STORE_DYN_BLK node. +// +GenTreeStoreDynBlk* Compiler::gtNewStoreDynBlkNode(GenTree* addr, + GenTree* data, + GenTree* dynamicSize, + GenTreeFlags indirFlags) +{ + assert((indirFlags & GTF_IND_INVARIANT) == 0); + assert(data->IsInitVal() || data->OperIs(GT_IND)); + + GenTreeStoreDynBlk* store = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(addr, data, dynamicSize); + store->gtFlags |= GTF_ASG; + gtInitializeIndirNode(store, indirFlags); + gtInitializeStoreNode(store, data); + + return store; +} + //------------------------------------------------------------------------------ // gtNewStoreIndNode : Create an indirect store node. // // Arguments: // type - Type of the store -// addr - Destination address +// addr - Destionation address // data - Value to store // indirFlags - Indirection flags // @@ -9106,9 +9143,7 @@ GenTree* Compiler::gtNewBitCastNode(var_types type, GenTree* arg) // can't be represented in jitted code. If this happens, this method will return // nullptr. // -GenTreeAllocObj* Compiler::gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, - bool useParent) +GenTreeAllocObj* Compiler::gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool useParent) { const bool mustRestoreHandle = true; bool* const pRuntimeLookup = nullptr; @@ -9124,7 +9159,7 @@ GenTreeAllocObj* Compiler::gtNewAllocObjNode(CORINFO_RESOLVED_TOKEN* pResolvedTo helper = CORINFO_HELP_READYTORUN_NEW; CORINFO_LOOKUP_KIND* const pGenericLookupKind = nullptr; usingReadyToRunHelper = - info.compCompHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, helper, callerHandle, &lookup); + info.compCompHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, helper, &lookup); } #endif @@ -9442,7 +9477,6 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) case GT_NO_OP: case GT_NOP: case GT_LABEL: - case GT_SWIFT_ERROR: copy = new (this, oper) GenTree(oper, tree->gtType); goto DONE; @@ -9756,6 +9790,12 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) gtCloneExpr(tree->AsCmpXchg()->Data()), gtCloneExpr(tree->AsCmpXchg()->Comparand())); break; + case GT_STORE_DYN_BLK: + copy = new (this, oper) GenTreeStoreDynBlk(gtCloneExpr(tree->AsStoreDynBlk()->Addr()), + gtCloneExpr(tree->AsStoreDynBlk()->Data()), + gtCloneExpr(tree->AsStoreDynBlk()->gtDynamicSize)); + break; + case GT_SELECT: copy = new (this, oper) GenTreeConditional(oper, tree->TypeGet(), gtCloneExpr(tree->AsConditional()->gtCond), @@ -10264,7 +10304,6 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: - case GT_SWIFT_ERROR: m_state = -1; return; @@ -10368,6 +10407,12 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) m_advance = &GenTreeUseEdgeIterator::AdvanceArrElem; return; + case GT_STORE_DYN_BLK: + m_edge = &m_node->AsStoreDynBlk()->Addr(); + assert(*m_edge != nullptr); + m_advance = &GenTreeUseEdgeIterator::AdvanceStoreDynBlk; + return; + case GT_CALL: m_statePtr = m_node->AsCall()->gtArgs.Args().begin().GetArg(); m_advance = &GenTreeUseEdgeIterator::AdvanceCall; @@ -10433,6 +10478,29 @@ void GenTreeUseEdgeIterator::AdvanceArrElem() } } +//------------------------------------------------------------------------ +// GenTreeUseEdgeIterator::AdvanceStoreDynBlk: produces the next operand of a StoreDynBlk node and advances the state. +// +void GenTreeUseEdgeIterator::AdvanceStoreDynBlk() +{ + GenTreeStoreDynBlk* const dynBlock = m_node->AsStoreDynBlk(); + switch (m_state) + { + case 0: + m_edge = &dynBlock->Data(); + m_state = 1; + break; + case 1: + m_edge = &dynBlock->gtDynamicSize; + m_advance = &GenTreeUseEdgeIterator::Terminate; + break; + default: + unreached(); + } + + assert(*m_edge != nullptr); +} + //------------------------------------------------------------------------ // GenTreeUseEdgeIterator::AdvanceFieldList: produces the next operand of a FieldList node and advances the state. // @@ -10788,7 +10856,7 @@ bool GenTree::Precedes(GenTree* other) // void GenTree::SetIndirExceptionFlags(Compiler* comp) { - assert(OperIsIndirOrArrMetaData() && (OperIsSimple() || OperIs(GT_CMPXCHG))); + assert(OperIsIndirOrArrMetaData() && (OperIsSimple() || OperIs(GT_CMPXCHG, GT_STORE_DYN_BLK))); if (IndirMayFault(comp)) { @@ -10810,27 +10878,11 @@ void GenTree::SetIndirExceptionFlags(Compiler* comp) gtFlags |= AsCmpXchg()->Data()->gtFlags & GTF_EXCEPT; gtFlags |= AsCmpXchg()->Comparand()->gtFlags & GTF_EXCEPT; } -} - -//------------------------------------------------------------------------------ -// HandleKindDataIsInvariant: Returns true if the data referred to by a handle -// address is guaranteed to be invariant. Note that GTF_ICON_FTN_ADDR handles may -// or may not point to invariant data. -// -// Arguments: -// flags - GenTree flags for handle. -// -/* static */ -bool GenTree::HandleKindDataIsInvariant(GenTreeFlags flags) -{ - GenTreeFlags handleKind = flags & GTF_ICON_HDL_MASK; - assert(handleKind != GTF_EMPTY); - - // All handle types are assumed invariant except those specifically listed here. - - return (handleKind != GTF_ICON_STATIC_HDL) && // Pointer to a mutable class Static variable - (handleKind != GTF_ICON_BBC_PTR) && // Pointer to a mutable basic block count value - (handleKind != GTF_ICON_GLOBAL_PTR); // Pointer to mutable data from the VM state + else if (OperIs(GT_STORE_DYN_BLK)) + { + gtFlags |= AsStoreDynBlk()->Data()->gtFlags & GTF_EXCEPT; + gtFlags |= AsStoreDynBlk()->gtDynamicSize->gtFlags & GTF_EXCEPT; + } } #ifdef DEBUG @@ -11253,6 +11305,7 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ case GT_IND: case GT_STOREIND: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: // We prefer printing V or U if ((tree->gtFlags & (GTF_IND_VOLATILE | GTF_IND_UNALIGNED)) == 0) { @@ -12099,34 +12152,13 @@ void Compiler::gtDispConst(GenTree* tree) printf(" scope"); break; case GTF_ICON_CLASS_HDL: - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf(" class"); - } - else - { - printf(" class %s", eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); - } + printf(" class %s", eeGetClassName((CORINFO_CLASS_HANDLE)iconVal)); break; case GTF_ICON_METHOD_HDL: - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf(" method"); - } - else - { - printf(" method %s", eeGetMethodFullName((CORINFO_METHOD_HANDLE)iconVal)); - } + printf(" method %s", eeGetMethodFullName((CORINFO_METHOD_HANDLE)iconVal)); break; case GTF_ICON_FIELD_HDL: - if (IsTargetAbi(CORINFO_NATIVEAOT_ABI) || opts.IsReadyToRun()) - { - printf(" field"); - } - else - { - printf(" field %s", eeGetFieldName((CORINFO_FIELD_HANDLE)iconVal, true)); - } + printf(" field %s", eeGetFieldName((CORINFO_FIELD_HANDLE)iconVal, true)); break; case GTF_ICON_STATIC_HDL: printf(" static"); @@ -12272,12 +12304,6 @@ void Compiler::gtDispConst(GenTree* tree) vecCon->gtSimdVal.u64[6], vecCon->gtSimdVal.u64[7]); break; } - - case TYP_MASK: - { - printf("<0x%08x, 0x%08x>", vecCon->gtSimdVal.u32[0], vecCon->gtSimdVal.u32[1]); - break; - } #endif // TARGET_XARCH default: @@ -12384,7 +12410,6 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) case GT_MEMORYBARRIER: case GT_PINVOKE_PROLOG: case GT_JMPTABLE: - case GT_SWIFT_ERROR: break; case GT_RET_EXPR: @@ -12678,6 +12703,11 @@ void Compiler::gtDispTree(GenTree* tree, case GenTreeBlk::BlkOpKindUnrollMemmove: printf(" (Memmove)"); break; +#ifndef TARGET_X86 + case GenTreeBlk::BlkOpKindHelper: + printf(" (Helper)"); + break; +#endif case GenTreeBlk::BlkOpKindLoop: printf(" (Loop)"); @@ -13086,6 +13116,28 @@ void Compiler::gtDispTree(GenTree* tree, } break; + case GT_STORE_DYN_BLK: + if (tree->OperIsCopyBlkOp()) + { + printf(" (copy)"); + } + else if (tree->OperIsInitBlkOp()) + { + printf(" (init)"); + } + gtDispCommonEndLine(tree); + + if (!topOnly) + { + gtDispChild(tree->AsStoreDynBlk()->Addr(), indentStack, IIArc, nullptr, topOnly); + if (tree->AsStoreDynBlk()->Data() != nullptr) + { + gtDispChild(tree->AsStoreDynBlk()->Data(), indentStack, IIArc, nullptr, topOnly); + } + gtDispChild(tree->AsStoreDynBlk()->gtDynamicSize, indentStack, IIArcBottom, nullptr, topOnly); + } + break; + case GT_SELECT: gtDispCommonEndLine(tree); @@ -13148,10 +13200,6 @@ const char* Compiler::gtGetWellKnownArgNameForArgMsg(WellKnownArg arg) case WellKnownArg::ValidateIndirectCallTarget: case WellKnownArg::DispatchIndirectCallTarget: return "cfg tgt"; - case WellKnownArg::SwiftError: - return "swift error"; - case WellKnownArg::SwiftSelf: - return "swift self"; default: return nullptr; } @@ -13342,45 +13390,48 @@ void Compiler::gtDispArgList(GenTreeCall* call, GenTree* lastCallOperand, Indent // void Compiler::gtDispStmt(Statement* stmt, const char* msg /* = nullptr */) { - if (msg != nullptr) - { - printf("%s ", msg); - } - printStmtID(stmt); - printf(" ( "); - const DebugInfo& di = stmt->GetDebugInfo(); - // For statements in the root we display just the location without the - // inline context info. - if (di.GetInlineContext() == nullptr || di.GetInlineContext()->IsRoot()) + if (opts.compDbgInfo) { - di.GetLocation().Dump(); - } - else - { - stmt->GetDebugInfo().Dump(false); - } - printf(" ... "); + if (msg != nullptr) + { + printf("%s ", msg); + } + printStmtID(stmt); + printf(" ( "); + const DebugInfo& di = stmt->GetDebugInfo(); + // For statements in the root we display just the location without the + // inline context info. + if (di.GetInlineContext() == nullptr || di.GetInlineContext()->IsRoot()) + { + di.GetLocation().Dump(); + } + else + { + stmt->GetDebugInfo().Dump(false); + } + printf(" ... "); - IL_OFFSET lastILOffs = stmt->GetLastILOffset(); - if (lastILOffs == BAD_IL_OFFSET) - { - printf("???"); - } - else - { - printf("0x%03X", lastILOffs); - } + IL_OFFSET lastILOffs = stmt->GetLastILOffset(); + if (lastILOffs == BAD_IL_OFFSET) + { + printf("???"); + } + else + { + printf("0x%03X", lastILOffs); + } - printf(" )"); + printf(" )"); - DebugInfo par; - if (stmt->GetDebugInfo().GetParent(&par)) - { - printf(" <- "); - par.Dump(true); - } - printf("\n"); + DebugInfo par; + if (stmt->GetDebugInfo().GetParent(&par)) + { + printf(" <- "); + par.Dump(true); + } + printf("\n"); + } gtDispTree(stmt->GetRootNode()); } @@ -13518,6 +13569,22 @@ void Compiler::gtDispLIRNode(GenTree* node, const char* prefixMsg /* = nullptr * displayOperand(operand, buf, operandArc, indentStack, prefixIndent); } } + else if (node->OperIs(GT_STORE_DYN_BLK)) + { + if (operand == node->AsBlk()->Addr()) + { + displayOperand(operand, "lhs", operandArc, indentStack, prefixIndent); + } + else if (operand == node->AsBlk()->Data()) + { + displayOperand(operand, "rhs", operandArc, indentStack, prefixIndent); + } + else + { + assert(operand == node->AsStoreDynBlk()->gtDynamicSize); + displayOperand(operand, "size", operandArc, indentStack, prefixIndent); + } + } else { displayOperand(operand, "", operandArc, indentStack, prefixIndent); @@ -16874,6 +16941,22 @@ bool Compiler::gtSplitTree( } private: + bool IsLocation(const UseInfo& useInf) + { + if (useInf.User == nullptr) + { + return false; + } + + if (useInf.User->OperIs(GT_STORE_DYN_BLK) && !(*useInf.Use)->OperIs(GT_CNS_INT, GT_INIT_VAL) && + (useInf.Use == &useInf.User->AsStoreDynBlk()->Data())) + { + return true; + } + + return false; + } + bool IsReturned(const UseInfo& useInf, bool userIsReturned) { if (useInf.User != nullptr) @@ -16967,6 +17050,18 @@ bool Compiler::gtSplitTree( return; } + if (IsLocation(useInf)) + { + // Only a handful of nodes can be location, and they are all unary or nullary. + assert((*use)->OperIs(GT_IND, GT_BLK, GT_LCL_VAR, GT_LCL_FLD)); + if ((*use)->OperIsUnary()) + { + SplitOutUse(UseInfo{&(*use)->AsUnOp()->gtOp1, user}, false); + } + + return; + } + #ifndef TARGET_64BIT // GT_MUL with GTF_MUL_64RSLT is required to stay with casts on the // operands. Note that one operand may also be a constant, but we @@ -17164,6 +17259,8 @@ void Compiler::gtExtractSideEffList(GenTree* expr, colon->gtOp2 = (elseSideEffects != nullptr) ? elseSideEffects : m_compiler->gtNewNothingNode(); qmark->gtType = TYP_VOID; colon->gtType = TYP_VOID; + + qmark->gtFlags &= ~GTF_QMARK_CAST_INSTOF; Append(qmark); } @@ -17223,7 +17320,7 @@ void Compiler::gtExtractSideEffList(GenTree* expr, // Set the ValueNumber 'gtVNPair' for the new GT_COMMA node // - if ((m_compiler->vnStore != nullptr) && m_result->gtVNPair.BothDefined() && node->gtVNPair.BothDefined()) + if (m_result->gtVNPair.BothDefined() && node->gtVNPair.BothDefined()) { // The result of a GT_COMMA node is op2, the normal value number is op2vnp // But we also need to include the union of side effects from op1 and op2. @@ -18692,12 +18789,6 @@ CORINFO_CLASS_HANDLE Compiler::gtGetClassHandle(GenTree* tree, bool* pIsExact, b } } - if ((objClass == NO_CLASS_HANDLE) && (vnStore != nullptr)) - { - // Try VN if we haven't found a class handle yet - objClass = vnStore->GetObjectType(tree->gtVNPair.GetConservative(), pIsExact, pIsNonNull); - } - if ((objClass != NO_CLASS_HANDLE) && !*pIsExact && JitConfig.JitEnableExactDevirtualization()) { CORINFO_CLASS_HANDLE exactClass; @@ -26110,11 +26201,8 @@ bool GenTreeHWIntrinsic::OperIsMemoryLoad(GenTree** pAddr) const case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x2: case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x3: case NI_AdvSimd_Arm64_LoadAndInsertScalarVector128x4: - addr = Op(3); - break; - case NI_Sve_LoadVector: - addr = Op(2); + addr = Op(3); break; #endif // TARGET_ARM64 @@ -26884,14 +26972,6 @@ void ReturnTypeDesc::InitializeStructReturnType(Compiler* comp, { assert(varTypeIsStruct(returnType)); -#ifdef SWIFT_SUPPORT - if (callConv == CorInfoCallConvExtension::Swift) - { - InitializeSwiftReturnRegs(comp, retClsHnd); - break; - } -#endif - #ifdef UNIX_AMD64_ABI SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR structDesc; @@ -26995,31 +27075,6 @@ void ReturnTypeDesc::InitializeStructReturnType(Compiler* comp, #endif } -#ifdef SWIFT_SUPPORT -//--------------------------------------------------------------------------------------- -// InitializeSwiftReturnRegs: -// Initialize the Return Type Descriptor for a method that returns with the -// Swift calling convention. -// -// Parameters: -// comp - Compiler instance -// clsHnd - Struct type being returned -// -void ReturnTypeDesc::InitializeSwiftReturnRegs(Compiler* comp, CORINFO_CLASS_HANDLE clsHnd) -{ - const CORINFO_SWIFT_LOWERING* lowering = comp->GetSwiftLowering(clsHnd); - assert(!lowering->byReference); - - static_assert_no_msg(MAX_SWIFT_LOWERED_ELEMENTS <= MAX_RET_REG_COUNT); - assert(lowering->numLoweredElements <= MAX_RET_REG_COUNT); - - for (size_t i = 0; i < lowering->numLoweredElements; i++) - { - m_regType[i] = JITtype2varType(lowering->loweredElements[i]); - } -} -#endif - //--------------------------------------------------------------------------------------- // InitializeLongReturnType: // Initialize the Return Type Descriptor for a method that returns a TYP_LONG @@ -27083,9 +27138,8 @@ void ReturnTypeDesc::InitializeReturnType(Compiler* comp, // GetABIReturnReg: Return i'th return register as per target ABI // // Arguments: -// idx - Index of the return register. -// The first return register has an index of 0 and so on. -// callConv - Associated calling convention +// idx - Index of the return register. +// The first return register has an index of 0 and so on. // // Return Value: // Returns i'th return register as per target ABI. @@ -27094,44 +27148,13 @@ void ReturnTypeDesc::InitializeReturnType(Compiler* comp, // x86 and ARM return long in multiple registers. // ARM and ARM64 return HFA struct in multiple registers. // -regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx, CorInfoCallConvExtension callConv) const +regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx) const { unsigned count = GetReturnRegCount(); assert(idx < count); regNumber resultReg = REG_NA; -#ifdef SWIFT_SUPPORT - if (callConv == CorInfoCallConvExtension::Swift) - { - static const regNumber swiftIntReturnRegs[] = {REG_SWIFT_INTRET_ORDER}; - static const regNumber swiftFloatReturnRegs[] = {REG_SWIFT_FLOATRET_ORDER}; - assert((idx < ArrLen(swiftIntReturnRegs)) && (idx < ArrLen(swiftFloatReturnRegs))); - unsigned intRegIdx = 0; - unsigned floatRegIdx = 0; - for (unsigned i = 0; i < idx; i++) - { - if (varTypeUsesIntReg(GetReturnRegType(i))) - { - intRegIdx++; - } - else - { - floatRegIdx++; - } - } - - if (varTypeUsesIntReg(GetReturnRegType(idx))) - { - return swiftIntReturnRegs[intRegIdx]; - } - else - { - return swiftFloatReturnRegs[floatRegIdx]; - } - } -#endif - #ifdef UNIX_AMD64_ABI var_types regType0 = GetReturnRegType(0); @@ -27279,7 +27302,7 @@ regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx, CorInfoCallConvExtension // GetABIReturnRegs: get the mask of return registers as per target arch ABI. // // Arguments: -// callConv - The calling convention +// None // // Return Value: // reg mask of return registers in which the return type is returned. @@ -27289,14 +27312,14 @@ regNumber ReturnTypeDesc::GetABIReturnReg(unsigned idx, CorInfoCallConvExtension // of return registers and wants to know the set of return registers. // // static -regMaskTP ReturnTypeDesc::GetABIReturnRegs(CorInfoCallConvExtension callConv) const +regMaskTP ReturnTypeDesc::GetABIReturnRegs() const { regMaskTP resultMask = RBM_NONE; unsigned count = GetReturnRegCount(); for (unsigned i = 0; i < count; ++i) { - resultMask |= genRegMask(GetABIReturnReg(i, callConv)); + resultMask |= genRegMask(GetABIReturnReg(i)); } return resultMask; diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 4362f71c983578..1bd7ce27004fe6 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -194,13 +194,13 @@ inline AssertionIndex GetAssertionIndex(unsigned index) class AssertionInfo { - // true if the assertion holds on the false edge instead of the true edge (for GT_JTRUE nodes) - unsigned short m_assertionHoldsOnFalseEdge : 1; + // true if the assertion holds on the bbNext edge instead of the bbTarget edge (for GT_JTRUE nodes) + unsigned short m_isNextEdgeAssertion : 1; // 1-based index of the assertion unsigned short m_assertionIndex : 15; - AssertionInfo(bool assertionHoldsOnFalseEdge, AssertionIndex assertionIndex) - : m_assertionHoldsOnFalseEdge(assertionHoldsOnFalseEdge), m_assertionIndex(assertionIndex) + AssertionInfo(bool isNextEdgeAssertion, AssertionIndex assertionIndex) + : m_isNextEdgeAssertion(isNextEdgeAssertion), m_assertionIndex(assertionIndex) { assert(m_assertionIndex == assertionIndex); } @@ -223,8 +223,8 @@ class AssertionInfo void Clear() { - m_assertionHoldsOnFalseEdge = 0; - m_assertionIndex = NO_ASSERTION_INDEX; + m_isNextEdgeAssertion = 0; + m_assertionIndex = NO_ASSERTION_INDEX; } bool HasAssertion() const @@ -237,9 +237,9 @@ class AssertionInfo return m_assertionIndex; } - bool AssertionHoldsOnFalseEdge() const + bool IsNextEdgeAssertion() const { - return m_assertionHoldsOnFalseEdge; + return m_isNextEdgeAssertion; } }; @@ -507,6 +507,9 @@ enum GenTreeFlags : unsigned int GTF_RET_MERGED = 0x80000000, // GT_RETURN -- This is a return generated during epilog merging. + GTF_QMARK_CAST_INSTOF = 0x80000000, // GT_QMARK -- Is this a top (not nested) level qmark created for + // castclass or instanceof? + GTF_BOX_CLONED = 0x40000000, // GT_BOX -- this box and its operand has been cloned, cannot assume it to be single-use anymore GTF_BOX_VALUE = 0x80000000, // GT_BOX -- "box" is on a value type @@ -1216,7 +1219,7 @@ struct GenTree static bool OperIsStoreBlk(genTreeOps gtOper) { - return StaticOperIs(gtOper, GT_STORE_BLK); + return StaticOperIs(gtOper, GT_STORE_BLK, GT_STORE_DYN_BLK); } bool OperIsStoreBlk() const @@ -1545,7 +1548,7 @@ struct GenTree static bool OperIsIndir(genTreeOps gtOper) { static_assert_no_msg(AreContiguous(GT_LOCKADD, GT_XAND, GT_XORR, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_IND, - GT_STOREIND, GT_BLK, GT_STORE_BLK, GT_NULLCHECK)); + GT_STOREIND, GT_BLK, GT_STORE_BLK, GT_STORE_DYN_BLK, GT_NULLCHECK)); return (GT_LOCKADD <= gtOper) && (gtOper <= GT_NULLCHECK); } @@ -2249,7 +2252,6 @@ struct GenTree } #if defined(TARGET_XARCH) && defined(FEATURE_HW_INTRINSICS) - bool IsEmbMaskOp() { return OperIsHWIntrinsic() && ((gtFlags & GTF_HW_EM_OP) != 0); @@ -2264,8 +2266,6 @@ struct GenTree #endif // TARGET_XARCH && FEATURE_HW_INTRINSICS - static bool HandleKindDataIsInvariant(GenTreeFlags flags); - bool IsCall() const { return OperGet() == GT_CALL; @@ -2865,6 +2865,7 @@ class GenTreeUseEdgeIterator final // Advance functions for special nodes void AdvanceCmpXchg(); void AdvanceArrElem(); + void AdvanceStoreDynBlk(); void AdvanceFieldList(); void AdvancePhi(); void AdvanceConditional(); @@ -4205,8 +4206,6 @@ struct ReturnTypeDesc bool m_inited; #endif - void InitializeSwiftReturnRegs(Compiler* comp, CORINFO_CLASS_HANDLE retClsHnd); - public: ReturnTypeDesc() { @@ -4328,10 +4327,10 @@ struct ReturnTypeDesc } // Get i'th ABI return register - regNumber GetABIReturnReg(unsigned idx, CorInfoCallConvExtension callConv) const; + regNumber GetABIReturnReg(unsigned idx) const; // Get reg mask of ABI return registers - regMaskTP GetABIReturnRegs(CorInfoCallConvExtension callConv) const; + regMaskTP GetABIReturnRegs() const; }; class TailCallSiteInfo @@ -4402,7 +4401,7 @@ enum class CFGCallKind class CallArgs; -enum class WellKnownArg : unsigned +enum class WellKnownArg { None, ThisPointer, @@ -4420,8 +4419,6 @@ enum class WellKnownArg : unsigned R2RIndirectionCell, ValidateIndirectCallTarget, DispatchIndirectCallTarget, - SwiftError, - SwiftSelf, }; #ifdef DEBUG @@ -5222,15 +5219,6 @@ struct GenTreeCall final : public GenTree return (gtFlags & GTF_CALL_INLINE_CANDIDATE) != 0; } -#ifdef SWIFT_SUPPORT - bool HasSwiftErrorHandling() - { - // Most calls aren't Swift calls, so short-circuit this check by checking the calling convention first. - return (GetUnmanagedCallConv() == CorInfoCallConvExtension::Swift) && - (gtArgs.FindWellKnownArg(WellKnownArg::SwiftError) != nullptr); - } -#endif // SWIFT_SUPPORT - bool IsR2ROrVirtualStubRelativeIndir() { #if defined(FEATURE_READYTORUN) @@ -6515,9 +6503,8 @@ struct GenTreeVecCon : public GenTree simd16_t gtSimd16Val; #if defined(TARGET_XARCH) - simd32_t gtSimd32Val; - simd64_t gtSimd64Val; - simdmask_t gtSimdMaskVal; + simd32_t gtSimd32Val; + simd64_t gtSimd64Val; #endif // TARGET_XARCH simd_t gtSimdVal; @@ -6769,11 +6756,6 @@ struct GenTreeVecCon : public GenTree { return gtSimd64Val.IsAllBitsSet(); } - - case TYP_MASK: - { - return gtSimdMaskVal.IsAllBitsSet(); - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -6821,11 +6803,6 @@ struct GenTreeVecCon : public GenTree { return left->gtSimd64Val == right->gtSimd64Val; } - - case TYP_MASK: - { - return left->gtSimdMaskVal == right->gtSimdMaskVal; - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -6866,11 +6843,6 @@ struct GenTreeVecCon : public GenTree { return gtSimd64Val.IsZero(); } - - case TYP_MASK: - { - return gtSimdMaskVal.IsZero(); - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -7385,10 +7357,15 @@ struct GenTreeBlk : public GenTreeIndir public: ClassLayout* GetLayout() const { - assert(m_layout != nullptr); return m_layout; } + void SetLayout(ClassLayout* layout) + { + assert((layout != nullptr) || OperIs(GT_STORE_DYN_BLK)); + m_layout = layout; + } + // The data to be stored (null for GT_BLK) GenTree*& Data() { @@ -7402,7 +7379,8 @@ struct GenTreeBlk : public GenTreeIndir // The size of the buffer to be copied. unsigned Size() const { - return m_layout->GetSize(); + assert((m_layout != nullptr) || OperIs(GT_STORE_DYN_BLK)); + return (m_layout != nullptr) ? m_layout->GetSize() : 0; } // Instruction selection: during codegen time, what code sequence we will be using @@ -7414,6 +7392,9 @@ struct GenTreeBlk : public GenTreeIndir #ifdef TARGET_XARCH BlkOpKindCpObjRepInstr, #endif +#ifndef TARGET_X86 + BlkOpKindHelper, +#endif #ifdef TARGET_XARCH BlkOpKindRepInstr, #endif @@ -7428,7 +7409,7 @@ struct GenTreeBlk : public GenTreeIndir bool ContainsReferences() { - return m_layout->HasGCPtr(); + return (m_layout != nullptr) && m_layout->HasGCPtr(); } bool IsOnHeapAndContainsReferences() @@ -7459,8 +7440,8 @@ struct GenTreeBlk : public GenTreeIndir void Initialize(ClassLayout* layout) { - assert(layout != nullptr); - assert(layout->GetSize() != 0); + assert(OperIsBlk(OperGet()) && ((layout != nullptr) || OperIs(GT_STORE_DYN_BLK))); + assert((layout == nullptr) || (layout->GetSize() != 0)); m_layout = layout; gtBlkOpKind = BlkOpKindInvalid; @@ -7478,6 +7459,35 @@ struct GenTreeBlk : public GenTreeIndir #endif // DEBUGGABLE_GENTREE }; +// GenTreeStoreDynBlk -- 'dynamic block store' (GT_STORE_DYN_BLK). +// +// This node is used to represent stores that have a dynamic size - the "cpblk" and "initblk" +// IL instructions are implemented with it. Note that such stores assume the input has no GC +// pointers in it, and as such do not ever use write barriers. +// +// The "Data()" member of this node will either be a "dummy" IND(struct) node, for "cpblk", or +// the zero constant/INIT_VAL for "initblk". +// +struct GenTreeStoreDynBlk : public GenTreeBlk +{ +public: + GenTree* gtDynamicSize; + + GenTreeStoreDynBlk(GenTree* dstAddr, GenTree* data, GenTree* dynamicSize) + : GenTreeBlk(GT_STORE_DYN_BLK, TYP_VOID, dstAddr, data, nullptr), gtDynamicSize(dynamicSize) + { + gtFlags |= dynamicSize->gtFlags & GTF_ALL_EFFECT; + } + +#if DEBUGGABLE_GENTREE +protected: + friend GenTree; + GenTreeStoreDynBlk() : GenTreeBlk() + { + } +#endif // DEBUGGABLE_GENTREE +}; + // Read-modify-write status of a RMW memory op rooted at a storeInd enum RMWStatus { @@ -8892,6 +8902,10 @@ struct GenTreeCCMP final : public GenTreeOpCC inline bool GenTree::OperIsBlkOp() { + if (OperIs(GT_STORE_DYN_BLK)) + { + return true; + } if (OperIsStore()) { return varTypeIsStruct(this); @@ -9299,7 +9313,7 @@ inline GenTree* GenTree::gtGetOp2IfPresent() const inline GenTree*& GenTree::Data() { - assert(OperIsStore()); + assert(OperIsStore() || OperIs(GT_STORE_DYN_BLK)); return OperIsLocalStore() ? AsLclVarCommon()->Data() : AsIndir()->Data(); } diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 26513f0f3441be..597a9e471d5b1f 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -37,7 +37,6 @@ GTNODE(LABEL , GenTree ,0,0,GTK_LEAF) // Jump- GTNODE(JMP , GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE) // Jump to another function GTNODE(FTN_ADDR , GenTreeFptrVal ,0,0,GTK_LEAF) // Address of a function GTNODE(RET_EXPR , GenTreeRetExpr ,0,0,GTK_LEAF|DBK_NOTLIR) // Place holder for the return expression from an inline candidate -GTNODE(SWIFT_ERROR , GenTree ,0,0,GTK_LEAF) // Error register value post-Swift call //----------------------------------------------------------------------------- // Constant nodes: @@ -77,12 +76,13 @@ GTNODE(XAND , GenTreeOp ,0,1,GTK_BINOP) GTNODE(XORR , GenTreeOp ,0,1,GTK_BINOP) GTNODE(XADD , GenTreeOp ,0,1,GTK_BINOP) GTNODE(XCHG , GenTreeOp ,0,1,GTK_BINOP) -GTNODE(CMPXCHG , GenTreeCmpXchg ,0,1,GTK_SPECIAL) // atomic CAS, small types need the comparand to be zero extended +GTNODE(CMPXCHG , GenTreeCmpXchg ,0,1,GTK_SPECIAL) GTNODE(IND , GenTreeIndir ,0,1,GTK_UNOP) // Load indirection GTNODE(STOREIND , GenTreeStoreInd ,0,1,GTK_BINOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // Store indirection GTNODE(BLK , GenTreeBlk ,0,1,GTK_UNOP|GTK_EXOP) // Struct load GTNODE(STORE_BLK , GenTreeBlk ,0,1,GTK_BINOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // Struct store +GTNODE(STORE_DYN_BLK , GenTreeStoreDynBlk ,0,1,GTK_SPECIAL|GTK_NOVALUE) // Dynamically sized block store, with native uint size GTNODE(NULLCHECK , GenTreeIndir ,0,1,GTK_UNOP|GTK_NOVALUE) // Null checks the source GTNODE(ARR_LENGTH , GenTreeArrLen ,0,0,GTK_UNOP|GTK_EXOP) // single-dimension (SZ) array length diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index e6823478a3c9a5..989b6e554eae7b 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -88,8 +88,9 @@ GTSTRUCT_1(AddrMode , GT_LEA) GTSTRUCT_1(Qmark , GT_QMARK) GTSTRUCT_1(PhiArg , GT_PHI_ARG) GTSTRUCT_1(Phi , GT_PHI) -GTSTRUCT_N(Indir , GT_IND, GT_NULLCHECK, GT_BLK, GT_STORE_BLK, GT_LOCKADD, GT_XAND, GT_XORR, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_STOREIND) -GTSTRUCT_N(Blk , GT_BLK, GT_STORE_BLK) +GTSTRUCT_N(Indir , GT_IND, GT_NULLCHECK, GT_BLK, GT_STORE_BLK, GT_STORE_DYN_BLK, GT_LOCKADD, GT_XAND, GT_XORR, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_STOREIND) +GTSTRUCT_N(Blk , GT_BLK, GT_STORE_BLK, GT_STORE_DYN_BLK) +GTSTRUCT_1(StoreDynBlk , GT_STORE_DYN_BLK) GTSTRUCT_1(StoreInd , GT_STOREIND) GTSTRUCT_1(CmpXchg , GT_CMPXCHG) #ifdef TARGET_ARM64 diff --git a/src/coreclr/jit/helperexpansion.cpp b/src/coreclr/jit/helperexpansion.cpp index eb203cbd3f68c0..52b7d3d68cbce9 100644 --- a/src/coreclr/jit/helperexpansion.cpp +++ b/src/coreclr/jit/helperexpansion.cpp @@ -319,11 +319,21 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm // Fallback basic block GenTree* fallbackValueDef = gtNewStoreLclVarNode(rtLookupLcl->GetLclNum(), call); - BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fallbackValueDef, debugInfo, true); + BasicBlock* fallbackBb = + fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fallbackValueDef, debugInfo, nullcheckBb->Next(), true); + + assert(fallbackBb->JumpsToNext()); + fallbackBb->SetFlags(BBF_NONE_QUIRK); + + // Set nullcheckBb's true jump target + nullcheckBb->SetTrueTarget(fallbackBb); // Fast-path basic block GenTree* fastpathValueDef = gtNewStoreLclVarNode(rtLookupLcl->GetLclNum(), fastPathValueClone); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fastpathValueDef, debugInfo); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fastpathValueDef, debugInfo, block); + + // Set nullcheckBb's false jump target + nullcheckBb->SetFalseTarget(fastPathBb); BasicBlock* sizeCheckBb = nullptr; if (needsSizeCheck) @@ -365,64 +375,45 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm GenTree* jtrue = gtNewOperNode(GT_JTRUE, TYP_VOID, sizeCheck); // sizeCheckBb fails - jump to fallbackBb - sizeCheckBb = fgNewBBFromTreeAfter(BBJ_COND, prevBb, jtrue, debugInfo); + sizeCheckBb = fgNewBBFromTreeAfter(BBJ_COND, prevBb, jtrue, debugInfo, fallbackBb); + sizeCheckBb->SetFalseTarget(nullcheckBb); } // // Update preds in all new blocks // + fgRemoveRefPred(block, prevBb); + fgAddRefPred(block, fastPathBb); + fgAddRefPred(block, fallbackBb); assert(prevBb->KindIs(BBJ_ALWAYS)); - { - FlowEdge* const newEdge = fgAddRefPred(block, fastPathBb); - fastPathBb->SetTargetEdge(newEdge); - } - - { - FlowEdge* const newEdge = fgAddRefPred(block, fallbackBb); - fallbackBb->SetTargetEdge(newEdge); - assert(fallbackBb->JumpsToNext()); - fallbackBb->SetFlags(BBF_NONE_QUIRK); - } - if (needsSizeCheck) { // sizeCheckBb is the first block after prevBb - fgRedirectTargetEdge(prevBb, sizeCheckBb); - + prevBb->SetTarget(sizeCheckBb); + fgAddRefPred(sizeCheckBb, prevBb); // sizeCheckBb flows into nullcheckBb in case if the size check passes - { - FlowEdge* const trueEdge = fgAddRefPred(fallbackBb, sizeCheckBb); - FlowEdge* const falseEdge = fgAddRefPred(nullcheckBb, sizeCheckBb); - sizeCheckBb->SetTrueEdge(trueEdge); - sizeCheckBb->SetFalseEdge(falseEdge); - trueEdge->setLikelihood(0.2); - falseEdge->setLikelihood(0.8); - } - + fgAddRefPred(nullcheckBb, sizeCheckBb); // fallbackBb is reachable from both nullcheckBb and sizeCheckBb + fgAddRefPred(fallbackBb, nullcheckBb); + fgAddRefPred(fallbackBb, sizeCheckBb); // fastPathBb is only reachable from successful nullcheckBb + fgAddRefPred(fastPathBb, nullcheckBb); } else { // nullcheckBb is the first block after prevBb - fgRedirectTargetEdge(prevBb, nullcheckBb); - + prevBb->SetTarget(nullcheckBb); + fgAddRefPred(nullcheckBb, prevBb); // No size check, nullcheckBb jumps to fast path + fgAddRefPred(fastPathBb, nullcheckBb); // fallbackBb is only reachable from nullcheckBb (jump destination) + fgAddRefPred(fallbackBb, nullcheckBb); } - FlowEdge* const trueEdge = fgAddRefPred(fallbackBb, nullcheckBb); - FlowEdge* const falseEdge = fgAddRefPred(fastPathBb, nullcheckBb); - nullcheckBb->SetTrueEdge(trueEdge); - nullcheckBb->SetFalseEdge(falseEdge); - trueEdge->setLikelihood(0.2); - falseEdge->setLikelihood(0.8); - // // Re-distribute weights (see '[weight: X]' on the diagrams above) // TODO: consider marking fallbackBb as rarely-taken - // TODO: derive block weights from edge likelihoods. // block->inheritWeight(prevBb); if (needsSizeCheck) @@ -708,10 +699,11 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St // fallbackBb GenTree* fallbackValueDef = gtNewStoreLclVarNode(finalLclNum, slowHelper); - BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, tlsRootNullCondBB, fallbackValueDef, debugInfo, true); + BasicBlock* fallbackBb = + fgNewBBFromTreeAfter(BBJ_ALWAYS, tlsRootNullCondBB, fallbackValueDef, debugInfo, block, true); GenTree* fastPathValueDef = gtNewStoreLclVarNode(finalLclNum, gtCloneExpr(finalLcl)); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, true); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, block, true); *callUse = finalLcl; @@ -721,22 +713,14 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St // // Update preds in all new blocks // - FlowEdge* const trueEdge = fgAddRefPred(fastPathBb, tlsRootNullCondBB); - FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, tlsRootNullCondBB); - tlsRootNullCondBB->SetTrueEdge(trueEdge); - tlsRootNullCondBB->SetFalseEdge(falseEdge); - trueEdge->setLikelihood(1.0); - falseEdge->setLikelihood(0.0); + fgAddRefPred(fallbackBb, tlsRootNullCondBB); + fgAddRefPred(fastPathBb, tlsRootNullCondBB); - { - FlowEdge* const newEdge = fgAddRefPred(block, fallbackBb); - fallbackBb->SetTargetEdge(newEdge); - } + fgAddRefPred(block, fallbackBb); + fgAddRefPred(block, fastPathBb); - { - FlowEdge* const newEdge = fgAddRefPred(block, fastPathBb); - fastPathBb->SetTargetEdge(newEdge); - } + tlsRootNullCondBB->SetTrueTarget(fastPathBb); + tlsRootNullCondBB->SetFalseTarget(fallbackBb); // Inherit the weights block->inheritWeight(prevBb); @@ -746,7 +730,9 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St // fallback will just execute first time fallbackBb->bbSetRunRarely(); - fgRedirectTargetEdge(prevBb, tlsRootNullCondBB); + fgRemoveRefPred(block, prevBb); + fgAddRefPred(tlsRootNullCondBB, prevBb); + prevBb->SetTarget(tlsRootNullCondBB); // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); @@ -1070,7 +1056,7 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // fallbackBb GenTree* fallbackValueDef = gtNewStoreLclVarNode(threadStaticBlockLclNum, call); BasicBlock* fallbackBb = - fgNewBBFromTreeAfter(BBJ_ALWAYS, threadStaticBlockNullCondBB, fallbackValueDef, debugInfo, true); + fgNewBBFromTreeAfter(BBJ_ALWAYS, threadStaticBlockNullCondBB, fallbackValueDef, debugInfo, block, true); // fastPathBb if (isGCThreadStatic) @@ -1085,41 +1071,32 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* GenTree* fastPathValueDef = gtNewStoreLclVarNode(threadStaticBlockLclNum, gtCloneExpr(threadStaticBlockBaseLclValueUse)); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, true); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, block, true); + + // Set maxThreadStaticBlocksCondBB's jump targets + maxThreadStaticBlocksCondBB->SetTrueTarget(fallbackBb); + maxThreadStaticBlocksCondBB->SetFalseTarget(threadStaticBlockNullCondBB); + + // Set threadStaticBlockNullCondBB's jump targets + threadStaticBlockNullCondBB->SetTrueTarget(fastPathBb); + threadStaticBlockNullCondBB->SetFalseTarget(fallbackBb); // // Update preds in all new blocks // assert(prevBb->KindIs(BBJ_ALWAYS)); - fgRedirectTargetEdge(prevBb, maxThreadStaticBlocksCondBB); + prevBb->SetTarget(maxThreadStaticBlocksCondBB); + fgRemoveRefPred(block, prevBb); + fgAddRefPred(maxThreadStaticBlocksCondBB, prevBb); - { - FlowEdge* const trueEdge = fgAddRefPred(fallbackBb, maxThreadStaticBlocksCondBB); - FlowEdge* const falseEdge = fgAddRefPred(threadStaticBlockNullCondBB, maxThreadStaticBlocksCondBB); - maxThreadStaticBlocksCondBB->SetTrueEdge(trueEdge); - maxThreadStaticBlocksCondBB->SetFalseEdge(falseEdge); - trueEdge->setLikelihood(0.0); - falseEdge->setLikelihood(1.0); - } + fgAddRefPred(threadStaticBlockNullCondBB, maxThreadStaticBlocksCondBB); + fgAddRefPred(fallbackBb, maxThreadStaticBlocksCondBB); - { - FlowEdge* const trueEdge = fgAddRefPred(fastPathBb, threadStaticBlockNullCondBB); - FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, threadStaticBlockNullCondBB); - threadStaticBlockNullCondBB->SetTrueEdge(trueEdge); - threadStaticBlockNullCondBB->SetFalseEdge(falseEdge); - trueEdge->setLikelihood(1.0); - falseEdge->setLikelihood(0.0); - } - - { - FlowEdge* const newEdge = fgAddRefPred(block, fastPathBb); - fastPathBb->SetTargetEdge(newEdge); - } + fgAddRefPred(fastPathBb, threadStaticBlockNullCondBB); + fgAddRefPred(fallbackBb, threadStaticBlockNullCondBB); - { - FlowEdge* const newEdge = fgAddRefPred(block, fallbackBb); - fallbackBb->SetTargetEdge(newEdge); - } + fgAddRefPred(block, fastPathBb); + fgAddRefPred(block, fallbackBb); // Inherit the weights block->inheritWeight(prevBb); @@ -1399,12 +1376,14 @@ bool Compiler::fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, G GenTree* isInitedCmp = gtNewOperNode(GT_EQ, TYP_INT, isInitedActualValueNode, isInitedExpectedValue); isInitedCmp->gtFlags |= GTF_RELOP_JMP_USED; BasicBlock* isInitedBb = - fgNewBBFromTreeAfter(BBJ_COND, prevBb, gtNewOperNode(GT_JTRUE, TYP_VOID, isInitedCmp), debugInfo); + fgNewBBFromTreeAfter(BBJ_COND, prevBb, gtNewOperNode(GT_JTRUE, TYP_VOID, isInitedCmp), debugInfo, block); // Fallback basic block // TODO-CQ: for JIT we can replace the original call with CORINFO_HELP_INITCLASS // that only accepts a single argument - BasicBlock* helperCallBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, isInitedBb, call, debugInfo, true); + BasicBlock* helperCallBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, isInitedBb, call, debugInfo, isInitedBb->Next(), true); + assert(helperCallBb->JumpsToNext()); + helperCallBb->SetFlags(BBF_NONE_QUIRK); GenTree* replacementNode = nullptr; if (retValKind == SHRV_STATIC_BASE_PTR) @@ -1463,28 +1442,23 @@ bool Compiler::fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, G // Update preds in all new blocks // - // Redirect prevBb from block to isInitedBb - fgRedirectTargetEdge(prevBb, isInitedBb); + // Unlink block and prevBb + fgRemoveRefPred(block, prevBb); + + // Block has two preds now: either isInitedBb or helperCallBb + fgAddRefPred(block, isInitedBb); + fgAddRefPred(block, helperCallBb); + + // prevBb always flows into isInitedBb + assert(prevBb->KindIs(BBJ_ALWAYS)); + prevBb->SetTarget(isInitedBb); prevBb->SetFlags(BBF_NONE_QUIRK); assert(prevBb->JumpsToNext()); + fgAddRefPred(isInitedBb, prevBb); - { - // Block has two preds now: either isInitedBb or helperCallBb - FlowEdge* const newEdge = fgAddRefPred(block, helperCallBb); - helperCallBb->SetTargetEdge(newEdge); - assert(helperCallBb->JumpsToNext()); - helperCallBb->SetFlags(BBF_NONE_QUIRK); - } - - { - // Both fastPathBb and helperCallBb have a single common pred - isInitedBb - FlowEdge* const trueEdge = fgAddRefPred(block, isInitedBb); - FlowEdge* const falseEdge = fgAddRefPred(helperCallBb, isInitedBb); - isInitedBb->SetTrueEdge(trueEdge); - isInitedBb->SetFalseEdge(falseEdge); - trueEdge->setLikelihood(1.0); - falseEdge->setLikelihood(0.0); - } + // Both fastPathBb and helperCallBb have a single common pred - isInitedBb + isInitedBb->SetFalseTarget(helperCallBb); + fgAddRefPred(helperCallBb, isInitedBb); // // Re-distribute weights @@ -1713,7 +1687,7 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, // // Block 1: lengthCheckBb (we check that dstLen < srcLen) // - BasicBlock* const lengthCheckBb = fgNewBBafter(BBJ_COND, prevBb, true); + BasicBlock* lengthCheckBb = fgNewBBafter(BBJ_COND, prevBb, true, block); lengthCheckBb->SetFlags(BBF_INTERNAL); // Set bytesWritten -1 by default, if the fast path is not taken we'll return it as the result. @@ -1735,8 +1709,9 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, // In theory, we could just emit the const U8 data to the data section and use GT_BLK here // but that would be a bit less efficient since we would have to load the data from memory. // - BasicBlock* fastpathBb = fgNewBBafter(BBJ_ALWAYS, lengthCheckBb, true); - fastpathBb->SetFlags(BBF_INTERNAL); + BasicBlock* fastpathBb = fgNewBBafter(BBJ_ALWAYS, lengthCheckBb, true, lengthCheckBb->Next()); + assert(fastpathBb->JumpsToNext()); + fastpathBb->SetFlags(BBF_INTERNAL | BBF_NONE_QUIRK); // The widest type we can use for loads const var_types maxLoadType = roundDownMaxType(srcLenU8); @@ -1788,30 +1763,20 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, // // Update preds in all new blocks // - // Redirect prevBb to lengthCheckBb - fgRedirectTargetEdge(prevBb, lengthCheckBb); + // block is no longer a predecessor of prevBb + fgRemoveRefPred(block, prevBb); + // prevBb flows into lengthCheckBb + assert(prevBb->KindIs(BBJ_ALWAYS)); + prevBb->SetTarget(lengthCheckBb); prevBb->SetFlags(BBF_NONE_QUIRK); assert(prevBb->JumpsToNext()); - - { - // lengthCheckBb has two successors: block and fastpathBb - FlowEdge* const trueEdge = fgAddRefPred(block, lengthCheckBb); - FlowEdge* const falseEdge = fgAddRefPred(fastpathBb, lengthCheckBb); - lengthCheckBb->SetTrueEdge(trueEdge); - lengthCheckBb->SetFalseEdge(falseEdge); - - // review: we assume length check always succeeds?? - trueEdge->setLikelihood(1.0); - falseEdge->setLikelihood(0.0); - } - - { - // fastpathBb flows into block - FlowEdge* const newEdge = fgAddRefPred(block, fastpathBb); - fastpathBb->SetTargetEdge(newEdge); - assert(fastpathBb->JumpsToNext()); - fastpathBb->SetFlags(BBF_NONE_QUIRK); - } + fgAddRefPred(lengthCheckBb, prevBb); + // lengthCheckBb has two successors: block and fastpathBb + lengthCheckBb->SetFalseTarget(fastpathBb); + fgAddRefPred(fastpathBb, lengthCheckBb); + fgAddRefPred(block, lengthCheckBb); + // fastpathBb flows into block + fgAddRefPred(block, fastpathBb); // // Re-distribute weights @@ -1965,23 +1930,11 @@ static int PickCandidatesForTypeCheck(Compiler* comp, CORINFO_CLASS_HANDLE castToCls = comp->gtGetHelperArgClassHandle(clsArg); if (castToCls == NO_CLASS_HANDLE) { - // If we don't see the constant class handle, we still can speculatively expand it - // for castclass case (we'll just take the unknown tree as a type check tree) - switch (helper) - { - case CORINFO_HELP_CHKCASTCLASS: - case CORINFO_HELP_CHKCASTARRAY: - case CORINFO_HELP_CHKCASTANY: - likelihoods[0] = 50; // 50% speculative guess - candidates[0] = NO_CLASS_HANDLE; - return 1; - - default: - // Otherwise, bail out. We don't expect the constant handles to be CSE'd as they normally - // have GTF_DONT_CSE flag set on them for cast helpers. - // TODO-InlineCast: One missing case to handle is isinst against Class<_Canon> - return 0; - } + // We don't expect the constant handle to be CSE'd here because importer + // sets GTF_DONT_CSE for it. The only case when this arg is not a constant is RUNTIMELOOKUP + // TODO-InlineCast: we should be able to handle RUNTIMELOOKUP as well. + JITDUMP("clsArg is not a constant handle - bail out.\n"); + return 0; } if ((objArg->gtFlags & GTF_ALL_EFFECT) != 0 && comp->lvaHaveManyLocals()) @@ -2379,8 +2332,8 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, // it's too late to rely on upstream phases to do this for us (unless we do optRepeat). GenTree* nullcheckOp = gtNewOperNode(GT_EQ, TYP_INT, tmpNode, gtNewNull()); nullcheckOp->gtFlags |= GTF_RELOP_JMP_USED; - BasicBlock* nullcheckBb = - fgNewBBFromTreeAfter(BBJ_COND, firstBb, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), debugInfo, true); + BasicBlock* nullcheckBb = fgNewBBFromTreeAfter(BBJ_COND, firstBb, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), + debugInfo, lastBb, true); // The very first statement in the whole expansion is to assign obj to tmp. // We assume it's the value we're going to return in most cases. @@ -2396,15 +2349,11 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, BasicBlock* lastTypeCheckBb = nullcheckBb; for (int candidateId = 0; candidateId < numOfCandidates; candidateId++) { - const CORINFO_CLASS_HANDLE expectedCls = expectedExactClasses[candidateId]; - // if expectedCls is NO_CLASS_HANDLE, it means we should just use the original clsArg - GenTree* expectedClsNode = expectedCls != NO_CLASS_HANDLE - ? gtNewIconEmbClsHndNode(expectedCls) - : gtCloneExpr(call->gtArgs.GetUserArgByIndex(0)->GetNode()); + GenTree* expectedClsNode = gtNewIconEmbClsHndNode(expectedExactClasses[candidateId]); + GenTree* storeCseVal = nullptr; // Manually CSE the expectedClsNode for first type check if it's the same as the original clsArg // TODO-InlineCast: consider not doing this if the helper call is cold - GenTree* storeCseVal = nullptr; if (candidateId == 0) { GenTree*& castArg = call->gtArgs.GetUserArgByIndex(0)->LateNodeRef(); @@ -2420,7 +2369,7 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTree* mtCheck = gtNewOperNode(GT_EQ, TYP_INT, gtNewMethodTableLookup(gtCloneExpr(tmpNode)), expectedClsNode); mtCheck->gtFlags |= GTF_RELOP_JMP_USED; GenTree* jtrue = gtNewOperNode(GT_JTRUE, TYP_VOID, mtCheck); - typeChecksBbs[candidateId] = fgNewBBFromTreeAfter(BBJ_COND, lastTypeCheckBb, jtrue, debugInfo, true); + typeChecksBbs[candidateId] = fgNewBBFromTreeAfter(BBJ_COND, lastTypeCheckBb, jtrue, debugInfo, lastBb, true); lastTypeCheckBb = typeChecksBbs[candidateId]; // Insert the CSE node as the first statement in the block @@ -2442,13 +2391,13 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, { // fallback call is used only to throw InvalidCastException call->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; - fallbackBb = fgNewBBFromTreeAfter(BBJ_THROW, lastTypeCheckBb, call, debugInfo, true); + fallbackBb = fgNewBBFromTreeAfter(BBJ_THROW, lastTypeCheckBb, call, debugInfo, nullptr, true); } else if (typeCheckFailedAction == TypeCheckFailedAction::ReturnNull) { // if fallback call is not needed, we just assign null to tmp GenTree* fallbackTree = gtNewTempStore(tmpNum, gtNewNull()); - fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, true); + fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, lastBb, true); } else { @@ -2459,7 +2408,7 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, call->gtCallMethHnd = eeFindHelper(CORINFO_HELP_CHKCASTCLASS_SPECIAL); } GenTree* fallbackTree = gtNewTempStore(tmpNum, call); - fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, true); + fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, lastBb, true); } // Block 4: typeCheckSucceedBb @@ -2473,47 +2422,16 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, // No-op because tmp was already assigned to obj typeCheckSucceedTree = gtNewNothingNode(); } + BasicBlock* typeCheckSucceedBb = + typeCheckNotNeeded ? nullptr + : fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, typeCheckSucceedTree, debugInfo, lastBb); // // Wire up the blocks // - - // We assume obj is 50%/50% null/not-null (TODO-InlineCast: rely on PGO) - // and rely on profile for the slow path. - // - // Alternatively we could profile nulls in the reservoir sample and - // treat that as another "option". - // - // True out of the null check means obj is null. - // - const weight_t nullcheckTrueLikelihood = 0.5; - const weight_t nullcheckFalseLikelihood = 0.5; - BasicBlock* typeCheckSucceedBb; - - { - FlowEdge* const trueEdge = fgAddRefPred(lastBb, nullcheckBb); - nullcheckBb->SetTrueEdge(trueEdge); - trueEdge->setLikelihood(nullcheckTrueLikelihood); - } - - if (typeCheckNotNeeded) - { - FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, nullcheckBb); - nullcheckBb->SetFalseEdge(falseEdge); - falseEdge->setLikelihood(nullcheckFalseLikelihood); - - typeCheckSucceedBb = nullptr; - } - else - { - FlowEdge* const falseEdge = fgAddRefPred(typeChecksBbs[0], nullcheckBb); - nullcheckBb->SetFalseEdge(falseEdge); - falseEdge->setLikelihood(nullcheckFalseLikelihood); - - typeCheckSucceedBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, typeCheckSucceedTree, debugInfo); - FlowEdge* const newEdge = fgAddRefPred(lastBb, typeCheckSucceedBb); - typeCheckSucceedBb->SetTargetEdge(newEdge); - } + firstBb->SetTarget(nullcheckBb); + nullcheckBb->SetTrueTarget(lastBb); + nullcheckBb->SetFalseTarget(typeCheckNotNeeded ? fallbackBb : typeChecksBbs[0]); // Tricky case - wire up multiple type check blocks (in most cases there is only one) for (int candidateId = 0; candidateId < numOfCandidates; candidateId++) @@ -2521,94 +2439,64 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, BasicBlock* curTypeCheckBb = typeChecksBbs[candidateId]; // All type checks jump straight to the typeCheckSucceedBb on success - FlowEdge* const trueEdge = fgAddRefPred(typeCheckSucceedBb, curTypeCheckBb); - curTypeCheckBb->SetTrueEdge(trueEdge); + curTypeCheckBb->SetTrueTarget(typeCheckSucceedBb); + fgAddRefPred(typeCheckSucceedBb, curTypeCheckBb); // or ... if (candidateId == numOfCandidates - 1) { // ... jump to the fallbackBb on last type check's failure - FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, curTypeCheckBb); - curTypeCheckBb->SetFalseEdge(falseEdge); + curTypeCheckBb->SetFalseTarget(fallbackBb); + fgAddRefPred(fallbackBb, curTypeCheckBb); } else { // ... jump to the next type check on failure - FlowEdge* const falseEdge = fgAddRefPred(typeChecksBbs[candidateId + 1], curTypeCheckBb); - curTypeCheckBb->SetFalseEdge(falseEdge); + curTypeCheckBb->SetFalseTarget(typeChecksBbs[candidateId + 1]); + fgAddRefPred(typeChecksBbs[candidateId + 1], curTypeCheckBb); } } - fgRedirectTargetEdge(firstBb, nullcheckBb); + fgRemoveRefPred(lastBb, firstBb); + fgAddRefPred(nullcheckBb, firstBb); + fgAddRefPred(lastBb, nullcheckBb); + if (typeCheckNotNeeded) + { + fgAddRefPred(fallbackBb, nullcheckBb); + } + else + { + fgAddRefPred(typeChecksBbs[0], nullcheckBb); + fgAddRefPred(lastBb, typeCheckSucceedBb); + } + + if (!fallbackBb->KindIs(BBJ_THROW)) + { + // if fallbackBb is BBJ_THROW then it has no successors + fgAddRefPred(lastBb, fallbackBb); + } // - // Re-distribute weights and set edge likelihoods. - // - // We have the likelihood estimate for each class to test against. - // As with multi-guess GDV, once we've failed the first check, the - // likelihood of the other checks will increase. - // - // For instance, suppose we have 3 classes A, B, C, with likelihoods 0.5, 0.2, 0.1, - // and a residual 0.2 likelihood for none of the above. - // - // We first test for A, this is a 0.5 likelihood of success. - // - // If the test for A fails, we test for B. Because we only reach this second - // test with relative likelihood 0.5, we need to divide (scale up) the remaining - // likelihoods by 0.5, so we end up with 0.4, 0.2, (none of the above: 0.4). - // - // If the test for B also fails, we test for C. Because we only reach - // this second test with relative likelihood 0.6, we need to divide (scale up) - // the remaining likelihoods by by 0.6, so we end up with 0.33, (none of the above: 0.67). - // - // In the code below, instead of updating all the likelihoods each time, - // we remember how much we need to divide by to fix the next likelihood. This divisor is - // 1.0 - (sumOfPreviousLikelihoods) - // - // So for the above, we have - // - // Test | Likelihood | Sum of Previous | Rel. Likelihood - // A | 0.5 | 0.0 | 0.5 / (1 - 0.0) = 0.50 - // B | 0.2 | 0.5 | 0.2 / (1 - 0.5) = 0.40 - // C | 0.1 | 0.7 | 0.1 / (1 - 0.7) = 0.33 - // n/a | 0.2 | 0.8 | 0.2 / (1 - 0.8) = 1.00 - // - // The same goes for inherited weights -- the block where we test for B will have - // the weight of A times the likelihood that A's test fails, etc. + // Re-distribute weights // nullcheckBb->inheritWeight(firstBb); - weight_t sumOfPreviousLikelihood = 0; + unsigned totalLikelihood = 0; for (int candidateId = 0; candidateId < numOfCandidates; candidateId++) { + unsigned likelihood = likelihoods[candidateId]; BasicBlock* curTypeCheckBb = typeChecksBbs[candidateId]; if (candidateId == 0) { - // Predecessor is the nullcheck, control reaches on false. - // - curTypeCheckBb->inheritWeight(nullcheckBb); - curTypeCheckBb->scaleBBWeight(nullcheckBb->GetFalseEdge()->getLikelihood()); + // We assume obj is 50%/50% null/not-null (TODO-InlineCast: rely on PGO) + // and rely on profile for the slow path. + curTypeCheckBb->inheritWeightPercentage(nullcheckBb, 50); } else { - // Predecessor is the prior type check, control reaches on false. - // - BasicBlock* const prevTypeCheckBb = typeChecksBbs[candidateId - 1]; - weight_t prevCheckFailedLikelihood = prevTypeCheckBb->GetFalseEdge()->getLikelihood(); - curTypeCheckBb->inheritWeight(prevTypeCheckBb); - curTypeCheckBb->scaleBBWeight(prevCheckFailedLikelihood); + BasicBlock* prevTypeCheckBb = typeChecksBbs[candidateId - 1]; + curTypeCheckBb->inheritWeightPercentage(prevTypeCheckBb, likelihood); } - - // Fix likelihood of block's outgoing edges. - // - weight_t likelihood = (weight_t)likelihoods[candidateId] / 100; - weight_t relLikelihood = likelihood / (1.0 - sumOfPreviousLikelihood); - - JITDUMP("Candidate %d: likelihood " FMT_WT " relative likelihood " FMT_WT "\n", candidateId, likelihood, - relLikelihood); - - curTypeCheckBb->GetTrueEdge()->setLikelihood(relLikelihood); - curTypeCheckBb->GetFalseEdge()->setLikelihood(1.0 - relLikelihood); - sumOfPreviousLikelihood += likelihood; + totalLikelihood += likelihood; } if (fallbackBb->KindIs(BBJ_THROW)) @@ -2617,20 +2505,12 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, } else { - assert(fallbackBb->KindIs(BBJ_ALWAYS)); - FlowEdge* const newEdge = fgAddRefPred(lastBb, fallbackBb); - fallbackBb->SetTargetEdge(newEdge); - fallbackBb->inheritWeight(lastTypeCheckBb); - weight_t lastTypeCheckFailedLikelihood = lastTypeCheckBb->GetFalseEdge()->getLikelihood(); - fallbackBb->scaleBBWeight(lastTypeCheckFailedLikelihood); + fallbackBb->inheritWeightPercentage(lastTypeCheckBb, 100 - totalLikelihood); } - if (!typeCheckNotNeeded) { - typeCheckSucceedBb->inheritWeight(typeChecksBbs[0]); - typeCheckSucceedBb->scaleBBWeight(sumOfPreviousLikelihood); + typeCheckSucceedBb->inheritWeightPercentage(typeChecksBbs[0], totalLikelihood); } - lastBb->inheritWeight(firstBb); // @@ -2641,12 +2521,12 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, assert(BasicBlock::sameEHRegion(firstBb, fallbackBb)); // call guarantees that obj is never null, we can drop the nullcheck - // by converting it to a BBJ_ALWAYS to its false target. + // by converting it to a BBJ_ALWAYS to typeCheckBb. if ((call->gtCallMoreFlags & GTF_CALL_M_CAST_OBJ_NONNULL) != 0) { fgRemoveStmt(nullcheckBb, nullcheckBb->lastStmt()); - fgRemoveRefPred(nullcheckBb->GetTrueEdge()); - nullcheckBb->SetKindAndTargetEdge(BBJ_ALWAYS, nullcheckBb->GetFalseEdge()); + nullcheckBb->SetKindAndTarget(BBJ_ALWAYS, typeCheckNotNeeded ? fallbackBb : typeChecksBbs[0]); + fgRemoveRefPred(lastBb, nullcheckBb); } // Bonus step: merge prevBb with nullcheckBb as they are likely to be mergeable diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 14c262524da2d8..f771a9ec978e2d 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1356,15 +1356,6 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, compFloatingPointUsed = true; } - var_types nodeRetType = retType; -#if defined(TARGET_ARM64) - if (HWIntrinsicInfo::ReturnsPerElementMask(intrinsic)) - { - // Ensure the result is generated to a mask. - nodeRetType = TYP_MASK; - } -#endif // defined(TARGET_ARM64) - // table-driven importer of simple intrinsics if (impIsTableDrivenHWIntrinsic(intrinsic, category)) { @@ -1401,7 +1392,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, case 0: { assert(!isScalar); - retNode = gtNewSimdHWIntrinsicNode(nodeRetType, intrinsic, simdBaseJitType, simdSize); + retNode = gtNewSimdHWIntrinsicNode(retType, intrinsic, simdBaseJitType, simdSize); break; } @@ -1419,8 +1410,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, } } - retNode = isScalar ? gtNewScalarHWIntrinsicNode(nodeRetType, op1, intrinsic) - : gtNewSimdHWIntrinsicNode(nodeRetType, op1, intrinsic, simdBaseJitType, simdSize); + retNode = isScalar ? gtNewScalarHWIntrinsicNode(retType, op1, intrinsic) + : gtNewSimdHWIntrinsicNode(retType, op1, intrinsic, simdBaseJitType, simdSize); #if defined(TARGET_XARCH) switch (intrinsic) @@ -1471,9 +1462,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, op2 = addRangeCheckIfNeeded(intrinsic, op2, mustExpand, immLowerBound, immUpperBound); op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); - retNode = isScalar - ? gtNewScalarHWIntrinsicNode(nodeRetType, op1, op2, intrinsic) - : gtNewSimdHWIntrinsicNode(nodeRetType, op1, op2, intrinsic, simdBaseJitType, simdSize); + retNode = isScalar ? gtNewScalarHWIntrinsicNode(retType, op1, op2, intrinsic) + : gtNewSimdHWIntrinsicNode(retType, op1, op2, intrinsic, simdBaseJitType, simdSize); #ifdef TARGET_XARCH if ((intrinsic == NI_SSE42_Crc32) || (intrinsic == NI_SSE42_X64_Crc32)) @@ -1553,9 +1543,9 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, op3 = addRangeCheckIfNeeded(intrinsic, op3, mustExpand, immLowerBound, immUpperBound); } - retNode = isScalar ? gtNewScalarHWIntrinsicNode(nodeRetType, op1, op2, op3, intrinsic) - : gtNewSimdHWIntrinsicNode(nodeRetType, op1, op2, op3, intrinsic, simdBaseJitType, - simdSize); + retNode = isScalar + ? gtNewScalarHWIntrinsicNode(retType, op1, op2, op3, intrinsic) + : gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, simdBaseJitType, simdSize); #ifdef TARGET_XARCH if ((intrinsic == NI_AVX2_GatherVector128) || (intrinsic == NI_AVX2_GatherVector256)) @@ -1576,8 +1566,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, op1 = getArgForHWIntrinsic(sigReader.GetOp1Type(), sigReader.op1ClsHnd); assert(!isScalar); - retNode = - gtNewSimdHWIntrinsicNode(nodeRetType, op1, op2, op3, op4, intrinsic, simdBaseJitType, simdSize); + retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, op4, intrinsic, simdBaseJitType, simdSize); break; } @@ -1587,26 +1576,8 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, } else { - retNode = impSpecialIntrinsic(intrinsic, clsHnd, method, sig, simdBaseJitType, nodeRetType, simdSize); - } - -#if defined(TARGET_ARM64) - if (HWIntrinsicInfo::IsMaskedOperation(intrinsic)) - { - // Op1 input is a vector. HWInstrinsic requires a mask, so convert to a mask. - assert(numArgs > 0); - GenTree* op1 = retNode->AsHWIntrinsic()->Op(1); - op1 = convertHWIntrinsicToMask(retType, op1, simdBaseJitType, simdSize); - retNode->AsHWIntrinsic()->Op(1) = op1; - } - - if (retType != nodeRetType) - { - // HWInstrinsic returns a mask, but all returns must be vectors, so convert mask to vector. - assert(HWIntrinsicInfo::ReturnsPerElementMask(intrinsic)); - retNode = convertHWIntrinsicFromMask(retNode->AsHWIntrinsic(), retType); + retNode = impSpecialIntrinsic(intrinsic, clsHnd, method, sig, simdBaseJitType, retType, simdSize); } -#endif // defined(TARGET_ARM64) if ((retNode != nullptr) && retNode->OperIs(GT_HWINTRINSIC)) { diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index cac041eb83ea6d..15256ea22e93b3 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -58,7 +58,6 @@ enum HWIntrinsicCategory : uint8_t HW_Category_ShiftLeftByImmediate, HW_Category_ShiftRightByImmediate, HW_Category_SIMDByIndexedElement, - HW_Category_EnumPattern, // Helper intrinsics // - do not directly correspond to a instruction, such as Vector64.AllBitsSet @@ -176,21 +175,6 @@ enum HWIntrinsicFlag : unsigned int // The intrinsic needs consecutive registers HW_Flag_NeedsConsecutiveRegisters = 0x4000, - - // The intrinsic uses scalable registers - HW_Flag_Scalable = 0x8000, - - // Returns Per-Element Mask - // the intrinsic returns a vector containing elements that are either "all bits set" or "all bits clear" - // this output can be used as a per-element mask - HW_Flag_ReturnsPerElementMask = 0x10000, - - // The intrinsic uses a mask in arg1 to select elements present in the result - HW_Flag_MaskedOperation = 0x20000, - - // The intrinsic uses a mask in arg1 to select elements present in the result, and must use a low register. - HW_Flag_LowMaskedOperation = 0x40000, - #else #error Unsupported platform #endif @@ -670,8 +654,10 @@ struct HWIntrinsicInfo static bool ReturnsPerElementMask(NamedIntrinsic id) { HWIntrinsicFlag flags = lookupFlags(id); -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) return (flags & HW_Flag_ReturnsPerElementMask) != 0; +#elif defined(TARGET_ARM64) + unreached(); #else #error Unsupported platform #endif @@ -862,25 +848,6 @@ struct HWIntrinsicInfo const HWIntrinsicFlag flags = lookupFlags(id); return (flags & HW_Flag_HasImmediateOperand) != 0; } - - static bool IsScalable(NamedIntrinsic id) - { - const HWIntrinsicFlag flags = lookupFlags(id); - return (flags & HW_Flag_Scalable) != 0; - } - - static bool IsMaskedOperation(NamedIntrinsic id) - { - const HWIntrinsicFlag flags = lookupFlags(id); - return ((flags & HW_Flag_MaskedOperation) != 0) || IsLowMaskedOperation(id); - } - - static bool IsLowMaskedOperation(NamedIntrinsic id) - { - const HWIntrinsicFlag flags = lookupFlags(id); - return (flags & HW_Flag_LowMaskedOperation) != 0; - } - #endif // TARGET_ARM64 static bool HasSpecialSideEffect(NamedIntrinsic id) @@ -940,7 +907,7 @@ struct HWIntrinsic final InitializeBaseType(node); } - bool codeGenIsTableDriven() const + bool IsTableDriven() const { // TODO-Arm64-Cleanup - make more categories to the table-driven framework bool isTableDrivenCategory = category != HW_Category_Helper; diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index 5c7f796c61c909..0561ac2adadd69 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -280,20 +280,6 @@ void HWIntrinsicInfo::lookupImmBounds( immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType) - 1; break; - case NI_Sve_CreateTrueMaskByte: - case NI_Sve_CreateTrueMaskDouble: - case NI_Sve_CreateTrueMaskInt16: - case NI_Sve_CreateTrueMaskInt32: - case NI_Sve_CreateTrueMaskInt64: - case NI_Sve_CreateTrueMaskSByte: - case NI_Sve_CreateTrueMaskSingle: - case NI_Sve_CreateTrueMaskUInt16: - case NI_Sve_CreateTrueMaskUInt32: - case NI_Sve_CreateTrueMaskUInt64: - immLowerBound = (int)SVE_PATTERN_POW2; - immUpperBound = (int)SVE_PATTERN_ALL; - break; - default: unreached(); } @@ -2193,7 +2179,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, retNode = gtNewSimdHWIntrinsicNode(retType, op1, op2, op3, intrinsic, simdBaseJitType, simdSize); break; } - default: { return nullptr; @@ -2203,43 +2188,4 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, return retNode; } -//------------------------------------------------------------------------ -// convertHWIntrinsicFromMask: Convert a HW instrinsic vector node to a mask -// -// Arguments: -// node -- The node to convert -// simdBaseJitType -- the base jit type of the converted node -// simdSize -- the simd size of the converted node -// -// Return Value: -// The node converted to the a mask type -// -GenTree* Compiler::convertHWIntrinsicToMask(var_types type, - GenTree* node, - CorInfoType simdBaseJitType, - unsigned simdSize) -{ - // ConvertVectorToMask uses cmpne which requires an embedded mask. - GenTree* embeddedMask = gtNewSimdHWIntrinsicNode(TYP_MASK, NI_Sve_CreateTrueMaskAll, simdBaseJitType, simdSize); - return gtNewSimdHWIntrinsicNode(TYP_MASK, embeddedMask, node, NI_Sve_ConvertVectorToMask, simdBaseJitType, - simdSize); -} - -//------------------------------------------------------------------------ -// convertHWIntrinsicFromMask: Convert a HW instrinsic mask node to a vector -// -// Arguments: -// node -- The node to convert -// type -- The type of the node to convert to -// -// Return Value: -// The node converted to the given type -// -GenTree* Compiler::convertHWIntrinsicFromMask(GenTreeHWIntrinsic* node, var_types type) -{ - assert(node->TypeGet() == TYP_MASK); - return gtNewSimdHWIntrinsicNode(type, node, NI_Sve_ConvertMaskToVector, node->GetSimdBaseJitType(), - node->GetSimdSize()); -} - #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 6418b72a8f3075..eba1b6f33a09c4 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -265,11 +265,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitSize = EA_UNKNOWN; opt = INS_OPTS_NONE; } - else if (HWIntrinsicInfo::IsScalable(intrin.id)) - { - emitSize = EA_SCALABLE; - opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType)); - } else { emitSize = emitActualTypeSize(Compiler::getSIMDTypeForSize(node->GetSimdSize())); @@ -281,7 +276,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) genConsumeMultiOpOperands(node); - if (intrin.codeGenIsTableDriven()) + if (intrin.IsTableDriven()) { const instruction ins = HWIntrinsicInfo::lookupIns(intrin.id, intrin.baseType); assert(ins != INS_invalid); @@ -377,27 +372,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitShift(intrin.op2, op1Reg); } } - else if (intrin.category == HW_Category_EnumPattern) - { - assert(hasImmediateOperand); - - switch (intrin.numOperands) - { - case 1: - { - HWIntrinsicImmOpHelper helper(this, intrin.op1, node); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) - { - const insSvePattern pattern = (insSvePattern)helper.ImmValue(); - GetEmitter()->emitIns_R_PATTERN(ins, emitSize, targetReg, opt, pattern); - } - }; - break; - - default: - unreached(); - } - } else { assert(!hasImmediateOperand); @@ -1280,23 +1254,6 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) GetEmitter()->emitIns_R_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, op3Reg); break; - case NI_Sve_ConvertMaskToVector: - // PMOV would be ideal here, but it is in SVE2.1. - // Instead, use a predicated move: MOV ., /Z, #1 - GetEmitter()->emitIns_R_R_I(ins, emitSize, targetReg, op1Reg, 1, opt); - break; - - case NI_Sve_ConvertVectorToMask: - // PMOV would be ideal here, but it is in SVE2.1. - // Instead, use a compare: CMPNE ., /Z, ., #0 - GetEmitter()->emitIns_R_R_R_I(ins, emitSize, targetReg, op1Reg, op2Reg, 0, opt); - break; - - case NI_Sve_CreateTrueMaskAll: - // Must use the pattern variant, as the non-pattern varient is SVE2.1. - GetEmitter()->emitIns_R_PATTERN(ins, emitSize, targetReg, opt, SVE_PATTERN_ALL); - break; - default: unreached(); } diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index ac110c2a0e1b5b..f8263c40bb0c66 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -16,32 +16,6 @@ // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // SVE Intrinsics -// Sve -HARDWARE_INTRINSIC(Sve, CreateTrueMaskByte, -1, 1, false, {INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskDouble, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskSByte, -1, 1, false, {INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskSingle, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt16, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt32, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) -HARDWARE_INTRINSIC(Sve, CreateTrueMaskUInt64, -1, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_ptrue, INS_invalid, INS_invalid}, HW_Category_EnumPattern, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_ReturnsPerElementMask) - -HARDWARE_INTRINSIC(Sve, LoadVector, -1, 2, true, {INS_sve_ld1b, INS_sve_ld1b, INS_sve_ld1h, INS_sve_ld1h, INS_sve_ld1w, INS_sve_ld1w, INS_sve_ld1d, INS_sve_ld1d, INS_sve_ld1w, INS_sve_ld1d}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_LowMaskedOperation) - - - -// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** -// ISA Function name SIMD size NumArg EncodesExtraTypeArg Instructions Category Flags -// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE} -// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** -// Special intrinsics that are generated during importing or lowering - -HARDWARE_INTRINSIC(Sve, ConvertMaskToVector, -1, 1, true, {INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov, INS_sve_mov}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_MaskedOperation) -HARDWARE_INTRINSIC(Sve, ConvertVectorToMask, -1, 2, true, {INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne, INS_sve_cmpne}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask|HW_Flag_LowMaskedOperation) - -HARDWARE_INTRINSIC(Sve, CreateTrueMaskAll, -1, -1, false, {INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue, INS_sve_ptrue}, HW_Category_Helper, HW_Flag_Scalable|HW_Flag_ReturnsPerElementMask) #endif // FEATURE_HW_INTRINSIC diff --git a/src/coreclr/jit/ifconversion.cpp b/src/coreclr/jit/ifconversion.cpp index 1e6a573aa7b0de..f9cb5af17925e7 100644 --- a/src/coreclr/jit/ifconversion.cpp +++ b/src/coreclr/jit/ifconversion.cpp @@ -739,7 +739,7 @@ bool OptIfConversionDsc::optIfConvert() // Update the flow from the original block. m_comp->fgRemoveAllRefPreds(m_startBlock->GetFalseTarget(), m_startBlock); - m_startBlock->SetKindAndTargetEdge(BBJ_ALWAYS, m_startBlock->GetTrueEdge()); + m_startBlock->SetKindAndTarget(BBJ_ALWAYS, m_startBlock->GetTrueTarget()); #ifdef DEBUG if (m_comp->verbose) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index 605c318ec98b86..196054a044a50b 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -1273,7 +1273,7 @@ GenTree* Compiler::impTokenToHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, assert(!fgGlobalMorph); CORINFO_GENERICHANDLE_RESULT embedInfo; - info.compCompHnd->embedGenericHandle(pResolvedToken, importParent, info.compMethodHnd, &embedInfo); + info.compCompHnd->embedGenericHandle(pResolvedToken, importParent, &embedInfo); if (pRuntimeLookup) { @@ -1485,7 +1485,7 @@ GenTreeCall* Compiler::impReadyToRunHelperToTree(CORINFO_RESOLVED_TOKEN* pResolv GenTree* arg1) { CORINFO_CONST_LOOKUP lookup; - if (!info.compCompHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, helper, info.compMethodHnd, &lookup)) + if (!info.compCompHnd->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, helper, &lookup)) { return nullptr; } @@ -1548,56 +1548,30 @@ GenTree* Compiler::impMethodPointer(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORI GenTree* Compiler::getRuntimeContextTree(CORINFO_RUNTIME_LOOKUP_KIND kind) { - GenTree* ctxTree; + GenTree* ctxTree = nullptr; // Collectible types requires that for shared generic code, if we use the generic context parameter - // that we report it. Conservatively mark the root method as using generic context, MARK_LOCAL_VARS phase - // will clean it up if it turns out to be unnecessary. - impInlineRoot()->lvaGenericsContextInUse = true; + // that we report it. (This is a conservative approach, we could detect some cases particularly when the + // context parameter is this that we don't need the eager reporting logic.) + lvaGenericsContextInUse = true; - // Always use generic context from the callsite if we're inlining and it's available. - if (compIsForInlining() && (impInlineInfo->inlInstParamArgInfo != nullptr)) - { - // Create a dummy lclInfo node, we know that nobody's going to do stloc or take address - // of the generic context, so we don't need to scan IL for it. - InlLclVarInfo lclInfo = {}; - lclInfo.lclTypeInfo = TYP_I_IMPL; - ctxTree = impInlineFetchArg(*impInlineInfo->inlInstParamArgInfo, lclInfo); - assert(ctxTree != nullptr); - assert(ctxTree->TypeIs(TYP_I_IMPL)); - // We don't need to worry about GTF_VAR_CONTEXT here, it should be set on the callsite anyway. - } - else if (kind == CORINFO_LOOKUP_THISOBJ) - { - // Use "this" from the callsite if we're inlining - if (compIsForInlining()) - { - // "this" is always the first argument in inlArgInfo - assert(impInlineInfo->argCnt > 0); - assert(impInlineInfo->inlArgInfo[0].argIsThis); - - ctxTree = impInlineFetchArg(impInlineInfo->inlArgInfo[0], impInlineInfo->lclVarInfo[0]); + Compiler* pRoot = impInlineRoot(); - // "this" is expected to be always a local, and we must mark it as a context - assert(ctxTree->OperIs(GT_LCL_VAR)); - ctxTree->gtFlags |= GTF_VAR_CONTEXT; - } - else - { - assert(info.compThisArg != BAD_VAR_NUM); - ctxTree = gtNewLclvNode(info.compThisArg, TYP_REF); - ctxTree->gtFlags |= GTF_VAR_CONTEXT; - } + if (kind == CORINFO_LOOKUP_THISOBJ) + { + // this Object + ctxTree = gtNewLclvNode(pRoot->info.compThisArg, TYP_REF); + ctxTree->gtFlags |= GTF_VAR_CONTEXT; // context is the method table pointer of the this object ctxTree = gtNewMethodTableLookup(ctxTree); } else { - assert((kind == CORINFO_LOOKUP_METHODPARAM) || (kind == CORINFO_LOOKUP_CLASSPARAM)); + assert(kind == CORINFO_LOOKUP_METHODPARAM || kind == CORINFO_LOOKUP_CLASSPARAM); // Exact method descriptor as passed in - ctxTree = gtNewLclvNode(impInlineRoot()->info.compTypeCtxtArg, TYP_I_IMPL); + ctxTree = gtNewLclvNode(pRoot->info.compTypeCtxtArg, TYP_I_IMPL); ctxTree->gtFlags |= GTF_VAR_CONTEXT; } return ctxTree; @@ -1669,9 +1643,6 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken GenTree* slotPtrTree = ctxTree; GenTree* indOffTree = nullptr; - // TODO-CQ: consider relaxing where it's safe to do so - const bool ctxTreeIsInvariant = !compIsForInlining(); - // Applied repeated indirections for (WORD i = 0; i < pRuntimeLookup->indirections; i++) { @@ -1683,8 +1654,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken if (i != 0) { - slotPtrTree = gtNewIndir(TYP_I_IMPL, slotPtrTree, - ctxTreeIsInvariant ? (GTF_IND_NONFAULTING | GTF_IND_INVARIANT) : GTF_EMPTY); + slotPtrTree = gtNewIndir(TYP_I_IMPL, slotPtrTree, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); } if ((i == 1 && pRuntimeLookup->indirectFirstOffset) || (i == 2 && pRuntimeLookup->indirectSecondOffset)) @@ -1707,8 +1677,7 @@ GenTree* Compiler::impRuntimeLookupToTree(CORINFO_RESOLVED_TOKEN* pResolvedToken return slotPtrTree; } - slotPtrTree = - gtNewIndir(TYP_I_IMPL, slotPtrTree, ctxTreeIsInvariant ? (GTF_IND_NONFAULTING | GTF_IND_INVARIANT) : GTF_EMPTY); + slotPtrTree = gtNewIndir(TYP_I_IMPL, slotPtrTree, GTF_IND_NONFAULTING | GTF_IND_INVARIANT); return slotPtrTree; } @@ -2051,13 +2020,13 @@ BasicBlock* Compiler::impPushCatchArgOnStack(BasicBlock* hndBlk, CORINFO_CLASS_H { // Create extra basic block for the spill // - BasicBlock* newBlk = fgNewBBbefore(BBJ_ALWAYS, hndBlk, /* extendRegion */ true); + BasicBlock* newBlk = fgNewBBbefore(BBJ_ALWAYS, hndBlk, /* extendRegion */ true, /* jumpDest */ hndBlk); newBlk->SetFlags(BBF_IMPORTED | BBF_DONT_REMOVE | BBF_NONE_QUIRK); newBlk->inheritWeight(hndBlk); newBlk->bbCodeOffs = hndBlk->bbCodeOffs; FlowEdge* const newEdge = fgAddRefPred(hndBlk, newBlk); - newBlk->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); // Spill into a temp. unsigned tempNum = lvaGrabTemp(false DEBUGARG("SpillCatchArg")); @@ -2524,7 +2493,7 @@ GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) void Compiler::verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)) { - block->SetKindAndTargetEdge(BBJ_THROW); + block->SetKindAndTarget(BBJ_THROW); block->SetFlags(BBF_FAILED_VERIFICATION); block->RemoveFlags(BBF_IMPORTED); @@ -3267,7 +3236,7 @@ void Compiler::impImportAndPushBox(CORINFO_RESOLVED_TOKEN* pResolvedToken) Statement* const cursor = impLastStmt; const bool useParent = false; - op1 = gtNewAllocObjNode(pResolvedToken, info.compMethodHnd, useParent); + op1 = gtNewAllocObjNode(pResolvedToken, useParent); if (op1 == nullptr) { // If we fail to create the newobj node, we must be inlining @@ -4434,10 +4403,11 @@ void Compiler::impImportLeave(BasicBlock* block) assert(step == DUMMY_INIT(NULL)); callBlock = block; - // callBlock calls the finally handler assert(callBlock->HasInitializedTarget()); - fgRedirectTargetEdge(callBlock, HBtab->ebdHndBeg); - callBlock->SetKind(BBJ_CALLFINALLY); + fgRemoveRefPred(callBlock->GetTarget(), callBlock); + + // callBlock will call the finally handler. Convert the BBJ_LEAVE to BBJ_CALLFINALLY. + callBlock->SetKindAndTarget(BBJ_CALLFINALLY, HBtab->ebdHndBeg); if (endCatches) { @@ -4459,22 +4429,16 @@ void Compiler::impImportLeave(BasicBlock* block) // Calling the finally block. - // callBlock calls the finally handler - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step); - - { - FlowEdge* const newEdge = fgAddRefPred(HBtab->ebdHndBeg, callBlock); - callBlock->SetTargetEdge(newEdge); - } + // callBlock will call the finally handler + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step, HBtab->ebdHndBeg); // step's jump target shouldn't be set yet assert(!step->HasInitializedTarget()); - { - // the previous call to a finally returns to this call (to the next finally in the chain) - FlowEdge* const newEdge = fgAddRefPred(callBlock, step); - step->SetTargetEdge(newEdge); - } + // the previous call to a finally returns to this call (to the next finally in the chain) + step->SetTarget(callBlock); + FlowEdge* const newEdge = fgAddRefPred(callBlock, step); + newEdge->setLikelihood(1.0); // The new block will inherit this block's weight. callBlock->inheritWeight(block); @@ -4504,9 +4468,6 @@ void Compiler::impImportLeave(BasicBlock* block) impEndTreeList(callBlock, endLFinStmt, lastStmt); } - // callBlock should be set up at this point - assert(callBlock->TargetIs(HBtab->ebdHndBeg)); - // Note: we don't know the jump target yet step = fgNewBBafter(BBJ_CALLFINALLYRET, callBlock, true); // The new block will inherit this block's weight. @@ -4525,6 +4486,11 @@ void Compiler::impImportLeave(BasicBlock* block) unsigned finallyNesting = compHndBBtab[XTnum].ebdHandlerNestingLevel; assert(finallyNesting <= compHndBBtabCount); + assert(callBlock->KindIs(BBJ_CALLFINALLY)); + assert(callBlock->TargetIs(HBtab->ebdHndBeg)); + FlowEdge* const newEdge = fgAddRefPred(callBlock->GetTarget(), callBlock); + newEdge->setLikelihood(1.0); + GenTree* endLFin = new (this, GT_END_LFIN) GenTreeVal(GT_END_LFIN, TYP_VOID, finallyNesting); endLFinStmt = gtNewStmt(endLFin); endCatches = NULL; @@ -4566,15 +4532,16 @@ void Compiler::impImportLeave(BasicBlock* block) // Insert a new BB either in the try region indicated by tryIndex or // the handler region indicated by leaveTarget->bbHndIndex, // depending on which is the inner region. - BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step); + BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step, leaveTarget); finalStep->SetFlags(BBF_KEEP_BBJ_ALWAYS); // step's jump target shouldn't be set yet assert(!step->HasInitializedTarget()); + step->SetTarget(finalStep); { FlowEdge* const newEdge = fgAddRefPred(finalStep, step); - step->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); } // The new block will inherit this block's weight. @@ -4606,7 +4573,7 @@ void Compiler::impImportLeave(BasicBlock* block) // this is the ultimate destination of the LEAVE { FlowEdge* const newEdge = fgAddRefPred(leaveTarget, finalStep); - finalStep->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); } // Queue up the jump target for importing @@ -4721,15 +4688,12 @@ void Compiler::impImportLeave(BasicBlock* block) assert((step == block) || !step->HasInitializedTarget()); if (step == block) { - fgRedirectTargetEdge(step, exitBlock); - } - else - { - FlowEdge* const newEdge = fgAddRefPred(exitBlock, step); - step->SetTargetEdge(newEdge); // the previous step (maybe a call to a nested finally, or a nested - // catch - // exit) returns to this block + fgRemoveRefPred(step->GetTarget(), step); } + step->SetTarget(exitBlock); // the previous step (maybe a call to a nested finally, or a nested catch + // exit) returns to this block + FlowEdge* const newEdge = fgAddRefPred(exitBlock, step); + newEdge->setLikelihood(1.0); // The new block will inherit this block's weight. exitBlock->inheritWeight(block); @@ -4764,23 +4728,22 @@ void Compiler::impImportLeave(BasicBlock* block) (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; unsigned callFinallyHndIndex = (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block); + callBlock = + fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block, HBtab->ebdHndBeg); // Convert the BBJ_LEAVE to BBJ_ALWAYS, jumping to the new BBJ_CALLFINALLY. This is because // the new BBJ_CALLFINALLY is in a different EH region, thus it can't just replace the BBJ_LEAVE, // which might be in the middle of the "try". In most cases, the BBJ_ALWAYS will jump to the // next block, and flow optimizations will remove it. - fgRedirectTargetEdge(block, callBlock); - block->SetKind(BBJ_ALWAYS); + fgRemoveRefPred(block->GetTarget(), block); + block->SetKindAndTarget(BBJ_ALWAYS, callBlock); + FlowEdge* const newEdge = fgAddRefPred(callBlock, block); + newEdge->setLikelihood(1.0); // The new block will inherit this block's weight. callBlock->inheritWeight(block); callBlock->SetFlags(BBF_IMPORTED); - // callBlock calls the finally handler - FlowEdge* const newEdge = fgAddRefPred(HBtab->ebdHndBeg, callBlock); - callBlock->SetKindAndTargetEdge(BBJ_CALLFINALLY, newEdge); - #ifdef DEBUG if (verbose) { @@ -4794,10 +4757,11 @@ void Compiler::impImportLeave(BasicBlock* block) callBlock = block; - // callBlock calls the finally handler assert(callBlock->HasInitializedTarget()); - fgRedirectTargetEdge(callBlock, HBtab->ebdHndBeg); - callBlock->SetKind(BBJ_CALLFINALLY); + fgRemoveRefPred(callBlock->GetTarget(), callBlock); + + // callBlock will call the finally handler. Convert the BBJ_LEAVE to BBJ_CALLFINALLY + callBlock->SetKindAndTarget(BBJ_CALLFINALLY, HBtab->ebdHndBeg); #ifdef DEBUG if (verbose) @@ -4840,14 +4804,11 @@ void Compiler::impImportLeave(BasicBlock* block) BasicBlock* step2 = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); if (step == block) { - fgRedirectTargetEdge(step, step2); - } - else - { - FlowEdge* const newEdge = fgAddRefPred(step2, step); - step->SetTargetEdge(newEdge); + fgRemoveRefPred(step->GetTarget(), step); } - + step->SetTarget(step2); + FlowEdge* const newEdge = fgAddRefPred(step2, step); + newEdge->setLikelihood(1.0); step2->inheritWeight(block); step2->CopyFlags(block, BBF_RUN_RARELY); step2->SetFlags(BBF_IMPORTED); @@ -4880,26 +4841,21 @@ void Compiler::impImportLeave(BasicBlock* block) assert((step == block) || !step->HasInitializedTarget()); // callBlock will call the finally handler - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step); + callBlock = + fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step, HBtab->ebdHndBeg); if (step == block) { - fgRedirectTargetEdge(step, callBlock); - } - else - { - FlowEdge* const newEdge = fgAddRefPred(callBlock, step); - step->SetTargetEdge(newEdge); // the previous call to a finally returns to this call (to the next - // finally in the chain) + fgRemoveRefPred(step->GetTarget(), step); } + step->SetTarget(callBlock); // the previous call to a finally returns to this call (to the next + // finally in the chain) + FlowEdge* const newEdge = fgAddRefPred(callBlock, step); + newEdge->setLikelihood(1.0); // The new block will inherit this block's weight. callBlock->inheritWeight(block); callBlock->SetFlags(BBF_IMPORTED); - // callBlock calls the finally handler - FlowEdge* const newEdge = fgAddRefPred(HBtab->ebdHndBeg, callBlock); - callBlock->SetKindAndTargetEdge(BBJ_CALLFINALLY, newEdge); - #ifdef DEBUG if (verbose) { @@ -4910,9 +4866,6 @@ void Compiler::impImportLeave(BasicBlock* block) #endif } - // callBlock should be set up at this point - assert(callBlock->TargetIs(HBtab->ebdHndBeg)); - // Note: we don't know the jump target yet step = fgNewBBafter(BBJ_CALLFINALLYRET, callBlock, true); stepType = ST_FinallyReturn; @@ -4930,6 +4883,11 @@ void Compiler::impImportLeave(BasicBlock* block) XTnum, step->bbNum); } #endif + + assert(callBlock->KindIs(BBJ_CALLFINALLY)); + assert(callBlock->TargetIs(HBtab->ebdHndBeg)); + FlowEdge* const newEdge = fgAddRefPred(callBlock->GetTarget(), callBlock); + newEdge->setLikelihood(1.0); } else if (HBtab->HasCatchHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && !jitIsBetween(jmpAddr, tryBeg, tryEnd)) @@ -4993,13 +4951,11 @@ void Compiler::impImportLeave(BasicBlock* block) if (step == block) { - fgRedirectTargetEdge(step, catchStep); - } - else - { - FlowEdge* const newEdge = fgAddRefPred(catchStep, step); - step->SetTargetEdge(newEdge); + fgRemoveRefPred(step->GetTarget(), step); } + step->SetTarget(catchStep); + FlowEdge* const newEdge = fgAddRefPred(catchStep, step); + newEdge->setLikelihood(1.0); // The new block will inherit this block's weight. catchStep->inheritWeight(block); @@ -5048,16 +5004,13 @@ void Compiler::impImportLeave(BasicBlock* block) { assert((step == block) || !step->HasInitializedTarget()); - // leaveTarget is the ultimate destination of the LEAVE if (step == block) { - fgRedirectTargetEdge(step, leaveTarget); - } - else - { - FlowEdge* const newEdge = fgAddRefPred(leaveTarget, step); - step->SetTargetEdge(newEdge); + fgRemoveRefPred(step->GetTarget(), step); } + step->SetTarget(leaveTarget); // this is the ultimate destination of the LEAVE + FlowEdge* const newEdge = fgAddRefPred(leaveTarget, step); + newEdge->setLikelihood(1.0); #ifdef DEBUG if (verbose) @@ -5116,10 +5069,10 @@ void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) // will be treated as pair and handled correctly. if (block->KindIs(BBJ_CALLFINALLY)) { - BasicBlock* dupBlock = BasicBlock::New(this); + BasicBlock* dupBlock = BasicBlock::New(this, BBJ_CALLFINALLY, block->GetTarget()); dupBlock->CopyFlags(block); - FlowEdge* const newEdge = fgAddRefPred(block->GetTarget(), dupBlock); - dupBlock->SetKindAndTargetEdge(BBJ_CALLFINALLY, newEdge); + FlowEdge* const newEdge = fgAddRefPred(dupBlock->GetTarget(), dupBlock); + newEdge->setLikelihood(1.0); dupBlock->copyEHRegion(block); dupBlock->bbCatchTyp = block->bbCatchTyp; @@ -5148,8 +5101,10 @@ void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) fgInitBBLookup(); - fgRedirectTargetEdge(block, fgLookupBB(jmpAddr)); - block->SetKind(BBJ_LEAVE); + fgRemoveRefPred(block->GetTarget(), block); + block->SetKindAndTarget(BBJ_LEAVE, fgLookupBB(jmpAddr)); + FlowEdge* const newEdge = fgAddRefPred(block->GetTarget(), block); + newEdge->setLikelihood(1.0); // We will leave the BBJ_ALWAYS block we introduced. When it's reimported // the BBJ_ALWAYS block will be unreachable, and will be removed after. The @@ -5465,14 +5420,15 @@ GenTree* Compiler::impOptimizeCastClassOrIsInst(GenTree* op1, CORINFO_RESOLVED_T // // Notes: // May expand into a series of runtime checks or a helper call. -// + GenTree* Compiler::impCastClassOrIsInstToTree( GenTree* op1, GenTree* op2, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool isCastClass, IL_OFFSET ilOffset) { assert(op1->TypeGet() == TYP_REF); // Optimistically assume the jit should expand this as an inline test - bool isClassExact = info.compCompHnd->isExactType(pResolvedToken->hClass); + bool shouldExpandInline = true; + bool isClassExact = info.compCompHnd->isExactType(pResolvedToken->hClass); // ECMA-335 III.4.3: If typeTok is a nullable type, Nullable, it is interpreted as "boxed" T // We can convert constant-ish tokens of nullable to its underlying type. @@ -5481,6 +5437,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( if (isClassExact && !(info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_SHAREDINST)) { CORINFO_CLASS_HANDLE hClass = info.compCompHnd->getTypeForBox(pResolvedToken->hClass); + if (hClass != pResolvedToken->hClass) { bool runtimeLookup; @@ -5490,34 +5447,93 @@ GenTree* Compiler::impCastClassOrIsInstToTree( } } - const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); + // Profitability check. + // + // Don't bother with inline expansion when jit is trying to generate code quickly + if (opts.OptimizationDisabled()) + { + // not worth the code expansion if jitting fast or in a rarely run block + shouldExpandInline = false; + } + else if ((op1->gtFlags & GTF_GLOB_EFFECT) && lvaHaveManyLocals()) + { + // not worth creating an untracked local variable + shouldExpandInline = false; + } + else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitProfileCasts() == 1)) + { + // Optimizations are enabled but we're still instrumenting (including casts) + if (isCastClass && !isClassExact) + { + // Usually, we make a speculative assumption that it makes sense to expand castclass + // even for non-sealed classes, but let's rely on PGO in this specific case + shouldExpandInline = false; + } + } + + if (shouldExpandInline && compCurBB->isRunRarely()) + { + // For cold blocks we only expand castclass against exact classes because it's cheap + shouldExpandInline = isCastClass && isClassExact; + } + + // Pessimistically assume the jit cannot expand this as an inline test + bool canExpandInline = false; + bool reversedMTCheck = false; + const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); + + CORINFO_CLASS_HANDLE exactCls = NO_CLASS_HANDLE; + + // By default, we assume it's 50/50 with the slow path. + unsigned fastPathLikelihood = 50; - bool shouldExpandEarly = false; - const bool tooManyLocals = (((op1->gtFlags & GTF_GLOB_EFFECT) != 0) && lvaHaveManyLocals()); - if (isClassExact && opts.OptimizationEnabled() && !compCurBB->isRunRarely() && !tooManyLocals) + // Legality check. + // + // Not all classclass/isinst operations can be inline expanded. + // Check legality only if an inline expansion is desirable. + if (shouldExpandInline) { - // TODO-InlineCast: Fix size regressions for these two cases if they're moved to the - // late cast expansion path and remove this early expansion entirely. - if (helper == CORINFO_HELP_ISINSTANCEOFCLASS) + if (isCastClass) { - shouldExpandEarly = true; + // Jit can only inline expand CHKCASTCLASS and CHKCASTARRAY helpers. + canExpandInline = (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY); + + // For ChkCastAny we ignore cases where the class is known to be abstract or is an interface. + if (helper == CORINFO_HELP_CHKCASTANY) + { + const bool isAbstract = (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & + (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) != 0; + canExpandInline = !isAbstract; + } } - else if (helper == CORINFO_HELP_ISINSTANCEOFARRAY && !op2->IsIconHandle(GTF_ICON_CLASS_HDL)) + else if ((helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY)) { - shouldExpandEarly = true; + // If the class is exact, the jit can expand the IsInst check inline. + canExpandInline = isClassExact; } } - if (!shouldExpandEarly) + bool expandInline = canExpandInline && shouldExpandInline; + + if (op2->IsIconHandle(GTF_ICON_CLASS_HDL) && (helper != CORINFO_HELP_ISINSTANCEOFCLASS || !isClassExact)) { - JITDUMP("\nImporting %s as call\n", isCastClass ? "castclass" : "isinst"); + // TODO-InlineCast: move these to the late cast expansion phase as well: + // 1) isinst + // 2) op2 being GT_RUNTIMELOOKUP + expandInline = false; + } + + if (!expandInline) + { + JITDUMP("\nExpanding %s as call because %s\n", isCastClass ? "castclass" : "isinst", + canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); // If we CSE this class handle we prevent assertionProp from making SubType assertions // so instead we force the CSE logic to not consider CSE-ing this class handle. // op2->gtFlags |= GTF_DONT_CSE; - GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); - call->gtCastHelperILOffset = ilOffset; + + GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); // Instrument this castclass/isinst if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !isClassExact && @@ -5543,41 +5559,129 @@ GenTree* Compiler::impCastClassOrIsInstToTree( return call; } - JITDUMP("\nExpanding isinst inline\n"); + JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling ")); + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark2")); + + GenTree* temp; + GenTree* condMT; + // + // expand the methodtable match: + // + // condMT ==> GT_NE + // / \. + // GT_IND op2 (typically CNS_INT) + // | + // op1Copy + // - // Now we import it as two QMark nodes representing this: + // This can replace op1 with a GT_COMMA that evaluates op1 into a local // - // tmp = op1; - // if (tmp != null) // qmarkNull - // { - // if (tmp->pMT == op2) // qmarkMT - // result = tmp; - // else - // result = null; - // } - // else - // result = null; + op1 = impCloneExpr(op1, &temp, CHECK_SPILL_ALL, nullptr DEBUGARG("CASTCLASS eval op1")); + // + // op1 is now known to be a non-complex tree + // thus we can use gtClone(op1) from now on // - // Spill op1 if it's a complex expression - GenTree* op1Clone; - op1 = impCloneExpr(op1, &op1Clone, CHECK_SPILL_ALL, nullptr DEBUGARG("ISINST eval op1")); + GenTree* op2Var = op2; + if (isCastClass && (exactCls == NO_CLASS_HANDLE)) + { + // if exactCls is not null we won't have to clone op2 (it will be used only for the fallback) + op2Var = fgInsertCommaFormTemp(&op2); + lvaTable[op2Var->AsLclVarCommon()->GetLclNum()].lvIsCSE = true; + } + temp = gtNewMethodTableLookup(temp); + condMT = + gtNewOperNode(GT_NE, TYP_INT, temp, (exactCls != NO_CLASS_HANDLE) ? gtNewIconEmbClsHndNode(exactCls) : op2); - GenTreeOp* condMT = gtNewOperNode(GT_NE, TYP_INT, gtNewMethodTableLookup(op1Clone), op2); - GenTreeOp* condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewNull()); - GenTreeQmark* qmarkMT = gtNewQmarkNode(TYP_REF, condMT, gtNewColonNode(TYP_REF, gtNewNull(), gtClone(op1))); - GenTreeQmark* qmarkNull = gtNewQmarkNode(TYP_REF, condNull, gtNewColonNode(TYP_REF, gtNewNull(), qmarkMT)); + GenTree* condNull; + // + // expand the null check: + // + // condNull ==> GT_EQ + // / \. + // op1Copy CNS_INT + // null + // + condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewNull()); + + // + // expand the true and false trees for the condMT + // + GenTree* condFalse = reversedMTCheck ? gtNewNull() : gtClone(op1); + GenTree* condTrue; + if (isCastClass) + { + assert((helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY) || + (helper == CORINFO_HELP_CHKCASTANY) || (helper == CORINFO_HELP_CHKCASTINTERFACE)); + + CorInfoHelpFunc specialHelper = helper; + if ((helper == CORINFO_HELP_CHKCASTCLASS) && + ((exactCls == nullptr) || (exactCls == gtGetHelperArgClassHandle(op2)))) + { + // use the special helper that skips the cases checked by our inlined cast + specialHelper = CORINFO_HELP_CHKCASTCLASS_SPECIAL; + } + condTrue = gtNewHelperCallNode(specialHelper, TYP_REF, op2Var, gtClone(op1)); + } + else + { + condTrue = gtNewIconNode(0, TYP_REF); + } + + GenTreeQmark* qmarkMT; + // + // Generate first QMARK - COLON tree + // + // qmarkMT ==> GT_QMARK + // / \. + // condMT GT_COLON + // / \. + // condFalse condTrue + // + temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); + qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); + qmarkMT->SetThenNodeLikelihood(fastPathLikelihood); + + if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) + { + if (helper == CORINFO_HELP_CHKCASTCLASS) + { + // condTrue is used only for throwing InvalidCastException in case of casting to an exact class. + condTrue->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; + + // defer calling setMethodHasNoReturnCalls until qmark expasion + } + } + + GenTree* qmarkNull; + // + // Generate second QMARK - COLON tree + // + // qmarkNull ==> GT_QMARK + // / \. + // condNull GT_COLON + // / \. + // qmarkMT op1Copy + // + temp = new (this, GT_COLON) GenTreeColon(TYP_REF, reversedMTCheck ? gtNewNull() : gtClone(op1), qmarkMT); + qmarkNull = gtNewQmarkNode(TYP_REF, condNull, temp->AsColon()); + qmarkNull->gtFlags |= GTF_QMARK_CAST_INSTOF; // Make QMark node a top level node by spilling it. - const unsigned result = lvaGrabTemp(true DEBUGARG("spilling qmarkNull")); - impStoreTemp(result, qmarkNull, CHECK_SPILL_NONE); + unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling QMark2")); + impStoreTemp(tmp, qmarkNull, CHECK_SPILL_NONE); + // TODO-CQ: Is it possible op1 has a better type? + // // See also gtGetHelperCallClassHandle where we make the same // determination for the helper call variants. - lvaSetClass(result, pResolvedToken->hClass); - return gtNewLclvNode(result, TYP_REF); + LclVarDsc* lclDsc = lvaGetDesc(tmp); + assert(lclDsc->lvSingleDef == 0); + lclDsc->lvSingleDef = 1; + JITDUMP("Marked V%02u as a single def temp\n", tmp); + lvaSetClass(tmp, pResolvedToken->hClass); + return gtNewLclvNode(tmp, TYP_REF); } #ifndef DEBUG @@ -5910,7 +6014,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) // Change block to BBJ_THROW so we won't trigger importation of successors. // - block->SetKindAndTargetEdge(BBJ_THROW); + block->SetKindAndTarget(BBJ_THROW); // If this method has a explicit generic context, the only uses of it may be in // the IL for this block. So assume it's used. @@ -6317,7 +6421,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (compIsForInlining()) { - op1 = impInlineFetchArg(impInlineInfo->inlArgInfo[lclNum], impInlineInfo->lclVarInfo[lclNum]); + op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); noway_assert(op1->gtOper == GT_LCL_VAR); lclNum = op1->AsLclVar()->GetLclNum(); @@ -6522,7 +6626,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) // In IL, LDARGA(_S) is used to load the byref managed pointer of struct argument, // followed by a ldfld to load the field. - op1 = impInlineFetchArg(impInlineInfo->inlArgInfo[lclNum], impInlineInfo->lclVarInfo[lclNum]); + op1 = impInlineFetchArg(lclNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo); if (op1->gtOper != GT_LCL_VAR) { compInlineResult->NoteFatal(InlineObservation::CALLSITE_LDARGA_NOT_LOCAL_VAR); @@ -7226,14 +7330,14 @@ void Compiler::impImportBlockCode(BasicBlock* block) // We may have already modified `block`'s jump kind, if this is a re-importation. // bool jumpToNextOptimization = false; - if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) + if (block->KindIs(BBJ_COND) && block->TrueTargetIs(block->GetFalseTarget())) { JITDUMP(FMT_BB " always branches to " FMT_BB ", changing to BBJ_ALWAYS\n", block->bbNum, block->GetFalseTarget()->bbNum); - fgRemoveRefPred(block->GetFalseEdge()); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetTrueEdge()); + fgRemoveRefPred(block->GetFalseTarget(), block); + block->SetKind(BBJ_ALWAYS); - // TODO-NoFallThrough: Once false target can diverge from bbNext, it may not make sense to + // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, it may not make sense to // set BBF_NONE_QUIRK block->SetFlags(BBF_NONE_QUIRK); @@ -7305,18 +7409,18 @@ void Compiler::impImportBlockCode(BasicBlock* block) { JITDUMP("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", block->GetTrueTarget()->bbNum); - fgRemoveRefPred(block->GetFalseEdge()); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetTrueEdge()); + fgRemoveRefPred(block->GetFalseTarget(), block); + block->SetKind(BBJ_ALWAYS); } else { - // TODO-NoFallThrough: Update once false target can diverge from bbNext + // TODO-NoFallThrough: Update once bbFalseTarget can diverge from bbNext assert(block->NextIs(block->GetFalseTarget())); JITDUMP("\nThe block jumps to the next " FMT_BB "\n", block->Next()->bbNum); - fgRemoveRefPred(block->GetTrueEdge()); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetFalseEdge()); + fgRemoveRefPred(block->GetTrueTarget(), block); + block->SetKindAndTarget(BBJ_ALWAYS, block->Next()); - // TODO-NoFallThrough: Once false target can diverge from bbNext, it may not make sense + // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, it may not make sense // to set BBF_NONE_QUIRK block->SetFlags(BBF_NONE_QUIRK); } @@ -7488,14 +7592,14 @@ void Compiler::impImportBlockCode(BasicBlock* block) // We may have already modified `block`'s jump kind, if this is a re-importation. // bool jumpToNextOptimization = false; - if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) + if (block->KindIs(BBJ_COND) && block->TrueTargetIs(block->GetFalseTarget())) { JITDUMP(FMT_BB " always branches to " FMT_BB ", changing to BBJ_ALWAYS\n", block->bbNum, block->GetFalseTarget()->bbNum); - fgRemoveRefPred(block->GetFalseEdge()); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetTrueEdge()); + fgRemoveRefPred(block->GetFalseTarget(), block); + block->SetKind(BBJ_ALWAYS); - // TODO-NoFallThrough: Once false target can diverge from bbNext, it may not make sense to + // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, it may not make sense to // set BBF_NONE_QUIRK block->SetFlags(BBF_NONE_QUIRK); @@ -7563,16 +7667,16 @@ void Compiler::impImportBlockCode(BasicBlock* block) if (opts.OptimizationEnabled() && (op1->gtOper == GT_CNS_INT)) { // Find the jump target - size_t switchVal = (size_t)op1->AsIntCon()->gtIconVal; - unsigned jumpCnt = block->GetSwitchTargets()->bbsCount; - FlowEdge** jumpTab = block->GetSwitchTargets()->bbsDstTab; - bool foundVal = false; + size_t switchVal = (size_t)op1->AsIntCon()->gtIconVal; + unsigned jumpCnt = block->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTab = block->GetSwitchTargets()->bbsDstTab; + bool foundVal = false; for (unsigned val = 0; val < jumpCnt; val++, jumpTab++) { - FlowEdge* curEdge = *jumpTab; + BasicBlock* curJump = *jumpTab; - assert(curEdge->getDestinationBlock()->countOfInEdges() > 0); + assert(curJump->countOfInEdges() > 0); // If val matches switchVal or we are at the last entry and // we never found the switch value then set the new jump dest @@ -7580,13 +7684,13 @@ void Compiler::impImportBlockCode(BasicBlock* block) if ((val == switchVal) || (!foundVal && (val == jumpCnt - 1))) { // transform the basic block into a BBJ_ALWAYS - block->SetKindAndTargetEdge(BBJ_ALWAYS, curEdge); + block->SetKindAndTarget(BBJ_ALWAYS, curJump); foundVal = true; } else { - // Remove 'curEdge' - fgRemoveRefPred(curEdge); + // Remove 'block' from the predecessor list of 'curJump' + fgRemoveRefPred(curJump, block); } } @@ -8518,7 +8622,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) } const bool useParent = true; - op1 = gtNewAllocObjNode(&resolvedToken, info.compMethodHnd, useParent); + op1 = gtNewAllocObjNode(&resolvedToken, useParent); if (op1 == nullptr) { return; @@ -10174,19 +10278,9 @@ void Compiler::impImportBlockCode(BasicBlock* block) case CEE_CPBLK: { GenTreeFlags indirFlags = impPrefixFlagsToIndirFlags(prefixFlags); - const bool isVolatile = (indirFlags & GTF_IND_VOLATILE) != 0; -#ifndef TARGET_X86 - if (isVolatile && !impStackTop(0).val->IsCnsIntOrI()) - { - // We're going to emit a helper call surrounded by memory barriers, so we need to spill any side - // effects. - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("spilling side-effects")); - } -#endif - - op3 = gtFoldExpr(impPopStack().val); // Size - op2 = gtFoldExpr(impPopStack().val); // Value / Src addr - op1 = impPopStack().val; // Dst addr + op3 = impPopStack().val; // Size + op2 = impPopStack().val; // Value / Src addr + op1 = impPopStack().val; // Dst addr if (op3->IsCnsIntOrI()) { @@ -10223,41 +10317,24 @@ void Compiler::impImportBlockCode(BasicBlock* block) } else { - if (TARGET_POINTER_SIZE == 8) - { - // Cast size to TYP_LONG on 64-bit targets - op3 = gtNewCastNode(TYP_LONG, op3, /* fromUnsigned */ true, TYP_LONG); - } - - GenTreeCall* call; if (opcode == CEE_INITBLK) { - // value is zero -> memzero, otherwise -> memset - if (op2->IsIntegralConst(0)) - { - call = gtNewHelperCallNode(CORINFO_HELP_MEMZERO, TYP_VOID, op1, op3); - } - else + if (!op2->IsIntegralConst(0)) { - call = gtNewHelperCallNode(CORINFO_HELP_MEMSET, TYP_VOID, op1, op2, op3); + op2 = gtNewOperNode(GT_INIT_VAL, TYP_INT, op2); } } else { - call = gtNewHelperCallNode(CORINFO_HELP_MEMCPY, TYP_VOID, op1, op2, op3); + op2 = gtNewIndir(TYP_STRUCT, op2); } - if (isVolatile) - { - // Wrap with memory barriers: full-barrier + call + load-barrier - impAppendTree(gtNewMemoryBarrier(), CHECK_SPILL_ALL, impCurStmtDI); - impAppendTree(call, CHECK_SPILL_ALL, impCurStmtDI); - op1 = gtNewMemoryBarrier(true); - } - else - { - op1 = call; - } +#ifdef TARGET_64BIT + // STORE_DYN_BLK takes a native uint size as it turns into call to memcpy. + op3 = gtNewCastNode(TYP_I_IMPL, op3, /* fromUnsigned */ true, TYP_I_IMPL); +#endif + + op1 = gtNewStoreDynBlkNode(op1, op2, op3, indirFlags); } goto SPILL_APPEND; } @@ -10496,8 +10573,7 @@ void Compiler::impLoadArg(unsigned ilArgNum, IL_OFFSET offset) tiRetVal = typeInfo(type); } - impPushOnStack(impInlineFetchArg(impInlineInfo->inlArgInfo[ilArgNum], impInlineInfo->lclVarInfo[ilArgNum]), - tiRetVal); + impPushOnStack(impInlineFetchArg(ilArgNum, impInlineInfo->inlArgInfo, impInlineInfo->lclVarInfo), tiRetVal); } else { @@ -12820,27 +12896,9 @@ void Compiler::impInlineInitVars(InlineInfo* pInlineInfo) inlArgInfo[ilArgCnt].argIsThis = true; break; case WellKnownArg::RetBuffer: - // This does not appear in the table of inline arg info; do not include them - continue; case WellKnownArg::InstParam: - { - InlArgInfo* ctxInfo = new (this, CMK_Inlining) InlArgInfo{}; - ctxInfo->arg = &arg; - ctxInfo->argTmpNum = BAD_VAR_NUM; - ctxInfo->argIsLclVar = arg.GetNode()->OperIs(GT_LCL_VAR); - if (arg.GetNode()->IsCnsIntOrI()) - { - ctxInfo->argIsInvariant = true; - } - else - { - // Conservative approach - ctxInfo->argHasSideEff = true; - ctxInfo->argHasGlobRef = true; - } - pInlineInfo->inlInstParamArgInfo = ctxInfo; + // These do not appear in the table of inline arg info; do not include them continue; - } default: break; } @@ -13211,8 +13269,9 @@ unsigned Compiler::impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reas // impInlineFetchArg: return tree node for argument value in an inlinee // // Arguments: -// argInfo -- argument info for inlinee -// lclInfo -- var info for inlinee +// lclNum -- argument number in inlinee IL +// inlArgInfo -- argument info for inlinee +// lclVarInfo -- var info for inlinee // // Returns: // Tree for the argument's value. Often an inlinee-scoped temp @@ -13239,13 +13298,15 @@ unsigned Compiler::impInlineFetchLocal(unsigned lclNum DEBUGARG(const char* reas // This method will side effect inlArgInfo. It should only be called // for actual uses of the argument in the inlinee. -GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& lclInfo) +GenTree* Compiler::impInlineFetchArg(unsigned lclNum, InlArgInfo* inlArgInfo, InlLclVarInfo* lclVarInfo) { // Cache the relevant arg and lcl info for this argument. // We will modify argInfo but not lclVarInfo. - const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp; - const var_types lclTyp = lclInfo.lclTypeInfo; - GenTree* op1 = nullptr; + InlArgInfo& argInfo = inlArgInfo[lclNum]; + const InlLclVarInfo& lclInfo = lclVarInfo[lclNum]; + const bool argCanBeModified = argInfo.argHasLdargaOp || argInfo.argHasStargOp; + const var_types lclTyp = lclInfo.lclTypeInfo; + GenTree* op1 = nullptr; GenTree* argNode = argInfo.arg->GetNode(); assert(!argNode->OperIs(GT_RET_EXPR)); @@ -13296,6 +13357,7 @@ GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& l if (argInfo.argIsUsed || ((lclTyp == TYP_BYREF) && (op1->TypeGet() != TYP_BYREF))) { assert(op1->gtOper == GT_LCL_VAR); + assert(lclNum == op1->AsLclVar()->gtLclILoffs); // Create a new lcl var node - remember the argument lclNum op1 = impCreateLocalNode(argLclNum DEBUGARG(op1->AsLclVar()->gtLclILoffs)); @@ -13408,7 +13470,7 @@ GenTree* Compiler::impInlineFetchArg(InlArgInfo& argInfo, const InlLclVarInfo& l !argInfo.argHasCallerLocalRef)) { /* Get a *LARGE* LCL_VAR node */ - op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp)); + op1 = gtNewLclLNode(tmpNum, genActualType(lclTyp) DEBUGARG(lclNum)); /* Record op1 as the very first use of this argument. If there are no further uses of the arg, we may be diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index f24750613054a9..b263285c42b545 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -98,10 +98,6 @@ var_types Compiler::impImportCall(OPCODE opcode, CORINFO_SIG_INFO calliSig; NewCallArg extraArg; - // Swift calls that might throw use a SwiftError* arg that requires additional IR to handle, - // so if we're importing a Swift call, look for this type in the signature - CallArg* swiftErrorArg = nullptr; - /*------------------------------------------------------------------------- * First create the call node */ @@ -655,8 +651,6 @@ var_types Compiler::impImportCall(OPCODE opcode, if (call->gtFlags & GTF_CALL_UNMANAGED) { - assert(call->IsCall()); - // We set up the unmanaged call by linking the frame, disabling GC, etc // This needs to be cleaned up on return. // In addition, native calls have different normalization rules than managed code @@ -669,7 +663,7 @@ var_types Compiler::impImportCall(OPCODE opcode, checkForSmallType = true; - impPopArgsForUnmanagedCall(call->AsCall(), sig, &swiftErrorArg); + impPopArgsForUnmanagedCall(call->AsCall(), sig); goto DONE; } @@ -1296,7 +1290,7 @@ var_types Compiler::impImportCall(OPCODE opcode, impAppendTree(call, verCurrentState.esStackDepth - 1, impCurStmtDI); } else if (JitConfig.JitProfileValues() && call->IsCall() && - call->AsCall()->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove)) + call->AsCall()->IsSpecialIntrinsic(this, NI_System_Buffer_Memmove)) { if (opts.IsOptimizedWithProfile()) { @@ -1491,15 +1485,6 @@ var_types Compiler::impImportCall(OPCODE opcode, impPushOnStack(call, tiRetVal); } -#ifdef SWIFT_SUPPORT - // If call is a Swift call with error handling, append additional IR - // to handle storing the error register's value post-call. - if (swiftErrorArg != nullptr) - { - impAppendSwiftErrorStore(call->AsCall(), swiftErrorArg); - } -#endif // SWIFT_SUPPORT - return callRetTyp; } #ifdef _PREFAST_ @@ -1570,7 +1555,7 @@ GenTree* Compiler::impDuplicateWithProfiledArg(GenTreeCall* call, IL_OFFSET ilOf unsigned argNum = 0; ssize_t minValue = 0; ssize_t maxValue = 0; - if (call->IsSpecialIntrinsic(this, NI_System_SpanHelpers_Memmove)) + if (call->IsSpecialIntrinsic(this, NI_System_Buffer_Memmove)) { // dst(0), src(1), len(2) argNum = 2; @@ -1765,7 +1750,12 @@ GenTree* Compiler::impFixupCallStructReturn(GenTreeCall* call, CORINFO_CLASS_HAN assert(returnType == TYP_STRUCT); assert((howToReturnStruct == SPK_ByValueAsHfa) || (howToReturnStruct == SPK_ByValue)); +#ifdef UNIX_AMD64_ABI + // must be a struct returned in two registers + assert(retRegCount == 2); +#else // not UNIX_AMD64_ABI assert(retRegCount >= 2); +#endif // not UNIX_AMD64_ABI if (!call->CanTailCall() && !call->IsInlineCandidate()) { @@ -1830,27 +1820,12 @@ GenTreeCall* Compiler::impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugI return call; } -//------------------------------------------------------------------------ -// impPopArgsForUnmanagedCall: Pop arguments from IL stack to a pinvoke call. -// -// Arguments: -// call - The unmanaged call -// sig - The signature of the call site -// swiftErrorArg - [out] If this is a Swift call with a SwiftError* argument, then the argument is returned here. -// Otherwise left at its existing value. -// -void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, CallArg** swiftErrorArg) +/*****************************************************************************/ + +void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig) { assert(call->gtFlags & GTF_CALL_UNMANAGED); -#ifdef SWIFT_SUPPORT - if (call->unmgdCallConv == CorInfoCallConvExtension::Swift) - { - impPopArgsForSwiftCall(call, sig, swiftErrorArg); - return; - } -#endif - /* Since we push the arguments in reverse order (i.e. right -> left) * spill any side effects from the stack * @@ -1867,7 +1842,7 @@ void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* s if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) { - assert(argsToReverse != 0); + assert(argsToReverse); argsToReverse--; } @@ -1917,23 +1892,6 @@ void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* s assert(thisPtr->TypeGet() == TYP_I_IMPL || thisPtr->TypeGet() == TYP_BYREF); } - impRetypeUnmanagedCallArgs(call); -} - -//------------------------------------------------------------------------ -// impRetypeUnmanagedCallArgs: Retype unmanaged call arguments from managed -// pointers to unmanaged ones. -// -// Arguments: -// call - The call -// -// Remarks: -// This makes use of the fact that TYP_I_IMPL <-> TYP_BYREF casts are -// implicit in JIT IR, allowing us to change the types directly without -// inserting a cast node. -// -void Compiler::impRetypeUnmanagedCallArgs(GenTreeCall* call) -{ for (CallArg& arg : call->gtArgs.Args()) { GenTree* argNode = arg.GetEarlyNode(); @@ -1959,366 +1917,6 @@ void Compiler::impRetypeUnmanagedCallArgs(GenTreeCall* call) } } -#ifdef SWIFT_SUPPORT - -//------------------------------------------------------------------------ -// GetSwiftLowering: Get the CORINFO_SWIFT_LOWERING associated with a struct. -// -// Arguments: -// call - The class -// -// Return Value: -// Pointer to lowering -// -const CORINFO_SWIFT_LOWERING* Compiler::GetSwiftLowering(CORINFO_CLASS_HANDLE hClass) -{ - if (m_swiftLoweringCache == nullptr) - { - m_swiftLoweringCache = new (this, CMK_CallArgs) SwiftLoweringMap(getAllocator(CMK_CallArgs)); - } - - CORINFO_SWIFT_LOWERING* lowering; - if (!m_swiftLoweringCache->Lookup(hClass, &lowering)) - { - lowering = new (this, CMK_CallArgs) CORINFO_SWIFT_LOWERING; - info.compCompHnd->getSwiftLowering(hClass, lowering); - m_swiftLoweringCache->Set(hClass, lowering); - } - - return lowering; -} - -//------------------------------------------------------------------------ -// impPopArgsForSwiftCall: Pop arguments from IL stack to a Swift pinvoke node. -// -// Arguments: -// call - The Swift call -// sig - The signature of the call site -// swiftErrorArg - [out] An argument that represents the SwiftError* -// argument. Left at its existing value if no such argument exists. -// -void Compiler::impPopArgsForSwiftCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, CallArg** swiftErrorArg) -{ - JITDUMP("Creating args for Swift call [%06u]\n", dspTreeID(call)); - - unsigned short swiftErrorIndex = sig->numArgs; - unsigned short swiftSelfIndex = sig->numArgs; - - // We are importing an unmanaged Swift call, which might require special parameter handling - bool checkEntireStack = false; - - // Check the signature of the Swift call for the special types - CORINFO_ARG_LIST_HANDLE sigArg = sig->args; - - for (unsigned short argIndex = 0; argIndex < sig->numArgs; - sigArg = info.compCompHnd->getArgNext(sigArg), argIndex++) - { - CORINFO_CLASS_HANDLE argClass; - CorInfoType argType = strip(info.compCompHnd->getArgType(sig, sigArg, &argClass)); - bool argIsByrefOrPtr = false; - - if ((argType == CORINFO_TYPE_BYREF) || (argType == CORINFO_TYPE_PTR)) - { - argClass = info.compCompHnd->getArgClass(sig, sigArg); - argType = info.compCompHnd->getChildType(argClass, &argClass); - argIsByrefOrPtr = true; - } - - if (argType != CORINFO_TYPE_VALUECLASS) - { - continue; - } - - if (info.compCompHnd->isIntrinsicType(argClass)) - { - const char* namespaceName; - const char* className = info.compCompHnd->getClassNameFromMetadata(argClass, &namespaceName); - - if ((strcmp(className, "SwiftError") == 0) && - (strcmp(namespaceName, "System.Runtime.InteropServices.Swift") == 0)) - { - // For error handling purposes, we expect a pointer/reference to a SwiftError to be passed - if (!argIsByrefOrPtr) - { - BADCODE("Expected SwiftError pointer/reference, got struct"); - } - - if (swiftErrorIndex != sig->numArgs) - { - BADCODE("Duplicate SwiftError* parameter"); - } - - swiftErrorIndex = argIndex; - checkEntireStack = true; - } - else if ((strcmp(className, "SwiftSelf") == 0) && - (strcmp(namespaceName, "System.Runtime.InteropServices.Swift") == 0)) - { - // We expect a SwiftSelf struct to be passed, not a pointer/reference - if (argIsByrefOrPtr) - { - BADCODE("Expected SwiftSelf struct, got pointer/reference"); - } - - if (swiftSelfIndex != sig->numArgs) - { - BADCODE("Duplicate SwiftSelf parameter"); - } - - swiftSelfIndex = argIndex; - // Fall through to make sure the struct value becomes a local. - } - // TODO: Handle SwiftAsync - } - - if (argIsByrefOrPtr) - { - continue; - } - - if (argIndex != swiftSelfIndex) - { - // This is a struct type. Check if it needs to be lowered. - // TODO-Bug: SIMD types are not handled correctly by this. - } - - // We must spill this struct to a local to be able to expand it into primitives. - GenTree* node = impStackTop(sig->numArgs - 1 - argIndex).val; - if (!node->OperIsLocalRead()) - { - // TODO-CQ: If we enable FEATURE_IMPLICIT_BYREFS on all platforms - // where we support Swift we can probably let normal implicit byref - // handling handle the unlowered case. - impSpillStackEntry(verCurrentState.esStackDepth - sig->numArgs + argIndex, - BAD_VAR_NUM DEBUGARG(false) DEBUGARG("Swift struct arg with lowering")); - } - } - - // If using SwiftError*, spill entire stack as we will need to reuse the - // error argument after the call. - if (checkEntireStack) - { - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("Spill for swift call")); - } - - impPopCallArgs(sig, call); - - JITDUMP("Node after popping args:\n"); - DISPTREE(call); - JITDUMP("\n"); - - if (swiftErrorIndex != sig->numArgs) - { - *swiftErrorArg = call->gtArgs.GetArgByIndex(swiftErrorIndex); - } - - // Now expand struct args that must be lowered into primitives - unsigned argIndex = 0; - for (CallArg* arg = call->gtArgs.Args().begin().GetArg(); arg != nullptr; argIndex++) - { - if (!varTypeIsStruct(arg->GetSignatureType())) - { - arg = arg->GetNext(); - continue; - } - - if (varTypeIsSIMD(arg->GetSignatureType())) - { - IMPL_LIMITATION("SIMD types are currently unsupported in Swift calls"); - } - - JITDUMP(" Argument %u is a struct [%06u]\n", argIndex, dspTreeID(arg->GetNode())); - - assert(arg->GetNode()->OperIsLocalRead()); - GenTreeLclVarCommon* structVal = arg->GetNode()->AsLclVarCommon(); - - CallArg* insertAfter = arg; - // For the self arg, change it from the SwiftSelf struct to a - // TYP_I_IMPL primitive directly. It must also be marked as a well - // known arg because it has a non-standard calling convention. - if (argIndex == swiftSelfIndex) - { - assert(arg->GetNode()->OperIsLocalRead()); - GenTree* primitiveSelf = gtNewLclFldNode(structVal->GetLclNum(), TYP_I_IMPL, structVal->GetLclOffs()); - NewCallArg newArg = NewCallArg::Primitive(primitiveSelf, TYP_I_IMPL).WellKnown(WellKnownArg::SwiftSelf); - insertAfter = call->gtArgs.InsertAfter(this, insertAfter, newArg); - } - else - { - const CORINFO_SWIFT_LOWERING* lowering = GetSwiftLowering(arg->GetSignatureClassHandle()); - if (lowering->byReference) - { - JITDUMP(" Argument %d of type %s must be passed by reference\n", argIndex, - typGetObjLayout(arg->GetSignatureClassHandle())->GetClassName()); - } - else - { - JITDUMP(" Argument %d of type %s must be passed as %d primitive(s)\n", argIndex, - typGetObjLayout(arg->GetSignatureClassHandle())->GetClassName(), lowering->numLoweredElements); - for (size_t i = 0; i < lowering->numLoweredElements; i++) - { - JITDUMP(" [%zu] @ +%02u: %s\n", i, lowering->offsets[i], - varTypeName(JitType2PreciseVarType(lowering->loweredElements[i]))); - } - } - - if (lowering->byReference) - { - GenTree* addrNode = gtNewLclAddrNode(structVal->GetLclNum(), structVal->GetLclOffs()); - JITDUMP(" Passing by reference\n"); - - insertAfter = call->gtArgs.InsertAfter(this, insertAfter, NewCallArg::Primitive(addrNode, TYP_I_IMPL)); - } - else - { - for (size_t i = 0; i < lowering->numLoweredElements; i++) - { - var_types loweredType = JITtype2varType(lowering->loweredElements[i]); - unsigned offset = lowering->offsets[i]; - - GenTree* loweredNode = nullptr; - - // It's possible for the lowering to require us to pass the - // tail of the sequence as a 64-bit value, even if the tail - // of the struct is smaller than 8 bytes. In that case we - // reconstruct the value using bitwise operations. - // Alternatively we could create IND(LCL_ADDR), assuming - // that the upper bits are undefined. This would result in - // address exposure instead. - unsigned sizeToRead = min(structVal->GetLayout(this)->GetSize() - offset, genTypeSize(loweredType)); - assert(sizeToRead > 0); - - if (sizeToRead == genTypeSize(loweredType)) - { - loweredNode = - gtNewLclFldNode(structVal->GetLclNum(), loweredType, structVal->GetLclOffs() + offset); - } - else - { - unsigned relOffset = 0; - auto addSegment = [=, &loweredNode, &relOffset](var_types type) { - GenTree* val = gtNewLclFldNode(structVal->GetLclNum(), type, - structVal->GetLclOffs() + offset + relOffset); - - if (loweredType == TYP_LONG) - { - val = gtNewCastNode(TYP_LONG, val, true, TYP_LONG); - } - - if (relOffset > 0) - { - val = gtNewOperNode(GT_LSH, genActualType(loweredType), val, - gtNewIconNode(relOffset * 8)); - } - - if (loweredNode == nullptr) - { - loweredNode = val; - } - else - { - loweredNode = gtNewOperNode(GT_OR, genActualType(loweredType), loweredNode, val); - } - - relOffset += genTypeSize(type); - }; - - if (sizeToRead - relOffset >= 4) - { - addSegment(TYP_INT); - } - if (sizeToRead - relOffset >= 2) - { - addSegment(TYP_USHORT); - } - if (sizeToRead - relOffset >= 1) - { - addSegment(TYP_UBYTE); - } - - assert(relOffset == sizeToRead); - } - - JITDUMP(" Adding expanded primitive argument [%06u]\n", dspTreeID(loweredNode)); - DISPTREE(loweredNode); - - insertAfter = - call->gtArgs.InsertAfter(this, insertAfter, NewCallArg::Primitive(loweredNode, loweredType)); - } - } - } - - JITDUMP(" Removing plain struct argument [%06u]\n", dspTreeID(structVal)); - call->gtArgs.Remove(arg); - arg = insertAfter->GetNext(); - } - -#ifdef DEBUG - if (verbose && call->TypeIs(TYP_STRUCT) && (sig->retTypeClass != NO_CLASS_HANDLE)) - { - const CORINFO_SWIFT_LOWERING* lowering = GetSwiftLowering(sig->retTypeClass); - if (lowering->byReference) - { - printf(" Call returns %s by reference\n", typGetObjLayout(sig->retTypeClass)->GetClassName()); - } - else - { - printf(" Call returns %s as %d primitive(s) in registers\n", - typGetObjLayout(sig->retTypeClass)->GetClassName(), lowering->numLoweredElements); - for (size_t i = 0; i < lowering->numLoweredElements; i++) - { - printf(" [%zu] @ +%02u: %s\n", i, lowering->offsets[i], - varTypeName(JitType2PreciseVarType(lowering->loweredElements[i]))); - } - } - } -#endif - - JITDUMP("Final result after Swift call lowering:\n"); - DISPTREE(call); - JITDUMP("\n"); - - impRetypeUnmanagedCallArgs(call); -} - -//------------------------------------------------------------------------ -// impAppendSwiftErrorStore: Append IR to store the Swift error register value -// to the SwiftError* argument specified by swiftErrorArg, post-Swift call -// -// Arguments: -// call - the Swift call -// swiftErrorArg - the SwiftError* argument passed to call -// -void Compiler::impAppendSwiftErrorStore(GenTreeCall* call, CallArg* const swiftErrorArg) -{ - assert(call != nullptr); - assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); - assert(swiftErrorArg != nullptr); - - GenTree* const argNode = swiftErrorArg->GetNode(); - assert(argNode != nullptr); - - // Store the error register value to where the SwiftError* points to - GenTree* errorRegNode = new (this, GT_SWIFT_ERROR) GenTree(GT_SWIFT_ERROR, TYP_I_IMPL); - errorRegNode->SetHasOrderingSideEffect(); - errorRegNode->gtFlags |= (GTF_CALL | GTF_GLOB_REF); - - GenTreeStoreInd* swiftErrorStore = gtNewStoreIndNode(argNode->TypeGet(), argNode, errorRegNode); - impAppendTree(swiftErrorStore, CHECK_SPILL_ALL, impCurStmtDI, false); - - // Before calling a Swift method that may throw, the error register must be cleared for the error check to work. - // By adding a well-known "sentinel" argument that uses the error register, - // the JIT will emit code for clearing the error register before the call, and will mark the error register as busy - // so that it isn't used to hold the function call's address. - GenTree* errorSentinelValueNode = gtNewIconNode(0); - call->gtArgs.InsertAfter(this, swiftErrorArg, - NewCallArg::Primitive(errorSentinelValueNode).WellKnown(WellKnownArg::SwiftError)); - - // Swift call isn't going to use the SwiftError* arg, so don't bother emitting it - call->gtArgs.Remove(swiftErrorArg); -} -#endif // SWIFT_SUPPORT - //------------------------------------------------------------------------ // impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray // with a GT_COPYBLK node. @@ -3163,7 +2761,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, betterToExpand = true; break; - case NI_System_SpanHelpers_Memmove: + case NI_System_Buffer_Memmove: case NI_System_SpanHelpers_SequenceEqual: // We're going to instrument these betterToExpand = opts.IsInstrumented(); @@ -3229,38 +2827,26 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, case NI_System_String_Equals: { - retNode = impUtf16StringComparison(StringComparisonKind::Equals, sig, methodFlags); + retNode = impStringEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); break; } case NI_System_MemoryExtensions_Equals: case NI_System_MemoryExtensions_SequenceEqual: { - retNode = impUtf16SpanComparison(StringComparisonKind::Equals, sig, methodFlags); + retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ false, sig, methodFlags); break; } case NI_System_String_StartsWith: { - retNode = impUtf16StringComparison(StringComparisonKind::StartsWith, sig, methodFlags); - break; - } - - case NI_System_String_EndsWith: - { - retNode = impUtf16StringComparison(StringComparisonKind::EndsWith, sig, methodFlags); + retNode = impStringEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); break; } case NI_System_MemoryExtensions_StartsWith: { - retNode = impUtf16SpanComparison(StringComparisonKind::StartsWith, sig, methodFlags); - break; - } - - case NI_System_MemoryExtensions_EndsWith: - { - retNode = impUtf16SpanComparison(StringComparisonKind::EndsWith, sig, methodFlags); + retNode = impSpanEqualsOrStartsWith(/*startsWith:*/ true, sig, methodFlags); break; } @@ -3403,7 +2989,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, resolvedToken.tokenType = CORINFO_TOKENKIND_Method; CORINFO_GENERICHANDLE_RESULT embedInfo; - info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, info.compMethodHnd, &embedInfo); + info.compCompHnd->expandRawHandleIntrinsic(&resolvedToken, &embedInfo); GenTree* rawHandle = impLookupToTree(&resolvedToken, &embedInfo.lookup, gtTokenToIconFlags(memberRef), embedInfo.compileTimeHandle); @@ -3884,11 +3470,6 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, assert(sig->numArgs == 3); GenTree* op3 = impPopStack().val; // comparand - if (varTypeIsSmall(callType)) - { - // small types need the comparand to have its upper bits zeroed - op3 = gtNewCastNode(genActualType(callType), op3, /* uns */ false, varTypeToUnsigned(callType)); - } GenTree* op2 = impPopStack().val; // value GenTree* op1 = impPopStack().val; // location retNode = gtNewAtomicNode(GT_CMPXCHG, callType, op1, op2, op3); @@ -3945,9 +3526,18 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, case NI_System_Threading_Interlocked_ReadMemoryBarrier: { assert(sig->numArgs == 0); + + GenTree* op1 = new (this, GT_MEMORYBARRIER) GenTree(GT_MEMORYBARRIER, TYP_VOID); + op1->gtFlags |= GTF_GLOB_REF | GTF_ASG; + // On XARCH `NI_System_Threading_Interlocked_ReadMemoryBarrier` fences need not be emitted. // However, we still need to capture the effect on reordering. - retNode = gtNewMemoryBarrier(ni == NI_System_Threading_Interlocked_ReadMemoryBarrier); + if (ni == NI_System_Threading_Interlocked_ReadMemoryBarrier) + { + op1->gtFlags |= GTF_MEMORYBARRIER_LOAD; + } + + retNode = op1; break; } @@ -4384,8 +3974,7 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, case NI_System_Text_UTF8Encoding_UTF8EncodingSealed_ReadUtf8: case NI_System_SpanHelpers_SequenceEqual: - case NI_System_SpanHelpers_ClearWithoutReferences: - case NI_System_SpanHelpers_Memmove: + case NI_System_Buffer_Memmove: { if (sig->sigInst.methInstCount == 0) { @@ -4396,16 +3985,6 @@ GenTree* Compiler::impIntrinsic(GenTree* newobjThis, break; } - case NI_System_SpanHelpers_Fill: - { - if (sig->sigInst.methInstCount == 1) - { - // We'll try to unroll this in lower for constant input. - isSpecial = true; - } - break; - } - case NI_System_BitConverter_DoubleToInt64Bits: { GenTree* op1 = impStackTop().val; @@ -6031,7 +5610,8 @@ void Compiler::impCheckForPInvokeCall( // return here without inlining the native call. if (unmanagedCallConv == CorInfoCallConvExtension::Managed || unmanagedCallConv == CorInfoCallConvExtension::Fastcall || - unmanagedCallConv == CorInfoCallConvExtension::FastcallMemberFunction) + unmanagedCallConv == CorInfoCallConvExtension::FastcallMemberFunction || + unmanagedCallConv == CorInfoCallConvExtension::Swift) { return; } @@ -9275,6 +8855,13 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) result = NI_System_BitConverter_Int64BitsToDouble; } } + else if (strcmp(className, "Buffer") == 0) + { + if (strcmp(methodName, "Memmove") == 0) + { + result = NI_System_Buffer_Memmove; + } + } break; } @@ -9345,10 +8932,6 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_MemoryExtensions_StartsWith; } - else if (strcmp(methodName, "EndsWith") == 0) - { - result = NI_System_MemoryExtensions_EndsWith; - } } break; } @@ -9426,18 +9009,6 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_SpanHelpers_SequenceEqual; } - else if (strcmp(methodName, "Fill") == 0) - { - result = NI_System_SpanHelpers_Fill; - } - else if (strcmp(methodName, "ClearWithoutReferences") == 0) - { - result = NI_System_SpanHelpers_ClearWithoutReferences; - } - else if (strcmp(methodName, "Memmove") == 0) - { - result = NI_System_SpanHelpers_Memmove; - } } else if (strcmp(className, "String") == 0) { @@ -9461,10 +9032,6 @@ NamedIntrinsic Compiler::lookupNamedIntrinsic(CORINFO_METHOD_HANDLE method) { result = NI_System_String_StartsWith; } - else if (strcmp(methodName, "EndsWith") == 0) - { - result = NI_System_String_EndsWith; - } } break; } diff --git a/src/coreclr/jit/importervectorization.cpp b/src/coreclr/jit/importervectorization.cpp index af7a2f2791d169..01ee4916d4eef2 100644 --- a/src/coreclr/jit/importervectorization.cpp +++ b/src/coreclr/jit/importervectorization.cpp @@ -25,10 +25,6 @@ // 8) MemoryExtensions.StartsWith(ROS, ROS) // 9) MemoryExtensions.StartsWith(ROS, ROS, Ordinal or OrdinalIgnoreCase) // -// 10) str.EndsWith(string, Ordinal or OrdinalIgnoreCase) -// 11) MemoryExtensions.EndsWith(ROS, ROS) -// 12) MemoryExtensions.EndsWith(ROS, ROS, Ordinal or OrdinalIgnoreCase) -// // When one of the arguments is a constant string of a [0..32] size so we can inline // a vectorized comparison against it using SWAR or SIMD techniques (e.g. via two V256 vectors) // @@ -282,12 +278,12 @@ GenTree* Compiler::impCreateCompareInd(GenTreeLclVarCommon* obj, } GenTree* valueTree = gtNewIconNode(value, actualType); - if (joint == StringComparisonJoint::Xor) + if (joint == Xor) { // XOR is better than CMP if we want to join multiple comparisons return gtNewOperNode(GT_XOR, actualType, indirTree, valueTree); } - assert(joint == StringComparisonJoint::Eq); + assert(joint == Eq); return gtNewOperNode(GT_EQ, TYP_INT, indirTree, valueTree); } @@ -346,13 +342,11 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // // where offset for value2 is 2 bytes (1 char) // - UINT32 value1 = MAKEINT32(cns[0], cns[1]); - UINT32 value2 = MAKEINT32(cns[1], cns[2]); - GenTree* firstIndir = - impCreateCompareInd(data, TYP_INT, dataOffset, value1, cmpMode, StringComparisonJoint::Xor); - GenTree* secondIndir = - impCreateCompareInd(gtClone(data)->AsLclVarCommon(), TYP_INT, dataOffset + sizeof(USHORT), value2, cmpMode, - StringComparisonJoint::Xor); + UINT32 value1 = MAKEINT32(cns[0], cns[1]); + UINT32 value2 = MAKEINT32(cns[1], cns[2]); + GenTree* firstIndir = impCreateCompareInd(data, TYP_INT, dataOffset, value1, cmpMode, Xor); + GenTree* secondIndir = impCreateCompareInd(gtClone(data)->AsLclVarCommon(), TYP_INT, + dataOffset + sizeof(USHORT), value2, cmpMode, Xor); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { @@ -383,13 +377,12 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // For 5..6 the overlapping part is 4 bytes if (len <= 6) { - UINT32 value2 = MAKEINT32(cns[len - 2], cns[len - 1]); - GenTree* firstIndir = - impCreateCompareInd(data, TYP_LONG, dataOffset, value1, cmpMode, StringComparisonJoint::Xor); + UINT32 value2 = MAKEINT32(cns[len - 2], cns[len - 1]); + GenTree* firstIndir = impCreateCompareInd(data, TYP_LONG, dataOffset, value1, cmpMode, Xor); - ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT32); - GenTree* secondIndir = impCreateCompareInd(gtClone(data)->AsLclVarCommon(), TYP_INT, offset, value2, cmpMode, - StringComparisonJoint::Xor); + ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT32); + GenTree* secondIndir = + impCreateCompareInd(gtClone(data)->AsLclVarCommon(), TYP_INT, offset, value2, cmpMode, Xor); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { @@ -405,11 +398,10 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( assert((len == 7) || (len == 8)); UINT64 value2 = MAKEINT64(cns[len - 4], cns[len - 3], cns[len - 2], cns[len - 1]); - GenTree* firstIndir = impCreateCompareInd(data, TYP_LONG, dataOffset, value1, cmpMode, StringComparisonJoint::Xor); + GenTree* firstIndir = impCreateCompareInd(data, TYP_LONG, dataOffset, value1, cmpMode, Xor); ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT64); - GenTree* secondIndir = impCreateCompareInd(gtClone(data)->AsLclVarCommon(), TYP_LONG, offset, value2, cmpMode, - StringComparisonJoint::Xor); + GenTree* secondIndir = impCreateCompareInd(gtClone(data)->AsLclVarCommon(), TYP_LONG, offset, value2, cmpMode, Xor); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { @@ -434,7 +426,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // data - Pointer (LCL_VAR) to a data to vectorize // lengthFld - Pointer (LCL_VAR or GT_IND) to Length field // checkForNull - Check data for null -// kind - Is it StartsWith, Equals or EndsWith? +// startsWith - Is it StartsWith or Equals? // cns - Constant data (array of 2-byte chars) // len - Number of 2-byte chars in the cns // dataOffset - Offset for data @@ -447,7 +439,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVarCommon* data, GenTree* lengthFld, bool checkForNull, - StringComparisonKind kind, + bool startsWith, WCHAR* cnsData, int len, int dataOffset, @@ -462,14 +454,14 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVarCommon* data, return nullptr; } - const genTreeOps cmpOp = kind == StringComparisonKind::Equals ? GT_EQ : GT_GE; + const genTreeOps cmpOp = startsWith ? GT_GE : GT_EQ; GenTree* elementsCount = gtNewIconNode(len); GenTree* lenCheckNode; if (len == 0) { // For zero length we don't need to compare content, the following expression is enough: // - // varData != null && lengthFld cmpOp 0 + // varData != null && lengthFld == 0 // lenCheckNode = gtNewOperNode(cmpOp, TYP_INT, lengthFld, elementsCount); } @@ -477,26 +469,15 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVarCommon* data, { assert(cnsData != nullptr); - GenTreeLclVarCommon* dataAddr = gtClone(data)->AsLclVarCommon(); - - if (kind == StringComparisonKind::EndsWith) - { - // For EndsWith we need to adjust dataAddr to point to the end of the string minus value's length - // We spawn a local that we're going to set below - unsigned dataTmp = lvaGrabTemp(true DEBUGARG("clonning data ptr")); - lvaTable[dataTmp].lvType = TYP_BYREF; - dataAddr = gtNewLclvNode(dataTmp, TYP_BYREF); - } - GenTree* indirCmp = nullptr; if (len < 8) // SWAR impl supports len == 8 but we'd better give it to SIMD { - indirCmp = impExpandHalfConstEqualsSWAR(dataAddr, cnsData, len, dataOffset, cmpMode); + indirCmp = impExpandHalfConstEqualsSWAR(gtClone(data)->AsLclVarCommon(), cnsData, len, dataOffset, cmpMode); } #if defined(FEATURE_HW_INTRINSICS) else if (IsBaselineSimdIsaSupported()) { - indirCmp = impExpandHalfConstEqualsSIMD(dataAddr, cnsData, len, dataOffset, cmpMode); + indirCmp = impExpandHalfConstEqualsSIMD(gtClone(data)->AsLclVarCommon(), cnsData, len, dataOffset, cmpMode); } #endif @@ -507,24 +488,9 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVarCommon* data, } assert(indirCmp->TypeIs(TYP_INT, TYP_UBYTE)); - if (kind == StringComparisonKind::EndsWith) - { - // len is expected to be small, so no overflow is possible - assert(!CheckedOps::MulOverflows(len, 2, CheckedOps::Signed)); - - // dataAddr = dataAddr + (length * 2 - len * 2) - GenTree* castedLen = gtNewCastNode(TYP_I_IMPL, gtCloneExpr(lengthFld), false, TYP_I_IMPL); - GenTree* byteLen = gtNewOperNode(GT_MUL, TYP_I_IMPL, castedLen, gtNewIconNode(2, TYP_I_IMPL)); - GenTreeOp* cmpStart = gtNewOperNode(GT_ADD, TYP_BYREF, gtClone(data), - gtNewOperNode(GT_SUB, TYP_I_IMPL, byteLen, - gtNewIconNode((ssize_t)(len * 2), TYP_I_IMPL))); - GenTree* storeTmp = gtNewTempStore(dataAddr->GetLclNum(), cmpStart); - indirCmp = gtNewOperNode(GT_COMMA, indirCmp->TypeGet(), storeTmp, indirCmp); - } - GenTreeColon* lenCheckColon = gtNewColonNode(TYP_INT, indirCmp, gtNewFalse()); - // For StartsWith/EndsWith we use GT_GE, e.g.: `x.Length >= 10` + // For StartsWith we use GT_GE, e.g.: `x.Length >= 10` lenCheckNode = gtNewQmarkNode(TYP_INT, gtNewOperNode(cmpOp, TYP_INT, lengthFld, elementsCount), lenCheckColon); } @@ -590,7 +556,7 @@ GenTreeStrCon* Compiler::impGetStrConFromSpan(GenTree* span) } //------------------------------------------------------------------------ -// impUtf16StringComparison: The main entry-point for String methods +// impStringEqualsOrStartsWith: The main entry-point for String methods // We're going to unroll & vectorize the following cases: // 1) String.Equals(obj, "cns") // 2) String.Equals(obj, "cns", Ordinal or OrdinalIgnoreCase) @@ -604,21 +570,18 @@ GenTreeStrCon* Compiler::impGetStrConFromSpan(GenTree* span) // 9) obj.StartsWith("cns", Ordinal or OrdinalIgnoreCase) // 10) "cns".StartsWith(obj, Ordinal or OrdinalIgnoreCase) // -// 11) obj.EndsWith("cns", Ordinal or OrdinalIgnoreCase) -// 12) "cns".EndsWith(obj, Ordinal or OrdinalIgnoreCase) -// // For cases 5, 6 and 9 we don't emit "obj != null" // NOTE: String.Equals(object) is not supported currently // // Arguments: -// kind - Is it StartsWith, EndsWith or Equals? -// sig - signature of StartsWith, EndsWith or Equals method +// startsWith - Is it StartsWith or Equals? +// sig - signature of StartsWith or Equals method // methodFlags - its flags // // Returns: // GenTree representing vectorized comparison or nullptr // -GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_SIG_INFO* sig, unsigned methodFlags) +GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags) { const bool isStatic = methodFlags & CORINFO_FLG_STATIC; const int argsCount = sig->numArgs + (isStatic ? 0 : 1); @@ -626,7 +589,7 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S // This optimization spawns several temps so make sure we have a room if (lvaHaveManyLocals(0.75)) { - JITDUMP("impUtf16StringComparison: Method has too many locals - bail out.\n") + JITDUMP("impSpanEqualsOrStartsWith: Method has too many locals - bail out.\n") return nullptr; } @@ -667,9 +630,9 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S } else { - if (kind != StringComparisonKind::Equals) + if (startsWith) { - // StartsWith and EndsWith are not commutative + // StartsWith is not commutative return nullptr; } cnsStr = op1->AsStrCon(); @@ -684,7 +647,6 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S // obj.Equals("cns") // obj.Equals("cns", Ordinal or OrdinalIgnoreCase) // obj.StartsWith("cns", Ordinal or OrdinalIgnoreCase) - // obj.EndsWith("cns", Ordinal or OrdinalIgnoreCase) // // instead, it should throw NRE if it's null needsNullcheck = false; @@ -696,7 +658,7 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S { // check for fake "" first cnsLength = 0; - JITDUMP("Trying to unroll String.Equals|StartsWith|EndsWith(op1, \"\")...\n", str) + JITDUMP("Trying to unroll String.Equals|StartsWith(op1, \"\")...\n", str) } else { @@ -706,7 +668,7 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S // We were unable to get the literal (e.g. dynamic context) return nullptr; } - JITDUMP("Trying to unroll String.Equals|StartsWith|EndsWith(op1, \"cns\")...\n") + JITDUMP("Trying to unroll String.Equals|StartsWith(op1, \"cns\")...\n") } // Create a temp which is safe to gtClone for varStr @@ -720,7 +682,7 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S GenTree* lenNode = gtNewArrLen(TYP_INT, varStrLcl, strLenOffset, compCurBB); varStrLcl = gtClone(varStrLcl)->AsLclVar(); - GenTree* unrolled = impExpandHalfConstEquals(varStrLcl, lenNode, needsNullcheck, kind, (WCHAR*)str, cnsLength, + GenTree* unrolled = impExpandHalfConstEquals(varStrLcl, lenNode, needsNullcheck, startsWith, (WCHAR*)str, cnsLength, strLenOffset + sizeof(int), cmpMode); if (unrolled != nullptr) { @@ -744,7 +706,7 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S } //------------------------------------------------------------------------ -// impUtf16SpanComparison: The main entry-point for [ReadOnly]Span methods +// impSpanEqualsOrStartsWith: The main entry-point for [ReadOnly]Span methods // We're going to unroll & vectorize the following cases: // 1) MemoryExtensions.SequenceEqual(var, "cns") // 2) MemoryExtensions.SequenceEqual("cns", var) @@ -755,20 +717,15 @@ GenTree* Compiler::impUtf16StringComparison(StringComparisonKind kind, CORINFO_S // 7) MemoryExtensions.StartsWith("cns", var, Ordinal or OrdinalIgnoreCase) // 8) MemoryExtensions.StartsWith(var, "cns", Ordinal or OrdinalIgnoreCase) // -// 9) MemoryExtensions.EndsWith("cns", var) -// 10) MemoryExtensions.EndsWith(var, "cns") -// 11) MemoryExtensions.EndsWith("cns", var, Ordinal or OrdinalIgnoreCase) -// 12) MemoryExtensions.EndsWith(var, "cns", Ordinal or OrdinalIgnoreCase) -// // Arguments: -// kind - Is it StartsWith, EndsWith or Equals? -// sig - signature of StartsWith, EndsWith or Equals method +// startsWith - Is it StartsWith or Equals? +// sig - signature of StartsWith or Equals method // methodFlags - its flags // // Returns: // GenTree representing vectorized comparison or nullptr // -GenTree* Compiler::impUtf16SpanComparison(StringComparisonKind kind, CORINFO_SIG_INFO* sig, unsigned methodFlags) +GenTree* Compiler::impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags) { const bool isStatic = methodFlags & CORINFO_FLG_STATIC; const int argsCount = sig->numArgs + (isStatic ? 0 : 1); @@ -776,7 +733,7 @@ GenTree* Compiler::impUtf16SpanComparison(StringComparisonKind kind, CORINFO_SIG // This optimization spawns several temps so make sure we have a room if (lvaHaveManyLocals(0.75)) { - JITDUMP("impUtf16SpanComparison: Method has too many locals - bail out.\n") + JITDUMP("impSpanEqualsOrStartsWith: Method has too many locals - bail out.\n") return nullptr; } @@ -803,7 +760,7 @@ GenTree* Compiler::impUtf16SpanComparison(StringComparisonKind kind, CORINFO_SIG op2 = impStackTop(0).val; } - // For generic StartsWith, EndsWith and Equals we need to make sure T is char + // For generic StartsWith and Equals we need to make sure T is char if (sig->sigInst.methInstCount != 0) { assert(sig->sigInst.methInstCount == 1); @@ -833,9 +790,9 @@ GenTree* Compiler::impUtf16SpanComparison(StringComparisonKind kind, CORINFO_SIG } else { - if (kind != StringComparisonKind::Equals) + if (startsWith) { - // StartsWith and EndsWith are not commutative + // StartsWith is not commutative return nullptr; } cnsStr = op1Str; @@ -878,8 +835,8 @@ GenTree* Compiler::impUtf16SpanComparison(StringComparisonKind kind, CORINFO_SIG GenTreeLclFld* spanReferenceFld = gtNewLclFldNode(spanLclNum, TYP_BYREF, OFFSETOF__CORINFO_Span__reference); GenTreeLclFld* spanLengthFld = gtNewLclFldNode(spanLclNum, TYP_INT, OFFSETOF__CORINFO_Span__length); - GenTree* unrolled = - impExpandHalfConstEquals(spanReferenceFld, spanLengthFld, false, kind, (WCHAR*)str, cnsLength, 0, cmpMode); + GenTree* unrolled = impExpandHalfConstEquals(spanReferenceFld, spanLengthFld, false, startsWith, (WCHAR*)str, + cnsLength, 0, cmpMode); if (unrolled != nullptr) { diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index 9220df5e3dc855..fbbe08d66fc3c6 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -207,10 +207,6 @@ class IndirectCallTransformer { remainderBlock = compiler->fgSplitBlockAfterStatement(currBlock, stmt); remainderBlock->SetFlags(BBF_INTERNAL); - - // We will be adding more blocks after currBlock, so remove edge to remainderBlock. - // - compiler->fgRemoveRefPred(currBlock->GetTargetEdge()); } virtual void CreateCheck(uint8_t checkIdx) = 0; @@ -222,12 +218,13 @@ class IndirectCallTransformer // Arguments: // jumpKind - jump kind for the new basic block // insertAfter - basic block, after which compiler has to insert the new one. + // jumpDest - jump target for the new basic block. Defaults to nullptr. // // Return Value: // new basic block. - BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter) + BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter, BasicBlock* jumpDest = nullptr) { - BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true); + BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true, jumpDest); block->SetFlags(BBF_IMPORTED); return block; } @@ -270,35 +267,37 @@ class IndirectCallTransformer assert(compiler->fgPredsComputed); // currBlock + compiler->fgRemoveRefPred(remainderBlock, currBlock); + if (checkBlock != currBlock) { assert(currBlock->KindIs(BBJ_ALWAYS)); + currBlock->SetTarget(checkBlock); FlowEdge* const newEdge = compiler->fgAddRefPred(checkBlock, currBlock); - currBlock->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); } // checkBlock // Todo: get likelihoods right // assert(checkBlock->KindIs(BBJ_ALWAYS)); + checkBlock->SetCond(elseBlock, thenBlock); FlowEdge* const thenEdge = compiler->fgAddRefPred(thenBlock, checkBlock); thenEdge->setLikelihood(0.5); FlowEdge* const elseEdge = compiler->fgAddRefPred(elseBlock, checkBlock); elseEdge->setLikelihood(0.5); - checkBlock->SetCond(elseEdge, thenEdge); // thenBlock + assert(thenBlock->TargetIs(remainderBlock)); { - assert(thenBlock->KindIs(BBJ_ALWAYS)); FlowEdge* const newEdge = compiler->fgAddRefPred(remainderBlock, thenBlock); - thenBlock->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); } // elseBlock { - assert(elseBlock->KindIs(BBJ_ALWAYS)); FlowEdge* const newEdge = compiler->fgAddRefPred(remainderBlock, elseBlock); - elseBlock->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); } } @@ -377,7 +376,7 @@ class IndirectCallTransformer { assert(checkIdx == 0); - checkBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, currBlock); + checkBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, currBlock, currBlock->Next()); checkBlock->SetFlags(BBF_NONE_QUIRK); GenTree* fatPointerMask = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, FAT_POINTER_MASK); GenTree* fptrAddressCopy = compiler->gtCloneExpr(fptrAddress); @@ -396,7 +395,7 @@ class IndirectCallTransformer virtual void CreateThen(uint8_t checkIdx) { assert(remainderBlock != nullptr); - thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); + thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock, remainderBlock); Statement* copyOfOriginalStmt = compiler->gtCloneStmt(stmt); compiler->fgInsertStmtAtEnd(thenBlock, copyOfOriginalStmt); } @@ -406,7 +405,7 @@ class IndirectCallTransformer // virtual void CreateElse() { - elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock); + elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock, thenBlock->Next()); elseBlock->SetFlags(BBF_NONE_QUIRK); GenTree* fixedFptrAddress = GetFixedFptrAddress(); @@ -568,7 +567,10 @@ class IndirectCallTransformer { assert(compiler->fgPredsComputed); - // Chaining is done in-place. + // currBlock + compiler->fgRemoveRefPred(remainderBlock, currBlock); + + // The rest of chaining is done in-place. } virtual void SetWeights() @@ -599,25 +601,23 @@ class IndirectCallTransformer checkBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock); checkFallsThrough = false; - // We computed the "then" likelihood in CreateThen, so we - // just use that to figure out the "else" likelihood. - // - assert(prevCheckBlock->KindIs(BBJ_ALWAYS)); - assert(prevCheckBlock->JumpsToNext()); - FlowEdge* const prevCheckThenEdge = prevCheckBlock->GetTargetEdge(); - assert(prevCheckThenEdge->hasLikelihood()); - weight_t checkLikelihood = max(0.0, 1.0 - prevCheckThenEdge->getLikelihood()); + // Calculate the total likelihood for this check as a sum of likelihoods + // of all previous candidates (thenBlocks) + unsigned checkLikelihood = 100; + for (uint8_t previousCandidate = 0; previousCandidate < checkIdx; previousCandidate++) + { + checkLikelihood -= origCall->GetGDVCandidateInfo(previousCandidate)->likelihood; + } - JITDUMP("Level %u Check block " FMT_BB " success likelihood " FMT_WT "\n", checkIdx, checkBlock->bbNum, - checkLikelihood); + // Make sure we didn't overflow + assert(checkLikelihood <= 100); + weight_t checkLikelihoodWt = ((weight_t)checkLikelihood) / 100.0; // prevCheckBlock is expected to jump to this new check (if its type check doesn't succeed) - // - FlowEdge* const prevCheckCheckEdge = compiler->fgAddRefPred(checkBlock, prevCheckBlock); - prevCheckCheckEdge->setLikelihood(checkLikelihood); - checkBlock->inheritWeight(prevCheckBlock); - checkBlock->scaleBBWeight(checkLikelihood); - prevCheckBlock->SetCond(prevCheckCheckEdge, prevCheckThenEdge); + prevCheckBlock->SetCond(checkBlock, prevCheckBlock->Next()); + FlowEdge* const checkEdge = compiler->fgAddRefPred(checkBlock, prevCheckBlock); + checkEdge->setLikelihood(checkLikelihoodWt); + checkBlock->inheritWeightPercentage(currBlock, checkLikelihood); } // Find last arg with a side effect. All args with any effect @@ -1016,64 +1016,25 @@ class IndirectCallTransformer { // Compute likelihoods // - // If this is the first check the likelihood is just the candidate likelihood. - // If there are multiple checks things are a bit more complicated. - // - // Say we had three candidates with likelihoods 0.5, 0.3, and 0.1. - // - // The first one's likelihood is 0.5. - // - // The second one (given that we've already checked the first and failed) - // is (0.3) / (1.0 - 0.5) = 0.6. - // - // The third one is (0.1) / (1.0 - (0.5 + 0.3)) = (0.1)/(0.2) = 0.5 - // - // So to figure out the proper divisor, we start with 1.0 and subtract off each - // preceeding test's likelihood of success. - // - unsigned const thenLikelihood = origCall->GetGDVCandidateInfo(checkIdx)->likelihood; - unsigned baseLikelihood = 0; - - for (uint8_t i = 0; i < checkIdx; i++) - { - baseLikelihood += origCall->GetGDVCandidateInfo(i)->likelihood; - } - assert(baseLikelihood < 100); - baseLikelihood = 100 - baseLikelihood; - - weight_t adjustedThenLikelihood = min(((weight_t)thenLikelihood) / baseLikelihood, 100.0); - JITDUMP("For check in " FMT_BB ": orig likelihood " FMT_WT ", base likelihood " FMT_WT - ", adjusted likelihood " FMT_WT "\n", - checkBlock->bbNum, (weight_t)thenLikelihood / 100.0, (weight_t)baseLikelihood / 100.0, - adjustedThenLikelihood); + unsigned const thenLikelihood = origCall->GetGDVCandidateInfo(checkIdx)->likelihood; + weight_t thenLikelihoodWt = min(((weight_t)thenLikelihood) / 100.0, 100.0); + weight_t elseLikelihoodWt = max(1.0 - thenLikelihoodWt, 0.0); // thenBlock always jumps to remainderBlock - // - thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); + thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock, remainderBlock); thenBlock->CopyFlags(currBlock, BBF_SPLIT_GAINED); - thenBlock->inheritWeight(currBlock); - thenBlock->scaleBBWeight(adjustedThenLikelihood); - FlowEdge* const thenRemainderEdge = compiler->fgAddRefPred(remainderBlock, thenBlock); - thenBlock->SetTargetEdge(thenRemainderEdge); + thenBlock->inheritWeightPercentage(currBlock, thenLikelihood); - // thenBlock has a single pred - last checkBlock. - // + // Also, thenBlock has a single pred - last checkBlock assert(checkBlock->KindIs(BBJ_ALWAYS)); - FlowEdge* const checkThenEdge = compiler->fgAddRefPred(thenBlock, checkBlock); - checkBlock->SetTargetEdge(checkThenEdge); + checkBlock->SetTarget(thenBlock); checkBlock->SetFlags(BBF_NONE_QUIRK); assert(checkBlock->JumpsToNext()); + FlowEdge* const thenEdge = compiler->fgAddRefPred(thenBlock, checkBlock); + thenEdge->setLikelihood(thenLikelihoodWt); + FlowEdge* const elseEdge = compiler->fgAddRefPred(remainderBlock, thenBlock); + elseEdge->setLikelihood(elseLikelihoodWt); - // SetTargetEdge() gave checkThenEdge a (correct) likelihood of 1.0. - // Later on, we might convert this checkBlock into a BBJ_COND. - // Since we have the adjusted likelihood calculated here, set it prematurely. - // If we leave this block as a BBJ_ALWAYS, we'll assert later that the likelihood is 1.0. - // - checkThenEdge->setLikelihood(adjustedThenLikelihood); - - // We will set the "else edge" likelihood in CreateElse later, - // based on the thenEdge likelihood. - // DevirtualizeCall(thenBlock, checkIdx); } @@ -1082,27 +1043,28 @@ class IndirectCallTransformer // virtual void CreateElse() { - elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock); + // Calculate the likelihood of the else block as a remainder of the sum + // of all the other likelihoods. + unsigned elseLikelihood = 100; + for (uint8_t i = 0; i < origCall->GetInlineCandidatesCount(); i++) + { + elseLikelihood -= origCall->GetGDVCandidateInfo(i)->likelihood; + } + // Make sure it didn't overflow + assert(elseLikelihood <= 100); + weight_t elseLikelihoodDbl = ((weight_t)elseLikelihood) / 100.0; + + elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock, thenBlock->Next()); elseBlock->CopyFlags(currBlock, BBF_SPLIT_GAINED); elseBlock->SetFlags(BBF_NONE_QUIRK); - // We computed the "then" likelihood in CreateThen, so we - // just use that to figure out the "else" likelihood. - // - assert(checkBlock->KindIs(BBJ_ALWAYS)); - FlowEdge* const checkThenEdge = checkBlock->GetTargetEdge(); - assert(checkThenEdge->hasLikelihood()); - weight_t elseLikelihood = max(0.0, 1.0 - checkThenEdge->getLikelihood()); - // CheckBlock flows into elseBlock unless we deal with the case // where we know the last check is always true (in case of "exact" GDV) - // if (!checkFallsThrough) { - assert(checkBlock->JumpsToNext()); - FlowEdge* const checkElseEdge = compiler->fgAddRefPred(elseBlock, checkBlock); - checkElseEdge->setLikelihood(elseLikelihood); - checkBlock->SetCond(checkElseEdge, checkThenEdge); + checkBlock->SetCond(elseBlock, checkBlock->Next()); + FlowEdge* const checkEdge = compiler->fgAddRefPred(elseBlock, checkBlock); + checkEdge->setLikelihood(elseLikelihoodDbl); } else { @@ -1110,21 +1072,16 @@ class IndirectCallTransformer // and is NativeAOT-only, we just assume the unreached block will be removed // by other phases. assert(origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_EXACT); - - // We aren't converting checkBlock to a BBJ_COND. Its successor edge likelihood should remain 1.0. - // - assert(checkThenEdge->getLikelihood() == 1.0); } // elseBlock always flows into remainderBlock - FlowEdge* const elseRemainderEdge = compiler->fgAddRefPred(remainderBlock, elseBlock); - elseBlock->SetTargetEdge(elseRemainderEdge); + FlowEdge* const elseEdge = compiler->fgAddRefPred(remainderBlock, elseBlock); + elseEdge->setLikelihood(1.0); // Remove everything related to inlining from the original call origCall->ClearInlineInfo(); - elseBlock->inheritWeight(checkBlock); - elseBlock->scaleBBWeight(elseLikelihood); + elseBlock->inheritWeightPercentage(currBlock, elseLikelihood); GenTreeCall* call = origCall; Statement* newStmt = compiler->gtNewStmt(call, stmt->GetDebugInfo()); @@ -1219,10 +1176,12 @@ class IndirectCallTransformer // Finally, rewire the cold block to jump to the else block, // not fall through to the check block. // - compiler->fgRedirectTargetEdge(coldBlock, elseBlock); + FlowEdge* const oldEdge = compiler->fgRemoveRefPred(checkBlock, coldBlock); + coldBlock->SetKindAndTarget(BBJ_ALWAYS, elseBlock); + compiler->fgAddRefPred(elseBlock, coldBlock, oldEdge); } - // When the current candidate has sufficiently high likelihood, scan + // When the current candidate hads sufficiently high likelihood, scan // the remainer block looking for another GDV candidate. // // (also consider: if currBlock has sufficiently high execution frequency) diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp deleted file mode 100644 index 59e5b6a0d497de..00000000000000 --- a/src/coreclr/jit/inductionvariableopts.cpp +++ /dev/null @@ -1,686 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// This file contains code to optimize induction variables in loops based on -// scalar evolution analysis (see scev.h and scev.cpp for more information -// about the scalar evolution analysis). -// -// Currently the only optimization done is widening of primary induction -// variables from 32 bits into 64 bits. This is generally only profitable on -// x64 that does not allow zero extension of 32-bit values in addressing modes -// (in contrast, arm64 does have the capability of including zero extensions in -// addressing modes). For x64 this saves a zero extension for every array -// access inside the loop, in exchange for some widening or narrowing stores -// outside the loop: -// - To make sure the new widened IV starts at the right value it is -// initialized to the value of the narrow IV outside the loop (either in the -// preheader or at the def location of the narrow IV). Usually the start -// value is a constant, in which case the widened IV is just initialized to -// the constant value. -// - If the narrow IV is used after the loop we need to store it back from -// the widened IV in the exits. We depend on liveness sets to figure out -// which exits to insert IR into. -// -// These steps ensure that the wide IV has the right value to begin with and -// the old narrow IV still has the right value after the loop. Additionally, -// we must replace every use of the narrow IV inside the loop with the widened -// IV. This is done by a traversal of the IR inside the loop. We do not -// actually widen the uses of the IV; rather, we keep all uses and defs as -// 32-bit, which the backend is able to handle efficiently on x64. Because of -// this we do not need to worry about overflow. -// - -#include "jitpch.h" -#include "scev.h" - -//------------------------------------------------------------------------ -// optCanSinkWidenedIV: Check to see if we are able to sink a store to the old -// local into the exits of a loop if we decide to widen. -// -// Parameters: -// lclNum - The primary induction variable -// loop - The loop -// -// Returns: -// True if we can sink a store to the old local after widening. -// -// Remarks: -// This handles the situation where the primary induction variable is used -// after the loop. In those cases we need to store the widened local back -// into the old one in the exits where the IV variable is live. -// -// We are able to sink when none of the exits are critical blocks, in the -// sense that all their predecessors must come from inside the loop. Loop -// exit canonicalization guarantees this for regular exit blocks. It is not -// guaranteed for exceptional exits, but we do not expect to widen IVs that -// are live into exceptional exits since those are marked DNER which makes it -// unprofitable anyway. -// -// Note that there may be natural loops that have not had their regular exits -// canonicalized at the time when IV opts run, in particular if RBO/assertion -// prop makes a previously unnatural loop natural. This function accounts for -// and rejects these cases. -// -bool Compiler::optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop) -{ - LclVarDsc* dsc = lvaGetDesc(lclNum); - - BasicBlockVisit result = loop->VisitRegularExitBlocks([=](BasicBlock* exit) { - - if (!VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) - { - JITDUMP(" Exit " FMT_BB " does not need a sink; V%02u is not live-in\n", exit->bbNum, lclNum); - return BasicBlockVisit::Continue; - } - - for (BasicBlock* pred : exit->PredBlocks()) - { - if (!loop->ContainsBlock(pred)) - { - JITDUMP(" Cannot safely sink widened version of V%02u into exit " FMT_BB " of " FMT_LP - "; it has a non-loop pred " FMT_BB "\n", - lclNum, exit->bbNum, loop->GetIndex(), pred->bbNum); - return BasicBlockVisit::Abort; - } - } - - return BasicBlockVisit::Continue; - }); - -#ifdef DEBUG - // We currently do not expect to ever widen IVs that are live into - // exceptional exits. Such IVs are expected to have been marked DNER - // previously (EH write-thru is only for single def locals) which makes it - // unprofitable. If this ever changes we need some more expansive handling - // here. - loop->VisitLoopBlocks([=](BasicBlock* block) { - - block->VisitAllSuccs(this, [=](BasicBlock* succ) { - if (!loop->ContainsBlock(succ) && bbIsHandlerBeg(succ)) - { - assert(!VarSetOps::IsMember(this, succ->bbLiveIn, dsc->lvVarIndex) && - "Candidate IV for widening is live into exceptional exit"); - } - - return BasicBlockVisit::Continue; - }); - - return BasicBlockVisit::Continue; - }); -#endif - - return result != BasicBlockVisit::Abort; -} - -//------------------------------------------------------------------------ -// optIsIVWideningProfitable: Check to see if IV widening is profitable. -// -// Parameters: -// lclNum - The primary induction variable -// initBlock - The block in where the new IV would be initialized -// initedToConstant - Whether or not the new IV will be initialized to a constant -// loop - The loop -// ivUses - Statements in which "lclNum" appears will be added to this list -// -// -// Returns: -// True if IV widening is profitable. -// -// Remarks: -// IV widening is generally profitable when it allows us to remove casts -// inside the loop. However, it may also introduce other reg-reg moves: -// 1. We may need to store the narrow IV into the wide one in the -// preheader. This is necessary when the start value is not constant. If -// the start value _is_ constant then we assume that the constant store to -// the narrow local will be a DCE'd. -// 2. We need to store the wide IV back into the narrow one in each of -// the exits where the narrow IV is live-in. -// -bool Compiler::optIsIVWideningProfitable(unsigned lclNum, - BasicBlock* initBlock, - bool initedToConstant, - FlowGraphNaturalLoop* loop, - ArrayStack& ivUses) -{ - for (FlowGraphNaturalLoop* otherLoop : m_loops->InReversePostOrder()) - { - if (otherLoop == loop) - continue; - - for (Statement* stmt : otherLoop->GetHeader()->Statements()) - { - if (!stmt->IsPhiDefnStmt()) - break; - - if (stmt->GetRootNode()->AsLclVarCommon()->GetLclNum() == lclNum) - { - JITDUMP(" V%02u has a phi [%06u] in " FMT_LP "'s header " FMT_BB "\n", lclNum, - dspTreeID(stmt->GetRootNode()), otherLoop->GetIndex(), otherLoop->GetHeader()->bbNum); - // TODO-CQ: We can legally widen these cases, but LSRA is - // unhappy about some of the lifetimes we create when we do - // this. This particularly affects cloned loops. - return false; - } - } - } - - const weight_t ExtensionCost = 2; - const int ExtensionSize = 3; - - weight_t savedCost = 0; - int savedSize = 0; - - loop->VisitLoopBlocks([&](BasicBlock* block) { - for (Statement* stmt : block->NonPhiStatements()) - { - bool hasUse = false; - int numExtensions = 0; - for (GenTree* node : stmt->TreeList()) - { - if (!node->OperIs(GT_CAST)) - { - hasUse |= node->OperIsLocal() && (node->AsLclVarCommon()->GetLclNum() == lclNum); - continue; - } - - GenTreeCast* cast = node->AsCast(); - if ((cast->gtCastType != TYP_LONG) || !cast->IsUnsigned() || cast->gtOverflow()) - { - continue; - } - - GenTree* op = cast->CastOp(); - if (!op->OperIs(GT_LCL_VAR) || (op->AsLclVarCommon()->GetLclNum() != lclNum)) - { - continue; - } - - // If this is already the source of a store then it is going to be - // free in our backends regardless. - GenTree* parent = node->gtGetParent(nullptr); - if ((parent != nullptr) && parent->OperIs(GT_STORE_LCL_VAR)) - { - continue; - } - - numExtensions++; - } - - if (hasUse) - { - ivUses.Push(stmt); - } - - if (numExtensions > 0) - { - JITDUMP(" Found %d zero extensions in " FMT_STMT "\n", numExtensions, stmt->GetID()); - - savedSize += numExtensions * ExtensionSize; - savedCost += numExtensions * block->getBBWeight(this) * ExtensionCost; - } - } - - return BasicBlockVisit::Continue; - }); - - if (!initedToConstant) - { - // We will need to store the narrow IV into the wide one in the init - // block. We only cost this when init value is not a constant since - // otherwise we assume that constant initialization of the narrow local - // will be DCE'd. - savedSize -= ExtensionSize; - savedCost -= initBlock->getBBWeight(this) * ExtensionCost; - } - - // Now account for the cost of sinks. - LclVarDsc* dsc = lvaGetDesc(lclNum); - loop->VisitRegularExitBlocks([&](BasicBlock* exit) { - if (VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) - { - savedSize -= ExtensionSize; - savedCost -= exit->getBBWeight(this) * ExtensionCost; - } - return BasicBlockVisit::Continue; - }); - - const weight_t ALLOWED_SIZE_REGRESSION_PER_CYCLE_IMPROVEMENT = 2; - weight_t cycleImprovementPerInvoc = savedCost / fgFirstBB->getBBWeight(this); - - JITDUMP(" Estimated cycle improvement: " FMT_WT " cycles per invocation\n", cycleImprovementPerInvoc); - JITDUMP(" Estimated size improvement: %d bytes\n", savedSize); - - if ((cycleImprovementPerInvoc > 0) && - ((cycleImprovementPerInvoc * ALLOWED_SIZE_REGRESSION_PER_CYCLE_IMPROVEMENT) >= -savedSize)) - { - JITDUMP(" Widening is profitable (cycle improvement)\n"); - return true; - } - - const weight_t ALLOWED_CYCLE_REGRESSION_PER_SIZE_IMPROVEMENT = 0.01; - - if ((savedSize > 0) && ((savedSize * ALLOWED_CYCLE_REGRESSION_PER_SIZE_IMPROVEMENT) >= -cycleImprovementPerInvoc)) - { - JITDUMP(" Widening is profitable (size improvement)\n"); - return true; - } - - JITDUMP(" Widening is not profitable\n"); - return false; -} - -//------------------------------------------------------------------------ -// optSinkWidenedIV: Create stores back to the narrow IV in the exits where -// that is necessary. -// -// Parameters: -// lclNum - Narrow version of primary induction variable -// newLclNum - Wide version of primary induction variable -// loop - The loop -// -// Returns: -// True if any store was created in any exit block. -// -void Compiler::optSinkWidenedIV(unsigned lclNum, unsigned newLclNum, FlowGraphNaturalLoop* loop) -{ - LclVarDsc* dsc = lvaGetDesc(lclNum); - loop->VisitRegularExitBlocks([=](BasicBlock* exit) { - if (!VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) - { - return BasicBlockVisit::Continue; - } - - GenTree* narrowing = gtNewCastNode(TYP_INT, gtNewLclvNode(newLclNum, TYP_LONG), false, TYP_INT); - GenTree* store = gtNewStoreLclVarNode(lclNum, narrowing); - Statement* newStmt = fgNewStmtFromTree(store); - JITDUMP("Narrow IV local V%02u live into exit block " FMT_BB "; sinking a narrowing\n", lclNum, exit->bbNum); - DISPSTMT(newStmt); - fgInsertStmtAtBeg(exit, newStmt); - - return BasicBlockVisit::Continue; - }); -} - -//------------------------------------------------------------------------ -// optReplaceWidenedIV: Replace uses of the narrow IV with the wide IV in the -// specified statement. -// -// Parameters: -// lclNum - Narrow version of primary induction variable -// newLclNum - Wide version of primary induction variable -// stmt - The statement to replace uses in. -// -void Compiler::optReplaceWidenedIV(unsigned lclNum, unsigned ssaNum, unsigned newLclNum, Statement* stmt) -{ - struct ReplaceVisitor : GenTreeVisitor - { - private: - unsigned m_lclNum; - unsigned m_ssaNum; - unsigned m_newLclNum; - - bool IsLocal(GenTreeLclVarCommon* tree) - { - return (tree->GetLclNum() == m_lclNum) && - ((m_ssaNum == SsaConfig::RESERVED_SSA_NUM) || (tree->GetSsaNum() == m_ssaNum)); - } - - public: - bool MadeChanges = false; - - enum - { - DoPreOrder = true, - }; - - ReplaceVisitor(Compiler* comp, unsigned lclNum, unsigned ssaNum, unsigned newLclNum) - : GenTreeVisitor(comp), m_lclNum(lclNum), m_ssaNum(ssaNum), m_newLclNum(newLclNum) - { - } - - fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) - { - GenTree* node = *use; - if (node->OperIs(GT_CAST)) - { - GenTreeCast* cast = node->AsCast(); - if ((cast->gtCastType == TYP_LONG) && cast->IsUnsigned() && !cast->gtOverflow()) - { - GenTree* op = cast->CastOp(); - if (op->OperIs(GT_LCL_VAR) && IsLocal(op->AsLclVarCommon())) - { - *use = m_compiler->gtNewLclvNode(m_newLclNum, TYP_LONG); - MadeChanges = true; - return fgWalkResult::WALK_SKIP_SUBTREES; - } - } - } - else if (node->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_LCL_FLD, GT_STORE_LCL_FLD) && - IsLocal(node->AsLclVarCommon())) - { - switch (node->OperGet()) - { - case GT_LCL_VAR: - node->AsLclVarCommon()->SetLclNum(m_newLclNum); - // No cast needed -- the backend allows TYP_INT uses of TYP_LONG locals. - break; - case GT_STORE_LCL_VAR: - { - node->AsLclVarCommon()->SetLclNum(m_newLclNum); - node->gtType = TYP_LONG; - node->AsLclVarCommon()->Data() = - m_compiler->gtNewCastNode(TYP_LONG, node->AsLclVarCommon()->Data(), true, TYP_LONG); - break; - } - case GT_LCL_FLD: - case GT_STORE_LCL_FLD: - assert(!"Unexpected field use for local not marked as DNER"); - break; - default: - break; - } - - MadeChanges = true; - } - - return fgWalkResult::WALK_CONTINUE; - } - }; - - ReplaceVisitor visitor(this, lclNum, ssaNum, newLclNum); - visitor.WalkTree(stmt->GetRootNodePointer(), nullptr); - if (visitor.MadeChanges) - { - gtSetStmtInfo(stmt); - fgSetStmtSeq(stmt); - JITDUMP("New tree:\n", dspTreeID(stmt->GetRootNode())); - DISPTREE(stmt->GetRootNode()); - JITDUMP("\n"); - } - else - { - JITDUMP("No replacements made\n"); - } -} - -//------------------------------------------------------------------------ -// optBestEffortReplaceNarrowIVUses: Try to find and replace uses of the specified -// SSA def with a new local. -// -// Parameters: -// lclNum - Previous local -// ssaNum - Previous local SSA num -// newLclNum - New local to replace with -// block - Block to replace in -// firstStmt - First statement in "block" to start replacing in -// -// Remarks: -// This function is best effort; it might not find all uses of the provided -// SSA num, particularly because it does not follow into joins. Note that we -// only use this to replace uses of the narrow IV outside the loop; inside -// the loop we do ensure that all uses/defs are replaced. -// Keeping it best-effort outside the loop is ok; there is no correctness -// issue since we do not invalidate the value of the old narrow IV in any -// way, but it may mean we end up leaving the narrow IV live concurrently -// with the new widened IV, increasing register pressure. -// -void Compiler::optBestEffortReplaceNarrowIVUses( - unsigned lclNum, unsigned ssaNum, unsigned newLclNum, BasicBlock* block, Statement* firstStmt) -{ - JITDUMP("Replacing V%02u -> V%02u in " FMT_BB " starting at " FMT_STMT "\n", lclNum, newLclNum, block->bbNum, - firstStmt == nullptr ? 0 : firstStmt->GetID()); - - for (Statement* stmt = firstStmt; stmt != nullptr; stmt = stmt->GetNextStmt()) - { - JITDUMP("Replacing V%02u -> V%02u in [%06u]\n", lclNum, newLclNum, dspTreeID(stmt->GetRootNode())); - DISPSTMT(stmt); - JITDUMP("\n"); - - optReplaceWidenedIV(lclNum, ssaNum, newLclNum, stmt); - } - - block->VisitRegularSuccs(this, [=](BasicBlock* succ) { - if (succ->GetUniquePred(this) == block) - { - optBestEffortReplaceNarrowIVUses(lclNum, ssaNum, newLclNum, succ, succ->firstStmt()); - } - - return BasicBlockVisit::Continue; - }); -} - -//------------------------------------------------------------------------ -// optInductionVariables: Try and optimize induction variables in the method. -// -// Returns: -// PhaseStatus indicating if anything changed. -// -PhaseStatus Compiler::optInductionVariables() -{ - JITDUMP("*************** In optInductionVariables()\n"); - -#ifdef DEBUG - static ConfigMethodRange s_range; - s_range.EnsureInit(JitConfig.JitEnableInductionVariableOptsRange()); - - if (!s_range.Contains(info.compMethodHash())) - { - return PhaseStatus::MODIFIED_NOTHING; - } -#endif - - if (!fgMightHaveNaturalLoops) - { - JITDUMP(" Skipping since this method has no natural loops\n"); - return PhaseStatus::MODIFIED_NOTHING; - } - - bool changed = false; - - // Currently we only do IV widening which generally is only profitable for - // x64 because arm64 addressing modes can include the zero/sign-extension - // of the index for free. - CLANG_FORMAT_COMMENT_ANCHOR; -#if defined(TARGET_XARCH) && defined(TARGET_64BIT) - m_dfsTree = fgComputeDfs(); - m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); - - ScalarEvolutionContext scevContext(this); - JITDUMP("Widening primary induction variables:\n"); - ArrayStack ivUses(getAllocator(CMK_LoopIVOpts)); - for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) - { - JITDUMP("Processing "); - DBEXEC(verbose, FlowGraphNaturalLoop::Dump(loop)); - scevContext.ResetForLoop(loop); - - int numWidened = 0; - - for (Statement* stmt : loop->GetHeader()->Statements()) - { - if (!stmt->IsPhiDefnStmt()) - { - break; - } - - JITDUMP("\n"); - - DISPSTMT(stmt); - - GenTreeLclVarCommon* lcl = stmt->GetRootNode()->AsLclVarCommon(); - LclVarDsc* lclDsc = lvaGetDesc(lcl); - if (lclDsc->TypeGet() != TYP_INT) - { - JITDUMP(" Type is %s, no widening to be done\n", varTypeName(lclDsc->TypeGet())); - continue; - } - - // If the IV is not enregisterable then uses/defs are going to go - // to stack regardless. This check also filters out IVs that may be - // live into exceptional exits since those are always marked DNER. - if (lclDsc->lvDoNotEnregister) - { - JITDUMP(" V%02u is marked DNER\n", lcl->GetLclNum()); - continue; - } - - Scev* scev = scevContext.Analyze(loop->GetHeader(), stmt->GetRootNode()); - if (scev == nullptr) - { - JITDUMP(" Could not analyze header PHI\n"); - continue; - } - - scev = scevContext.Simplify(scev); - JITDUMP(" => "); - DBEXEC(verbose, scev->Dump(this)); - JITDUMP("\n"); - if (!scev->OperIs(ScevOper::AddRec)) - { - JITDUMP(" Not an addrec\n"); - continue; - } - - ScevAddRec* addRec = (ScevAddRec*)scev; - - JITDUMP(" V%02u is a primary induction variable in " FMT_LP "\n", lcl->GetLclNum(), loop->GetIndex()); - - if (!optCanSinkWidenedIV(lcl->GetLclNum(), loop)) - { - continue; - } - - // Start value should always be an SSA use from outside the loop - // since we only widen primary IVs. - assert(addRec->Start->OperIs(ScevOper::Local)); - ScevLocal* startLocal = (ScevLocal*)addRec->Start; - int64_t startConstant = 0; - bool initToConstant = startLocal->GetConstantValue(this, &startConstant); - LclSsaVarDsc* startSsaDsc = lclDsc->GetPerSsaData(startLocal->SsaNum); - - BasicBlock* preheader = loop->EntryEdge(0)->getSourceBlock(); - BasicBlock* initBlock = preheader; - if ((startSsaDsc->GetBlock() != nullptr) && (startSsaDsc->GetDefNode() != nullptr)) - { - initBlock = startSsaDsc->GetBlock(); - } - - ivUses.Reset(); - if (!optIsIVWideningProfitable(lcl->GetLclNum(), initBlock, initToConstant, loop, ivUses)) - { - continue; - } - - changed = true; - - Statement* insertInitAfter = nullptr; - if (initBlock != preheader) - { - GenTree* narrowInitRoot = startSsaDsc->GetDefNode(); - while (true) - { - GenTree* parent = narrowInitRoot->gtGetParent(nullptr); - if (parent == nullptr) - break; - - narrowInitRoot = parent; - } - - for (Statement* stmt : initBlock->Statements()) - { - if (stmt->GetRootNode() == narrowInitRoot) - { - insertInitAfter = stmt; - break; - } - } - - assert(insertInitAfter != nullptr); - - if (insertInitAfter->IsPhiDefnStmt()) - { - while ((insertInitAfter->GetNextStmt() != nullptr) && - insertInitAfter->GetNextStmt()->IsPhiDefnStmt()) - { - insertInitAfter = insertInitAfter->GetNextStmt(); - } - } - } - - Statement* initStmt = nullptr; - unsigned newLclNum = lvaGrabTemp(false DEBUGARG(printfAlloc("Widened IV V%02u", lcl->GetLclNum()))); - INDEBUG(lclDsc = nullptr); - assert(startLocal->LclNum == lcl->GetLclNum()); - - if (initBlock != preheader) - { - JITDUMP("Adding initialization of new widened local to same block as reaching def outside loop, " FMT_BB - "\n", - initBlock->bbNum); - } - else - { - JITDUMP("Adding initialization of new widened local to preheader " FMT_BB "\n", initBlock->bbNum); - } - - GenTree* initVal; - if (initToConstant) - { - initVal = gtNewIconNode((int64_t)(uint32_t)startConstant, TYP_LONG); - } - else - { - initVal = gtNewCastNode(TYP_LONG, gtNewLclvNode(lcl->GetLclNum(), TYP_INT), true, TYP_LONG); - } - - GenTree* widenStore = gtNewTempStore(newLclNum, initVal); - initStmt = fgNewStmtFromTree(widenStore); - if (insertInitAfter != nullptr) - { - fgInsertStmtAfter(initBlock, insertInitAfter, initStmt); - } - else - { - fgInsertStmtNearEnd(initBlock, initStmt); - } - - DISPSTMT(initStmt); - JITDUMP("\n"); - - JITDUMP(" Replacing uses of V%02u with widened version V%02u\n", lcl->GetLclNum(), newLclNum); - - if (initStmt != nullptr) - { - JITDUMP(" Replacing on the way to the loop\n"); - optBestEffortReplaceNarrowIVUses(lcl->GetLclNum(), startLocal->SsaNum, newLclNum, initBlock, - initStmt->GetNextStmt()); - } - - JITDUMP(" Replacing in the loop; %d statements with appearances\n", ivUses.Height()); - for (int i = 0; i < ivUses.Height(); i++) - { - Statement* stmt = ivUses.Bottom(i); - JITDUMP("Replacing V%02u -> V%02u in [%06u]\n", lcl->GetLclNum(), newLclNum, - dspTreeID(stmt->GetRootNode())); - DISPSTMT(stmt); - JITDUMP("\n"); - optReplaceWidenedIV(lcl->GetLclNum(), SsaConfig::RESERVED_SSA_NUM, newLclNum, stmt); - } - - optSinkWidenedIV(lcl->GetLclNum(), newLclNum, loop); - - numWidened++; - } - - Metrics.WidenedIVs += numWidened; - if (numWidened > 0) - { - Metrics.LoopsIVWidened++; - } - } - - fgInvalidateDfsTree(); -#endif - - return changed ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; -} diff --git a/src/coreclr/jit/inline.h b/src/coreclr/jit/inline.h index 342dc3fca5d238..dca92e39241e49 100644 --- a/src/coreclr/jit/inline.h +++ b/src/coreclr/jit/inline.h @@ -686,7 +686,6 @@ struct InlineInfo unsigned argCnt; InlArgInfo inlArgInfo[MAX_INL_ARGS + 1]; - InlArgInfo* inlInstParamArgInfo; int lclTmpNum[MAX_INL_LCLS]; // map local# -> temp# (-1 if unused) InlLclVarInfo lclVarInfo[MAX_INL_LCLS + MAX_INL_ARGS + 1]; // type information from local sig diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 75e712504a19f2..caee21bff8d869 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -916,14 +916,18 @@ CodeGen::OperandDesc CodeGen::genOperandDesc(GenTree* op) #endif // FEATURE_HW_INTRINSICS } - if (addr->isContained() && addr->OperIs(GT_LCL_ADDR)) + switch (addr->OperGet()) { - varNum = addr->AsLclFld()->GetLclNum(); - offset = addr->AsLclFld()->GetLclOffs(); - } - else - { - return (memIndir != nullptr) ? OperandDesc(memIndir) : OperandDesc(op->TypeGet(), addr); + case GT_LCL_ADDR: + { + assert(addr->isContained()); + varNum = addr->AsLclFld()->GetLclNum(); + offset = addr->AsLclFld()->GetLclOffs(); + break; + } + + default: + return (memIndir != nullptr) ? OperandDesc(memIndir) : OperandDesc(op->TypeGet(), addr); } } else @@ -989,13 +993,6 @@ CodeGen::OperandDesc CodeGen::genOperandDesc(GenTree* op) memcpy(&constValue, &op->AsVecCon()->gtSimdVal, sizeof(simd64_t)); return OperandDesc(emit->emitSimd64Const(constValue)); } - - case TYP_MASK: - { - simdmask_t constValue; - memcpy(&constValue, &op->AsVecCon()->gtSimdVal, sizeof(simdmask_t)); - return OperandDesc(emit->emitSimdMaskConst(constValue)); - } #endif // TARGET_XARCH #endif // FEATURE_SIMD diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index c23cb258d46315..bdbddc60160cf0 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -370,22 +370,16 @@ enum insScalableOpts : unsigned INS_SCALABLE_OPTS_WITH_PREDICATE_PAIR, // Variants with {., .} predicate pair (eg whilege) INS_SCALABLE_OPTS_VL_2X, // Variants with a vector length specifier of 2x (eg whilege) INS_SCALABLE_OPTS_VL_4X, // Variants with a vector length specifier of 4x (eg whilege) + INS_SCALABLE_OPTS_SHIFT, // Variants with an optional shift operation (eg dup) INS_SCALABLE_OPTS_LSL_N, // Variants with a LSL #N (eg {.}, , [, , LSL #2]) INS_SCALABLE_OPTS_MOD_N, // Variants with a #N (eg {.S }, , [, .S, #2]) INS_SCALABLE_OPTS_WITH_VECTOR_PAIR, // Variants with {., .} sve register pair (eg splice) - INS_SCALABLE_OPTS_IMM_BITMASK, // Variants with an immediate that is a bitmask - - INS_SCALABLE_OPTS_IMM_FIRST, // Variants with an immediate and a register, where the immediate comes first - // Removable once REG_V0 and REG_P0 are distinct INS_SCALABLE_OPTS_UNPREDICATED, // Variants without a predicate (eg add) INS_SCALABLE_OPTS_UNPREDICATED_WIDE, // Variants without a predicate and wide elements (eg asr) - INS_SCALABLE_OPTS_TO_PREDICATE, // Variants moving to a predicate from a vector (e.g. pmov) - INS_SCALABLE_OPTS_TO_VECTOR, // Variants moving to a vector from a predicate (e.g. pmov) - INS_SCALABLE_OPTS_BROADCAST // Used to distinguish mov from cpy, where mov is an alias for both }; // Maps directly to the pattern used in SVE instructions such as cntb. @@ -405,33 +399,11 @@ enum insSvePattern : unsigned SVE_PATTERN_VL64 = 11, // 64 elements. SVE_PATTERN_VL128 = 12, // 128 elements. SVE_PATTERN_VL256 = 13, // 256 elements. - SVE_PATTERN_MUL4 = 29, // The largest multiple of 4. - SVE_PATTERN_MUL3 = 30, // The largest multiple of 3. + SVE_PATTERN_MUL4 = 29, // The largest multiple of 3. + SVE_PATTERN_MUL3 = 30, // The largest multiple of 4. SVE_PATTERN_ALL = 31 // All available (implicitly a multiple of two). }; -// Prefetch operation specifier for SVE instructions such as prfb. -enum insSvePrfop : unsigned -{ - SVE_PRFOP_PLDL1KEEP = 0b0000, - SVE_PRFOP_PLDL1STRM = 0b0001, - SVE_PRFOP_PLDL2KEEP = 0b0010, - SVE_PRFOP_PLDL2STRM = 0b0011, - SVE_PRFOP_PLDL3KEEP = 0b0100, - SVE_PRFOP_PLDL3STRM = 0b0101, - SVE_PRFOP_PSTL1KEEP = 0b1000, - SVE_PRFOP_PSTL1STRM = 0b1001, - SVE_PRFOP_PSTL2KEEP = 0b1010, - SVE_PRFOP_PSTL2STRM = 0b1011, - SVE_PRFOP_PSTL3KEEP = 0b1100, - SVE_PRFOP_PSTL3STRM = 0b1101, - - SVE_PRFOP_CONST6 = 0b0110, - SVE_PRFOP_CONST7 = 0b0111, - SVE_PRFOP_CONST14 = 0b1110, - SVE_PRFOP_CONST15 = 0b1111 -}; - enum insCond : unsigned { INS_COND_EQ, diff --git a/src/coreclr/jit/instrsarm64sve.h b/src/coreclr/jit/instrsarm64sve.h index f7209c6df98a58..0c0a4019237149 100644 --- a/src/coreclr/jit/instrsarm64sve.h +++ b/src/coreclr/jit/instrsarm64sve.h @@ -239,6 +239,7 @@ INST7(ld1sw, "ld1sw", 0, IF_SV // LD1SW {.D }, /Z, [, .D] SVE_IU_4B_B 11000101010mmmmm 100gggnnnnnttttt C540 8000 // LD1SW {.D }, /Z, [.D{, #}] SVE_IV_3A 11000101001iiiii 100gggnnnnnttttt C520 8000 + // enum name info SVE_AE_3A SVE_BD_3A SVE_EE_1A SVE_FD_3A SVE_FD_3B SVE_FD_3C INST6(mul, "mul", 0, IF_SVE_6A, 0x04100000, 0x04206000, 0x2530C000, 0x4420F800, 0x44A0F800, 0x44E0F800 ) // MUL ., /M, ., . SVE_AE_3A 00000100xx010000 000gggmmmmmddddd 0410 0000 @@ -284,7 +285,7 @@ INST6(prfb, "prfb", 0, IF_SV // PRFB , , [, .S, ] SVE_HY_3A 100001000h1mmmmm 000gggnnnnn0oooo 8420 0000 // PRFB , , [, .D, ] SVE_HY_3A_A 110001000h1mmmmm 000gggnnnnn0oooo C420 0000 // PRFB , , [, .D] SVE_HY_3B 11000100011mmmmm 100gggnnnnn0oooo C460 8000 - // PRFB , , [.S{, #}] SVE_HZ_2A_B 10000100000iiiii 111gggnnnnn0oooo 8400 E000 + // PRFB , , [.D{, #}] SVE_HZ_2A_B 10000100000iiiii 111gggnnnnn0oooo 8400 E000 // PRFB , , [{, #, MUL VL}] SVE_IA_2A 1000010111iiiiii 000gggnnnnn0oooo 85C0 0000 // PRFB , , [, ] SVE_IB_3A 10000100000mmmmm 110gggnnnnn0oooo 8400 C000 @@ -292,7 +293,7 @@ INST6(prfd, "prfd", 0, IF_SV // PRFD , , [, .S, #3] SVE_HY_3A 100001000h1mmmmm 011gggnnnnn0oooo 8420 6000 // PRFD , , [, .D, #3] SVE_HY_3A_A 110001000h1mmmmm 011gggnnnnn0oooo C420 6000 // PRFD , , [, .D, LSL #3] SVE_HY_3B 11000100011mmmmm 111gggnnnnn0oooo C460 E000 - // PRFD , , [.S{, #}] SVE_HZ_2A_B 10000101100iiiii 111gggnnnnn0oooo 8580 E000 + // PRFD , , [.D{, #}] SVE_HZ_2A_B 10000101100iiiii 111gggnnnnn0oooo 8580 E000 // PRFD , , [{, #, MUL VL}] SVE_IA_2A 1000010111iiiiii 011gggnnnnn0oooo 85C0 6000 // PRFD , , [, , LSL #3] SVE_IB_3A 10000101100mmmmm 110gggnnnnn0oooo 8580 C000 @@ -300,7 +301,7 @@ INST6(prfh, "prfh", 0, IF_SV // PRFH , , [, .S, #1] SVE_HY_3A 100001000h1mmmmm 001gggnnnnn0oooo 8420 2000 // PRFH , , [, .D, #1] SVE_HY_3A_A 110001000h1mmmmm 001gggnnnnn0oooo C420 2000 // PRFH , , [, .D, LSL #1] SVE_HY_3B 11000100011mmmmm 101gggnnnnn0oooo C460 A000 - // PRFH , , [.S{, #}] SVE_HZ_2A_B 10000100100iiiii 111gggnnnnn0oooo 8480 E000 + // PRFH , , [.D{, #}] SVE_HZ_2A_B 10000100100iiiii 111gggnnnnn0oooo 8480 E000 // PRFH , , [{, #, MUL VL}] SVE_IA_2A 1000010111iiiiii 001gggnnnnn0oooo 85C0 2000 // PRFH , , [, , LSL #1] SVE_IB_3A 10000100100mmmmm 110gggnnnnn0oooo 8480 C000 @@ -308,7 +309,7 @@ INST6(prfw, "prfw", 0, IF_SV // PRFW , , [, .S, #2] SVE_HY_3A 100001000h1mmmmm 010gggnnnnn0oooo 8420 4000 // PRFW , , [, .D, #2] SVE_HY_3A_A 110001000h1mmmmm 010gggnnnnn0oooo C420 4000 // PRFW , , [, .D, LSL #2] SVE_HY_3B 11000100011mmmmm 110gggnnnnn0oooo C460 C000 - // PRFW , , [.S{, #}] SVE_HZ_2A_B 10000101000iiiii 111gggnnnnn0oooo 8500 E000 + // PRFW , , [.D{, #}] SVE_HZ_2A_B 10000101000iiiii 111gggnnnnn0oooo 8500 E000 // PRFW , , [{, #, MUL VL}] SVE_IA_2A 1000010111iiiiii 010gggnnnnn0oooo 85C0 4000 // PRFW , , [, , LSL #2] SVE_IB_3A 10000101000mmmmm 110gggnnnnn0oooo 8500 C000 @@ -441,6 +442,32 @@ INST4(fmov, "fmov", 0, IF_SV // FMOV ., #0.0 SVE_EB_1B 00100101xx111000 11000000000ddddd 2538 C000 +// enum name info SVE_HS_3A SVE_HS_3A_H SVE_HS_3A_I SVE_HS_3A_J +INST4(scvtf, "scvtf", 0, IF_SVE_4C, 0x6594A000, 0x65D0A000, 0x65D4A000, 0x65D6A000 ) + // SCVTF .S, /M, .S SVE_HS_3A 0110010110010100 101gggnnnnnddddd 6594 A000 + // SCVTF .D, /M, .S SVE_HS_3A_H 0110010111010000 101gggnnnnnddddd 65D0 A000 + // SCVTF .S, /M, .D SVE_HS_3A_I 0110010111010100 101gggnnnnnddddd 65D4 A000 + // SCVTF .D, /M, .D SVE_HS_3A_J 0110010111010110 101gggnnnnnddddd 65D6 A000 + +INST4(ucvtf, "ucvtf", 0, IF_SVE_4C, 0x6595A000, 0x65D1A000, 0x65D5A000, 0x65D7A000 ) + // UCVTF .S, /M, .S SVE_HS_3A 0110010110010101 101gggnnnnnddddd 6595 A000 + // UCVTF .D, /M, .S SVE_HS_3A_H 0110010111010001 101gggnnnnnddddd 65D1 A000 + // UCVTF .S, /M, .D SVE_HS_3A_I 0110010111010101 101gggnnnnnddddd 65D5 A000 + // UCVTF .D, /M, .D SVE_HS_3A_J 0110010111010111 101gggnnnnnddddd 65D7 A000 + + +// enum name info SVE_HP_3B SVE_HP_3B_H SVE_HP_3B_I SVE_HP_3B_J +INST4(fcvtzs, "fcvtzs", 0, IF_SVE_4D, 0x659CA000, 0x65DCA000, 0x65D8A000, 0x65DEA000 ) + // FCVTZS .S, /M, .S SVE_HP_3B 0110010110011100 101gggnnnnnddddd 659C A000 + // FCVTZS .D, /M, .S SVE_HP_3B_H 0110010111011100 101gggnnnnnddddd 65DC A000 + // FCVTZS .S, /M, .D SVE_HP_3B_I 0110010111011000 101gggnnnnnddddd 65D8 A000 + // FCVTZS .D, /M, .D SVE_HP_3B_J 0110010111011110 101gggnnnnnddddd 65DE A000 + +INST4(fcvtzu, "fcvtzu", 0, IF_SVE_4D, 0x659DA000, 0x65DDA000, 0x65D9A000, 0x65DFA000 ) + // FCVTZU .S, /M, .S SVE_HP_3B 0110010110011101 101gggnnnnnddddd 659D A000 + // FCVTZU .D, /M, .S SVE_HP_3B_H 0110010111011101 101gggnnnnnddddd 65DD A000 + // FCVTZU .S, /M, .D SVE_HP_3B_I 0110010111011001 101gggnnnnnddddd 65D9 A000 + // FCVTZU .D, /M, .D SVE_HP_3B_J 0110010111011111 101gggnnnnnddddd 65DF A000 // enum name info SVE_BE_3A SVE_FI_3A SVE_FI_3B SVE_FI_3C @@ -1094,6 +1121,12 @@ INST2(not, "not", 0, IF_SV // NOT .B, /Z, .B SVE_CZ_4A 001001010000MMMM 01gggg1NNNN0DDDD 2500 4200 +// enum name info SVE_HO_3A SVE_HO_3A_B +INST2(fcvt, "fcvt", 0, IF_SVE_2AS, 0x65CBA000, 0x65CAA000 ) + // FCVT .D, /M, .S SVE_HO_3A 0110010111001011 101gggnnnnnddddd 65CB A000 + // FCVT .S, /M, .D SVE_HO_3A_B 0110010111001010 101gggnnnnnddddd 65CA A000 + + // enum name info SVE_AB_3A SVE_EC_1A INST2(subr, "subr", 0, IF_SVE_2AT, 0x04030000, 0x2523C000 ) // SUBR ., /M, ., . SVE_AB_3A 00000100xx000011 000gggmmmmmddddd 0403 0000 @@ -1790,6 +1823,7 @@ INST1(frsqrts, "frsqrts", 0, IF_SV INST1(ftsmul, "ftsmul", 0, IF_SVE_HK_3A, 0x65000C00 ) // FTSMUL ., ., . SVE_HK_3A 01100101xx0mmmmm 000011nnnnnddddd 6500 0C00 + // enum name info SVE_HT_4A INST1(facge, "facge", 0, IF_SVE_HT_4A, 0x6500C010 ) // FACGE ., /Z, ., . SVE_HT_4A 01100101xx0mmmmm 110gggnnnnn1DDDD 6500 C010 @@ -2023,6 +2057,14 @@ INST1(xar, "xar", 0, IF_SV // XAR ., ., ., # SVE_AW_2A 00000100xx1xxiii 001101mmmmmddddd 0420 3400 +// enum name info SVE_HO_3A +INST1(bfcvt, "bfcvt", 0, IF_SVE_HO_3A, 0x658AA000 ) + // BFCVT .H, /M, .S SVE_HO_3A 0110010110001010 101gggnnnnnddddd 658A A000 + +INST1(fcvtx, "fcvtx", 0, IF_SVE_HO_3A, 0x650AA000 ) + // FCVTX .S, /M, .D SVE_HO_3A 0110010100001010 101gggnnnnnddddd 650A A000 + + // enum name info SVE_AF_3A INST1(andv, "andv", 0, IF_SVE_AF_3A, 0x041A2000 ) // ANDV , , . SVE_AF_3A 00000100xx011010 001gggnnnnnddddd 041A 2000 @@ -2716,35 +2758,10 @@ INST1(ftmad, "ftmad", 0, IF_SV // FTMAD ., ., ., # SVE_HN_2A 01100101xx010iii 100000mmmmmddddd 6510 8000 -// enum name info SVE_HO_3A -INST1(bfcvt, "bfcvt", 0, IF_SVE_HO_3A, 0x658AA000 ) - // BFCVT .H, /M, .S SVE_HO_3A 0110010110001010 101gggnnnnnddddd 658A A000 - -// enum name info SVE_HO_3B -INST1(fcvt, "fcvt", 0, IF_SVE_HO_3B, 0x6588A000) - // FCVT .D, /M, .S SVE_HO_3B 0110010110001000 101gggnnnnnddddd 6588 A000 - -// enum name info SVE_HO_3C -INST1(fcvtx, "fcvtx", 0, IF_SVE_HO_3C, 0x650AA000 ) - // FCVTX .S, /M, .D SVE_HO_3C 0110010100001010 101gggnnnnnddddd 650A A000 - // enum name info SVE_HP_3A INST1(flogb, "flogb", 0, IF_SVE_HP_3A, 0x6518A000 ) // FLOGB ., /M, . SVE_HP_3A 0110010100011xx0 101gggnnnnnddddd 6518 A000 -// enum name info SVE_HP_3B -INST1(fcvtzs, "fcvtzs", 0, IF_SVE_HP_3B, 0x6518A000) - // FCVTZS ., /M, . SVE_HP_3B 0110010100011000 101gggnnnnnddddd 6518 A000 - -INST1(fcvtzu, "fcvtzu", 0, IF_SVE_HP_3B, 0x6519A000) - // FCVTZU ., /M, . SVE_HP_3B 0110010100011001 101gggnnnnnddddd 6519 A000 - -// enum name info SVE_HS_3A -INST1(scvtf, "scvtf", 0, IF_SVE_HS_3A, 0x6510A000) - // SCVTF ., /M, . SVE_HS_3A 0110010100010000 101gggnnnnnddddd 6594 A000 - -INST1(ucvtf, "ucvtf", 0, IF_SVE_HS_3A, 0x6511A000) - // UCVTF ., /M, . SVE_HS_3A 0110010100010001 101gggnnnnnddddd 6595 A000 // enum name info SVE_HU_4A INST1(fnmla, "fnmla", 0, IF_SVE_HU_4A, 0x65204000 ) diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index 7c16c29b0972f9..abc510d967a80d 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -144,11 +144,9 @@ CONFIG_INTEGER(JitPrintInlinedMethodsVerbose, W("JitPrintInlinedMethodsVerboseLe CONFIG_METHODSET(JitPrintInlinedMethods, W("JitPrintInlinedMethods")) CONFIG_METHODSET(JitPrintDevirtualizedMethods, W("JitPrintDevirtualizedMethods")) - -// -1: just do internal checks (CHECK_HASLIKELIHOOD | CHECK_LIKELIHOODSUM | RAISE_ASSERT) -// Else bitflag of ProfileChecks enum. +// -1: just do internal checks +// Else bitflag: 0x1 check classic, 0x2 check likely, 0x4 enable asserts CONFIG_INTEGER(JitProfileChecks, W("JitProfileChecks"), -1) - CONFIG_INTEGER(JitRequired, W("JITRequired"), -1) CONFIG_INTEGER(JitRoundFloat, W("JITRoundFloat"), DEFAULT_ROUND_LEVEL) CONFIG_INTEGER(JitStackAllocToLocalSize, W("JitStackAllocToLocalSize"), DEFAULT_MAX_LOCALLOC_TO_LOCAL_SIZE) @@ -379,14 +377,6 @@ CONFIG_INTEGER(JitConstCSE, W("JitConstCSE"), 0) #define CONST_CSE_ENABLE_ALL 3 #define CONST_CSE_ENABLE_ALL_NO_SHARING 4 -// If nonzero, use the greedy RL policy. -// -CONFIG_INTEGER(JitRLCSEGreedy, W("JitRLCSEGreedy"), 0) - -// If nonzero, dump out details of parameterized policy evaluation and -// gradient updates -CONFIG_INTEGER(JitRLCSEVerbose, W("JitRLCSEVerbose"), 0) - #if defined(DEBUG) // Allow fine-grained controls of CSEs done in a particular method // @@ -425,7 +415,7 @@ CONFIG_STRING(JitReplayCSE, W("JitReplayCSE")) CONFIG_STRING(JitReplayCSEReward, W("JitReplayCSEReward")) // When set, specifies the initial parameter string for -// the reinforcement-learning based CSE heuristic. +// a reinforcement-learning based CSE heuristic. // // Note you can also set JitReplayCSE and JitReplayCSEPerfScore // along with this, in which case we are asking for a policy @@ -436,9 +426,16 @@ CONFIG_STRING(JitRLCSE, W("JitRLCSE")) // use in learning. CONFIG_STRING(JitRLCSEAlpha, W("JitRLCSEAlpha")) +// If nonzero, dump out details of policy evaluation and +// gradient updates +CONFIG_INTEGER(JitRLCSEVerbose, W("JitRLCSEVerbose"), 0) + // If nonzero, dump candidate feature values CONFIG_INTEGER(JitRLCSECandidateFeatures, W("JitRLCSECandidateFeatures"), 0) +// If nonzero, use the greedy policy with current parameters. +CONFIG_INTEGER(JitRLCSEGreedy, W("JitRLCSEGreedy"), 0) + #endif /// @@ -483,9 +480,8 @@ CONFIG_INTEGER(JitNoRngChks, W("JitNoRngChks"), 0) // If 1, don't generate range #if defined(OPT_CONFIG) CONFIG_INTEGER(JitDoAssertionProp, W("JitDoAssertionProp"), 1) // Perform assertion propagation optimization -CONFIG_INTEGER(JitDoCopyProp, W("JitDoCopyProp"), 1) // Perform copy propagation on variables that appear redundant -CONFIG_INTEGER(JitDoOptimizeIVs, W("JitDoOptimizeIVs"), 1) // Perform optimization of induction variables -CONFIG_INTEGER(JitDoEarlyProp, W("JitDoEarlyProp"), 1) // Perform Early Value Propagation +CONFIG_INTEGER(JitDoCopyProp, W("JitDoCopyProp"), 1) // Perform copy propagation on variables that appear redundant +CONFIG_INTEGER(JitDoEarlyProp, W("JitDoEarlyProp"), 1) // Perform Early Value Propagation CONFIG_INTEGER(JitDoLoopHoisting, W("JitDoLoopHoisting"), 1) // Perform loop hoisting on loop invariant values CONFIG_INTEGER(JitDoLoopInversion, W("JitDoLoopInversion"), 1) // Perform loop inversion on "for/while" loops CONFIG_INTEGER(JitDoRangeAnalysis, W("JitDoRangeAnalysis"), 1) // Perform range check analysis @@ -500,7 +496,6 @@ CONFIG_STRING(JitOnlyOptimizeRange, W("JitOnlyOptimizeRange")) // If set, all methods that do _not_ match are forced into MinOpts CONFIG_STRING(JitEnablePhysicalPromotionRange, W("JitEnablePhysicalPromotionRange")) CONFIG_STRING(JitEnableCrossBlockLocalAssertionPropRange, W("JitEnableCrossBlockLocalAssertionPropRange")) -CONFIG_STRING(JitEnableInductionVariableOptsRange, W("JitEnableInductionVariableOptsRange")) CONFIG_INTEGER(JitDoSsa, W("JitDoSsa"), 1) // Perform Static Single Assignment (SSA) numbering on the variables CONFIG_INTEGER(JitDoValueNumber, W("JitDoValueNumber"), 1) // Perform value numbering on method expressions diff --git a/src/coreclr/jit/jiteh.cpp b/src/coreclr/jit/jiteh.cpp index f4b651559ee0fd..a4eacd9069db42 100644 --- a/src/coreclr/jit/jiteh.cpp +++ b/src/coreclr/jit/jiteh.cpp @@ -1984,10 +1984,10 @@ bool Compiler::fgNormalizeEHCase1() { // ...then we want to insert an empty, non-removable block outside the try to be the new first block of the // handler. - BasicBlock* newHndStart = BasicBlock::New(this); + BasicBlock* newHndStart = BasicBlock::New(this, BBJ_ALWAYS, handlerStart); fgInsertBBbefore(handlerStart, newHndStart); FlowEdge* newEdge = fgAddRefPred(handlerStart, newHndStart); - newHndStart->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); // Handler begins have an extra implicit ref count. // BasicBlock::New has already handled this for newHndStart. @@ -2154,11 +2154,11 @@ bool Compiler::fgNormalizeEHCase2() // We've got multiple 'try' blocks starting at the same place! // Add a new first 'try' block for 'ehOuter' that will be outside 'eh'. - BasicBlock* newTryStart = BasicBlock::New(this); + BasicBlock* newTryStart = BasicBlock::New(this, BBJ_ALWAYS, insertBeforeBlk); newTryStart->bbRefs = 0; fgInsertBBbefore(insertBeforeBlk, newTryStart); FlowEdge* const newEdge = fgAddRefPred(insertBeforeBlk, newTryStart); - newTryStart->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); // It's possible for a try to start at the beginning of a method. If so, we need // to adjust the implicit ref counts as we've just created a new first bb @@ -2337,9 +2337,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions() info.compCompHnd->resolveToken(&resolvedToken); CORINFO_GENERICHANDLE_RESULT embedInfo; - // NOTE: inlining is done at this point, so we don't know which method contained this token. - // It's fine because currently this is never used for something that belongs to an inlinee. - info.compCompHnd->embedGenericHandle(&resolvedToken, true, info.compMethodHnd, &embedInfo); + info.compCompHnd->embedGenericHandle(&resolvedToken, true, &embedInfo); if (!embedInfo.lookup.lookupKind.needsRuntimeLookup) { // Exception type does not need runtime lookup @@ -2348,7 +2346,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions() // Create a new bb for the fake filter BasicBlock* handlerBb = eh->ebdHndBeg; - BasicBlock* filterBb = BasicBlock::New(this); + BasicBlock* filterBb = BasicBlock::New(this, BBJ_EHFILTERRET, handlerBb); // Now we need to spill CATCH_ARG (it should be the first thing evaluated) GenTree* arg = new (this, GT_CATCH_ARG) GenTree(GT_CATCH_ARG, TYP_REF); @@ -2377,7 +2375,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions() // Insert it right before the handler (and make it a pred of the handler) fgInsertBBbefore(handlerBb, filterBb); FlowEdge* const newEdge = fgAddRefPred(handlerBb, filterBb); - filterBb->SetKindAndTargetEdge(BBJ_EHFILTERRET, newEdge); + newEdge->setLikelihood(1.0); fgNewStmtAtEnd(filterBb, retFilt, handlerBb->firstStmt()->GetDebugInfo()); filterBb->bbCatchTyp = BBCT_FILTER; @@ -2634,7 +2632,7 @@ bool Compiler::fgNormalizeEHCase3() // Add a new last block for 'ehOuter' that will be outside the EH region with which it encloses and // shares a 'last' pointer - BasicBlock* newLast = BasicBlock::New(this); + BasicBlock* newLast = BasicBlock::New(this, BBJ_ALWAYS, insertAfterBlk->Next()); newLast->bbRefs = 0; assert(insertAfterBlk != nullptr); fgInsertBBafter(insertAfterBlk, newLast); @@ -2684,7 +2682,7 @@ bool Compiler::fgNormalizeEHCase3() newLast->inheritWeight(insertAfterBlk); newLast->SetFlags(BBF_INTERNAL | BBF_NONE_QUIRK); FlowEdge* const newEdge = fgAddRefPred(newLast, insertAfterBlk); - insertAfterBlk->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + newEdge->setLikelihood(1.0); // Move the insert pointer. More enclosing equivalent 'last' blocks will be inserted after this. insertAfterBlk = newLast; @@ -4327,8 +4325,8 @@ void Compiler::fgExtendEHRegionBefore(BasicBlock* block) #endif // FEATURE_EH_FUNCLETS // If this is a handler for a filter, the last block of the filter will end with - // a BBJ_EHFILTERRET block that jumps to the first block of its handler. - // So we need to update it to keep things in sync. + // a BBJ_EHFILTERRET block that has a bbTarget that jumps to the first block of + // its handler. So we need to update it to keep things in sync. // if (HBtab->HasFilter()) { @@ -4339,12 +4337,15 @@ void Compiler::fgExtendEHRegionBefore(BasicBlock* block) #ifdef DEBUG if (verbose) { - printf("EH#%u: Updating target for filter ret block: " FMT_BB " => " FMT_BB "\n", ehGetIndex(HBtab), - bFilterLast->bbNum, bPrev->bbNum); + printf("EH#%u: Updating bbTarget for filter ret block: " FMT_BB " => " FMT_BB "\n", + ehGetIndex(HBtab), bFilterLast->bbNum, bPrev->bbNum); } #endif // DEBUG - // Change the target for bFilterLast from the old first 'block' to the new first 'bPrev' - fgRedirectTargetEdge(bFilterLast, bPrev); + // Change the bbTarget for bFilterLast from the old first 'block' to the new first 'bPrev' + fgRemoveRefPred(bFilterLast->GetTarget(), bFilterLast); + bFilterLast->SetTarget(bPrev); + FlowEdge* const newEdge = fgAddRefPred(bPrev, bFilterLast); + newEdge->setLikelihood(1.0); } } diff --git a/src/coreclr/jit/jitmetadata.cpp b/src/coreclr/jit/jitmetadata.cpp deleted file mode 100644 index 905cdb7317d8bb..00000000000000 --- a/src/coreclr/jit/jitmetadata.cpp +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#include "jitpch.h" -#include "jitmetadata.h" - -#ifdef DEBUG - -//------------------------------------------------------------------------ -// JitMetadata::report: Report metadata back to the EE. -// -// Parameters: -// comp - Compiler instance -// key - Key name of metadata -// data - Pointer to the value to report back -// -void JitMetadata::report(Compiler* comp, const char* key, const void* data, size_t length) -{ - comp->info.compCompHnd->reportMetadata(key, data, length); -} - -//------------------------------------------------------------------------ -// reportValue: Report a specific value back to the EE. -// -// Parameters: -// comp - Compiler instance -// key - The key -// value - Value to report back -// -template -static void reportValue(Compiler* comp, const char* key, T value) -{ - JitMetadata::report(comp, key, &value, sizeof(value)); -} - -//------------------------------------------------------------------------ -// JitMetrics::report: Report all metrics and their values back to the EE. -// -// Parameters: -// comp - Compiler instance -// -void JitMetrics::report(Compiler* comp) -{ -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) reportValue(comp, #name, name); -#include "jitmetadatalist.h" -} - -//------------------------------------------------------------------------ -// printMetric: Print a double metric value to jitstdout. -// -// Parameters: -// value - The value -// -static void printMetric(double value) -{ - printf("%f", value); -} - -//------------------------------------------------------------------------ -// printMetric: Print an int metric value to jitstdout. -// -// Parameters: -// value - The value -// -static void printMetric(int value) -{ - printf("%d", value); -} - -//------------------------------------------------------------------------ -// printMetric: Print an int64_t metric value to jitstdout. -// -// Parameters: -// value - The value -// -static void printMetric(int64_t value) -{ - printf("%lld", value); -} - -//------------------------------------------------------------------------ -// JitMetrics::dump: Print the values of all metrics to jitstdout. -// -void JitMetrics::dump() -{ - int nameMaxWidth = 0; -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) nameMaxWidth = max(nameMaxWidth, (int)strlen(#name)); -#include "jitmetadatalist.h" - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) \ - printf("%-*s: ", nameMaxWidth + 5, #name); \ - printMetric(name); \ - printf("\n"); -#include "jitmetadatalist.h" -} - -#endif diff --git a/src/coreclr/jit/jitmetadata.h b/src/coreclr/jit/jitmetadata.h deleted file mode 100644 index 3b4b324497cc21..00000000000000 --- a/src/coreclr/jit/jitmetadata.h +++ /dev/null @@ -1,26 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#pragma once - -class Compiler; - -class JitMetadata -{ -public: -#define JITMETADATA(name, type, flags) static constexpr const char* name = #name; -#include "jitmetadatalist.h" - - static void report(Compiler* comp, const char* name, const void* data, size_t length); -}; - -class JitMetrics -{ -public: -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) type name = 0; -#include "jitmetadatalist.h" - - void report(Compiler* comp); - void dump(); -}; diff --git a/src/coreclr/jit/jitmetadatalist.h b/src/coreclr/jit/jitmetadatalist.h deleted file mode 100644 index f36c15ab9991d6..00000000000000 --- a/src/coreclr/jit/jitmetadatalist.h +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// clang-format off - -#ifdef JITMETADATA -#define JITMETADATAINFO(name, type, flags) JITMETADATA(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) JITMETADATA(name, type, flags) -#endif - -#if !defined(JITMETADATAINFO) || !defined(JITMETADATAMETRIC) -#error Define JITMETADATAINFO and JITMETADATAMETRIC before including this file. -#endif - -// List of metadata that the JIT can report. There are two categories: -// -// - JITMETADATAINFO: General info that can be of any type and that cannot be -// aggregated in straightforward ways. These properties are not handled -// automatically; the JIT must explicitly report them using -// JitMetadata::report, and the SPMI side needs to manually handle (or ignore) -// them in ICorJitInfo::reportMetadata. -// -// - JITMETADATAMETRIC: Metrics which are numeric types (currently int, double -// and int64_t types supported). Their reporting is handled automatically and -// they will be propagated all the way into SPMI replay/diff results. - -// Name, type flags -JITMETADATAINFO(MethodFullName, const char*, 0) -JITMETADATAINFO(TieringName, const char*, 0) -JITMETADATAMETRIC(PhysicallyPromotedFields, int, 0) -JITMETADATAMETRIC(LoopsFoundDuringOpts, int, 0) -JITMETADATAMETRIC(LoopsCloned, int, 0) -JITMETADATAMETRIC(LoopsUnrolled, int, 0) -JITMETADATAMETRIC(LoopAlignmentCandidates, int, 0) -JITMETADATAMETRIC(LoopsAligned, int, 0) -JITMETADATAMETRIC(LoopsIVWidened, int, 0) -JITMETADATAMETRIC(WidenedIVs, int, 0) -JITMETADATAMETRIC(VarsInSsa, int, 0) -JITMETADATAMETRIC(HoistedExpressions, int, 0) -JITMETADATAMETRIC(RedundantBranchesEliminated, int, JIT_METADATA_HIGHER_IS_BETTER) -JITMETADATAMETRIC(JumpThreadingsPerformed, int, JIT_METADATA_HIGHER_IS_BETTER) -JITMETADATAMETRIC(CseCount, int, 0) -JITMETADATAMETRIC(BasicBlocksAtCodegen, int, 0) -JITMETADATAMETRIC(PerfScore, double, JIT_METADATA_LOWER_IS_BETTER) -JITMETADATAMETRIC(BytesAllocated, int64_t, JIT_METADATA_LOWER_IS_BETTER) - -#undef JITMETADATA -#undef JITMETADATAINFO -#undef JITMETADATAMETRIC - -// clang-format on diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index 44b0afe1caf927..7ebba3c14bdca7 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -719,7 +719,7 @@ class LocalAddressVisitor final : public GenTreeVisitor GenTreeCall* callUser = user->IsCall() ? user->AsCall() : nullptr; bool hasHiddenStructArg = false; if (m_compiler->opts.compJitOptimizeStructHiddenBuffer && (callUser != nullptr) && - m_compiler->IsValidLclAddr(lclNum, val.Offset())) + IsValidLclAddr(lclNum, val.Offset())) { // We will only attempt this optimization for locals that are: // a) Not susceptible to liveness bugs (see "lvaSetHiddenBufferStructArg"). @@ -805,7 +805,6 @@ class LocalAddressVisitor final : public GenTreeVisitor unsigned indirSize = node->AsIndir()->Size(); bool isWide; - // TODO-Cleanup: delete "indirSize == 0", use "Compiler::IsValidLclAddr". if ((indirSize == 0) || ((offset + indirSize) > UINT16_MAX)) { // If we can't figure out the indirection size then treat it as a wide indirection. @@ -824,6 +823,15 @@ class LocalAddressVisitor final : public GenTreeVisitor else { isWide = endOffset.Value() > m_compiler->lvaLclExactSize(lclNum); + + if ((varDsc->TypeGet() == TYP_STRUCT) && varDsc->GetLayout()->IsBlockLayout()) + { + // TODO-CQ: TYP_BLK used to always be exposed here. This is in principle not necessary, but + // not doing so would require VN changes. For now, exposing gets better CQ as otherwise the + // variable ends up untracked and VN treats untracked-not-exposed locals more conservatively + // than exposed ones. + m_compiler->lvaSetVarAddrExposed(lclNum DEBUGARG(AddressExposedReason::TOO_CONSERVATIVE)); + } } } @@ -857,7 +865,7 @@ class LocalAddressVisitor final : public GenTreeVisitor assert(addr->TypeIs(TYP_BYREF, TYP_I_IMPL)); assert(m_compiler->lvaVarAddrExposed(lclNum) || m_compiler->lvaGetDesc(lclNum)->IsHiddenBufferStructArg()); - if (m_compiler->IsValidLclAddr(lclNum, offset)) + if (IsValidLclAddr(lclNum, offset)) { addr->ChangeOper(GT_LCL_ADDR); addr->AsLclFld()->SetLclNum(lclNum); @@ -1450,6 +1458,24 @@ class LocalAddressVisitor final : public GenTreeVisitor } private: + //------------------------------------------------------------------------ + // IsValidLclAddr: Can the given local address be represented as "LCL_FLD_ADDR"? + // + // Local address nodes cannot point beyond the local and can only store + // 16 bits worth of offset. + // + // Arguments: + // lclNum - The local's number + // offset - The address' offset + // + // Return Value: + // Whether "LCL_FLD_ADDR [+offset]" would be valid IR. + // + bool IsValidLclAddr(unsigned lclNum, unsigned offset) const + { + return (offset < UINT16_MAX) && (offset < m_compiler->lvaLclExactSize(lclNum)); + } + //------------------------------------------------------------------------ // IsUnused: is the given node unused? // diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index f2e47ae2ddac44..55787580cc1df7 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -1540,7 +1540,7 @@ unsigned Compiler::compMapILvarNum(unsigned ILvarNum) else if (ILvarNum == (unsigned)ICorDebugInfo::TYPECTXT_ILNUM) { noway_assert(info.compTypeCtxtArg >= 0); - varNum = info.compTypeCtxtArg; + varNum = unsigned(info.compTypeCtxtArg); } else if (ILvarNum < info.compILargsCount) { @@ -1593,7 +1593,7 @@ unsigned Compiler::compMap2ILvarNum(unsigned varNum) const // We create an extra argument for the type context parameter // needed for shared generic code. - if ((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_PARAMTYPE) && varNum == info.compTypeCtxtArg) + if ((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_PARAMTYPE) && varNum == (unsigned)info.compTypeCtxtArg) { return (unsigned)ICorDebugInfo::TYPECTXT_ILNUM; } @@ -1606,7 +1606,7 @@ unsigned Compiler::compMap2ILvarNum(unsigned varNum) const #endif // FEATURE_FIXED_OUT_ARGS // Now mutate varNum to remove extra parameters from the count. - if ((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_PARAMTYPE) && varNum > info.compTypeCtxtArg) + if ((info.compMethodInfo->args.callConv & CORINFO_CALLCONV_PARAMTYPE) && varNum > (unsigned)info.compTypeCtxtArg) { varNum--; } @@ -4357,7 +4357,7 @@ PhaseStatus Compiler::lvaMarkLocalVars() else if (lvaReportParamTypeArg()) { // We should have a context arg. - assert(info.compTypeCtxtArg != BAD_VAR_NUM); + assert(info.compTypeCtxtArg != (int)BAD_VAR_NUM); lvaGetDesc(info.compTypeCtxtArg)->lvImplicitlyReferenced = reportParamTypeArg; } @@ -5498,7 +5498,7 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() //@GENERICS: extra argument for instantiation info if (info.compMethodInfo->args.callConv & CORINFO_CALLCONV_PARAMTYPE) { - noway_assert(lclNum == info.compTypeCtxtArg); + noway_assert(lclNum == (unsigned)info.compTypeCtxtArg); argOffs = lvaAssignVirtualFrameOffsetToArg(lclNum++, REGSIZE_BYTES, argOffs UNIX_AMD64_ABI_ONLY_ARG(&callerArgOffset)); } @@ -5607,7 +5607,7 @@ void Compiler::lvaAssignVirtualFrameOffsetsToArgs() //@GENERICS: extra argument for instantiation info if (info.compMethodInfo->args.callConv & CORINFO_CALLCONV_PARAMTYPE) { - noway_assert(lclNum == info.compTypeCtxtArg); + noway_assert(lclNum == (unsigned)info.compTypeCtxtArg); argOffs = lvaAssignVirtualFrameOffsetToArg(lclNum++, REGSIZE_BYTES, argOffs UNIX_AMD64_ABI_ONLY_ARG(&callerArgOffset)); } diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index 78fb96fe3d77d2..c4a4d44489f0d2 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -245,6 +245,7 @@ void Compiler::fgPerNodeLocalVarLiveness(GenTree* tree) case GT_STOREIND: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_MEMORYBARRIER: // Similar to Volatile indirections, we must handle this as a memory def. fgCurMemoryDef |= memoryKindSet(GcHeap, ByrefExposed); break; @@ -1936,6 +1937,7 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR case GT_STOREIND: case GT_BOUNDS_CHECK: case GT_STORE_BLK: + case GT_STORE_DYN_BLK: case GT_JCMP: case GT_JTEST: case GT_JCC: diff --git a/src/coreclr/jit/loopcloning.cpp b/src/coreclr/jit/loopcloning.cpp index 5f51c77eb2384b..ca4c2572fa41d9 100644 --- a/src/coreclr/jit/loopcloning.cpp +++ b/src/coreclr/jit/loopcloning.cpp @@ -853,40 +853,24 @@ BasicBlock* LoopCloneContext::CondToStmtInBlock(Compiler* noway_assert(conds.Size() > 0); assert(slowPreheader != nullptr); - // For now assume high likelihood for the fast path, - // uniformly spread across the gating branches. - // - // For "normal" cloning this is probably ok. For GDV cloning this - // may be inaccurate. We should key off the type test likelihood(s). - // - const weight_t fastLikelihood = fastPathWeightScaleFactor; - // Choose how to generate the conditions const bool generateOneConditionPerBlock = true; if (generateOneConditionPerBlock) { - // N = conds.Size() branches must all be true to execute the fast loop. - // Use the N'th root.... - // - const weight_t fastLikelihoodPerBlock = exp(log(fastLikelihood) / (weight_t)conds.Size()); - for (unsigned i = 0; i < conds.Size(); ++i) { - BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true); + BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true, slowPreheader); newBlk->inheritWeight(insertAfter); - JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, slowPreheader->bbNum); - FlowEdge* const trueEdge = comp->fgAddRefPred(slowPreheader, newBlk); - newBlk->SetTrueEdge(trueEdge); - trueEdge->setLikelihood(1 - fastLikelihoodPerBlock); + JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, newBlk->GetTrueTarget()->bbNum); + comp->fgAddRefPred(newBlk->GetTrueTarget(), newBlk); if (insertAfter->KindIs(BBJ_COND)) { JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", insertAfter->bbNum, newBlk->bbNum); - FlowEdge* const falseEdge = comp->fgAddRefPred(newBlk, insertAfter); - insertAfter->SetFalseEdge(falseEdge); - falseEdge->setLikelihood(fastLikelihoodPerBlock); + insertAfter->SetFalseTarget(newBlk); + comp->fgAddRefPred(newBlk, insertAfter); } JITDUMP("Adding conditions %u to " FMT_BB "\n", i, newBlk->bbNum); @@ -910,20 +894,16 @@ BasicBlock* LoopCloneContext::CondToStmtInBlock(Compiler* } else { - BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true); + BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true, slowPreheader); newBlk->inheritWeight(insertAfter); - JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, slowPreheader->bbNum); - FlowEdge* const trueEdge = comp->fgAddRefPred(slowPreheader, newBlk); - newBlk->SetTrueEdge(trueEdge); - trueEdge->setLikelihood(1.0 - fastLikelihood); + JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, newBlk->GetTrueTarget()->bbNum); + comp->fgAddRefPred(newBlk->GetTrueTarget(), newBlk); - if (insertAfter->KindIs(BBJ_COND)) + if (insertAfter->bbFallsThrough()) { JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", insertAfter->bbNum, newBlk->bbNum); - FlowEdge* const falseEdge = comp->fgAddRefPred(newBlk, insertAfter); - insertAfter->SetFalseEdge(falseEdge); - falseEdge->setLikelihood(fastLikelihood); + comp->fgAddRefPred(newBlk, insertAfter); } JITDUMP("Adding conditions to " FMT_BB "\n", newBlk->bbNum); @@ -1958,6 +1938,13 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // taking the max with the head block's weight. ambientWeight = max(ambientWeight, preheader->bbWeight); + // We assume that the fast path will run 99% of the time, and thus should get 99% of the block weights. + // The slow path will, correspondingly, get only 1% of the block weights. It could be argued that we should + // mark the slow path as "run rarely", since it really shouldn't execute (given the currently optimized loop + // conditions) except under exceptional circumstances. + const weight_t fastPathWeightScaleFactor = 0.99; + const weight_t slowPathWeightScaleFactor = 1.0 - fastPathWeightScaleFactor; + // We're going to transform this loop: // // preheader --> header @@ -1972,11 +1959,12 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // Make a new pre-header block for the fast loop. JITDUMP("Create new preheader block for fast loop\n"); - BasicBlock* fastPreheader = fgNewBBafter(BBJ_ALWAYS, preheader, /*extendRegion*/ true); + BasicBlock* fastPreheader = + fgNewBBafter(BBJ_ALWAYS, preheader, /*extendRegion*/ true, /*jumpDest*/ loop->GetHeader()); JITDUMP("Adding " FMT_BB " after " FMT_BB "\n", fastPreheader->bbNum, preheader->bbNum); fastPreheader->bbWeight = fastPreheader->isRunRarely() ? BB_ZERO_WEIGHT : ambientWeight; - if (fastPreheader->NextIs(loop->GetHeader())) + if (fastPreheader->JumpsToNext()) { fastPreheader->SetFlags(BBF_NONE_QUIRK); } @@ -1984,10 +1972,7 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex assert(preheader->KindIs(BBJ_ALWAYS)); assert(preheader->TargetIs(loop->GetHeader())); - FlowEdge* const oldEdge = preheader->GetTargetEdge(); - fgReplacePred(oldEdge, fastPreheader); - fastPreheader->SetTargetEdge(oldEdge); - + fgReplacePred(loop->GetHeader(), preheader, fastPreheader); JITDUMP("Replace " FMT_BB " -> " FMT_BB " with " FMT_BB " -> " FMT_BB "\n", preheader->bbNum, loop->GetHeader()->bbNum, fastPreheader->bbNum, loop->GetHeader()->bbNum); @@ -2013,18 +1998,18 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex BasicBlock* slowPreheader = fgNewBBafter(BBJ_ALWAYS, newPred, /*extendRegion*/ true); JITDUMP("Adding " FMT_BB " after " FMT_BB "\n", slowPreheader->bbNum, newPred->bbNum); slowPreheader->bbWeight = newPred->isRunRarely() ? BB_ZERO_WEIGHT : ambientWeight; - slowPreheader->scaleBBWeight(LoopCloneContext::slowPathWeightScaleFactor); + slowPreheader->scaleBBWeight(slowPathWeightScaleFactor); newPred = slowPreheader; // Now we'll clone the blocks of the loop body. These cloned blocks will be the slow path. BlockToBlockMap* blockMap = new (getAllocator(CMK_LoopClone)) BlockToBlockMap(getAllocator(CMK_LoopClone)); - loop->Duplicate(&newPred, blockMap, LoopCloneContext::slowPathWeightScaleFactor); + loop->Duplicate(&newPred, blockMap, slowPathWeightScaleFactor); // Scale old blocks to the fast path weight. loop->VisitLoopBlocks([=](BasicBlock* block) { - block->scaleBBWeight(LoopCloneContext::fastPathWeightScaleFactor); + block->scaleBBWeight(fastPathWeightScaleFactor); return BasicBlockVisit::Continue; }); @@ -2054,12 +2039,9 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // We haven't set the jump target yet assert(slowPreheader->KindIs(BBJ_ALWAYS)); assert(!slowPreheader->HasInitializedTarget()); + slowPreheader->SetTarget(slowHeader); - { - FlowEdge* const newEdge = fgAddRefPred(slowHeader, slowPreheader); - slowPreheader->SetTargetEdge(newEdge); - } - + fgAddRefPred(slowHeader, slowPreheader); JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", slowPreheader->bbNum, slowHeader->bbNum); BasicBlock* condLast = optInsertLoopChoiceConditions(context, loop, slowPreheader, preheader); @@ -2067,20 +2049,14 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // Now redirect the old preheader to jump to the first new condition that // was inserted by the above function. assert(preheader->KindIs(BBJ_ALWAYS)); - - { - FlowEdge* const newEdge = fgAddRefPred(preheader->Next(), preheader); - preheader->SetTargetEdge(newEdge); - } - + preheader->SetTarget(preheader->Next()); + fgAddRefPred(preheader->Next(), preheader); preheader->SetFlags(BBF_NONE_QUIRK); // And make sure we insert a pred link for the final fallthrough into the fast preheader. assert(condLast->NextIs(fastPreheader)); - FlowEdge* const falseEdge = fgAddRefPred(fastPreheader, condLast); - condLast->SetFalseEdge(falseEdge); - FlowEdge* const trueEdge = condLast->GetTrueEdge(); - falseEdge->setLikelihood(max(0, 1.0 - trueEdge->getLikelihood())); + condLast->SetFalseTarget(fastPreheader); + fgAddRefPred(fastPreheader, condLast); } //------------------------------------------------------------------------- @@ -2979,19 +2955,19 @@ PhaseStatus Compiler::optCloneLoops() #endif #endif - assert(Metrics.LoopsCloned == 0); // It should be initialized, but not yet changed. + assert(optLoopsCloned == 0); // It should be initialized, but not yet changed. for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) { if (context.GetLoopOptInfo(loop->GetIndex()) != nullptr) { - Metrics.LoopsCloned++; + optLoopsCloned++; context.OptimizeConditions(loop->GetIndex() DEBUGARG(verbose)); context.OptimizeBlockConditions(loop->GetIndex() DEBUGARG(verbose)); optCloneLoop(loop, &context); } } - if (Metrics.LoopsCloned > 0) + if (optLoopsCloned > 0) { fgInvalidateDfsTree(); m_dfsTree = fgComputeDfs(); @@ -3010,7 +2986,7 @@ PhaseStatus Compiler::optCloneLoops() #ifdef DEBUG if (verbose) { - printf("Loops cloned: %d\n", Metrics.LoopsCloned); + printf("Loops cloned: %d\n", optLoopsCloned); printf("Loops statically optimized: %d\n", optStaticallyOptimizedLoops); printf("After loop cloning:\n"); fgDispBasicBlocks(/*dumpTrees*/ true); diff --git a/src/coreclr/jit/loopcloning.h b/src/coreclr/jit/loopcloning.h index 64e810be6ff424..2333d491764cd7 100644 --- a/src/coreclr/jit/loopcloning.h +++ b/src/coreclr/jit/loopcloning.h @@ -814,14 +814,6 @@ struct NaturalLoopIterInfo; */ struct LoopCloneContext { - // We assume that the fast path will run 99% of the time, and thus should get 99% of the block weights. - // The slow path will, correspondingly, get only 1% of the block weights. It could be argued that we should - // mark the slow path as "run rarely", since it really shouldn't execute (given the currently optimized loop - // conditions) except under exceptional circumstances. - // - static constexpr weight_t fastPathWeightScaleFactor = 0.99; - static constexpr weight_t slowPathWeightScaleFactor = 1.0 - fastPathWeightScaleFactor; - CompAllocator alloc; // The allocator // The array of optimization opportunities found in each loop. (loop x optimization-opportunities) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 591c2b9165879c..591db3a78a22c0 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -570,6 +570,8 @@ GenTree* Lowering::LowerNode(GenTree* node) LowerStoreSingleRegCallStruct(node->AsBlk()); break; } + FALLTHROUGH; + case GT_STORE_DYN_BLK: LowerBlockStoreCommon(node->AsBlk()); break; @@ -825,6 +827,10 @@ GenTree* Lowering::LowerArrLength(GenTreeArrCommon* node) GenTree* Lowering::LowerSwitch(GenTree* node) { + unsigned jumpCnt; + unsigned targetCnt; + BasicBlock** jumpTab; + assert(node->gtOper == GT_SWITCH); // The first step is to build the default case conditional construct that is @@ -838,9 +844,9 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // jumpCnt is the number of elements in the jump table array. // jumpTab is the actual pointer to the jump table array. // targetCnt is the number of unique targets in the jump table array. - const unsigned jumpCnt = originalSwitchBB->GetSwitchTargets()->bbsCount; - FlowEdge** const jumpTab = originalSwitchBB->GetSwitchTargets()->bbsDstTab; - const unsigned targetCnt = originalSwitchBB->NumSucc(comp); + jumpCnt = originalSwitchBB->GetSwitchTargets()->bbsCount; + jumpTab = originalSwitchBB->GetSwitchTargets()->bbsDstTab; + targetCnt = originalSwitchBB->NumSucc(comp); // GT_SWITCH must be a top-level node with no use. #ifdef DEBUG @@ -859,7 +865,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) { JITDUMP("Lowering switch " FMT_BB ": single target; converting to BBJ_ALWAYS\n", originalSwitchBB->bbNum); noway_assert(comp->opts.OptimizationDisabled()); - originalSwitchBB->SetKindAndTargetEdge(BBJ_ALWAYS, jumpTab[0]); + originalSwitchBB->SetKindAndTarget(BBJ_ALWAYS, jumpTab[0]); if (originalSwitchBB->JumpsToNext()) { @@ -869,7 +875,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // Remove extra predecessor links if there was more than one case. for (unsigned i = 1; i < jumpCnt; ++i) { - comp->fgRemoveRefPred(jumpTab[i]); + (void)comp->fgRemoveRefPred(jumpTab[i], originalSwitchBB); } // We have to get rid of the GT_SWITCH node but a child might have side effects so just assign @@ -903,11 +909,11 @@ GenTree* Lowering::LowerSwitch(GenTree* node) unsigned tempLclNum = temp->AsLclVarCommon()->GetLclNum(); var_types tempLclType = temp->TypeGet(); - BasicBlock* defaultBB = jumpTab[jumpCnt - 1]->getDestinationBlock(); + BasicBlock* defaultBB = jumpTab[jumpCnt - 1]; BasicBlock* followingBB = originalSwitchBB->Next(); /* Is the number of cases right for a test and jump switch? */ - const bool fFirstCaseFollows = (followingBB == jumpTab[0]->getDestinationBlock()); + const bool fFirstCaseFollows = (followingBB == jumpTab[0]); const bool fDefaultFollows = (followingBB == defaultBB); unsigned minSwitchTabJumpCnt = 2; // table is better than just 2 cmp/jcc @@ -949,34 +955,21 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // originalSwitchBB is now a BBJ_ALWAYS, and there is a predecessor edge in afterDefaultCondBlock // representing the fall-through flow from originalSwitchBB. assert(originalSwitchBB->KindIs(BBJ_ALWAYS)); - assert(originalSwitchBB->TargetIs(afterDefaultCondBlock)); - assert(originalSwitchBB->JumpsToNext()); + assert(originalSwitchBB->NextIs(afterDefaultCondBlock)); assert(afterDefaultCondBlock->KindIs(BBJ_SWITCH)); assert(afterDefaultCondBlock->GetSwitchTargets()->bbsHasDefault); assert(afterDefaultCondBlock->isEmpty()); // Nothing here yet. // The GT_SWITCH code is still in originalSwitchBB (it will be removed later). + // Turn originalSwitchBB into a BBJ_COND. + originalSwitchBB->SetCond(jumpTab[jumpCnt - 1], afterDefaultCondBlock); + // Fix the pred for the default case: the default block target still has originalSwitchBB // as a predecessor, but the fgSplitBlockAfterStatement() moved all predecessors to point // to afterDefaultCondBlock. - - // Note defaultEdge may also be the edge for some switch cases. We only probe edges, - // so assume each possibility is equally likely. - FlowEdge* const defaultEdge = jumpTab[jumpCnt - 1]; - weight_t const defaultLikelihood = defaultEdge->getLikelihood() / defaultEdge->getDupCount(); - comp->fgRemoveRefPred(defaultEdge); - FlowEdge* const trueEdge = comp->fgAddRefPred(defaultBB, originalSwitchBB); - trueEdge->setLikelihood(defaultLikelihood); - defaultEdge->setLikelihood(defaultEdge->getLikelihood() - defaultLikelihood); - - // Turn originalSwitchBB into a BBJ_COND. - FlowEdge* const falseEdge = originalSwitchBB->GetTargetEdge(); - weight_t const switchLikelihood = 1.0 - defaultLikelihood; - falseEdge->setLikelihood(switchLikelihood); - originalSwitchBB->SetCond(trueEdge, falseEdge); - afterDefaultCondBlock->inheritWeight(originalSwitchBB); - afterDefaultCondBlock->scaleBBWeight(switchLikelihood); + FlowEdge* oldEdge = comp->fgRemoveRefPred(jumpTab[jumpCnt - 1], afterDefaultCondBlock); + comp->fgAddRefPred(jumpTab[jumpCnt - 1], originalSwitchBB, oldEdge); bool useJumpSequence = jumpCnt < minSwitchTabJumpCnt; @@ -996,7 +989,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // If we originally had 2 unique successors, check to see whether there is a unique // non-default case, in which case we can eliminate the switch altogether. // Note that the single unique successor case is handled above. - FlowEdge* uniqueSucc = nullptr; + BasicBlock* uniqueSucc = nullptr; if (targetCnt == 2) { uniqueSucc = jumpTab[0]; @@ -1015,17 +1008,17 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // If the unique successor immediately follows this block, we have nothing to do - // it will simply fall-through after we remove the switch, below. // Otherwise, make this a BBJ_ALWAYS. - // Now, fixup the predecessor links to uniqueSucc's target block. In the original jumpTab: + // Now, fixup the predecessor links to uniqueSucc. In the original jumpTab: // jumpTab[i-1] was the default target, which we handled above, // jumpTab[0] is the first target, and we'll leave that predecessor link. - // Remove any additional predecessor links to uniqueSucc's target block. + // Remove any additional predecessor links to uniqueSucc. for (unsigned i = 1; i < jumpCnt - 1; ++i) { assert(jumpTab[i] == uniqueSucc); - comp->fgRemoveRefPred(uniqueSucc); + (void)comp->fgRemoveRefPred(uniqueSucc, afterDefaultCondBlock); } - afterDefaultCondBlock->SetKindAndTargetEdge(BBJ_ALWAYS, uniqueSucc); + afterDefaultCondBlock->SetKindAndTarget(BBJ_ALWAYS, uniqueSucc); if (afterDefaultCondBlock->JumpsToNext()) { @@ -1058,38 +1051,15 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // If no case target follows, the last one doesn't need to be a compare/branch: it can be an // unconditional branch. bool fAnyTargetFollows = false; - - // We need to track how much of the original switch's likelihood has already been - // tested for. We'll use this to adjust the likelihood of the branches we're adding. - // So far we've tested for the default case, so we'll start with that. - weight_t totalTestLikelihood = defaultLikelihood; for (unsigned i = 0; i < jumpCnt - 1; ++i) { assert(currentBlock != nullptr); - BasicBlock* const targetBlock = jumpTab[i]->getDestinationBlock(); // Remove the switch from the predecessor list of this case target's block. // We'll add the proper new predecessor edge later. - FlowEdge* const oldEdge = jumpTab[i]; + FlowEdge* oldEdge = comp->fgRemoveRefPred(jumpTab[i], afterDefaultCondBlock); - // Compute the likelihood that this test is successful. - // Divide by number of cases still sharing this edge (reduces likelihood) - // Divide by likelihood of reaching this test (increases likelihood). - // But if there is little chance of reaching this test, set the likelihood to 0.5 - // - weight_t const edgeLikelihood = oldEdge->getLikelihood(); - weight_t const caseLikelihood = edgeLikelihood / oldEdge->getDupCount(); - bool const unlikelyToReachThisCase = Compiler::fgProfileWeightsEqual(totalTestLikelihood, 1.0, 0.001); - weight_t const adjustedCaseLikelihood = - unlikelyToReachThisCase ? 0.5 : min(1.0, caseLikelihood / (1.0 - totalTestLikelihood)); - comp->fgRemoveRefPred(oldEdge); - - // Decrement the likelihood on the old edge, so if other cases are sharing it, - // they get the right values later. - // - oldEdge->setLikelihood(edgeLikelihood - caseLikelihood); - - if (targetBlock == followingBB) + if (jumpTab[i] == followingBB) { // This case label follows the switch; let it fall through. fAnyTargetFollows = true; @@ -1098,56 +1068,23 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // We need a block to put in the new compare and/or branch. // If we haven't used the afterDefaultCondBlock yet, then use that. - // if (fUsedAfterDefaultCondBlock) { - BasicBlock* newBlock = comp->fgNewBBafter(BBJ_ALWAYS, currentBlock, true); + BasicBlock* newBlock = comp->fgNewBBafter(BBJ_ALWAYS, currentBlock, true, currentBlock->Next()); newBlock->SetFlags(BBF_NONE_QUIRK); - FlowEdge* const falseEdge = comp->fgAddRefPred(newBlock, currentBlock); // The fall-through predecessor. - - // We set the true edge likelihood earlier, use that to figure out the false edge likelihood - // and the block weight. - // - FlowEdge* const trueEdge = currentBlock->GetTrueEdge(); - weight_t const falseLikelihood = 1.0 - trueEdge->getLikelihood(); - falseEdge->setLikelihood(falseLikelihood); - currentBlock->SetFalseEdge(falseEdge); - newBlock->inheritWeight(currentBlock); - newBlock->scaleBBWeight(falseLikelihood); + currentBlock->SetFalseTarget(newBlock); + comp->fgAddRefPred(newBlock, currentBlock); // The fall-through predecessor. currentBlock = newBlock; currentBBRange = &LIR::AsRange(currentBlock); } else { assert(currentBlock == afterDefaultCondBlock); - - // If the first switch case we peel off has the same target as - // other cases (that is, it has nonzero dup count, it's simpler to - // just make a new here block, so that as we peel off cases, - // we're not sharing edges with the original switch. - // - // That is, the call to fgAddRefPred below always creates a new edge. - // - if (oldEdge->getDupCount() > 0) - { - BasicBlock* const newBlock = comp->fgNewBBafter(BBJ_ALWAYS, currentBlock, true); - newBlock->SetFlags(BBF_NONE_QUIRK); - FlowEdge* const newEdge = comp->fgAddRefPred(newBlock, currentBlock); - currentBlock = newBlock; - currentBBRange = &LIR::AsRange(currentBlock); - afterDefaultCondBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); - } - fUsedAfterDefaultCondBlock = true; } - // Update the total test case likelihood. - totalTestLikelihood += caseLikelihood; - // Wire up the predecessor list for the "branch" case. - FlowEdge* const newEdge = comp->fgAddRefPred(targetBlock, currentBlock, oldEdge); - // This should truly be a new edge. - assert(newEdge->getDupCount() == 1); + comp->fgAddRefPred(jumpTab[i], currentBlock, oldEdge); if (!fAnyTargetFollows && (i == jumpCnt - 2)) { @@ -1156,15 +1093,13 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // case: there is no need to compare against the case index, since it's // guaranteed to be taken (since the default case was handled first, above). - currentBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + currentBlock->SetKindAndTarget(BBJ_ALWAYS, jumpTab[i]); } else { // Otherwise, it's a conditional branch. Set the branch kind, then add the // condition statement. - // We will set the false edge in a later iteration of the loop, or after. - currentBlock->SetCond(newEdge); - newEdge->setLikelihood(adjustedCaseLikelihood); + currentBlock->SetCond(jumpTab[i], currentBlock->Next()); // Now, build the conditional statement for the current case that is // being evaluated: @@ -1186,14 +1121,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // There is a fall-through to the following block. In the loop // above, we deleted all the predecessor edges from the switch. // In this case, we need to add one back. - FlowEdge* const falseEdge = comp->fgAddRefPred(currentBlock->Next(), currentBlock); - currentBlock->SetFalseEdge(falseEdge); - FlowEdge* const trueEdge = currentBlock->GetTrueEdge(); - weight_t const falseLikelihood = 1.0 - trueEdge->getLikelihood(); - falseEdge->setLikelihood(falseLikelihood); - - // The following block weight should remain unchanged. All we've done - // is alter the various paths that can reach it. + comp->fgAddRefPred(currentBlock->Next(), currentBlock); } if (!fUsedAfterDefaultCondBlock) @@ -1204,8 +1132,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) JITDUMP("Lowering switch " FMT_BB ": all switch cases were fall-through\n", originalSwitchBB->bbNum); assert(currentBlock == afterDefaultCondBlock); assert(currentBlock->KindIs(BBJ_SWITCH)); - FlowEdge* const newEdge = comp->fgAddRefPred(currentBlock->Next(), currentBlock); - currentBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + currentBlock->SetKindAndTarget(BBJ_ALWAYS, currentBlock->Next()); currentBlock->RemoveFlags(BBF_DONT_REMOVE); comp->fgRemoveBlock(currentBlock, /* unreachable */ false); // It's an empty block. } @@ -1219,14 +1146,9 @@ GenTree* Lowering::LowerSwitch(GenTree* node) LIR::Range& switchBlockRange = LIR::AsRange(afterDefaultCondBlock); switchBlockRange.InsertAtEnd(switchValue); - // We are going to modify the switch, invalidate any desc map. - // - comp->fgInvalidateSwitchDescMapEntry(afterDefaultCondBlock); - // Try generating a bit test based switch first, // if that's not possible a jump table based switch will be generated. - if (!TryLowerSwitchToBitTest(jumpTab, jumpCnt, targetCnt, afterDefaultCondBlock, switchValue, - defaultLikelihood)) + if (!TryLowerSwitchToBitTest(jumpTab, jumpCnt, targetCnt, afterDefaultCondBlock, switchValue)) { JITDUMP("Lowering switch " FMT_BB ": using jump table expansion\n", originalSwitchBB->bbNum); @@ -1246,49 +1168,9 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // this block no longer branches to the default block afterDefaultCondBlock->GetSwitchTargets()->removeDefault(); - - // We need to scale up the likelihood of the remaining switch edges, now that we've peeled off - // the default case. But if the remaining likelihood is zero (default likelihood was 1.0), - // we don't know the case likelihoods. Instead, divide likelihood evenly among all cases. - // - // First, rebuild the unique succ set - // - Compiler::SwitchUniqueSuccSet successors = comp->GetDescriptorForSwitch(afterDefaultCondBlock); - - // Then fix each successor edge - // - if (Compiler::fgProfileWeightsEqual(defaultLikelihood, 1.0, 0.001)) - { - JITDUMP("Zero weight switch block " FMT_BB ", distributing likelihoods equally per case\n", - afterDefaultCondBlock->bbNum); - // jumpCnt-1 here because we peeled the default after copying this value. - weight_t const newLikelihood = 1.0 / (jumpCnt - 1); - for (unsigned i = 0; i < successors.numDistinctSuccs; i++) - { - FlowEdge* const edge = successors.nonDuplicates[i]; - edge->setLikelihood(newLikelihood * edge->getDupCount()); - } - } - else - { - weight_t const scaleFactor = 1.0 / (1.0 - defaultLikelihood); - JITDUMP("Scaling switch block " FMT_BB " likelihoods by " FMT_WT "\n", afterDefaultCondBlock->bbNum, - scaleFactor); - for (unsigned i = 0; i < successors.numDistinctSuccs; i++) - { - FlowEdge* const edge = successors.nonDuplicates[i]; - weight_t newLikelihood = scaleFactor * edge->getLikelihood(); - - if (newLikelihood > 1.0) - { - // tolerate small overflows - assert(Compiler::fgProfileWeightsEqual(newLikelihood, 1.0, 0.001)); - newLikelihood = 1.0; - } - edge->setLikelihood(newLikelihood); - } - } } + + comp->fgInvalidateSwitchDescMapEntry(afterDefaultCondBlock); } GenTree* next = node->gtNext; @@ -1309,7 +1191,6 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // targetCount - The number of distinct blocks in the jump table // bbSwitch - The switch block // switchValue - A LclVar node that provides the switch value -// defaultLikelihood - likelihood control flow took the default case (already checked) // // Return value: // true if the switch has been lowered to a bit test @@ -1330,12 +1211,8 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // than the traditional jump table base code. And of course, it also avoids the need // to emit the jump table itself that can reach up to 256 bytes (for 64 entries). // -bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[], - unsigned jumpCount, - unsigned targetCount, - BasicBlock* bbSwitch, - GenTree* switchValue, - weight_t defaultLikelihood) +bool Lowering::TryLowerSwitchToBitTest( + BasicBlock* jumpTable[], unsigned jumpCount, unsigned targetCount, BasicBlock* bbSwitch, GenTree* switchValue) { assert(jumpCount >= 2); assert(targetCount >= 2); @@ -1372,32 +1249,29 @@ bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[], // table and/or swap the blocks if it's beneficial. // - FlowEdge* case0Edge = nullptr; - FlowEdge* case1Edge = jumpTable[0]; - size_t bitTable = 1; + BasicBlock* bbCase0 = nullptr; + BasicBlock* bbCase1 = jumpTable[0]; + size_t bitTable = 1; for (unsigned bitIndex = 1; bitIndex < bitCount; bitIndex++) { - if (jumpTable[bitIndex] == case1Edge) + if (jumpTable[bitIndex] == bbCase1) { bitTable |= (size_t(1) << bitIndex); } - else if (case0Edge == nullptr) + else if (bbCase0 == nullptr) { - case0Edge = jumpTable[bitIndex]; + bbCase0 = jumpTable[bitIndex]; } - else if (jumpTable[bitIndex] != case0Edge) + else if (jumpTable[bitIndex] != bbCase0) { - // If it's neither case0Edge nor case`Edge then it means we have 3 targets. There can't be more + // If it's neither bbCase0 nor bbCase1 then it means we have 3 targets. There can't be more // than 3 because of the check at the start of the function. assert(targetCount == 3); return false; } } - BasicBlock* bbCase0 = case0Edge->getDestinationBlock(); - BasicBlock* bbCase1 = case1Edge->getDestinationBlock(); - // // One of the case blocks has to follow the switch block. This requirement could be avoided // by adding a BBJ_ALWAYS block after the switch block but doing that sometimes negatively @@ -1409,8 +1283,6 @@ bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[], return false; } - JITDUMP("Lowering switch " FMT_BB " to bit test\n", bbSwitch->bbNum); - #if defined(TARGET_64BIT) && defined(TARGET_XARCH) // // See if we can avoid a 8 byte immediate on 64 bit targets. If all upper 32 bits are 1 @@ -1435,33 +1307,11 @@ bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[], comp->fgRemoveAllRefPreds(bbCase1, bbSwitch); comp->fgRemoveAllRefPreds(bbCase0, bbSwitch); - case0Edge = comp->fgAddRefPred(bbCase0, bbSwitch, case0Edge); - case1Edge = comp->fgAddRefPred(bbCase1, bbSwitch, case1Edge); - - // If defaultLikelihood is not ~ 1.0 - // up-scale case likelihoods by 1.0 / (1.0 - defaultLikelihood) - // else switch block weight should be zero - // edge likelihoods are unknown, use 0.5 - // - bool const likelyToReachSwitch = !Compiler::fgProfileWeightsEqual(defaultLikelihood, 1.0, 0.001); - - if (likelyToReachSwitch) - { - weight_t const scaleFactor = 1.0 / (1.0 - defaultLikelihood); - case0Edge->setLikelihood(min(1.0, scaleFactor * case0Edge->getLikelihood())); - case1Edge->setLikelihood(min(1.0, scaleFactor * case1Edge->getLikelihood())); - } - else - { - case0Edge->setLikelihood(0.5); - case1Edge->setLikelihood(0.5); - } - if (bbSwitch->NextIs(bbCase0)) { // GenCondition::C generates JC so we jump to bbCase1 when the bit is set bbSwitchCondition = GenCondition::C; - bbSwitch->SetCond(case1Edge, case0Edge); + bbSwitch->SetCond(bbCase1, bbCase0); } else { @@ -1469,9 +1319,12 @@ bool Lowering::TryLowerSwitchToBitTest(FlowEdge* jumpTable[], // GenCondition::NC generates JNC so we jump to bbCase0 when the bit is not set bbSwitchCondition = GenCondition::NC; - bbSwitch->SetCond(case0Edge, case1Edge); + bbSwitch->SetCond(bbCase0, bbCase1); } + comp->fgAddRefPred(bbCase0, bbSwitch); + comp->fgAddRefPred(bbCase1, bbSwitch); + var_types bitTableType = (bitCount <= (genTypeSize(TYP_INT) * 8)) ? TYP_INT : TYP_LONG; GenTree* bitTableIcon = comp->gtNewIconNode(bitTable, bitTableType); @@ -1988,160 +1841,8 @@ GenTree* Lowering::AddrGen(void* addr) return AddrGen((ssize_t)addr); } -// LowerCallMemset: Replaces the following memset-like special intrinsics: -// -// SpanHelpers.Fill(ref dstRef, CNS_SIZE, CNS_VALUE) -// CORINFO_HELP_MEMSET(ref dstRef, CNS_VALUE, CNS_SIZE) -// SpanHelpers.ClearWithoutReferences(ref dstRef, CNS_SIZE) -// -// with a GT_STORE_BLK node: -// -// * STORE_BLK struct (init) (Unroll) -// +--* LCL_VAR byref dstRef -// \--* CNS_INT int 0 -// -// Arguments: -// tree - GenTreeCall node to replace with STORE_BLK -// next - [out] Next node to lower if this function returns true -// -// Return Value: -// false if no changes were made -// -bool Lowering::LowerCallMemset(GenTreeCall* call, GenTree** next) -{ - assert(call->IsSpecialIntrinsic(comp, NI_System_SpanHelpers_Fill) || - call->IsSpecialIntrinsic(comp, NI_System_SpanHelpers_ClearWithoutReferences) || - call->IsHelperCall(comp, CORINFO_HELP_MEMSET)); - - JITDUMP("Considering Memset-like call [%06d] for unrolling.. ", comp->dspTreeID(call)) - - if (comp->info.compHasNextCallRetAddr) - { - JITDUMP("compHasNextCallRetAddr=true so we won't be able to remove the call - bail out.\n"); - return false; - } - - GenTree* dstRefArg = call->gtArgs.GetUserArgByIndex(0)->GetNode(); - GenTree* lengthArg; - GenTree* valueArg; - - // Fill's length is not in bytes, so we need to scale it depending on the signature - unsigned lengthScale; - - if (call->IsSpecialIntrinsic(comp, NI_System_SpanHelpers_Fill)) - { - // void SpanHelpers::Fill(ref T refData, nuint numElements, T value) - // - assert(call->gtArgs.CountUserArgs() == 3); - lengthArg = call->gtArgs.GetUserArgByIndex(1)->GetNode(); - CallArg* valueCallArg = call->gtArgs.GetUserArgByIndex(2); - valueArg = valueCallArg->GetNode(); - - // Get that from the signature - lengthScale = genTypeSize(valueCallArg->GetSignatureType()); - // NOTE: structs and TYP_REF will be ignored by the "Value is not a constant" check - // Some of those cases can be enabled in future, e.g. s - } - else if (call->IsHelperCall(comp, CORINFO_HELP_MEMSET)) - { - // void CORINFO_HELP_MEMSET(ref T refData, byte value, nuint numElements) - // - assert(call->gtArgs.CountUserArgs() == 3); - lengthArg = call->gtArgs.GetUserArgByIndex(2)->GetNode(); - valueArg = call->gtArgs.GetUserArgByIndex(1)->GetNode(); - lengthScale = 1; // it's always in bytes - } - else - { - // void SpanHelpers::ClearWithoutReferences(ref byte b, nuint byteLength) - // - assert(call->IsSpecialIntrinsic(comp, NI_System_SpanHelpers_ClearWithoutReferences)); - assert(call->gtArgs.CountUserArgs() == 2); - - // Simple zeroing - lengthArg = call->gtArgs.GetUserArgByIndex(1)->GetNode(); - valueArg = comp->gtNewZeroConNode(TYP_INT); - lengthScale = 1; // it's always in bytes - } - - if (!lengthArg->IsIntegralConst()) - { - JITDUMP("Length is not a constant - bail out.\n"); - return false; - } - - if (!valueArg->IsCnsIntOrI() || !valueArg->TypeIs(TYP_INT)) - { - JITDUMP("Value is not a constant - bail out.\n"); - return false; - } - - // If value is not zero, we can only unroll for single-byte values - if (!valueArg->IsIntegralConst(0) && (lengthScale != 1)) - { - JITDUMP("Value is not unroll-friendly - bail out.\n"); - return false; - } - - // Convert lenCns to bytes - ssize_t lenCns = lengthArg->AsIntCon()->IconValue(); - if (CheckedOps::MulOverflows((target_ssize_t)lenCns, (target_ssize_t)lengthScale, CheckedOps::Signed)) - { - // lenCns overflows - JITDUMP("lenCns * lengthScale overflows - bail out.\n") - return false; - } - lenCns *= (ssize_t)lengthScale; - - // TODO-CQ: drop the whole thing in case of lenCns = 0 - if ((lenCns <= 0) || (lenCns > (ssize_t)comp->getUnrollThreshold(Compiler::UnrollKind::Memset))) - { - JITDUMP("Size is either 0 or too big to unroll - bail out.\n") - return false; - } - - JITDUMP("Accepted for unrolling!\nOld tree:\n"); - DISPTREERANGE(BlockRange(), call); - - if (!valueArg->IsIntegralConst(0)) - { - // Non-zero (byte) value, wrap value with GT_INIT_VAL - GenTree* initVal = valueArg; - valueArg = comp->gtNewOperNode(GT_INIT_VAL, TYP_INT, initVal); - BlockRange().InsertAfter(initVal, valueArg); - } - - GenTreeBlk* storeBlk = - comp->gtNewStoreBlkNode(comp->typGetBlkLayout((unsigned)lenCns), dstRefArg, valueArg, GTF_IND_UNALIGNED); - storeBlk->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; - - // Insert/Remove trees into LIR - BlockRange().InsertBefore(call, storeBlk); - if (call->IsSpecialIntrinsic(comp, NI_System_SpanHelpers_ClearWithoutReferences)) - { - // Value didn't exist in LIR previously - BlockRange().InsertBefore(storeBlk, valueArg); - } - - // Remove the call and mark everything as unused ... - BlockRange().Remove(call, true); - // ... except the args we're going to re-use - dstRefArg->ClearUnusedValue(); - valueArg->ClearUnusedValue(); - if (valueArg->OperIs(GT_INIT_VAL)) - { - valueArg->gtGetOp1()->ClearUnusedValue(); - } - - JITDUMP("\nNew tree:\n"); - DISPTREERANGE(BlockRange(), storeBlk); - *next = storeBlk; - return true; -} - //------------------------------------------------------------------------ // LowerCallMemmove: Replace Buffer.Memmove(DST, SRC, CNS_SIZE) with a GT_STORE_BLK: -// Do the same for CORINFO_HELP_MEMCPY(DST, SRC, CNS_SIZE) // // * STORE_BLK struct (copy) (Unroll) // +--* LCL_VAR byref dst @@ -2158,8 +1859,7 @@ bool Lowering::LowerCallMemset(GenTreeCall* call, GenTree** next) bool Lowering::LowerCallMemmove(GenTreeCall* call, GenTree** next) { JITDUMP("Considering Memmove [%06d] for unrolling.. ", comp->dspTreeID(call)) - assert(call->IsHelperCall(comp, CORINFO_HELP_MEMCPY) || - (comp->lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_SpanHelpers_Memmove)); + assert(comp->lookupNamedIntrinsic(call->gtCallMethHnd) == NI_System_Buffer_Memmove); assert(call->gtArgs.CountUserArgs() == 3); @@ -2193,8 +1893,7 @@ bool Lowering::LowerCallMemmove(GenTreeCall* call, GenTree** next) // TODO-CQ: Use GenTreeBlk::BlkOpKindUnroll here if srcAddr and dstAddr don't overlap, thus, we can // unroll this memmove as memcpy - it doesn't require lots of temp registers - storeBlk->gtBlkOpKind = call->IsHelperCall(comp, CORINFO_HELP_MEMCPY) ? GenTreeBlk::BlkOpKindUnroll - : GenTreeBlk::BlkOpKindUnrollMemmove; + storeBlk->gtBlkOpKind = GenTreeBlk::BlkOpKindUnrollMemmove; BlockRange().InsertBefore(call, srcBlk); BlockRange().InsertBefore(call, storeBlk); @@ -2515,49 +2214,16 @@ GenTree* Lowering::LowerCall(GenTree* node) } #if defined(TARGET_AMD64) || defined(TARGET_ARM64) - GenTree* nextNode = nullptr; if (call->gtCallMoreFlags & GTF_CALL_M_SPECIAL_INTRINSIC) { - switch (comp->lookupNamedIntrinsic(call->gtCallMethHnd)) + GenTree* nextNode = nullptr; + NamedIntrinsic ni = comp->lookupNamedIntrinsic(call->gtCallMethHnd); + if (((ni == NI_System_Buffer_Memmove) && LowerCallMemmove(call, &nextNode)) || + ((ni == NI_System_SpanHelpers_SequenceEqual) && LowerCallMemcmp(call, &nextNode))) { - case NI_System_SpanHelpers_Memmove: - if (LowerCallMemmove(call, &nextNode)) - { - return nextNode; - } - break; - - case NI_System_SpanHelpers_SequenceEqual: - if (LowerCallMemcmp(call, &nextNode)) - { - return nextNode; - } - break; - - case NI_System_SpanHelpers_Fill: - case NI_System_SpanHelpers_ClearWithoutReferences: - if (LowerCallMemset(call, &nextNode)) - { - return nextNode; - } - break; - - default: - break; + return nextNode; } } - - // Try to lower CORINFO_HELP_MEMCPY to unrollable STORE_BLK - if (call->IsHelperCall(comp, CORINFO_HELP_MEMCPY) && LowerCallMemmove(call, &nextNode)) - { - return nextNode; - } - - // Try to lower CORINFO_HELP_MEMSET to unrollable STORE_BLK - if (call->IsHelperCall(comp, CORINFO_HELP_MEMSET) && LowerCallMemset(call, &nextNode)) - { - return nextNode; - } #endif call->ClearOtherRegs(); @@ -3432,7 +3098,23 @@ void Lowering::LowerCFGCall(GenTreeCall* call) LowerNode(regNode); // Finally move all GT_PUTARG_* nodes - MoveCFGCallArgs(call); + for (CallArg& arg : call->gtArgs.EarlyArgs()) + { + GenTree* node = arg.GetEarlyNode(); + // Non-value nodes in early args are setup nodes for late args. + if (node->IsValue()) + { + assert(node->OperIsPutArg() || node->OperIsFieldList()); + MoveCFGCallArg(call, node); + } + } + + for (CallArg& arg : call->gtArgs.LateArgs()) + { + GenTree* node = arg.GetLateNode(); + assert(node->OperIsPutArg() || node->OperIsFieldList()); + MoveCFGCallArg(call, node); + } break; } case CFGCallKind::Dispatch: @@ -3579,38 +3261,6 @@ void Lowering::MoveCFGCallArg(GenTreeCall* call, GenTree* node) BlockRange().InsertBefore(call, node); } -//------------------------------------------------------------------------ -// MoveCFGCallArgs: Given a call that will be CFG transformed using the -// validate+call scheme, move all GT_PUTARG_* or GT_FIELD_LIST nodes right before the call. -// -// Arguments: -// call - The call that is being CFG transformed -// -// Remarks: -// See comments in MoveCFGCallArg for more details. -// -void Lowering::MoveCFGCallArgs(GenTreeCall* call) -{ - // Finally move all GT_PUTARG_* nodes - for (CallArg& arg : call->gtArgs.EarlyArgs()) - { - GenTree* node = arg.GetEarlyNode(); - // Non-value nodes in early args are setup nodes for late args. - if (node->IsValue()) - { - assert(node->OperIsPutArg() || node->OperIsFieldList()); - MoveCFGCallArg(call, node); - } - } - - for (CallArg& arg : call->gtArgs.LateArgs()) - { - GenTree* node = arg.GetLateNode(); - assert(node->OperIsPutArg() || node->OperIsFieldList()); - MoveCFGCallArg(call, node); - } -} - #ifndef TARGET_64BIT //------------------------------------------------------------------------ // Lowering::DecomposeLongCompare: Decomposes a TYP_LONG compare node. @@ -6143,26 +5793,6 @@ GenTree* Lowering::LowerNonvirtPinvokeCall(GenTreeCall* call) InsertPInvokeCallEpilog(call); } -#ifdef SWIFT_SUPPORT - // For Swift calls that require error handling, ensure the GT_SWIFT_ERROR node - // that consumes the error register is the call node's successor. - // This is to simplify logic for marking the error register as busy in LSRA. - if (call->HasSwiftErrorHandling()) - { - GenTree* swiftErrorNode = call->gtNext; - assert(swiftErrorNode != nullptr); - - while (!swiftErrorNode->OperIs(GT_SWIFT_ERROR)) - { - swiftErrorNode = swiftErrorNode->gtNext; - assert(swiftErrorNode != nullptr); - } - - BlockRange().Remove(swiftErrorNode); - BlockRange().InsertAfter(call, swiftErrorNode); - } -#endif // SWIFT_SUPPORT - return result; } @@ -8214,141 +7844,6 @@ void Lowering::ContainCheckBitCast(GenTree* node) } } -//------------------------------------------------------------------------ -// LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call -// -// Arguments: -// blkNode - The block store node to lower -// -void Lowering::LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode) -{ - // We shouldn't be using helper calls for blocks on heap containing GC pointers. - // due to atomicity guarantees. - assert(!blkNode->IsZeroingGcPointersOnHeap()); - - LIR::Use use; - assert(!BlockRange().TryGetUse(blkNode, &use)); - - const bool isVolatile = blkNode->IsVolatile(); - - GenTree* dest = blkNode->Addr(); - GenTree* data = blkNode->Data(); - GenTree* size; - - CorInfoHelpFunc helper; - - // Is it Memset ... - if (blkNode->OperIsInitBlkOp()) - { - helper = CORINFO_HELP_MEMSET; - - // Drop GT_INIT_VAL nodes - if (data->OperIsInitVal()) - { - BlockRange().Remove(data); - data = data->gtGetOp1(); - } - } - else - { - // ... or Memcpy? - helper = CORINFO_HELP_MEMCPY; - - if (data->OperIs(GT_IND)) - { - // Drop GT_IND nodes - BlockRange().Remove(data); - data = data->AsIndir()->Addr(); - } - else - { - assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); - - // Convert local to LCL_ADDR - unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); - - data->ChangeOper(GT_LCL_ADDR); - data->ChangeType(TYP_I_IMPL); - data->AsLclFld()->SetLclOffs(lclOffset); - data->ClearContained(); - } - } - - // Size is a constant - size = comp->gtNewIconNode(blkNode->Size(), TYP_I_IMPL); - BlockRange().InsertBefore(data, size); - - // A hacky way to safely call fgMorphTree in Lower - GenTree* destPlaceholder = comp->gtNewZeroConNode(dest->TypeGet()); - GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); - GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); - - const bool isMemzero = helper == CORINFO_HELP_MEMSET ? data->IsIntegralConst(0) : false; - - GenTreeCall* call; - if (isMemzero) - { - BlockRange().Remove(data); - call = comp->gtNewHelperCallNode(CORINFO_HELP_MEMZERO, TYP_VOID, destPlaceholder, sizePlaceholder); - } - else - { - call = comp->gtNewHelperCallNode(helper, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder); - } - comp->fgMorphArgs(call); - - LIR::Range range = LIR::SeqTree(comp, call); - GenTree* rangeStart = range.FirstNode(); - GenTree* rangeEnd = range.LastNode(); - - BlockRange().InsertBefore(blkNode, std::move(range)); - blkNode->gtBashToNOP(); - - LIR::Use destUse; - LIR::Use sizeUse; - BlockRange().TryGetUse(destPlaceholder, &destUse); - BlockRange().TryGetUse(sizePlaceholder, &sizeUse); - destUse.ReplaceWith(dest); - sizeUse.ReplaceWith(size); - destPlaceholder->SetUnusedValue(); - sizePlaceholder->SetUnusedValue(); - - if (!isMemzero) - { - LIR::Use dataUse; - BlockRange().TryGetUse(dataPlaceholder, &dataUse); - dataUse.ReplaceWith(data); - dataPlaceholder->SetUnusedValue(); - } - - LowerRange(rangeStart, rangeEnd); - - // Finally move all GT_PUTARG_* nodes - // Re-use the existing logic for CFG call args here - MoveCFGCallArgs(call); - - BlockRange().Remove(destPlaceholder); - BlockRange().Remove(sizePlaceholder); - if (!isMemzero) - { - BlockRange().Remove(dataPlaceholder); - } - -// Wrap with memory barriers on weak memory models -// if the block store was volatile -#ifndef TARGET_XARCH - if (isVolatile) - { - GenTree* firstBarrier = comp->gtNewMemoryBarrier(); - GenTree* secondBarrier = comp->gtNewMemoryBarrier(/*loadOnly*/ true); - BlockRange().InsertBefore(call, firstBarrier); - BlockRange().InsertAfter(call, secondBarrier); - LowerNode(firstBarrier); - LowerNode(secondBarrier); - } -#endif -} - struct StoreCoalescingData { var_types targetType; @@ -9008,7 +8503,7 @@ bool Lowering::TryMakeIndirsAdjacent(GenTreeIndir* prevIndir, GenTreeIndir* indi // We can reorder indirs with some calls, but introducing a LIR edge // that spans a call can introduce spills (or callee-saves). - if (cur->IsCall()) + if (cur->IsCall() || (cur->OperIsStoreBlk() && (cur->AsBlk()->gtBlkOpKind == GenTreeBlk::BlkOpKindHelper))) { JITDUMP(" ..but they are separated by node [%06u] that kills registers\n", Compiler::dspTreeID(cur)); return false; @@ -9241,36 +8736,6 @@ void Lowering::UnmarkTree(GenTree* node) #endif // TARGET_ARM64 -//------------------------------------------------------------------------ -// IsContainableLclAddr: Can a given local address be contained? -// -// Most local addresses can be contained, however, there are two edge cases -// where this is not true: -// 1. When the resulting memory access will go beyond the local's location. -// 2. When the resulting access may go past a UINT16_MAX. -// Both of these requirements are imposed by the emitter. -// -// Arguments: -// lclAddr - The local address node -// accessSize - The access size (of an indirection) -// -// Return Value: -// Whether an indirection of "accessSize" may contain "lclAddr". -// -bool Lowering::IsContainableLclAddr(GenTreeLclFld* lclAddr, unsigned accessSize) const -{ - if (CheckedOps::AddOverflows(lclAddr->GetLclOffs(), accessSize, CheckedOps::Unsigned) || - !comp->IsValidLclAddr(lclAddr->GetLclNum(), lclAddr->GetLclOffs() + accessSize - 1)) - { - // We depend on containment for correctness of liveness updates in codegen. Therefore, all - // locals that may "return false" here MUST be address-exposed. Local morph ensures this. - assert(comp->lvaGetDesc(lclAddr)->IsAddressExposed()); - return false; - } - - return true; -} - //------------------------------------------------------------------------ // TransformUnusedIndirection: change the opcode and the type of the unused indirection. // @@ -9375,6 +8840,7 @@ void Lowering::LowerLclHeap(GenTree* node) GenTreeBlk(GT_STORE_BLK, TYP_STRUCT, heapLcl, zero, comp->typGetBlkLayout((unsigned)alignedSize)); storeBlk->gtFlags |= (GTF_IND_UNALIGNED | GTF_ASG | GTF_EXCEPT | GTF_GLOB_REF); BlockRange().InsertAfter(use.Def(), heapLcl, zero, storeBlk); + LowerNode(storeBlk); } else { @@ -9395,10 +8861,13 @@ void Lowering::LowerLclHeap(GenTree* node) // void Lowering::LowerBlockStoreCommon(GenTreeBlk* blkNode) { - assert(blkNode->OperIs(GT_STORE_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); if (blkNode->ContainsReferences() && !blkNode->OperIsCopyBlkOp()) { + // Make sure we don't use GT_STORE_DYN_BLK + assert(blkNode->OperIs(GT_STORE_BLK)); + // and we only zero it (and that zero is better to be not hoisted/CSE'd) assert(blkNode->Data()->IsIntegralConst(0)); } @@ -9434,12 +8903,17 @@ void Lowering::LowerBlockStoreCommon(GenTreeBlk* blkNode) // bool Lowering::TryTransformStoreObjAsStoreInd(GenTreeBlk* blkNode) { - assert(blkNode->OperIs(GT_STORE_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); if (!comp->opts.OptimizationEnabled()) { return false; } + if (blkNode->OperIs(GT_STORE_DYN_BLK)) + { + return false; + } + var_types regType = blkNode->GetLayout()->GetRegisterType(); if (regType == TYP_UNDEF) { diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 76124820944f3c..d35738c944dcf2 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -114,7 +114,7 @@ class Lowering final : public Phase void ContainCheckIntrinsic(GenTreeOp* node); #endif // TARGET_XARCH #ifdef FEATURE_HW_INTRINSICS - void ContainCheckHWIntrinsicAddr(GenTreeHWIntrinsic* node, GenTree* addr, unsigned size); + void ContainCheckHWIntrinsicAddr(GenTreeHWIntrinsic* node, GenTree* addr); void ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node); #ifdef TARGET_XARCH void TryFoldCnsVecForEmbeddedBroadcast(GenTreeHWIntrinsic* parentNode, GenTreeVecCon* childNode); @@ -140,9 +140,7 @@ class Lowering final : public Phase GenTree* LowerCall(GenTree* call); bool LowerCallMemmove(GenTreeCall* call, GenTree** next); bool LowerCallMemcmp(GenTreeCall* call, GenTree** next); - bool LowerCallMemset(GenTreeCall* call, GenTree** next); void LowerCFGCall(GenTreeCall* call); - void MoveCFGCallArgs(GenTreeCall* call); void MoveCFGCallArg(GenTreeCall* call, GenTree* node); #ifndef TARGET_64BIT GenTree* DecomposeLongCompare(GenTree* cmp); @@ -333,7 +331,6 @@ class Lowering final : public Phase GenTree* LowerSignedDivOrMod(GenTree* node); void LowerBlockStore(GenTreeBlk* blkNode); void LowerBlockStoreCommon(GenTreeBlk* blkNode); - void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); void LowerLclHeap(GenTree* node); void ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent); void LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode); @@ -351,12 +348,8 @@ class Lowering final : public Phase void TryRetypingFloatingPointStoreToIntegerStore(GenTree* store); GenTree* LowerSwitch(GenTree* node); - bool TryLowerSwitchToBitTest(FlowEdge* jumpTable[], - unsigned jumpCount, - unsigned targetCount, - BasicBlock* bbSwitch, - GenTree* switchValue, - weight_t defaultLikelihood); + bool TryLowerSwitchToBitTest( + BasicBlock* jumpTable[], unsigned jumpCount, unsigned targetCount, BasicBlock* bbSwitch, GenTree* switchValue); void LowerCast(GenTree* node); @@ -496,8 +489,6 @@ class Lowering final : public Phase return false; } - bool IsContainableLclAddr(GenTreeLclFld* lclAddr, unsigned accessSize) const; - #ifdef TARGET_ARM64 bool IsContainableUnaryOrBinaryOp(GenTree* parentNode, GenTree* childNode) const; #endif // TARGET_ARM64 diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 9d28135c92a1a0..5c0acbbdc40115 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -585,7 +585,8 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if ((size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT)) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && + src->OperIs(GT_CNS_INT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -633,8 +634,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - LowerBlockStoreAsHelperCall(blkNode); - return; + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; } } else @@ -650,7 +650,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = layout->HasGCPtr(); + bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy); if (doCpObj && (size <= copyBlockUnrollLimit)) @@ -685,8 +685,9 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK)); - LowerBlockStoreAsHelperCall(blkNode); + assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; } } } @@ -705,7 +706,7 @@ void Lowering::ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenT assert(blkNode->OperIs(GT_STORE_BLK) && (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)); assert(size < INT32_MAX); - if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), size)) + if (addr->OperIs(GT_LCL_ADDR)) { addr->SetContained(); return; @@ -2060,7 +2061,7 @@ void Lowering::ContainCheckIndir(GenTreeIndir* indirNode) MakeSrcContained(indirNode, addr); } } - else if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), indirNode->Size())) + else if (addr->OperIs(GT_LCL_ADDR)) { // These nodes go into an addr mode: // - GT_LCL_ADDR is a stack addr mode. @@ -3187,24 +3188,6 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) break; } - case NI_Sve_CreateTrueMaskByte: - case NI_Sve_CreateTrueMaskDouble: - case NI_Sve_CreateTrueMaskInt16: - case NI_Sve_CreateTrueMaskInt32: - case NI_Sve_CreateTrueMaskInt64: - case NI_Sve_CreateTrueMaskSByte: - case NI_Sve_CreateTrueMaskSingle: - case NI_Sve_CreateTrueMaskUInt16: - case NI_Sve_CreateTrueMaskUInt32: - case NI_Sve_CreateTrueMaskUInt64: - assert(hasImmediateOperand); - assert(varTypeIsIntegral(intrin.op1)); - if (intrin.op1->IsCnsIntOrI()) - { - MakeSrcContained(node, intrin.op1); - } - break; - default: unreached(); } diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index f0b61aeba66305..5110442eda10d1 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -296,7 +296,8 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if ((size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT)) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && + src->OperIs(GT_CNS_INT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -333,8 +334,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - LowerBlockStoreAsHelperCall(blkNode); - return; + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; } } else @@ -350,7 +350,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = layout->HasGCPtr(); + bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy); if (doCpObj && (size <= copyBlockUnrollLimit)) @@ -386,8 +386,9 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK)); - LowerBlockStoreAsHelperCall(blkNode); + assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; } } } @@ -407,7 +408,7 @@ void Lowering::ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenT assert(blkNode->OperIs(GT_STORE_BLK) && (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)); assert(size < INT32_MAX); - if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), size)) + if (addr->OperIs(GT_LCL_ADDR)) { addr->SetContained(); return; @@ -485,8 +486,7 @@ void Lowering::LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode) } // Codegen supports containment of local addresses under BLKs. - if (src->OperIs(GT_BLK) && src->AsBlk()->Addr()->IsLclVarAddr() && - IsContainableLclAddr(src->AsBlk()->Addr()->AsLclFld(), src->AsBlk()->Size())) + if (src->OperIs(GT_BLK) && src->AsBlk()->Addr()->IsLclVarAddr()) { // TODO-LOONGARCH64-CQ: support containment of LCL_ADDR with non-zero offset too. MakeSrcContained(src, src->AsBlk()->Addr()); @@ -705,7 +705,7 @@ void Lowering::ContainCheckIndir(GenTreeIndir* indirNode) { MakeSrcContained(indirNode, addr); } - else if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), indirNode->Size())) + else if (addr->OperIs(GT_LCL_ADDR)) { // These nodes go into an addr mode: // - GT_LCL_ADDR is a stack addr mode. diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index 22830e92ba25c2..4f60458fd25161 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -245,7 +245,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if ((size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT)) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= INITBLK_UNROLL_LIMIT) && src->OperIs(GT_CNS_INT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -282,8 +282,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - LowerBlockStoreAsHelperCall(blkNode); - return; + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; } } else @@ -298,11 +297,10 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) comp->lvaSetVarDoNotEnregister(srcLclNum DEBUGARG(DoNotEnregisterReason::BlockOp)); } - ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = layout->HasGCPtr(); - unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy); + ClassLayout* layout = blkNode->GetLayout(); + bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); - if (doCpObj && (size <= copyBlockUnrollLimit)) + if (doCpObj && (size <= CPBLK_UNROLL_LIMIT)) { // No write barriers are needed on the stack. // If the layout contains a byref, then we know it must live on the stack. @@ -322,7 +320,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } - else if (blkNode->OperIs(GT_STORE_BLK) && (size <= copyBlockUnrollLimit)) + else if (blkNode->OperIs(GT_STORE_BLK) && (size <= CPBLK_UNROLL_LIMIT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -335,8 +333,9 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK)); - LowerBlockStoreAsHelperCall(blkNode); + assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; } } } @@ -355,7 +354,7 @@ void Lowering::ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenT assert(blkNode->OperIs(GT_STORE_BLK) && (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)); assert(size < INT32_MAX); - if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), size)) + if (addr->OperIs(GT_LCL_ADDR)) { addr->SetContained(); return; @@ -616,7 +615,7 @@ void Lowering::ContainCheckIndir(GenTreeIndir* indirNode) { MakeSrcContained(indirNode, addr); } - else if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), indirNode->Size())) + else if (addr->OperIs(GT_LCL_ADDR)) { // These nodes go into an addr mode: // - GT_LCL_ADDR is a stack addr mode. diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 5c4f05a04ad57a..6709a57a49e8d7 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -349,7 +349,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset))) { if (!src->OperIs(GT_CNS_INT)) { @@ -407,20 +407,14 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) else { TOO_BIG_TO_UNROLL: - if (blkNode->IsZeroingGcPointersOnHeap()) - { - blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindLoop; - } - else - { #ifdef TARGET_AMD64 - LowerBlockStoreAsHelperCall(blkNode); - return; + blkNode->gtBlkOpKind = + blkNode->IsZeroingGcPointersOnHeap() ? GenTreeBlk::BlkOpKindLoop : GenTreeBlk::BlkOpKindHelper; #else - // TODO-X86-CQ: Investigate whether a helper call would be beneficial on x86 - blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindRepInstr; + // TODO-X86-CQ: Investigate whether a helper call would be beneficial on x86 + blkNode->gtBlkOpKind = + blkNode->IsZeroingGcPointersOnHeap() ? GenTreeBlk::BlkOpKindLoop : GenTreeBlk::BlkOpKindRepInstr; #endif - } } } else @@ -436,7 +430,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = layout->HasGCPtr(); + bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy, false); #ifndef JIT32_GCENCODER @@ -516,11 +510,10 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); #ifdef TARGET_AMD64 - LowerBlockStoreAsHelperCall(blkNode); - return; + blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindHelper; #else // TODO-X86-CQ: Investigate whether a helper call would be beneficial on x86 blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindRepInstr; @@ -529,6 +522,13 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } assert(blkNode->gtBlkOpKind != GenTreeBlk::BlkOpKindInvalid); + +#ifndef TARGET_X86 + if ((MIN_ARG_AREA_FOR_CALL > 0) && (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindHelper)) + { + RequireOutgoingArgSpace(blkNode, MIN_ARG_AREA_FOR_CALL); + } +#endif } //------------------------------------------------------------------------ @@ -545,7 +545,7 @@ void Lowering::ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenT assert(blkNode->OperIs(GT_STORE_BLK) && (blkNode->gtBlkOpKind == GenTreeBlk::BlkOpKindUnroll)); assert(size < INT32_MAX); - if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), size)) + if (addr->OperIs(GT_LCL_ADDR)) { addr->SetContained(); return; @@ -6687,10 +6687,12 @@ void Lowering::ContainCheckIndir(GenTreeIndir* node) // The address of an indirection that requires its address in a reg. // Skip any further processing that might otherwise make it contained. } - else if (addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), node->Size())) + else if (addr->OperIs(GT_LCL_ADDR)) { // These nodes go into an addr mode: // - GT_LCL_ADDR is a stack addr mode. + + // make this contained, it turns into a constant that goes into an addr mode MakeSrcContained(node, addr); } else if (addr->IsCnsIntOrI()) @@ -7628,8 +7630,7 @@ bool Lowering::LowerRMWMemOp(GenTreeIndir* storeInd) // If it is a GT_LCL_VAR, it still needs the reg to hold the address. // We would still need a reg for GT_CNS_INT if it doesn't fit within addressing mode base. - if (indirCandidateChild->OperIs(GT_LCL_ADDR) && - IsContainableLclAddr(indirCandidateChild->AsLclFld(), storeInd->Size())) + if (indirCandidateChild->OperIs(GT_LCL_ADDR)) { indirDst->SetContained(); } @@ -8760,8 +8761,6 @@ void Lowering::TryFoldCnsVecForEmbeddedBroadcast(GenTreeHWIntrinsic* parentNode, void Lowering::TryCompressConstVecData(GenTreeStoreInd* node) { assert(node->Data()->IsCnsVec()); - assert(node->Data()->AsVecCon()->TypeIs(TYP_SIMD32, TYP_SIMD64)); - GenTreeVecCon* vecCon = node->Data()->AsVecCon(); GenTreeHWIntrinsic* broadcast = nullptr; @@ -8831,24 +8830,16 @@ void Lowering::TryCompressConstVecData(GenTreeStoreInd* node) // Arguments: // node - The hardware intrinsic node // addr - The address node to try contain -// size - Size of the memory access (can be an overestimate) // -void Lowering::ContainCheckHWIntrinsicAddr(GenTreeHWIntrinsic* node, GenTree* addr, unsigned size) +void Lowering::ContainCheckHWIntrinsicAddr(GenTreeHWIntrinsic* node, GenTree* addr) { - assert((genActualType(addr) == TYP_I_IMPL) || (addr->TypeGet() == TYP_BYREF)); - if ((addr->OperIs(GT_LCL_ADDR) && IsContainableLclAddr(addr->AsLclFld(), size)) || - (addr->IsCnsIntOrI() && addr->AsIntConCommon()->FitsInAddrBase(comp))) + assert((addr->TypeGet() == TYP_I_IMPL) || (addr->TypeGet() == TYP_BYREF)); + TryCreateAddrMode(addr, true, node); + if ((addr->OperIs(GT_LCL_ADDR, GT_LEA) || (addr->IsCnsIntOrI() && addr->AsIntConCommon()->FitsInAddrBase(comp))) && + IsInvariantInRange(addr, node)) { MakeSrcContained(node, addr); } - else - { - TryCreateAddrMode(addr, true, node); - if (addr->OperIs(GT_LEA) && IsInvariantInRange(addr, node)) - { - MakeSrcContained(node, addr); - } - } } //---------------------------------------------------------------------------------------------- @@ -8923,7 +8914,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) switch (category) { case HW_Category_MemoryLoad: - ContainCheckHWIntrinsicAddr(node, op1, simdSize); + ContainCheckHWIntrinsicAddr(node, op1); break; case HW_Category_SimpleSIMD: @@ -8981,7 +8972,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { if (node->OperIsMemoryLoad()) { - ContainCheckHWIntrinsicAddr(node, op1, /* conservative maximum */ 16); + ContainCheckHWIntrinsicAddr(node, op1); return; } break; @@ -8994,7 +8985,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { if (node->OperIsMemoryLoad()) { - ContainCheckHWIntrinsicAddr(node, op1, /* conservative maximum */ 8); + ContainCheckHWIntrinsicAddr(node, op1); return; } @@ -9033,7 +9024,7 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { if (node->OperIsMemoryLoad()) { - ContainCheckHWIntrinsicAddr(node, op1, /* conservative maximum */ 16); + ContainCheckHWIntrinsicAddr(node, op1); return; } @@ -9132,16 +9123,16 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case HW_Category_MemoryLoad: if ((intrinsicId == NI_AVX_MaskLoad) || (intrinsicId == NI_AVX2_MaskLoad)) { - ContainCheckHWIntrinsicAddr(node, op1, simdSize); + ContainCheckHWIntrinsicAddr(node, op1); } else { - ContainCheckHWIntrinsicAddr(node, op2, simdSize); + ContainCheckHWIntrinsicAddr(node, op2); } break; case HW_Category_MemoryStore: - ContainCheckHWIntrinsicAddr(node, op1, /* conservative maximum */ simdSize); + ContainCheckHWIntrinsicAddr(node, op1); break; case HW_Category_SimpleSIMD: @@ -9495,6 +9486,10 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) switch (category) { + case HW_Category_MemoryStore: + ContainCheckHWIntrinsicAddr(node, op1); + break; + case HW_Category_SimpleSIMD: case HW_Category_SIMDScalar: case HW_Category_Scalar: diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index b2d37b9becad9d..04ca6149c9fc36 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -788,7 +788,7 @@ LinearScan::LinearScan(Compiler* theCompiler) availableFloatRegs = RBM_ALLFLOAT; availableDoubleRegs = RBM_ALLDOUBLE; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) availableMaskRegs = RBM_ALLMASK; #endif @@ -1681,21 +1681,6 @@ bool LinearScan::isRegCandidate(LclVarDsc* varDsc) return false; } - // Avoid allocating parameters that are passed in float regs into integer - // registers. We currently home float registers before integer registers, - // so that kind of enregistration can trash integer registers containing - // other parameters. - // We assume that these cases will be homed to float registers if they are - // promoted. - // TODO-CQ: Combine integer and float register homing to handle these kinds - // of conflicts. - if ((varDsc->TypeGet() == TYP_STRUCT) && varDsc->lvIsRegArg && !varDsc->lvPromoted && - varTypeUsesIntReg(varDsc->GetRegisterType()) && genIsValidFloatReg(varDsc->GetArgReg())) - { - compiler->lvaSetVarDoNotEnregister(lclNum DEBUGARG(DoNotEnregisterReason::IsStructArg)); - return false; - } - // Are we not optimizing and we have exception handlers? // if so mark all args and locals as volatile, so that they // won't ever get enregistered. @@ -5124,13 +5109,6 @@ void LinearScan::allocateRegistersMinimal() } regsInUseThisLocation |= currentRefPosition.registerAssignment; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); - -#ifdef SWIFT_SUPPORT - if (currentRefPosition.delayRegFree) - { - regsInUseNextLocation |= currentRefPosition.registerAssignment; - } -#endif // SWIFT_SUPPORT } else { @@ -5840,13 +5818,6 @@ void LinearScan::allocateRegisters() } regsInUseThisLocation |= currentRefPosition.registerAssignment; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); - -#ifdef SWIFT_SUPPORT - if (currentRefPosition.delayRegFree) - { - regsInUseNextLocation |= currentRefPosition.registerAssignment; - } -#endif // SWIFT_SUPPORT } else { @@ -5951,62 +5922,9 @@ void LinearScan::allocateRegisters() assert(lclVarInterval->isLocalVar); if (refType == RefTypeUpperVectorSave) { - assert(currentInterval->recentRefPosition == ¤tRefPosition); - - // For a given RefTypeUpperVectorSave, there should be a matching RefTypeUpperVectorRestore - // If not, probably, this was an extra one we added conservatively and should not need a register. - RefPosition* nextRefPosition = currentRefPosition.nextRefPosition; - bool isExtraUpperVectorSave = currentRefPosition.IsExtraUpperVectorSave(); - - if ((lclVarInterval->physReg == REG_NA) || isExtraUpperVectorSave || + if ((lclVarInterval->physReg == REG_NA) || (lclVarInterval->isPartiallySpilled && (currentInterval->physReg == REG_STK))) { - if (!currentRefPosition.liveVarUpperSave) - { - if (isExtraUpperVectorSave) - { - // If this was just an extra upperVectorSave that do not have corresponding - // upperVectorRestore, we do not need to mark this as isPartiallySpilled - // or need to insert the save/restore. - currentRefPosition.skipSaveRestore = true; - } - - if (assignedRegister != REG_NA) - { - // If we ever assigned register to this interval, it was because in the past - // there were valid save/restore RefPositions associated. For non-live vars, - // we want to reduce the affect of their presence and hence, we will - // unassign register from this interval without spilling and free it. - - // We do not take similar action on upperVectorRestore below because here, we - // have already removed the register association with the interval. - // The "allocate = false" route, will do a no-op. - if (currentInterval->isActive) - { - RegRecord* physRegRecord = getRegisterRecord(assignedRegister); - unassignPhysRegNoSpill(physRegRecord); - } - else - { - updateNextIntervalRef(assignedRegister, currentInterval); - updateSpillCost(assignedRegister, currentInterval); - } - - regsToFree |= getRegMask(assignedRegister, currentInterval->registerType); - } - INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_NO_REG_ALLOCATED, nullptr, assignedRegister)); - currentRefPosition.registerAssignment = RBM_NONE; - lastAllocatedRefPosition = ¤tRefPosition; - - continue; - } - else - { - // We should never have an extra upperVectorSave for non-live var because there will - // always be a valid use for which we will add the restore. - assert(!isExtraUpperVectorSave); - } - allocate = false; } #if defined(TARGET_XARCH) @@ -8051,15 +7969,7 @@ void LinearScan::resolveRegisters() insertUpperVectorSave(treeNode, currentRefPosition, currentRefPosition->getInterval(), block); } - - if (!currentRefPosition->IsExtraUpperVectorSave()) - { - localVarInterval->isPartiallySpilled = true; - } - else - { - assert(!currentRefPosition->liveVarUpperSave); - } + localVarInterval->isPartiallySpilled = true; } } else @@ -10186,7 +10096,7 @@ void LinearScan::dumpLsraStatsCsv(FILE* file) { fprintf(file, ",%u", sumStats[statIndex]); } - fprintf(file, ",%.2f\n", compiler->Metrics.PerfScore); + fprintf(file, ",%.2f\n", compiler->info.compPerfScore); } // ----------------------------------------------------------- @@ -11835,7 +11745,8 @@ void LinearScan::verifyFinalAllocation() } } - currentLocation = currentRefPosition.nodeLocation; + LsraLocation newLocation = currentRefPosition.nodeLocation; + currentLocation = newLocation; switch (currentRefPosition.refType) { @@ -12177,8 +12088,7 @@ void LinearScan::verifyFinalAllocation() (currentRefPosition.refType == RefTypeUpperVectorRestore)) { Interval* lclVarInterval = interval->relatedInterval; - assert((lclVarInterval->physReg == REG_NA) || lclVarInterval->isPartiallySpilled || - currentRefPosition.IsExtraUpperVectorSave()); + assert((lclVarInterval->physReg == REG_NA) || lclVarInterval->isPartiallySpilled); } } #endif // FEATURE_PARTIAL_SIMD_CALLEE_SAVE diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 9620abbc5a7824..c0e0f5d2fdbd34 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -51,12 +51,12 @@ RegisterType regType(T type) { return IntRegisterType; } -#if (defined(TARGET_XARCH) || defined(TARGET_ARM64)) && defined(FEATURE_SIMD) +#if defined(TARGET_XARCH) && defined(FEATURE_SIMD) else if (varTypeUsesMaskReg(type)) { return MaskRegisterType; } -#endif // (TARGET_XARCH || TARGET_ARM64) && FEATURE_SIMD +#endif // TARGET_XARCH && FEATURE_SIMD else { assert(varTypeUsesFloatReg(type)); @@ -1662,12 +1662,12 @@ class LinearScan : public LinearScanInterface PhasedVar availableIntRegs; PhasedVar availableFloatRegs; PhasedVar availableDoubleRegs; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) PhasedVar availableMaskRegs; #endif PhasedVar* availableRegs[TYP_COUNT]; -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) +#if defined(TARGET_XARCH) #define allAvailableRegs (availableIntRegs | availableFloatRegs | availableMaskRegs) #else #define allAvailableRegs (availableIntRegs | availableFloatRegs) @@ -2517,10 +2517,6 @@ class RefPosition #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE // If upper vector save/restore can be avoided. unsigned char skipSaveRestore : 1; - // If upper vector save is related to live var - // or created just based on bbLiveIn/bbDefs and - // whose liveness is not entirely know. - unsigned char liveVarUpperSave : 1; #endif #ifdef DEBUG @@ -2724,11 +2720,6 @@ class RefPosition #endif #endif // TARGET_ARM64 - FORCEINLINE bool IsExtraUpperVectorSave() const - { - assert(refType == RefTypeUpperVectorSave); - return (nextRefPosition == nullptr) || (nextRefPosition->refType != RefTypeUpperVectorRestore); - } #ifdef DEBUG // operator= copies everything except 'rpNum', which must remain unique RefPosition& operator=(const RefPosition& rp) diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 2192265984d68e..30991778868d62 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -579,6 +579,7 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: + case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 1096d7f11701c5..ea3bc9d7fb37e0 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1076,6 +1076,7 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: + case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -1281,20 +1282,6 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = BuildSelect(tree->AsOp()); break; -#ifdef SWIFT_SUPPORT - case GT_SWIFT_ERROR: - srcCount = 0; - assert(dstCount == 1); - - // Any register should do here, but the error register value should immediately - // be moved from GT_SWIFT_ERROR's destination register to the SwiftError struct, - // and we know REG_SWIFT_ERROR should be busy up to this point, anyway. - // By forcing LSRA to use REG_SWIFT_ERROR as both the source and destination register, - // we can ensure the redundant move is elided. - BuildDef(tree, RBM_SWIFT_ERROR); - break; -#endif // SWIFT_SUPPORT - } // end switch (tree->OperGet()) if (tree->IsUnusedValue() && (dstCount != 0)) @@ -1329,9 +1316,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou const HWIntrinsic intrin(intrinsicTree); - int srcCount = 0; - int dstCount = 0; - regMaskTP dstCandidates = RBM_NONE; + int srcCount = 0; + int dstCount = 0; if (HWIntrinsicInfo::IsMultiReg(intrin.id)) { @@ -1444,19 +1430,6 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou assert(intrin.op4->isContainedIntOrIImmed()); break; - case NI_Sve_CreateTrueMaskByte: - case NI_Sve_CreateTrueMaskDouble: - case NI_Sve_CreateTrueMaskInt16: - case NI_Sve_CreateTrueMaskInt32: - case NI_Sve_CreateTrueMaskInt64: - case NI_Sve_CreateTrueMaskSByte: - case NI_Sve_CreateTrueMaskSingle: - case NI_Sve_CreateTrueMaskUInt16: - case NI_Sve_CreateTrueMaskUInt32: - case NI_Sve_CreateTrueMaskUInt64: - needBranchTargetReg = !intrin.op1->isContainedIntOrIImmed(); - break; - default: unreached(); } @@ -1545,11 +1518,6 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou srcCount++; } } - else if (HWIntrinsicInfo::IsMaskedOperation(intrin.id)) - { - regMaskTP predMask = HWIntrinsicInfo::IsLowMaskedOperation(intrin.id) ? RBM_LOWMASK : RBM_ALLMASK; - srcCount += BuildOperandUses(intrin.op1, predMask); - } else if (intrinsicTree->OperIsMemoryLoadOrStore()) { srcCount += BuildAddrUses(intrin.op1); @@ -1716,7 +1684,6 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou assert(intrinsicTree->OperIsMemoryLoadOrStore()); srcCount += BuildAddrUses(intrin.op3); - buildInternalRegisterUses(); FALLTHROUGH; } @@ -1749,7 +1716,6 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou } return srcCount; } - else if (intrin.op2 != nullptr) { // RMW intrinsic operands doesn't have to be delayFree when they can be assigned the same register as op1Reg @@ -1804,11 +1770,11 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if ((dstCount == 1) || (dstCount == 2)) { - BuildDef(intrinsicTree, dstCandidates); + BuildDef(intrinsicTree); if (dstCount == 2) { - BuildDef(intrinsicTree, dstCandidates, 1); + BuildDef(intrinsicTree, RBM_NONE, 1); } } else @@ -2058,30 +2024,17 @@ bool RefPosition::isLiveAtConsecutiveRegistersLoc(LsraLocation consecutiveRegist return true; } - bool atConsecutiveRegsLoc = consecutiveRegistersLocation == nodeLocation; - bool treeNeedsConsecutiveRegisters = false; - - if ((treeNode != nullptr) && treeNode->OperIsHWIntrinsic()) - { - const HWIntrinsic intrin(treeNode->AsHWIntrinsic()); - treeNeedsConsecutiveRegisters = HWIntrinsicInfo::NeedsConsecutiveRegisters(intrin.id); - } - if (refType == RefTypeDef) { - return treeNeedsConsecutiveRegisters; - } - else if (refType == RefTypeUse) - { - if (isIntervalRef() && getInterval()->isInternal) + if (treeNode->OperIsHWIntrinsic()) { - return treeNeedsConsecutiveRegisters; + const HWIntrinsic intrin(treeNode->AsHWIntrinsic()); + return HWIntrinsicInfo::NeedsConsecutiveRegisters(intrin.id); } - return atConsecutiveRegsLoc; } - else if (refType == RefTypeUpperVectorRestore) + else if ((refType == RefTypeUse) || (refType == RefTypeUpperVectorRestore)) { - return atConsecutiveRegsLoc; + return consecutiveRegistersLocation == nodeLocation; } return false; } diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index 4738fcf33725e6..a0f27ba65b3058 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -226,7 +226,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (hasMultiRegRetVal) { assert(retTypeDesc != nullptr); - dstCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); + dstCandidates = retTypeDesc->GetABIReturnRegs(); } else if (varTypeUsesFloatArgReg(registerType)) { @@ -393,29 +393,6 @@ int LinearScan::BuildCall(GenTreeCall* call) regMaskTP killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); -#ifdef SWIFT_SUPPORT - if (call->HasSwiftErrorHandling()) - { - // Tree is a Swift call with error handling; error register should have been killed - assert((killMask & RBM_SWIFT_ERROR) != 0); - - // After a Swift call that might throw returns, we expect the error register to be consumed - // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed - // before GT_SWIFT_ERROR can consume it. - // (For example, the PInvoke epilog comes before the error register store.) - // To do so, delay the freeing of the error register until the next node. - // This only works if the next node after the call is the GT_SWIFT_ERROR node. - // (InsertPInvokeCallEpilog should have moved the GT_SWIFT_ERROR node during lowering.) - assert(call->gtNext != nullptr); - assert(call->gtNext->OperIs(GT_SWIFT_ERROR)); - - // We could use RefTypeKill, but RefTypeFixedReg is used less commonly, so the check for delayRegFree - // during register allocation should be cheaper in terms of TP. - RefPosition* pos = newRefPosition(REG_SWIFT_ERROR, currentLoc, RefTypeFixedReg, call, RBM_SWIFT_ERROR); - setDelayFree(pos); - } -#endif // SWIFT_SUPPORT - // No args are placed in registers anymore. placedArgRegs = RBM_NONE; numPlacedArgLocals = 0; @@ -667,6 +644,13 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode, availableIntRegs); break; + case GenTreeBlk::BlkOpKindHelper: + assert(!src->isContained()); + dstAddrRegMask = RBM_ARG_0; + srcRegMask = RBM_ARG_1; + sizeRegMask = RBM_ARG_2; + break; + default: unreached(); } @@ -697,13 +681,6 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode, internalIntCandidates); } - if (size >= 4 * REGSIZE_BYTES && compiler->IsBaselineSimdIsaSupported()) - { - // We can use 128-bit SIMD ldp/stp for larger block sizes - buildInternalFloatRegisterDefForNode(blkNode, internalFloatRegCandidates()); - buildInternalFloatRegisterDefForNode(blkNode, internalFloatRegCandidates()); - } - // If we have a dest address we want it in RBM_WRITE_BARRIER_DST_BYREF. dstAddrRegMask = RBM_WRITE_BARRIER_DST_BYREF; @@ -806,12 +783,22 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } break; + case GenTreeBlk::BlkOpKindHelper: + dstAddrRegMask = RBM_ARG_0; + if (srcAddrOrFill != nullptr) + { + assert(!srcAddrOrFill->isContained()); + srcRegMask = RBM_ARG_1; + } + sizeRegMask = RBM_ARG_2; + break; + default: unreached(); } } - if (sizeRegMask != RBM_NONE) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -842,6 +829,12 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } + if (blkNode->OperIs(GT_STORE_DYN_BLK)) + { + useCount++; + BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); + } + buildInternalRegisterUses(); regMaskTP killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 8fd31c072ffed3..0c1d3f74475c8d 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -635,8 +635,7 @@ RefPosition* LinearScan::newRefPosition(Interval* theInterval, newRP->setRegOptional(false); #if FEATURE_PARTIAL_SIMD_CALLEE_SAVE - newRP->skipSaveRestore = false; - newRP->liveVarUpperSave = false; + newRP->skipSaveRestore = false; #endif associateRefPosWithInterval(newRP); @@ -881,16 +880,6 @@ regMaskTP LinearScan::getKillSetForCall(GenTreeCall* call) assert(!call->IsVirtualStub() || ((killMask & compiler->virtualStubParamInfo->GetRegMask()) == compiler->virtualStubParamInfo->GetRegMask())); #endif // !TARGET_ARM - -#ifdef SWIFT_SUPPORT - // Swift calls that throw may trash the callee-saved error register, - // so don't use the register post-call until it is consumed by SwiftError. - if (call->HasSwiftErrorHandling()) - { - killMask |= RBM_SWIFT_ERROR; - } -#endif // SWIFT_SUPPORT - return killMask; } @@ -918,6 +907,18 @@ regMaskTP LinearScan::getKillSetForBlockStore(GenTreeBlk* blkNode) killMask = compiler->compHelperCallKillSet(CORINFO_HELP_ASSIGN_BYREF); break; +#ifndef TARGET_X86 + case GenTreeBlk::BlkOpKindHelper: + if (isCopyBlk) + { + killMask = compiler->compHelperCallKillSet(CORINFO_HELP_MEMCPY); + } + else + { + killMask = compiler->compHelperCallKillSet(CORINFO_HELP_MEMSET); + } + break; +#endif #ifdef TARGET_XARCH case GenTreeBlk::BlkOpKindRepInstr: if (isCopyBlk) @@ -1054,6 +1055,7 @@ regMaskTP LinearScan::getKillSetForNode(GenTree* tree) break; case GT_STORE_BLK: + case GT_STORE_DYN_BLK: killMask = getKillSetForBlockStore(tree->AsBlk()); break; @@ -1489,25 +1491,9 @@ void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation cu assert((fpCalleeKillSet & RBM_FLT_CALLEE_TRASH) != RBM_NONE); assert((fpCalleeKillSet & RBM_FLT_CALLEE_SAVED) == RBM_NONE); - // We should only save the upper half of any large vector vars that are currently live. - // However, the liveness information may not be accurate, specially around the place where - // we load the LCL_VAR and the node that uses it. Hence, as a conservative approach, we will - // add all variables that are live-in/defined in the block. We need to add variable although - // it is not in the live-out set, because a variable may get defined before the call and - // (last) used after the call. - // - // This will create more UpperSave/UpperRestore RefPositions then needed, but we need to do - // this for correctness anyway. - VARSET_TP bbLiveDefs(VarSetOps::Union(compiler, compiler->compCurBB->bbLiveIn, compiler->compCurBB->bbVarDef)); - - VARSET_TP liveDefsLargeVectors(VarSetOps::Intersection(compiler, bbLiveDefs, largeVectorVars)); - - // Make sure that `liveLargeVectors` captures the currentLiveVars as well. - VARSET_TP liveLargeVectors(VarSetOps::Intersection(compiler, currentLiveVars, largeVectorVars)); - - assert(VarSetOps::IsSubset(compiler, liveLargeVectors, liveDefsLargeVectors)); - - VarSetOps::Iter iter(compiler, liveDefsLargeVectors); + // We only need to save the upper half of any large vector vars that are currently live. + VARSET_TP liveLargeVectors(VarSetOps::Intersection(compiler, currentLiveVars, largeVectorVars)); + VarSetOps::Iter iter(compiler, liveLargeVectors); unsigned varIndex = 0; bool blockAlwaysReturn = compiler->compCurBB->KindIs(BBJ_THROW, BBJ_EHFINALLYRET, BBJ_EHFAULTRET, BBJ_EHFILTERRET, BBJ_EHCATCHRET); @@ -1522,7 +1508,6 @@ void LinearScan::buildUpperVectorSaveRefPositions(GenTree* tree, LsraLocation cu newRefPosition(upperVectorInterval, currentLoc, RefTypeUpperVectorSave, tree, RBM_FLT_CALLEE_SAVED); varInterval->isPartiallySpilled = true; pos->skipSaveRestore = blockAlwaysReturn; - pos->liveVarUpperSave = VarSetOps::IsMember(compiler, liveLargeVectors, varIndex); #ifdef TARGET_XARCH pos->regOptional = true; #endif @@ -1600,21 +1585,12 @@ void LinearScan::buildUpperVectorRestoreRefPosition( { if (lclVarInterval->isPartiallySpilled) { - lclVarInterval->isPartiallySpilled = false; - unsigned varIndex = lclVarInterval->getVarIndex(compiler); - Interval* upperVectorInterval = getUpperVectorInterval(varIndex); - RefPosition* savePos = upperVectorInterval->recentRefPosition; - if (!isUse && !savePos->liveVarUpperSave) - { - // If we are just restoring upper vector at the block boundary and if this is not - // a upperVector related to the liveVar, then ignore creating restore for them. - // During allocation, we will detect that this was an extra save-upper and skip - // the save/restore altogether. - return; - } - + unsigned varIndex = lclVarInterval->getVarIndex(compiler); + Interval* upperVectorInterval = getUpperVectorInterval(varIndex); + RefPosition* savePos = upperVectorInterval->recentRefPosition; RefPosition* restorePos = newRefPosition(upperVectorInterval, currentLoc, RefTypeUpperVectorRestore, node, RBM_NONE); + lclVarInterval->isPartiallySpilled = false; restorePos->setMultiRegIdx(multiRegIdx); @@ -1622,14 +1598,12 @@ void LinearScan::buildUpperVectorRestoreRefPosition( { // If there was a use of the restore before end of the block restore, // then it is needed and cannot be eliminated - savePos->skipSaveRestore = false; - savePos->liveVarUpperSave = true; + savePos->skipSaveRestore = false; } else { // otherwise, just do the whatever was decided for save position - restorePos->skipSaveRestore = savePos->skipSaveRestore; - restorePos->liveVarUpperSave = savePos->liveVarUpperSave; + restorePos->skipSaveRestore = savePos->skipSaveRestore; } #ifdef TARGET_XARCH @@ -2519,7 +2493,7 @@ void LinearScan::buildIntervals() killed = RBM_EDI | RBM_ECX | RBM_EAX; #else // Poisoning uses REG_SCRATCH for small vars and memset helper for big vars. - killed = genRegMask(REG_SCRATCH) | compiler->compHelperCallKillSet(CORINFO_HELP_NATIVE_MEMSET); + killed = genRegMask(REG_SCRATCH) | compiler->compHelperCallKillSet(CORINFO_HELP_MEMSET); #endif addRefsForPhysRegMask(killed, currentLoc + 1, RefTypeKill, true); currentLoc += 2; @@ -3058,8 +3032,7 @@ void LinearScan::BuildDefs(GenTree* tree, int dstCount, regMaskTP dstCandidates) // For all other cases of multi-reg definitions, the registers must be in sequential order. if (retTypeDesc != nullptr) { - thisDstCandidates = genRegMask( - tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i, tree->AsCall()->GetUnmanagedCallConv())); + thisDstCandidates = genRegMask(tree->AsCall()->GetReturnTypeDesc()->GetABIReturnReg(i)); assert((dstCandidates & thisDstCandidates) != RBM_NONE); } else @@ -4004,8 +3977,7 @@ int LinearScan::BuildReturn(GenTree* tree) if (srcType != dstType) { hasMismatchedRegTypes = true; - regMaskTP dstRegMask = - genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)); + regMaskTP dstRegMask = genRegMask(retTypeDesc.GetABIReturnReg(i)); if (varTypeUsesIntReg(dstType)) { @@ -4032,7 +4004,7 @@ int LinearScan::BuildReturn(GenTree* tree) if (!hasMismatchedRegTypes || (regType(op1->AsLclVar()->GetFieldTypeByIndex(compiler, i)) == regType(retTypeDesc.GetReturnRegType(i)))) { - BuildUse(op1, genRegMask(retTypeDesc.GetABIReturnReg(i, compiler->info.compCallConv)), i); + BuildUse(op1, genRegMask(retTypeDesc.GetABIReturnReg(i)), i); } else { diff --git a/src/coreclr/jit/lsraloongarch64.cpp b/src/coreclr/jit/lsraloongarch64.cpp index 1ceb61e536255d..67b27aa51300c8 100644 --- a/src/coreclr/jit/lsraloongarch64.cpp +++ b/src/coreclr/jit/lsraloongarch64.cpp @@ -394,6 +394,7 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: + case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -748,7 +749,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (hasMultiRegRetVal) { assert(retTypeDesc != nullptr); - dstCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); + dstCandidates = retTypeDesc->GetABIReturnRegs(); } else if (varTypeUsesFloatArgReg(registerType)) { @@ -1103,6 +1104,13 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode, availableIntRegs); break; + case GenTreeBlk::BlkOpKindHelper: + assert(!src->isContained()); + dstAddrRegMask = RBM_ARG_0; + srcRegMask = RBM_ARG_1; + sizeRegMask = RBM_ARG_2; + break; + default: unreached(); } @@ -1151,12 +1159,22 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode); break; + case GenTreeBlk::BlkOpKindHelper: + dstAddrRegMask = RBM_ARG_0; + if (srcAddrOrFill != nullptr) + { + assert(!srcAddrOrFill->isContained()); + srcRegMask = RBM_ARG_1; + } + sizeRegMask = RBM_ARG_2; + break; + default: unreached(); } } - if (sizeRegMask != RBM_NONE) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -1187,6 +1205,12 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } + if (blkNode->OperIs(GT_STORE_DYN_BLK)) + { + useCount++; + BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); + } + buildInternalRegisterUses(); regMaskTP killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); diff --git a/src/coreclr/jit/lsrariscv64.cpp b/src/coreclr/jit/lsrariscv64.cpp index 6af21c06ab208d..ec4ca4a34972b8 100644 --- a/src/coreclr/jit/lsrariscv64.cpp +++ b/src/coreclr/jit/lsrariscv64.cpp @@ -309,7 +309,7 @@ int LinearScan::BuildNode(GenTree* tree) needTemp = true; } - if (!needTemp && tree->OperIs(GT_DIV, GT_MOD)) + if (!needTemp && (tree->gtOper == GT_DIV || tree->gtOper == GT_MOD)) { if ((exceptions & ExceptionSetFlags::ArithmeticException) != ExceptionSetFlags::None) needTemp = true; @@ -512,6 +512,7 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: + case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -911,7 +912,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (hasMultiRegRetVal) { assert(retTypeDesc != nullptr); - dstCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); + dstCandidates = retTypeDesc->GetABIReturnRegs(); } else if (varTypeUsesFloatArgReg(registerType)) { @@ -1259,6 +1260,13 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode, availableIntRegs); break; + case GenTreeBlk::BlkOpKindHelper: + assert(!src->isContained()); + dstAddrRegMask = RBM_ARG_0; + srcRegMask = RBM_ARG_1; + sizeRegMask = RBM_ARG_2; + break; + default: unreached(); } @@ -1307,12 +1315,22 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode); break; + case GenTreeBlk::BlkOpKindHelper: + dstAddrRegMask = RBM_ARG_0; + if (srcAddrOrFill != nullptr) + { + assert(!srcAddrOrFill->isContained()); + srcRegMask = RBM_ARG_1; + } + sizeRegMask = RBM_ARG_2; + break; + default: unreached(); } } - if (sizeRegMask != RBM_NONE) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -1343,6 +1361,12 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } + if (blkNode->OperIs(GT_STORE_DYN_BLK)) + { + useCount++; + BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); + } + buildInternalRegisterUses(); regMaskTP killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 1e7935ee5a2153..cb2b82d5d8ce92 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -507,6 +507,7 @@ int LinearScan::BuildNode(GenTree* tree) #endif // FEATURE_PUT_STRUCT_ARG_STK case GT_STORE_BLK: + case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -632,24 +633,11 @@ int LinearScan::BuildNode(GenTree* tree) } break; -#ifdef SWIFT_SUPPORT - case GT_SWIFT_ERROR: - srcCount = 0; - assert(dstCount == 1); - - // Any register should do here, but the error register value should immediately - // be moved from GT_SWIFT_ERROR's destination register to the SwiftError struct, - // and we know REG_SWIFT_ERROR should be busy up to this point, anyway. - // By forcing LSRA to use REG_SWIFT_ERROR as both the source and destination register, - // we can ensure the redundant move is elided. - BuildDef(tree, RBM_SWIFT_ERROR); - break; -#endif // SWIFT_SUPPORT - } // end switch (tree->OperGet()) // We need to be sure that we've set srcCount and dstCount appropriately. - assert((dstCount < 2) || tree->IsMultiRegNode()); + // Not that for XARCH, the maximum number of registers defined is 2. + assert((dstCount < 2) || ((dstCount == 2) && tree->IsMultiRegNode())); assert(isLocalDefUse == (tree->IsValue() && tree->IsUnusedValue())); assert(!tree->IsValue() || (dstCount != 0)); assert(dstCount == tree->GetRegisterDstCount(compiler)); @@ -1184,7 +1172,7 @@ int LinearScan::BuildCall(GenTreeCall* call) if (hasMultiRegRetVal) { assert(retTypeDesc != nullptr); - dstCandidates = retTypeDesc->GetABIReturnRegs(call->GetUnmanagedCallConv()); + dstCandidates = retTypeDesc->GetABIReturnRegs(); assert((int)genCountBits(dstCandidates) == dstCount); } else if (varTypeUsesFloatReg(registerType)) @@ -1369,29 +1357,6 @@ int LinearScan::BuildCall(GenTreeCall* call) regMaskTP killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); -#ifdef SWIFT_SUPPORT - if (call->HasSwiftErrorHandling()) - { - // Tree is a Swift call with error handling; error register should have been killed - assert((killMask & RBM_SWIFT_ERROR) != 0); - - // After a Swift call that might throw returns, we expect the error register to be consumed - // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed - // before GT_SWIFT_ERROR can consume it. - // (For example, the PInvoke epilog comes before the error register store.) - // To do so, delay the freeing of the error register until the next node. - // This only works if the next node after the call is the GT_SWIFT_ERROR node. - // (InsertPInvokeCallEpilog should have moved the GT_SWIFT_ERROR node during lowering.) - assert(call->gtNext != nullptr); - assert(call->gtNext->OperIs(GT_SWIFT_ERROR)); - - // We could use RefTypeKill, but RefTypeFixedReg is used less commonly, so the check for delayRegFree - // during register allocation should be cheaper in terms of TP. - RefPosition* pos = newRefPosition(REG_SWIFT_ERROR, currentLoc, RefTypeFixedReg, call, RBM_SWIFT_ERROR); - setDelayFree(pos); - } -#endif // SWIFT_SUPPORT - // No args are placed in registers anymore. placedArgRegs = RBM_NONE; numPlacedArgLocals = 0; @@ -1469,6 +1434,14 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) buildInternalIntRegisterDefForNode(blkNode, availableIntRegs); break; +#ifdef TARGET_AMD64 + case GenTreeBlk::BlkOpKindHelper: + dstAddrRegMask = RBM_ARG_0; + srcRegMask = RBM_ARG_1; + sizeRegMask = RBM_ARG_2; + break; +#endif + default: unreached(); } @@ -1589,6 +1562,14 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) sizeRegMask = RBM_RCX; break; +#ifdef TARGET_AMD64 + case GenTreeBlk::BlkOpKindHelper: + dstAddrRegMask = RBM_ARG_0; + srcRegMask = RBM_ARG_1; + sizeRegMask = RBM_ARG_2; + break; +#endif + default: unreached(); } @@ -1601,7 +1582,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (sizeRegMask != RBM_NONE) + if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -1632,6 +1613,12 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } + if (blkNode->OperIs(GT_STORE_DYN_BLK)) + { + useCount++; + BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); + } + #ifdef TARGET_X86 // If we require a byte register on x86, we may run into an over-constrained situation // if we have BYTE_REG_COUNT or more uses (currently, it can be at most 4, if both the diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 90d599e4984068..0c44bbad973ea6 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -765,10 +765,6 @@ const char* getWellKnownArgName(WellKnownArg arg) return "ValidateIndirectCallTarget"; case WellKnownArg::DispatchIndirectCallTarget: return "DispatchIndirectCallTarget"; - case WellKnownArg::SwiftError: - return "SwiftError"; - case WellKnownArg::SwiftSelf: - return "SwiftSelf"; } return "N/A"; @@ -2047,23 +2043,17 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call PushBack(comp, NewCallArg::Primitive(newArg).WellKnown(WellKnownArg::WrapperDelegateCell)); } #endif // defined(TARGET_ARM) - - bool addStubCellArg = true; - -#ifdef TARGET_X86 +#ifndef TARGET_X86 // TODO-X86-CQ: Currently RyuJIT/x86 passes args on the stack, so this is not needed. // If/when we change that, the following code needs to be changed to correctly support the (TBD) managed calling // convention for x86/SSE. - addStubCellArg = call->gtCallType != CT_INDIRECT && comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); -#endif - // We are allowed to have a ret buffer argument combined // with any of the remaining non-standard arguments // CLANG_FORMAT_COMMENT_ANCHOR; - if (call->IsVirtualStub() && addStubCellArg) + if (call->IsVirtualStub()) { if (!call->IsTailCallViaJitHelper()) { @@ -2080,7 +2070,9 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call // add as a non-standard arg. } } - else if (call->gtCallType == CT_INDIRECT && (call->gtCallCookie != nullptr)) + else +#endif // !TARGET_X86 + if (call->gtCallType == CT_INDIRECT && (call->gtCallCookie != nullptr)) { assert(!call->IsUnmanaged()); @@ -6102,11 +6094,19 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) // If this block has a flow successor, make suitable updates. // - if (compCurBB->KindIs(BBJ_ALWAYS)) + BasicBlock* nextBlock = compCurBB->GetUniqueSucc(); + + if (nextBlock == nullptr) { - // Flow no longer reaches the target from here. + // No unique successor. compCurBB should be a return. // - fgRemoveRefPred(compCurBB->GetTargetEdge()); + assert(compCurBB->KindIs(BBJ_RETURN)); + } + else + { + // Flow no longer reaches nextBlock from here. + // + fgRemoveRefPred(nextBlock, compCurBB); // Adjust profile weights of the successor blocks. // @@ -6116,8 +6116,7 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) BasicBlock* curBlock = compCurBB; if (curBlock->hasProfileWeight()) { - weight_t weightLoss = curBlock->bbWeight; - BasicBlock* nextBlock = curBlock->GetTarget(); + weight_t weightLoss = curBlock->bbWeight; while (nextBlock->hasProfileWeight()) { @@ -6146,22 +6145,15 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) nextBlock->bbNum, nextWeight, compCurBB->bbNum, weightLoss); } - if (!nextBlock->KindIs(BBJ_ALWAYS)) + curBlock = nextBlock; + nextBlock = curBlock->GetUniqueSucc(); + if (nextBlock == nullptr) { break; } - - curBlock = nextBlock; - nextBlock = curBlock->GetTarget(); } } } - else - { - // No unique successor. compCurBB should be a return. - // - assert(compCurBB->KindIs(BBJ_RETURN)); - } #if !FEATURE_TAILCALL_OPT_SHARED_RETURN // We enable shared-ret tail call optimization for recursive calls even if @@ -6172,7 +6164,8 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) // Many tailcalls will have call and ret in the same block, and thus be // BBJ_RETURN, but if the call falls through to a ret, and we are doing a // tailcall, change it here. - compCurBB->SetKindAndTargetEdge(BBJ_RETURN); + // (compCurBB may have a jump target, so use SetKind() to avoid nulling it) + compCurBB->SetKind(BBJ_RETURN); } GenTree* stmtExpr = fgMorphStmt->GetRootNode(); @@ -6323,7 +6316,7 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) { // We call CORINFO_HELP_TAILCALL which does not return, so we will // not need epilogue. - compCurBB->SetKindAndTargetEdge(BBJ_THROW); + compCurBB->SetKindAndTarget(BBJ_THROW); } if (isRootReplaced) @@ -7006,8 +6999,7 @@ GenTree* Compiler::getVirtMethodPointerTree(GenTree* thisPtr, } //------------------------------------------------------------------------ -// getTokenHandleTree: get a handle tree for a token. This method should never -// be called for tokens imported from inlinees. +// getTokenHandleTree: get a handle tree for a token // // Arguments: // pResolvedToken - token to get a handle for @@ -7019,14 +7011,7 @@ GenTree* Compiler::getVirtMethodPointerTree(GenTree* thisPtr, GenTree* Compiler::getTokenHandleTree(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool parent) { CORINFO_GENERICHANDLE_RESULT embedInfo; - - // NOTE: inlining is done at this point, so we don't know which method contained this token. - // It's fine because currently this is never used for something that belongs to an inlinee. - // Namely, we currently use it for: - // 1) Methods with EH are never inlined - // 2) Methods with explicit tail calls are never inlined - // - info.compCompHnd->embedGenericHandle(pResolvedToken, parent, info.compMethodHnd, &embedInfo); + info.compCompHnd->embedGenericHandle(pResolvedToken, parent, &embedInfo); GenTree* result = getLookupTree(pResolvedToken, &embedInfo.lookup, gtTokenToIconFlags(pResolvedToken->token), embedInfo.compileTimeHandle); @@ -7478,8 +7463,7 @@ void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCa { // Todo: this may not look like a viable loop header. // Might need the moral equivalent of a scratch BB. - FlowEdge* const newEdge = fgAddRefPred(fgEntryBB, block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + block->SetKindAndTarget(BBJ_ALWAYS, fgEntryBB); } else { @@ -7494,11 +7478,11 @@ void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCa // block removal on it. // fgFirstBB->SetFlags(BBF_DONT_REMOVE); - FlowEdge* const newEdge = fgAddRefPred(fgFirstBB->Next(), block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + block->SetKindAndTarget(BBJ_ALWAYS, fgFirstBB->Next()); } // Finish hooking things up. + fgAddRefPred(block->GetTarget(), block); block->RemoveFlags(BBF_HAS_JMP); } @@ -9231,11 +9215,11 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA // TODO #4104: there are a lot of other places where // this condition is not checked before transformations. - noway_assert(op2); - if (fgGlobalMorph && !op2->TypeIs(TYP_BYREF)) + if (fgGlobalMorph) { /* Check for "op1 - cns2" , we change it to "op1 + (-cns2)" */ + noway_assert(op2); if (op2->IsCnsIntOrI() && !op2->IsIconHandle()) { // Negate the constant and change the node to be "+", @@ -9253,7 +9237,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA noway_assert(op1); if (op1->IsCnsIntOrI()) { - noway_assert(varTypeIsIntegralOrI(tree)); + noway_assert(varTypeIsIntOrI(tree)); // The type of the new GT_NEG node cannot just be op2->TypeGet(). // Otherwise we may sign-extend incorrectly in cases where the GT_NEG @@ -12801,6 +12785,10 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) gtUpdateNodeSideEffects(tree); break; + case GT_STORE_DYN_BLK: + tree = fgMorphStoreDynBlock(tree->AsStoreDynBlk()); + break; + case GT_SELECT: tree->AsConditional()->gtCond = fgMorphTree(tree->AsConditional()->gtCond); tree->AsConditional()->gtOp1 = fgMorphTree(tree->AsConditional()->gtOp1); @@ -13004,7 +12992,7 @@ void Compiler::fgAssertionGen(GenTree* tree) AssertionIndex ifFalseAssertionIndex; AssertionIndex ifTrueAssertionIndex; - if (info.AssertionHoldsOnFalseEdge()) + if (info.IsNextEdgeAssertion()) { ifFalseAssertionIndex = info.GetAssertionIndex(); ifTrueAssertionIndex = optFindComplementary(ifFalseAssertionIndex); @@ -13204,31 +13192,20 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) // BasicBlock* bTaken; BasicBlock* bNotTaken; - FlowEdge* edgeTaken; if (cond->AsIntCon()->gtIconVal != 0) { // JTRUE 1 - transform the basic block into a BBJ_ALWAYS bTaken = block->GetTrueTarget(); bNotTaken = block->GetFalseTarget(); - - // Remove 'block' from the predecessor list of 'bNotTaken' */ - fgRemoveRefPred(block->GetFalseEdge()); - - edgeTaken = block->GetTrueEdge(); - block->SetKindAndTargetEdge(BBJ_ALWAYS, edgeTaken); + block->SetKind(BBJ_ALWAYS); } else { // JTRUE 0 - transform the basic block into a BBJ_ALWAYS bTaken = block->GetFalseTarget(); bNotTaken = block->GetTrueTarget(); - - // Remove 'block' from the predecessor list of 'bNotTaken' */ - fgRemoveRefPred(block->GetTrueEdge()); - - edgeTaken = block->GetFalseEdge(); - block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetFalseEdge()); + block->SetKindAndTarget(BBJ_ALWAYS, bTaken); block->SetFlags(BBF_NONE_QUIRK); } @@ -13238,7 +13215,8 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) // and we have already computed the edge weights, so // we will try to adjust some of the weights // - BasicBlock* bUpdated = nullptr; // non-NULL if we updated the weight of an internal block + FlowEdge* edgeTaken = fgGetPredForBlock(bTaken, block); + BasicBlock* bUpdated = nullptr; // non-NULL if we updated the weight of an internal block // We examine the taken edge (block -> bTaken) // if block has valid profile weight and bTaken does not we try to adjust bTaken's weight @@ -13284,19 +13262,19 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) switch (bUpdated->GetKind()) { case BBJ_COND: - edge = bUpdated->GetFalseEdge(); + edge = fgGetPredForBlock(bUpdated->GetFalseTarget(), bUpdated); newMaxWeight = bUpdated->bbWeight; newMinWeight = min(edge->edgeWeightMin(), newMaxWeight); edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->GetFalseTarget()); - edge = bUpdated->GetTrueEdge(); + edge = fgGetPredForBlock(bUpdated->GetTrueTarget(), bUpdated); newMaxWeight = bUpdated->bbWeight; newMinWeight = min(edge->edgeWeightMin(), newMaxWeight); edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->GetFalseTarget()); break; case BBJ_ALWAYS: - edge = bUpdated->GetTargetEdge(); + edge = fgGetPredForBlock(bUpdated->GetTarget(), bUpdated); newMaxWeight = bUpdated->bbWeight; newMinWeight = min(edge->edgeWeightMin(), newMaxWeight); edge->setEdgeWeights(newMinWeight, newMaxWeight, bUpdated->Next()); @@ -13309,6 +13287,11 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) } } + /* modify the flow graph */ + + /* Remove 'block' from the predecessor list of 'bNotTaken' */ + fgRemoveRefPred(bNotTaken, block); + #ifdef DEBUG if (verbose) { @@ -13374,29 +13357,29 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) // modify the flow graph // Find the actual jump target - size_t switchVal = (size_t)cond->AsIntCon()->gtIconVal; - unsigned jumpCnt = block->GetSwitchTargets()->bbsCount; - FlowEdge** jumpTab = block->GetSwitchTargets()->bbsDstTab; - bool foundVal = false; + size_t switchVal = (size_t)cond->AsIntCon()->gtIconVal; + unsigned jumpCnt = block->GetSwitchTargets()->bbsCount; + BasicBlock** jumpTab = block->GetSwitchTargets()->bbsDstTab; + bool foundVal = false; for (unsigned val = 0; val < jumpCnt; val++, jumpTab++) { - FlowEdge* curEdge = *jumpTab; + BasicBlock* curJump = *jumpTab; - assert(curEdge->getDestinationBlock()->countOfInEdges() > 0); + assert(curJump->countOfInEdges() > 0); // If val matches switchVal or we are at the last entry and // we never found the switch value then set the new jump dest if ((val == switchVal) || (!foundVal && (val == jumpCnt - 1))) { - block->SetKindAndTargetEdge(BBJ_ALWAYS, curEdge); + block->SetKindAndTarget(BBJ_ALWAYS, curJump); foundVal = true; } else { - // Remove 'curEdge' - fgRemoveRefPred(curEdge); + // Remove 'block' from the predecessor list of 'curJump' + fgRemoveRefPred(curJump, block); } } @@ -13578,7 +13561,11 @@ void Compiler::fgMorphStmtBlockOps(BasicBlock* block, Statement* stmt) { if ((*use)->OperIsBlkOp()) { - if ((*use)->OperIsInitBlkOp()) + if ((*use)->OperIs(GT_STORE_DYN_BLK)) + { + *use = m_compiler->fgMorphStoreDynBlock((*use)->AsStoreDynBlk()); + } + else if ((*use)->OperIsInitBlkOp()) { *use = m_compiler->fgMorphInitBlock(*use); } @@ -14142,8 +14129,8 @@ void Compiler::fgMergeBlockReturn(BasicBlock* block) else #endif // !TARGET_X86 { - FlowEdge* const newEdge = fgAddRefPred(genReturnBB, block); - block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + block->SetKindAndTarget(BBJ_ALWAYS, genReturnBB); + fgAddRefPred(genReturnBB, block); fgReturnCount--; } @@ -14518,6 +14505,241 @@ GenTreeQmark* Compiler::fgGetTopLevelQmark(GenTree* expr, GenTree** ppDst /* = N return topQmark; } +//------------------------------------------------------------------------ +// fgExpandQmarkForCastInstOf: expand qmark for cast +// +// Arguments: +// block - block containing the qmark +// stmt - statement containing the qmark +// +// Returns: +// true if the expansion introduced a throwing block +// +// Notes: +// +// For a castclass helper call, importer creates the following tree: +// tmp = (op1 == null) ? op1 : ((*op1 == (cse = op2, cse)) ? op1 : helper()); +// +// This method splits the qmark expression created by the importer into the +// following blocks: (block, asg, cond1, cond2, helper, remainder). +// Notice that op1 is the result for both the conditions. So we coalesce these +// assignments into a single block instead of two blocks resulting a nested diamond. +// +// +---------->-----------+ +// | | | +// ^ ^ v +// | | | +// block-->asg-->cond1--+-->cond2--+-->helper--+-->remainder +// +// We expect to achieve the following codegen: +// mov rsi, rdx tmp2 = op1 // asgBlock +// test rsi, rsi goto skip if tmp2 == null ? // cond1Block +// je SKIP +// mov rcx, 0x76543210 cns = op2 // cond2Block +// cmp qword ptr [rsi], rcx goto skip if *tmp2 == op2 +// je SKIP +// call CORINFO_HELP_CHKCASTCLASS_SPECIAL tmp2 = helper(cns, tmp2) // helperBlock +// mov rsi, rax +// SKIP: // remainderBlock +// mov rdi, rsi tmp = tmp2 +// tmp has the result. +// +// Note that we can't use `tmp` during the computation of the result: we must create a new temp, +// and only assign `tmp` to the final value. This is because `tmp` may already have been annotated +// via lvaSetClass/lvaUpdateClass as having a known type. This is only true after the full expansion, +// where any other type gets converted to null. If we used `tmp` during the expansion, then it would +// appear to subsequent optimizations that cond2Block (where the type is checked) is unnecessary. +// +bool Compiler::fgExpandQmarkForCastInstOf(BasicBlock* block, Statement* stmt) +{ +#ifdef DEBUG + if (verbose) + { + printf("\nExpanding CastInstOf qmark in " FMT_BB " (before)\n", block->bbNum); + fgDispBasicBlocks(block, block, true); + } +#endif // DEBUG + + bool introducedThrow = false; + GenTree* expr = stmt->GetRootNode(); + + GenTree* dst = nullptr; + GenTreeQmark* qmark = fgGetTopLevelQmark(expr, &dst); + + noway_assert(dst != nullptr); + assert(dst->OperIsLocalStore()); + assert(qmark->gtFlags & GTF_QMARK_CAST_INSTOF); + + // Get cond, true, false exprs for the qmark. + GenTree* condExpr = qmark->gtGetOp1(); + GenTree* trueExpr = qmark->gtGetOp2()->AsColon()->ThenNode(); + GenTree* falseExpr = qmark->gtGetOp2()->AsColon()->ElseNode(); + + // Get cond, true, false exprs for the nested qmark. + GenTree* nestedQmark = falseExpr; + GenTree* cond2Expr; + GenTree* true2Expr; + GenTree* false2Expr; + + unsigned nestedQmarkElseLikelihood = 50; + if (nestedQmark->gtOper == GT_QMARK) + { + cond2Expr = nestedQmark->gtGetOp1(); + true2Expr = nestedQmark->gtGetOp2()->AsColon()->ThenNode(); + false2Expr = nestedQmark->gtGetOp2()->AsColon()->ElseNode(); + nestedQmarkElseLikelihood = nestedQmark->AsQmark()->ElseNodeLikelihood(); + } + else + { + // This is a rare case that arises when we are doing minopts and encounter isinst of null. + // gtFoldExpr was still able to optimize away part of the tree (but not all). + // That means it does not match our pattern. + // + // Rather than write code to handle this case, just fake up some nodes to make it match the common + // case. Synthesize a comparison that is always true, and for the result-on-true, use the + // entire subtree we expected to be the nested question op. + + cond2Expr = gtNewOperNode(GT_EQ, TYP_INT, gtNewIconNode(0, TYP_I_IMPL), gtNewIconNode(0, TYP_I_IMPL)); + true2Expr = nestedQmark; + false2Expr = gtNewIconNode(0, TYP_I_IMPL); + } + assert(false2Expr->OperGet() == trueExpr->OperGet()); + + // Create the chain of blocks. See method header comment. + // The order of blocks after this is the following: + // block ... asgBlock ... cond1Block ... cond2Block ... helperBlock ... remainderBlock + // + // We need to remember flags that exist on 'block' that we want to propagate to 'remainderBlock', + // if they are going to be cleared by fgSplitBlockAfterStatement(). We currently only do this + // for the GC safe point bit, the logic being that if 'block' was marked gcsafe, then surely + // remainderBlock will still be GC safe. + BasicBlockFlags propagateFlags = block->GetFlagsRaw() & BBF_GC_SAFE_POINT; + BasicBlock* remainderBlock = fgSplitBlockAfterStatement(block, stmt); + fgRemoveRefPred(remainderBlock, block); // We're going to put more blocks between block and remainderBlock. + + BasicBlock* helperBlock = fgNewBBafter(BBJ_ALWAYS, block, true, block->Next()); + BasicBlock* cond2Block = fgNewBBafter(BBJ_COND, block, true, remainderBlock); + BasicBlock* cond1Block = fgNewBBafter(BBJ_COND, block, true, remainderBlock); + BasicBlock* asgBlock = fgNewBBafter(BBJ_ALWAYS, block, true, block->Next()); + + block->RemoveFlags(BBF_NEEDS_GCPOLL); + remainderBlock->SetFlags(propagateFlags); + helperBlock->SetFlags(BBF_NONE_QUIRK); + asgBlock->SetFlags(BBF_NONE_QUIRK); + + // These blocks are only internal if 'block' is (but they've been set as internal by fgNewBBafter). + // If they're not internal, mark them as imported to avoid asserts about un-imported blocks. + if (!block->HasFlag(BBF_INTERNAL)) + { + helperBlock->RemoveFlags(BBF_INTERNAL); + cond2Block->RemoveFlags(BBF_INTERNAL); + cond1Block->RemoveFlags(BBF_INTERNAL); + asgBlock->RemoveFlags(BBF_INTERNAL); + helperBlock->SetFlags(BBF_IMPORTED); + cond2Block->SetFlags(BBF_IMPORTED); + cond1Block->SetFlags(BBF_IMPORTED); + asgBlock->SetFlags(BBF_IMPORTED); + } + + // Chain the flow correctly. + assert(block->KindIs(BBJ_ALWAYS)); + block->SetTarget(asgBlock); + fgAddRefPred(asgBlock, block); + fgAddRefPred(cond1Block, asgBlock); + fgAddRefPred(remainderBlock, helperBlock); + + cond1Block->SetFalseTarget(cond2Block); + cond2Block->SetFalseTarget(helperBlock); + fgAddRefPred(cond2Block, cond1Block); + fgAddRefPred(helperBlock, cond2Block); + fgAddRefPred(remainderBlock, cond1Block); + fgAddRefPred(remainderBlock, cond2Block); + + // Set the weights; some are guesses. + asgBlock->inheritWeight(block); + cond1Block->inheritWeight(block); + + // We only have likelihood for the fast path (and fallback), but we don't know + // how often we have null in the root QMARK (although, we might be able to guess it too) + // so leave 50/50 for now. Thus, we have: + // + // [weight 1.0] + // if (obj != null) + // { + // [weight 0.5] + // if (obj.GetType() == typeof(FastType)) + // { + // [weight 0.5 * ] + // } + // else + // { + // [weight 0.5 * <100 - likelihood of FastType>] + // } + // } + // + cond2Block->inheritWeightPercentage(cond1Block, 50); + helperBlock->inheritWeightPercentage(cond2Block, nestedQmarkElseLikelihood); + + // Append cond1 as JTRUE to cond1Block + GenTree* jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, condExpr); + Statement* jmpStmt = fgNewStmtFromTree(jmpTree, stmt->GetDebugInfo()); + fgInsertStmtAtEnd(cond1Block, jmpStmt); + + // Append cond2 as JTRUE to cond2Block + jmpTree = gtNewOperNode(GT_JTRUE, TYP_VOID, cond2Expr); + jmpStmt = fgNewStmtFromTree(jmpTree, stmt->GetDebugInfo()); + fgInsertStmtAtEnd(cond2Block, jmpStmt); + + unsigned tmp2 = lvaGrabTemp(false DEBUGARG("CastInstOf QMark result")); + lvaGetDesc(tmp2)->lvType = dst->TypeGet(); + + // AsgBlock should get tmp2 = op1. + GenTree* trueExprStore = gtNewStoreLclVarNode(tmp2, trueExpr)->AsLclVarCommon(); + Statement* trueStmt = fgNewStmtFromTree(trueExprStore, stmt->GetDebugInfo()); + fgInsertStmtAtEnd(asgBlock, trueStmt); + + // Since we are adding helper in the JTRUE false path, reverse the cond2 and add the helper. + gtReverseCond(cond2Expr); + + if (true2Expr->OperIs(GT_CALL) && (true2Expr->AsCall()->gtCallMoreFlags & GTF_CALL_M_DOES_NOT_RETURN)) + { + Statement* helperStmt = fgNewStmtFromTree(true2Expr, stmt->GetDebugInfo()); + fgInsertStmtAtEnd(helperBlock, helperStmt); + fgConvertBBToThrowBB(helperBlock); + setMethodHasNoReturnCalls(); + introducedThrow = true; + } + else + { + GenTree* helperExprStore = gtNewStoreLclVarNode(tmp2, true2Expr)->AsLclVarCommon(); + Statement* helperStmt = fgNewStmtFromTree(helperExprStore, stmt->GetDebugInfo()); + fgInsertStmtAtEnd(helperBlock, helperStmt); + } + + // RemainderBlock should get tmp = tmp2. + GenTree* tmp2CopyLcl = gtNewLclvNode(tmp2, dst->TypeGet()); + unsigned dstLclNum = dst->AsLclVarCommon()->GetLclNum(); + GenTree* resultCopy = + dst->OperIs(GT_STORE_LCL_FLD) + ? gtNewStoreLclFldNode(dstLclNum, dst->TypeGet(), dst->AsLclFld()->GetLclOffs(), tmp2CopyLcl) + : gtNewStoreLclVarNode(dstLclNum, tmp2CopyLcl)->AsLclVarCommon(); + Statement* resultCopyStmt = fgNewStmtFromTree(resultCopy, stmt->GetDebugInfo()); + fgInsertStmtAtBeg(remainderBlock, resultCopyStmt); + + // Finally remove the nested qmark stmt. + fgRemoveStmt(block, stmt); + +#ifdef DEBUG + if (verbose) + { + printf("\nExpanding CastInstOf qmark in " FMT_BB " (after)\n", block->bbNum); + fgDispBasicBlocks(block, remainderBlock, true); + } +#endif // DEBUG + + return introducedThrow; +} + //------------------------------------------------------------------------ // fgExpandQmarkStmt: expand a qmark into control flow // @@ -14591,6 +14813,11 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) return false; } + if (qmark->gtFlags & GTF_QMARK_CAST_INSTOF) + { + return fgExpandQmarkForCastInstOf(block, stmt); + } + #ifdef DEBUG if (verbose) { @@ -14621,6 +14848,7 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // Conservatively propagate BBF_COPY_PROPAGATE flags to all blocks BasicBlockFlags propagateFlagsToAll = block->GetFlagsRaw() & BBF_COPY_PROPAGATE; BasicBlock* remainderBlock = fgSplitBlockAfterStatement(block, stmt); + fgRemoveRefPred(remainderBlock, block); // We're going to put more blocks between block and remainderBlock. BasicBlock* condBlock = fgNewBBafter(BBJ_ALWAYS, block, true); BasicBlock* elseBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true); @@ -14644,21 +14872,16 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) assert(condBlock->bbWeight == remainderBlock->bbWeight); assert(block->KindIs(BBJ_ALWAYS)); - fgRedirectTargetEdge(block, condBlock); - - { - FlowEdge* const newEdge = fgAddRefPred(elseBlock, condBlock); - condBlock->SetTargetEdge(newEdge); - } - - { - FlowEdge* const newEdge = fgAddRefPred(remainderBlock, elseBlock); - elseBlock->SetTargetEdge(newEdge); - } - + block->SetTarget(condBlock); + condBlock->SetTarget(elseBlock); + elseBlock->SetTarget(remainderBlock); assert(condBlock->JumpsToNext()); assert(elseBlock->JumpsToNext()); + fgAddRefPred(condBlock, block); + fgAddRefPred(elseBlock, condBlock); + fgAddRefPred(remainderBlock, elseBlock); + condBlock->SetFlags(propagateFlagsToAll | BBF_NONE_QUIRK); elseBlock->SetFlags(propagateFlagsToAll | BBF_NONE_QUIRK); @@ -14675,28 +14898,20 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // gtReverseCond(condExpr); - thenBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true); + thenBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true, remainderBlock); thenBlock->SetFlags(propagateFlagsToAll); + condBlock->SetCond(elseBlock, thenBlock); if (!block->HasFlag(BBF_INTERNAL)) { thenBlock->RemoveFlags(BBF_INTERNAL); thenBlock->SetFlags(BBF_IMPORTED); } - const unsigned thenLikelihood = qmark->ThenNodeLikelihood(); - const unsigned elseLikelihood = qmark->ElseNodeLikelihood(); - - FlowEdge* const newEdge = fgAddRefPred(remainderBlock, thenBlock); - thenBlock->SetTargetEdge(newEdge); + fgAddRefPred(thenBlock, condBlock); + fgAddRefPred(remainderBlock, thenBlock); - assert(condBlock->TargetIs(elseBlock)); - FlowEdge* const elseEdge = fgAddRefPred(thenBlock, condBlock); - FlowEdge* const thenEdge = condBlock->GetTargetEdge(); - condBlock->SetCond(thenEdge, elseEdge); - thenBlock->inheritWeightPercentage(condBlock, thenLikelihood); - elseBlock->inheritWeightPercentage(condBlock, elseLikelihood); - thenEdge->setLikelihood(thenLikelihood / 100.0); - elseEdge->setLikelihood(elseLikelihood / 100.0); + thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood()); + elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood()); } else if (hasTrueExpr) { @@ -14707,22 +14922,13 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // bbj_cond(true) // gtReverseCond(condExpr); - - const unsigned thenLikelihood = qmark->ThenNodeLikelihood(); - const unsigned elseLikelihood = qmark->ElseNodeLikelihood(); - - assert(condBlock->TargetIs(elseBlock)); - FlowEdge* const thenEdge = fgAddRefPred(remainderBlock, condBlock); - FlowEdge* const elseEdge = condBlock->GetTargetEdge(); - condBlock->SetCond(thenEdge, elseEdge); - + condBlock->SetCond(remainderBlock, elseBlock); + fgAddRefPred(remainderBlock, condBlock); // Since we have no false expr, use the one we'd already created. thenBlock = elseBlock; elseBlock = nullptr; - thenBlock->inheritWeightPercentage(condBlock, thenLikelihood); - thenEdge->setLikelihood(thenLikelihood / 100.0); - elseEdge->setLikelihood(elseLikelihood / 100.0); + thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood()); } else if (hasFalseExpr) { @@ -14732,17 +14938,10 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // +-->------------+ // bbj_cond(true) // - const unsigned thenLikelihood = qmark->ThenNodeLikelihood(); - const unsigned elseLikelihood = qmark->ElseNodeLikelihood(); - - assert(condBlock->TargetIs(elseBlock)); - FlowEdge* const thenEdge = fgAddRefPred(remainderBlock, condBlock); - FlowEdge* const elseEdge = condBlock->GetTargetEdge(); - condBlock->SetCond(thenEdge, elseEdge); + condBlock->SetCond(remainderBlock, elseBlock); + fgAddRefPred(remainderBlock, condBlock); - elseBlock->inheritWeightPercentage(condBlock, elseLikelihood); - thenEdge->setLikelihood(thenLikelihood / 100.0); - elseEdge->setLikelihood(elseLikelihood / 100.0); + elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood()); } assert(condBlock->KindIs(BBJ_COND)); diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index d7fa5821eb9dbc..94d10dd5887f55 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -45,6 +45,7 @@ class MorphInitBlockHelper GenTreeLclVarCommon* m_dstLclNode = nullptr; LclVarDsc* m_dstVarDsc = nullptr; unsigned m_dstLclOffset = 0; + bool m_dstUseLclFld = false; bool m_dstSingleStoreLclVar = false; enum class BlockTransformation @@ -608,6 +609,7 @@ class MorphCopyBlockHelper : public MorphInitBlockHelper unsigned m_srcLclNum = BAD_VAR_NUM; LclVarDsc* m_srcVarDsc = nullptr; GenTreeLclVarCommon* m_srcLclNode = nullptr; + bool m_srcUseLclFld = false; unsigned m_srcLclOffset = 0; bool m_srcSingleStoreLclVar = false; @@ -1114,7 +1116,9 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() } else if (m_dstDoFldStore) { - if (m_srcLclNum == BAD_VAR_NUM) + m_srcUseLclFld = m_srcVarDsc != nullptr; + + if (!m_srcUseLclFld) { addr = m_src->AsIndir()->Addr(); @@ -1139,7 +1143,8 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() else { assert(m_srcDoFldStore); - fieldCnt = m_srcVarDsc->lvFieldCnt; + fieldCnt = m_srcVarDsc->lvFieldCnt; + m_dstUseLclFld = m_dstVarDsc != nullptr; // Clear the def flags, we'll reuse the node below and reset them. if (m_dstLclNode != nullptr) @@ -1147,7 +1152,7 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() m_dstLclNode->gtFlags &= ~(GTF_VAR_DEF | GTF_VAR_USEASG); } - if (m_dstLclNum == BAD_VAR_NUM) + if (!m_dstUseLclFld) { addr = m_store->AsIndir()->Addr(); @@ -1237,7 +1242,7 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() { JITDUMP("All fields of destination of field-by-field copy are dying, skipping entirely\n"); - if (m_srcLclNum != BAD_VAR_NUM) + if (m_srcUseLclFld) { return m_comp->gtNewNothingNode(); } @@ -1311,7 +1316,12 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() } if (!done) { - if (m_srcLclNum != BAD_VAR_NUM) + if (!m_srcUseLclFld) + { + GenTree* fldAddr = grabAddr(fldOffset); + srcFld = m_comp->gtNewIndir(destType, fldAddr); + } + else { // If the src was a struct type field "B" in a struct "A" then we add // add offset of ("B" in "A") + current offset in "B". @@ -1321,11 +1331,6 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() // TODO-1stClassStructs: remove this and implement reading a field from a struct in a reg. m_comp->lvaSetVarDoNotEnregister(m_srcLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); } - else - { - GenTree* fldAddr = grabAddr(fldOffset); - srcFld = m_comp->gtNewIndir(destType, fldAddr); - } } } } @@ -1359,7 +1364,12 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() unsigned srcFieldOffset = srcFieldVarDsc->lvFldOffset; var_types srcType = srcFieldVarDsc->TypeGet(); - if (m_dstLclNum != BAD_VAR_NUM) + if (!m_dstUseLclFld) + { + GenTree* fldAddr = grabAddr(srcFieldOffset); + dstFldStore = m_comp->gtNewStoreIndNode(srcType, fldAddr, srcFld); + } + else { // If the dst was a struct type field "B" in a struct "A" then we add // add offset of ("B" in "A") + current offset in "B". @@ -1369,11 +1379,6 @@ GenTree* MorphCopyBlockHelper::CopyFieldByField() // TODO-1stClassStructs: remove this and implement storing to a field in a struct in a reg. m_comp->lvaSetVarDoNotEnregister(m_dstLclNum DEBUGARG(DoNotEnregisterReason::LocalField)); } - else - { - GenTree* fldAddr = grabAddr(srcFieldOffset); - dstFldStore = m_comp->gtNewStoreIndNode(srcType, fldAddr, srcFld); - } } } noway_assert(dstFldStore->TypeGet() == srcFld->TypeGet()); @@ -1507,3 +1512,70 @@ GenTree* Compiler::fgMorphInitBlock(GenTree* tree) { return MorphInitBlockHelper::MorphInitBlock(this, tree); } + +//------------------------------------------------------------------------ +// fgMorphStoreDynBlock: Morph a dynamic block store (GT_STORE_DYN_BLK). +// +// Performs full (pre-order and post-order) morphing for a STORE_DYN_BLK. +// +// Arguments: +// tree - The GT_STORE_DYN_BLK tree to morph. +// +// Return Value: +// In case the size turns into a constant - the store, transformed +// into an "ordinary" STORE_BLK one, and further morphed by +// "fgMorphInitBlock"/"fgMorphCopyBlock". Otherwise, the original +// tree (fully morphed). +// +GenTree* Compiler::fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree) +{ + if (!tree->Data()->OperIs(GT_CNS_INT, GT_INIT_VAL)) + { + // Data is a location and required to have GTF_DONT_CSE. + tree->Data()->gtFlags |= GTF_DONT_CSE; + } + + tree->Addr() = fgMorphTree(tree->Addr()); + tree->Data() = fgMorphTree(tree->Data()); + tree->gtDynamicSize = fgMorphTree(tree->gtDynamicSize); + + if (tree->gtDynamicSize->IsIntegralConst()) + { + int64_t size = tree->gtDynamicSize->AsIntConCommon()->IntegralValue(); + + if ((size != 0) && FitsIn(size)) + { + ClassLayout* layout = typGetBlkLayout(static_cast(size)); + GenTree* src = tree->Data(); + if (src->OperIs(GT_IND)) + { + assert(src->TypeIs(TYP_STRUCT)); + src->SetOper(GT_BLK); + src->AsBlk()->Initialize(layout); + } + + GenTree* store = gtNewStoreValueNode(layout, tree->Addr(), src, tree->gtFlags & GTF_IND_FLAGS); + store->AddAllEffectsFlags(tree); + INDEBUG(store->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED); + + JITDUMP("MorphStoreDynBlock: transformed STORE_DYN_BLK into STORE_BLK\n"); + + return tree->OperIsCopyBlkOp() ? fgMorphCopyBlock(store) : fgMorphInitBlock(store); + } + } + + tree->SetAllEffectsFlags(tree->Addr(), tree->Data(), tree->gtDynamicSize); + + if (tree->OperMayThrow(this)) + { + tree->gtFlags |= GTF_EXCEPT; + } + else + { + tree->gtFlags |= GTF_IND_NONFAULTING; + } + + tree->gtFlags |= GTF_ASG; + + return tree; +} diff --git a/src/coreclr/jit/namedintrinsiclist.h b/src/coreclr/jit/namedintrinsiclist.h index 9fa128c38f74ee..9c4f44e8e2f0bf 100644 --- a/src/coreclr/jit/namedintrinsiclist.h +++ b/src/coreclr/jit/namedintrinsiclist.h @@ -20,7 +20,7 @@ enum NamedIntrinsic : unsigned short NI_System_BitConverter_Int64BitsToDouble, NI_System_BitConverter_SingleToInt32Bits, - NI_System_SpanHelpers_Memmove, + NI_System_Buffer_Memmove, NI_SYSTEM_MATH_START, NI_System_Math_Abs, @@ -115,11 +115,8 @@ enum NamedIntrinsic : unsigned short NI_System_String_get_Length, NI_System_String_op_Implicit, NI_System_String_StartsWith, - NI_System_String_EndsWith, NI_System_Span_get_Item, NI_System_Span_get_Length, - NI_System_SpanHelpers_ClearWithoutReferences, - NI_System_SpanHelpers_Fill, NI_System_SpanHelpers_SequenceEqual, NI_System_ReadOnlySpan_get_Item, NI_System_ReadOnlySpan_get_Length, @@ -128,7 +125,6 @@ enum NamedIntrinsic : unsigned short NI_System_MemoryExtensions_Equals, NI_System_MemoryExtensions_SequenceEqual, NI_System_MemoryExtensions_StartsWith, - NI_System_MemoryExtensions_EndsWith, NI_System_Threading_Interlocked_And, NI_System_Threading_Interlocked_Or, diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp index cd554f02e56372..e4b3c9faeff4fb 100644 --- a/src/coreclr/jit/optcse.cpp +++ b/src/coreclr/jit/optcse.cpp @@ -18,22 +18,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "optcse.h" -#ifdef DEBUG -#define RLDUMP(...) \ - { \ - if (m_verbose) \ - logf(__VA_ARGS__); \ - } -#define RLDUMPEXEC(x) \ - { \ - if (m_verbose) \ - x; \ - } -#else -#define RLDUMP(...) -#define RLDUMPEXEC(x) -#endif - /* static */ const size_t Compiler::s_optCSEhashSizeInitial = EXPSET_SZ * 2; const size_t Compiler::s_optCSEhashGrowthFactor = 2; @@ -164,11 +148,6 @@ bool Compiler::optUnmarkCSE(GenTree* tree) // 2. Unmark the CSE information in the node tree->gtCSEnum = NO_CSE; - - // 3. Leave breadcrumbs so we know some dsc was altered - - optCSEunmarks++; - return true; } else @@ -2114,6 +2093,7 @@ void CSE_HeuristicCommon::DumpMetrics() CSE_HeuristicRandom::CSE_HeuristicRandom(Compiler* pCompiler) : CSE_HeuristicCommon(pCompiler) { m_cseRNG.Init(m_pCompiler->info.compMethodHash() ^ JitConfig.JitRandomCSE()); + Announce(); } //------------------------------------------------------------------------ @@ -2232,6 +2212,7 @@ void CSE_HeuristicRandom::ConsiderCandidates() // CSE_HeuristicReplay::CSE_HeuristicReplay(Compiler* pCompiler) : CSE_HeuristicCommon(pCompiler) { + Announce(); } //------------------------------------------------------------------------ @@ -2307,33 +2288,108 @@ void CSE_HeuristicReplay::ConsiderCandidates() } } -#endif // DEBUG - -// From PolicyGradient -// Greedy/Base: 35483 methods, 8669 better, 23752 same, 3061 worse, 1.0041 geomean - -double CSE_HeuristicParameterized::s_defaultParameters[CSE_HeuristicParameterized::numParameters] = - {0.2425, 0.2479, 0.1089, -0.2363, 0.2472, -0.0559, -0.8418, -0.0585, -0.2773, 0.0000, 0.0213, -0.4116, 0.0000, - -0.0922, 0.2593, -0.0315, -0.0745, 0.2607, 0.3475, -0.0590, -0.3177, -0.6883, -0.4998, -0.3220, -0.2268}; - //------------------------------------------------------------------------ -// CSE_HeuristicParameterized: CSE heuristic using parameterized, linear profitability model +// CSE_HeuristicRL: construct RL CSE heuristic // // Arguments; // pCompiler - compiler instance // -CSE_HeuristicParameterized::CSE_HeuristicParameterized(Compiler* pCompiler) : CSE_HeuristicCommon(pCompiler) +// Notes: +// This creates the RL CSE heuristic. It does CSEs based on a stochastic +// softmax policy, governed by a parameter vector. +// +// JitRLCSE specified the initial parameter values. +// JitRandomCSE can be used to supply salt for the RNG. +// JitReplayCSE can be used to supply a sequence to follow. +// JitReplayCSEReward can be used to supply the perf score for the sequence. +// +CSE_HeuristicRL::CSE_HeuristicRL(Compiler* pCompiler) + : CSE_HeuristicCommon(pCompiler), m_alpha(0.0), m_updateParameters(false), m_greedy(false), m_verbose(false) { - // Default parameter values... + // Set up the random state // - for (unsigned i = 0; i < numParameters; i++) + m_cseRNG.Init(m_pCompiler->info.compMethodHash() ^ JitConfig.JitRandomCSE()); + + // Parameters + // + ConfigDoubleArray initialParameters; + initialParameters.EnsureInit(JitConfig.JitRLCSE()); + const unsigned initialParamLength = initialParameters.GetLength(); + + for (unsigned i = 0; (i < initialParamLength) && (i < numParameters); i++) + { + m_parameters[i] = initialParameters.GetData()[i]; + } + + if (numParameters > initialParamLength) + { + JITDUMP("Too few parameters (expected %d), trailing will be zero\n", numParameters); + for (unsigned i = initialParamLength; i < numParameters; i++) + { + m_parameters[i] = 0; + } + } + else if (numParameters < initialParamLength) { - m_parameters[i] = s_defaultParameters[i]; + JITDUMP("Too many parameters (expected %d), trailing will be ignored\n", numParameters); } - // These get set during... + // Policy sub-behavior: explore / update / greedy + // + // We may be given a prior sequence and perf score to use to + // update the parameters .... if so, we will replay same sequence of CSEs + // (like the replay policy) and update the parameters via the policy + // gradient algorithm. + // + // For updates: + // + // m_alpha controls the "step size" or learning rate; when we want to adjust + // the parameters we only partially move them towards the gradient indicated values. + // + // m_rewards describes the reward associated with each step. + // + // This "two-pass" technique (first run the current policy and, obtain the perf score + // and CSE sequence, then rerun with the same sequence and update the policy + // parameters) ensures all the policy model logic is within the + // JIT, so the preference computation and its gradient can be kept in sync. // - m_localWeights = nullptr; + if ((JitConfig.JitReplayCSE() != nullptr) && (JitConfig.JitReplayCSEReward() != nullptr)) + { + m_updateParameters = true; + + // Reward + // + ConfigDoubleArray rewards; + rewards.EnsureInit(JitConfig.JitReplayCSEReward()); + const unsigned rewardsLength = rewards.GetLength(); + + for (unsigned i = 0; (i < rewardsLength) && (i < maxSteps); i++) + { + m_rewards[i] = rewards.GetData()[i]; + } + + for (unsigned i = rewardsLength; i < maxSteps; i++) + { + m_rewards[i] = 0; + } + + // Alpha + // + if (JitConfig.JitRLCSEAlpha() != nullptr) + { + ConfigDoubleArray JitRLCSEAlphaArray; + JitRLCSEAlphaArray.EnsureInit(JitConfig.JitRLCSEAlpha()); + m_alpha = JitRLCSEAlphaArray.GetData()[0]; + } + else + { + m_alpha = 0.001; + } + } + else if (JitConfig.JitRLCSEGreedy() > 0) + { + m_greedy = true; + } // Stopping "parameter" // @@ -2341,29 +2397,121 @@ CSE_HeuristicParameterized::CSE_HeuristicParameterized(Compiler* pCompiler) : CS // Verbose // - m_verbose = (JitConfig.JitRLCSEVerbose() > 0); + if (m_pCompiler->verbose || (JitConfig.JitRLCSEVerbose() > 0)) + { + m_verbose = true; + } #ifdef DEBUG - m_verbose |= m_pCompiler->verbose; CompAllocator allocator = m_pCompiler->getAllocator(CMK_CSE); m_likelihoods = new (allocator) jitstd::vector(allocator); + m_baseLikelihoods = new (allocator) jitstd::vector(allocator); + m_features = new (allocator) jitstd::vector(allocator); #endif + Announce(); } //------------------------------------------------------------------------ -// ConsiderCandidates: examine candidates and perform CSEs. +// Name: name this jit heuristic // -void CSE_HeuristicParameterized::ConsiderCandidates() +// Returns: +// descriptive name string +// +const char* CSE_HeuristicRL::Name() const { - const int numCandidates = m_pCompiler->optCSECandidateCount; - sortTab = new (m_pCompiler, CMK_CSE) CSEdsc*[numCandidates]; - sortSiz = numCandidates * sizeof(*sortTab); - memcpy(sortTab, m_pCompiler->optCSEtab, sortSiz); + if (m_updateParameters) + { + return "RL Policy Gradient Update"; + } + else if (m_greedy) + { + return "RL Policy Gradient Greedy"; + } + else + { + return "RL Policy Gradient Stochastic"; + } +} - // Capture distribution of enregisterable local var weights. - // - CaptureLocalWeights(); - GreedyPolicy(); +//------------------------------------------------------------------------ +// Announce: describe heuristic in jit dump +// +void CSE_HeuristicRL::Announce() +{ + JITDUMP("%s salt %d parameters ", Name(), JitConfig.JitRandomCSE()); + for (int i = 0; i < numParameters; i++) + { + JITDUMP("%s%f", (i == 0) ? "" : ",", m_parameters[i]); + } + JITDUMP("\n"); + + if (m_updateParameters) + { + JITDUMP("Operating in update mode with sequence %ls, rewards %ls, and alpha %f\n", JitConfig.JitReplayCSE(), + JitConfig.JitReplayCSEReward(), m_alpha); + } +} + +//------------------------------------------------------------------------ +// DumpMetrics: dump post-CSE metrics +// +void CSE_HeuristicRL::DumpMetrics() +{ + CSE_HeuristicCommon::DumpMetrics(); + + if (m_updateParameters) + { + // For update, dump the new parameter values + // + printf(" updatedparams "); + for (int i = 0; i < numParameters; i++) + { + printf("%s%f", (i == 0) ? "" : ",", m_parameters[i]); + } + + if (JitConfig.JitRLCSECandidateFeatures() > 0) + { + bool first = true; + printf(", features "); + for (char* f : *m_features) + { + printf("%s%s", first ? "" : ",", f); + first = false; + } + } + } + else if (m_greedy) + { + // Show the parameters used. + // + printf(" params "); + for (int i = 0; i < numParameters; i++) + { + printf("%s%f", (i == 0) ? "" : ",", m_parameters[i]); + } + } + else + { + // For evaluation, dump likelihood of the choices made + // + printf(" likelihoods "); + bool first = true; + for (double d : *m_likelihoods) + { + printf("%s%.3f", first ? "" : ",", d); + first = false; + } + + // For evaluation, dump initial likelihood each choice + // + printf(" baseLikelihoods "); + first = true; + for (double d : *m_baseLikelihoods) + { + printf("%s%.3f", first ? "" : ",", d); + first = false; + } + } } //------------------------------------------------------------------------ @@ -2376,7 +2524,7 @@ void CSE_HeuristicParameterized::ConsiderCandidates() // Returns: // true if this tree can be a CSE candidate // -bool CSE_HeuristicParameterized::ConsiderTree(GenTree* tree, bool isReturn) +bool CSE_HeuristicRL::ConsiderTree(GenTree* tree, bool isReturn) { return CanConsiderTree(tree, isReturn); } @@ -2389,7 +2537,7 @@ bool CSE_HeuristicParameterized::ConsiderTree(GenTree* tree, bool isReturn) // Used to estimate where the temp introduced by a CSE would rank compared // to other locals in the method, as they compete for registers. // -void CSE_HeuristicParameterized::CaptureLocalWeights() +void CSE_HeuristicRL::CaptureLocalWeights() { JITDUMP("Local weight table...\n"); CompAllocator allocator = m_pCompiler->getAllocator(CMK_SSA); @@ -2425,6 +2573,35 @@ void CSE_HeuristicParameterized::CaptureLocalWeights() } } +//------------------------------------------------------------------------ +// ConsiderCandidates: examine candidates and perform CSEs. +// +void CSE_HeuristicRL::ConsiderCandidates() +{ + const int numCandidates = m_pCompiler->optCSECandidateCount; + sortTab = new (m_pCompiler, CMK_CSE) CSEdsc*[numCandidates]; + sortSiz = numCandidates * sizeof(*sortTab); + memcpy(sortTab, m_pCompiler->optCSEtab, sortSiz); + + // Capture distribution of enregisterable local var weights. + // + CaptureLocalWeights(); + + if (m_updateParameters) + { + UpdateParameters(); + return; + } + + if (m_greedy) + { + GreedyPolicy(); + return; + } + + SoftmaxPolicy(); +} + //------------------------------------------------------------------------ // GreedyPolicy: use a greedy policy // @@ -2432,29 +2609,27 @@ void CSE_HeuristicParameterized::CaptureLocalWeights() // This always performs the most-preferred choice, using lower candidate number // as a tie-breaker. // -void CSE_HeuristicParameterized::GreedyPolicy() +void CSE_HeuristicRL::GreedyPolicy() { - RLDUMP("RL using greedy policy\n"); + if (m_verbose) + { + printf("RL using greedy policy\n"); + } // Number of choices is num candidates + 1, since // early stopping is also a choice. // const int numCandidates = m_pCompiler->optCSECandidateCount; ArrayStack choices(m_pCompiler->getAllocator(CMK_CSE), numCandidates + 1); - unsigned numUnmarked = m_pCompiler->optCSEunmarks; - bool recomputeFeatures = true; while (true) { - Choice& choice = ChooseGreedy(choices, recomputeFeatures); + Choice& choice = ChooseGreedy(choices); CSEdsc* const dsc = choice.m_dsc; -#ifdef DEBUG - m_likelihoods->push_back(choice.m_softmax); -#endif - if (dsc == nullptr) { + m_likelihoods->push_back(choice.m_softmax); break; } @@ -2479,38 +2654,109 @@ void CSE_HeuristicParameterized::GreedyPolicy() JITDUMP("\n"); PerformCSE(&candidate); - madeChanges = true; - choice.m_performed = true; - - // If performing this CSE impacted other CSEs, we need to - // recompute all cse features. - // - unsigned newNumUnmarked = m_pCompiler->optCSEunmarks; - assert(newNumUnmarked >= numUnmarked); - recomputeFeatures = (numUnmarked != newNumUnmarked); - numUnmarked = newNumUnmarked; + madeChanges = true; + m_likelihoods->push_back(choice.m_softmax); } return; } //------------------------------------------------------------------------ -// GetFeatures: extract features for this CSE -// -// Arguments: -// cse - cse descriptor -// features - array to fill in with feature values +// SoftmaxPolicy: use a randomized softmax policy // // Notes: -// Current set of features: +// This converts preferences to likelihoods using softmax, and then +// randomly selects a candidate proportional to its likelihood. // -// 0. cse costEx -// 1. cse use count weighted (log) -// 2. cse def count weighted (log) -// 3. cse costSz -// 4. cse use count -// 5. cse def count -// 6. cse live across call (0/1) +void CSE_HeuristicRL::SoftmaxPolicy() +{ + if (m_verbose) + { + printf("RL using softmax policy\n"); + } + + // Number of choices is num candidates + 1, since + // early stopping is also a choice. + // + const int numCandidates = m_pCompiler->optCSECandidateCount; + ArrayStack choices(m_pCompiler->getAllocator(CMK_CSE), numCandidates + 1); + bool first = true; + + while (true) + { + Choice& choice = ChooseSoftmax(choices); + + if (first) + { + for (int i = 0; i < choices.Height(); i++) + { + Choice& option = choices.TopRef(i); + if (option.m_dsc == nullptr) + { + m_baseLikelihoods->push_back(0); + } + else + { + m_baseLikelihoods->push_back(option.m_dsc->csdIndex); + } + m_baseLikelihoods->push_back(option.m_softmax); + } + first = false; + } + + CSEdsc* const dsc = choice.m_dsc; + + if (dsc == nullptr) + { + m_likelihoods->push_back(choice.m_softmax); + break; + } + + // purge this CSE from sortTab so we won't choose it again + // + assert(sortTab[dsc->csdIndex - 1] == dsc); + sortTab[dsc->csdIndex - 1] = nullptr; + + // ChooseCSE should only choose viable options + // + assert(dsc->IsViable()); + + CSE_Candidate candidate(this, dsc); + + if (m_verbose) + { + printf("\nRL attempting " FMT_CSE "\n", candidate.CseIndex()); + } + + JITDUMP("CSE Expression : \n"); + JITDUMPEXEC(m_pCompiler->gtDispTree(candidate.Expr())); + JITDUMP("\n"); + + PerformCSE(&candidate); + madeChanges = true; + m_likelihoods->push_back(choice.m_softmax); + } + + return; +} + +//------------------------------------------------------------------------ +// GetFeatures: extract features for this CSE +// +// Arguments: +// cse - cse descriptor +// features - array to fill in with feature values +// +// Notes: +// Current set of features: +// +// 0. cse costEx +// 1. cse use count weighted (log) +// 2. cse def count weighted (log) +// 3. cse costSz +// 4. cse use count +// 5. cse def count +// 6. cse live across call (0/1) // 7. cse is int (0/1) // 8. cse is a constant, but not shared (0/1) // 9. cse is a shared const (0/1) @@ -2533,7 +2779,7 @@ void CSE_HeuristicParameterized::GreedyPolicy() // // 24. log (pressure estimate weight) // -void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) +void CSE_HeuristicRL::GetFeatures(CSEdsc* cse, double* features) { for (int i = 0; i < numParameters; i++) { @@ -2591,7 +2837,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) unsigned maxPostorderNum = 0; BasicBlock* minPostorderBlock = nullptr; BasicBlock* maxPostorderBlock = nullptr; - for (treeStmtLst* treeList = cse->csdTreeList; treeList != nullptr; treeList = treeList->tslNext) + for (treeStmtLst* treeList = cse->csdTreeList; treeList != nullptr && !isMakeCse; treeList = treeList->tslNext) { BasicBlock* const treeBlock = treeList->tslBlock; unsigned postorderNum = treeBlock->bbPostorderNum; @@ -2632,6 +2878,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) // LSRA "is live across call" // bool isLiveAcrossCallLSRA = isLiveAcrossCall; + if (!isLiveAcrossCallLSRA) { unsigned count = 0; @@ -2645,6 +2892,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) } } } + features[23] = booleanScale * isLiveAcrossCallLSRA; } @@ -2663,7 +2911,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) // All boolean features are scaled up by booleanScale so their // numeric range is similar to the non-boolean features // -void CSE_HeuristicParameterized::GetStoppingFeatures(double* features) +void CSE_HeuristicRL::GetStoppingFeatures(double* features) { // Estimate the (log) weight at which a new CSE would cause a spill // if m_registerPressure registers were initially available. @@ -2697,632 +2945,141 @@ void CSE_HeuristicParameterized::GetStoppingFeatures(double* features) JITDUMP("Pressure count %u, pressure weight " FMT_WT "\n", currentPressure, spillAtWeight); - // Large frame...? - // todo: scan all vars, not just tracked? - // - - features[24] = deMinimusAdj + log(max(deMinimis, spillAtWeight)); -} - -//------------------------------------------------------------------------ -// Preference: determine a preference score for this CSE -// -// Arguments: -// cse - cse descriptor, or nullptr for the option to stop doing CSEs. -// -double CSE_HeuristicParameterized::Preference(CSEdsc* cse) -{ - double features[numParameters]; - GetFeatures(cse, features); - -#ifdef DEBUG - if (JitConfig.JitRLCSECandidateFeatures() > 0) - { - DumpFeatures(cse, features); - } -#endif - - double preference = 0; - for (int i = 0; i < numParameters; i++) - { - preference += features[i] * m_parameters[i]; - } - - return preference; -} - -//------------------------------------------------------------------------ -// StoppingPreference: determine a preference score for this stopping CSE -// -// Arguments: -// regAvail - number of registers threshold -// -double CSE_HeuristicParameterized::StoppingPreference() -{ - double features[numParameters]; - GetFeatures(nullptr, features); - -#ifdef DEBUG - if (JitConfig.JitRLCSECandidateFeatures() > 0) - { - DumpFeatures(nullptr, features); - } -#endif - - double preference = 0; - for (int i = 0; i < numParameters; i++) - { - preference += features[i] * m_parameters[i]; - } - - return preference; -} - -//------------------------------------------------------------------------ -// ChooseGreedy: examine candidates and choose the next CSE to perform -// via greedy policy -// -// Arguments: -// choices -- array of choices, possibly already filled in -// recompute -- if true, rebuild the choice array from scratch -// -// Returns: -// Choice of CSE to perform -// -// Notes: -// Picks the most-preferred candidate. -// If there is a tie, picks stop, or the lowest cse index. -// -CSE_HeuristicParameterized::Choice& CSE_HeuristicParameterized::ChooseGreedy(ArrayStack& choices, - bool recompute) -{ - if (recompute) - { - choices.Reset(); - BuildChoices(choices); - } - else - { - // Always recompute the stopping preference as this - // reflects ambient state after each CSE. - // - // By convention, this is at TopRef(0). - // - Choice& stopping = choices.TopRef(0); - assert(stopping.m_dsc == nullptr); - stopping.m_preference = StoppingPreference(); - } - - // Find the maximally preferred case. - // - int choiceNum = 0; - - for (int i = 1; i < choices.Height(); i++) - { - const Choice& choice = choices.TopRef(i); - - if (choice.m_performed == true) - { - continue; - } - - const Choice& bestChoice = choices.TopRef(choiceNum); - - const double delta = choice.m_preference - bestChoice.m_preference; - - bool update = false; - - if (delta > 0) - { - update = true; - } - else if (delta == 0) - { - if (choice.m_dsc == nullptr) - { - update = true; - } - else if ((bestChoice.m_dsc != nullptr) && (choice.m_dsc->csdIndex < bestChoice.m_dsc->csdIndex)) - { - update = true; - } - } - - if (update) - { - choiceNum = i; - } - } - - RLDUMP("Greedy candidate evaluation\n"); - RLDUMPEXEC(DumpChoices(choices, choiceNum)); - - return choices.TopRef(choiceNum); -} - -//------------------------------------------------------------------------ -// BuildChoices: fill in the choices currently available -// -// choices - array of choices to be filled in -// -// Notes: -// Also computes the preference for each choice. -// -void CSE_HeuristicParameterized::BuildChoices(ArrayStack& choices) -{ - JITDUMP("Building choice array...\n"); - - for (unsigned i = 0; i < m_pCompiler->optCSECandidateCount; i++) - { - CSEdsc* const dsc = sortTab[i]; - if ((dsc == nullptr) || !dsc->IsViable()) - { - // already did this cse, - // or the cse is not viable - continue; - } - - double preference = Preference(dsc); - choices.Emplace(dsc, preference); - } - - // Doing nothing is also an option. - // - const double stoppingPreference = StoppingPreference(); - choices.Emplace(nullptr, stoppingPreference); -} - -#ifdef DEBUG - -//------------------------------------------------------------------------ -// Announce: describe heuristic in jit dump -// -void CSE_HeuristicParameterized::Announce() -{ - JITDUMP("%s parameters ", Name()); - for (int i = 0; i < numParameters; i++) - { - JITDUMP("%s%f", (i == 0) ? "" : ",", m_parameters[i]); - } - JITDUMP("\n"); -} - -//------------------------------------------------------------------------ -// DumpMetrics: dump post-CSE metrics -// -void CSE_HeuristicParameterized::DumpMetrics() -{ - CSE_HeuristicCommon::DumpMetrics(); - - // Show the parameters used. - // - printf(" params "); - for (int i = 0; i < numParameters; i++) - { - printf("%s%f", (i == 0) ? "" : ",", m_parameters[i]); - } -} - -//------------------------------------------------------------------------ -// DumpFeatures: dump feature values for a CSE candidate -// -// Arguments: -// dsc - cse descriptor -// features - feature vector for that candidate -// -// Notes: -// Dumps a comma separated row of data, prefixed by method index. -// -void CSE_HeuristicParameterized::DumpFeatures(CSEdsc* dsc, double* features) -{ - printf("features,%d," FMT_CSE, m_pCompiler->info.compMethodSuperPMIIndex, dsc == nullptr ? 0 : dsc->csdIndex); - for (int i = 0; i < numParameters; i++) - { - printf(",%f", features[i]); - } - printf("\n"); -} - -//------------------------------------------------------------------------ -// DumpChoices: dump out information on current choices -// -// Arguments: -// choices - array of choices -// highlight - highlight this choice -// -void CSE_HeuristicParameterized::DumpChoices(ArrayStack& choices, int highlight) -{ - for (int i = 0; i < choices.Height(); i++) - { - const Choice& choice = choices.TopRef(i); - - if (choice.m_performed == true) - { - continue; - } - - CSEdsc* const cse = choice.m_dsc; - const char* msg = (i == highlight) ? "=>" : " "; - if (cse != nullptr) - { - printf("%s%2d: " FMT_CSE " preference %10.7f likelihood %10.7f\n", msg, i, cse->csdIndex, - choice.m_preference, choice.m_softmax); - } - else - { - printf("%s%2d: QUIT preference %10.7f likelihood %10.7f\n", msg, i, choice.m_preference, - choice.m_softmax); - } - } -} - -//------------------------------------------------------------------------ -// DumpChoices: dump out information on current choices -// -// Arguments: -// choices - array of choices -// highlight - highlight this choice -// -void CSE_HeuristicParameterized::DumpChoices(ArrayStack& choices, CSEdsc* highlight) -{ - for (int i = 0; i < choices.Height(); i++) - { - const Choice& choice = choices.TopRef(i); - - if (choice.m_performed == true) - { - continue; - } - - CSEdsc* const cse = choice.m_dsc; - const char* msg = (cse == highlight) ? "=>" : " "; - if (cse != nullptr) - { - printf("%s%2d: " FMT_CSE " preference %10.7f likelihood %10.7f\n", msg, i, cse->csdIndex, - choice.m_preference, choice.m_softmax); - } - else - { - printf("%s%2d: QUIT preference %10.7f likelihood %10.7f\n", msg, i, choice.m_preference, - choice.m_softmax); - } - } -} - -#endif // DEBUG - -#ifdef DEBUG - -//------------------------------------------------------------------------ -// CSE_HeuristicRL: construct RL CSE heuristic -// -// Arguments; -// pCompiler - compiler instance -// -// Notes: -// This creates the RL CSE heuristic, selected when JitRLCSE is set. -// It has 3 modes of operation: -// -// (1) Stochastic (default) softmax policy, governed by a parameter vector. -// * JitRLCSE specifies the initial parameter values. -// Missing values default to zero, extra values are ignored. -// * JitRandomCSE can be used to supply salt for the RNG. -// (2) Update: replay a sequence with known rewards, and compute updated -// parameters based on stochastic gradient ascent -// * JitReplayCSE specifies the sequence -// * JitReplayCSEReward the rewards per step (actor-critic style) -// (3) Greedy: -// Enable via JitRLCSEGreedy=1. -// Uses parameters from JitRLCSE to drive a deterministic greedy policy -// -CSE_HeuristicRL::CSE_HeuristicRL(Compiler* pCompiler) - : CSE_HeuristicParameterized(pCompiler), m_alpha(0.0), m_updateParameters(false), m_greedy(false) -{ - // Set up the random state - // - m_cseRNG.Init(m_pCompiler->info.compMethodHash() ^ JitConfig.JitRandomCSE()); - - // Parameters - // - ConfigDoubleArray initialParameters; - initialParameters.EnsureInit(JitConfig.JitRLCSE()); - const unsigned initialParamLength = initialParameters.GetLength(); - - for (unsigned i = 0; (i < initialParamLength) && (i < numParameters); i++) - { - m_parameters[i] = initialParameters.GetData()[i]; - } - - if (numParameters > initialParamLength) - { - JITDUMP("Too few parameters (expected %d), trailing will be zero\n", numParameters); - for (unsigned i = initialParamLength; i < numParameters; i++) - { - m_parameters[i] = 0; - } - } - else if (numParameters < initialParamLength) - { - JITDUMP("Too many parameters (expected %d), trailing will be ignored\n", numParameters); - } - - // Policy sub-behavior: explore / update / greedy - // - // We may be given a prior sequence and perf score to use to - // update the parameters .... if so, we will replay same sequence of CSEs - // (like the replay policy) and update the parameters via the policy - // gradient algorithm. - // - // For updates: - // - // m_alpha controls the "step size" or learning rate; when we want to adjust - // the parameters we only partially move them towards the gradient indicated values. - // - // m_rewards describes the reward associated with each step. - // - // This "two-pass" technique (first run the current policy and, obtain the perf score - // and CSE sequence, then rerun with the same sequence and update the policy - // parameters) ensures all the policy model logic is within the - // JIT, so the preference computation and its gradient can be kept in sync. - // - if ((JitConfig.JitReplayCSE() != nullptr) && (JitConfig.JitReplayCSEReward() != nullptr)) - { - m_updateParameters = true; - - // Reward - // - ConfigDoubleArray rewards; - rewards.EnsureInit(JitConfig.JitReplayCSEReward()); - const unsigned rewardsLength = rewards.GetLength(); - - for (unsigned i = 0; (i < rewardsLength) && (i < maxSteps); i++) - { - m_rewards[i] = rewards.GetData()[i]; - } - - for (unsigned i = rewardsLength; i < maxSteps; i++) - { - m_rewards[i] = 0; - } - - // Alpha - // - if (JitConfig.JitRLCSEAlpha() != nullptr) - { - ConfigDoubleArray JitRLCSEAlphaArray; - JitRLCSEAlphaArray.EnsureInit(JitConfig.JitRLCSEAlpha()); - m_alpha = JitRLCSEAlphaArray.GetData()[0]; - } - else - { - m_alpha = 0.001; - } - } - else if (JitConfig.JitRLCSEGreedy() > 0) - { - m_greedy = true; - } - - CompAllocator allocator = m_pCompiler->getAllocator(CMK_CSE); - m_baseLikelihoods = new (allocator) jitstd::vector(allocator); - m_features = new (allocator) jitstd::vector(allocator); -} - -//------------------------------------------------------------------------ -// Name: name this jit heuristic -// -// Returns: -// descriptive name string -// -const char* CSE_HeuristicRL::Name() const -{ - if (m_updateParameters) - { - return "RL Policy Gradient Update"; - } - else - { - return "RL Policy Gradient Stochastic"; - } -} - -//------------------------------------------------------------------------ -// Announce: describe heuristic in jit dump -// -void CSE_HeuristicRL::Announce() -{ - JITDUMP("%s salt %d parameters ", Name(), JitConfig.JitRandomCSE()); - for (int i = 0; i < numParameters; i++) - { - JITDUMP("%s%f", (i == 0) ? "" : ",", m_parameters[i]); - } - JITDUMP("\n"); - - if (m_updateParameters) - { - JITDUMP("Operating in update mode with sequence %ls, rewards %ls, and alpha %f\n", JitConfig.JitReplayCSE(), - JitConfig.JitReplayCSEReward(), m_alpha); - } -} - -//------------------------------------------------------------------------ -// DumpMetrics: dump post-CSE metrics -// -void CSE_HeuristicRL::DumpMetrics() -{ - CSE_HeuristicParameterized::DumpMetrics(); - - if (m_updateParameters) - { - // For update, dump the new parameter values - // - printf(" updatedparams "); - for (int i = 0; i < numParameters; i++) - { - printf("%s%f", (i == 0) ? "" : ",", m_parameters[i]); - } - - if (JitConfig.JitRLCSECandidateFeatures() > 0) - { - bool first = true; - printf(", features "); - for (char* f : *m_features) - { - printf("%s%s", first ? "" : ",", f); - first = false; - } - } - } - else if (m_greedy) - { - // handled by base class - } - else - { - // For evaluation, dump likelihood of the choices made - // - printf(" likelihoods "); - bool first = true; - for (double d : *m_likelihoods) - { - printf("%s%.3f", first ? "" : ",", d); - first = false; - } - - // For evaluation, dump initial likelihood each choice - // - printf(" baseLikelihoods "); - first = true; - for (double d : *m_baseLikelihoods) - { - printf("%s%.3f", first ? "" : ",", d); - first = false; - } - } + // Large frame...? + // todo: scan all vars, not just tracked? + // + + features[24] = deMinimusAdj + log(max(deMinimis, spillAtWeight)); } //------------------------------------------------------------------------ -// ConsiderTree: check if this tree can be a CSE candidate +// DumpFeatures: dump feature values for a CSE candidate // // Arguments: -// tree - tree in question -// isReturn - true if tree is part of a return statement +// dsc - cse descriptor +// features - feature vector for that candidate // -// Returns: -// true if this tree can be a CSE candidate +// Notes: +// Dumps a comma separated row of data, prefixed by method index. // -bool CSE_HeuristicRL::ConsiderTree(GenTree* tree, bool isReturn) +void CSE_HeuristicRL::DumpFeatures(CSEdsc* dsc, double* features) { - return CanConsiderTree(tree, isReturn); + printf("features,%d," FMT_CSE, m_pCompiler->info.compMethodSuperPMIIndex, dsc == nullptr ? 0 : dsc->csdIndex); + for (int i = 0; i < numParameters; i++) + { + printf(",%f", features[i]); + } + printf("\n"); } //------------------------------------------------------------------------ -// ConsiderCandidates: examine candidates and perform CSEs. +// Preference: determine a preference score for this CSE // -void CSE_HeuristicRL::ConsiderCandidates() +// Arguments: +// cse - cse descriptor, or nullptr for the option to stop doing CSEs. +// +double CSE_HeuristicRL::Preference(CSEdsc* cse) { - const int numCandidates = m_pCompiler->optCSECandidateCount; - sortTab = new (m_pCompiler, CMK_CSE) CSEdsc*[numCandidates]; - sortSiz = numCandidates * sizeof(*sortTab); - memcpy(sortTab, m_pCompiler->optCSEtab, sortSiz); + double features[numParameters]; + GetFeatures(cse, features); - // Capture distribution of enregisterable local var weights. - // - CaptureLocalWeights(); + if (JitConfig.JitRLCSECandidateFeatures() > 0) + { + DumpFeatures(cse, features); + } - if (m_updateParameters) + double preference = 0; + for (int i = 0; i < numParameters; i++) { - UpdateParameters(); - return; + preference += features[i] * m_parameters[i]; } - else if (m_greedy) + + return preference; +} + +//------------------------------------------------------------------------ +// StoppingPreference: determine a preference score for this stopping CSE +// +// Arguments: +// regAvail - number of registers threshold +// +double CSE_HeuristicRL::StoppingPreference() +{ + double features[numParameters]; + GetFeatures(nullptr, features); + + if (JitConfig.JitRLCSECandidateFeatures() > 0) { - GreedyPolicy(); - return; + DumpFeatures(nullptr, features); } - else + + double preference = 0; + for (int i = 0; i < numParameters; i++) { - SoftmaxPolicy(); + preference += features[i] * m_parameters[i]; } + + return preference; } //------------------------------------------------------------------------ -// SoftmaxPolicy: use a randomized softmax policy +// ChooseGreedy: examine candidates and choose the next CSE to perform +// via greedy policy +// +// Returns: +// Choice of CSE to perform // // Notes: -// This converts preferences to likelihoods using softmax, and then -// randomly selects a candidate proportional to its likelihood. +// Picks the most-preferred candidate. +// If there is a tie, picks stop, or the lowest cse index. // -void CSE_HeuristicRL::SoftmaxPolicy() +CSE_HeuristicRL::Choice& CSE_HeuristicRL::ChooseGreedy(ArrayStack& choices) { - if (m_verbose) - { - printf("RL using softmax policy\n"); - } + choices.Reset(); + BuildChoices(choices); - // Number of choices is num candidates + 1, since - // early stopping is also a choice. + // Find the maximally preferred case. // - const int numCandidates = m_pCompiler->optCSECandidateCount; - ArrayStack choices(m_pCompiler->getAllocator(CMK_CSE), numCandidates + 1); - bool first = true; + Choice& bestChoice = choices.TopRef(0); + int choiceNum = 0; - while (true) + for (int i = 1; i < choices.Height(); i++) { - Choice& choice = ChooseSoftmax(choices); + Choice& choice = choices.TopRef(i); + const double delta = choice.m_preference - bestChoice.m_preference; - if (first) + bool update = false; + + if (delta > 0) { - for (int i = 0; i < choices.Height(); i++) - { - Choice& option = choices.TopRef(i); - if (option.m_dsc == nullptr) - { - m_baseLikelihoods->push_back(0); - } - else - { - m_baseLikelihoods->push_back(option.m_dsc->csdIndex); - } - m_baseLikelihoods->push_back(option.m_softmax); - } - first = false; + update = true; } - - CSEdsc* const dsc = choice.m_dsc; - - if (dsc == nullptr) + else if (delta == 0) { - m_likelihoods->push_back(choice.m_softmax); - break; + if (choice.m_dsc == nullptr) + { + update = true; + } + else if ((bestChoice.m_dsc != nullptr) && (choice.m_dsc->csdIndex < bestChoice.m_dsc->csdIndex)) + { + update = true; + } } - // purge this CSE from sortTab so we won't choose it again - // - assert(sortTab[dsc->csdIndex - 1] == dsc); - sortTab[dsc->csdIndex - 1] = nullptr; - - // ChooseCSE should only choose viable options - // - assert(dsc->IsViable()); - - CSE_Candidate candidate(this, dsc); - - if (m_verbose) + if (update) { - printf("\nRL attempting " FMT_CSE "\n", candidate.CseIndex()); + bestChoice = choice; + choiceNum = i; } + } - JITDUMP("CSE Expression : \n"); - JITDUMPEXEC(m_pCompiler->gtDispTree(candidate.Expr())); - JITDUMP("\n"); - - PerformCSE(&candidate); - madeChanges = true; - m_likelihoods->push_back(choice.m_softmax); + if (m_verbose) + { + printf("Greedy candidate evaluation\n"); + DumpChoices(choices, choiceNum); } - return; + return bestChoice; } //------------------------------------------------------------------------ @@ -3383,6 +3140,36 @@ CSE_HeuristicRL::Choice& CSE_HeuristicRL::ChooseSoftmax(ArrayStack& choi return choices.TopRef(choiceNum); } +//------------------------------------------------------------------------ +// BuildChoices: fill in the choices currently available +// +// choices - array of choices to be filled in +// +// Notes: +// Also computes the preference for each choice. +// +void CSE_HeuristicRL::BuildChoices(ArrayStack& choices) +{ + for (unsigned i = 0; i < m_pCompiler->optCSECandidateCount; i++) + { + CSEdsc* const dsc = sortTab[i]; + if ((dsc == nullptr) || !dsc->IsViable()) + { + // already did this cse, + // or the cse is not viable + continue; + } + + double preference = Preference(dsc); + choices.Emplace(dsc, preference); + } + + // Doing nothing is also an option. + // + const double stoppingPreference = StoppingPreference(); + choices.Emplace(nullptr, stoppingPreference); +} + //------------------------------------------------------------------------ // Softmax: fill in likelihoods for each choice vis softmax // @@ -3421,6 +3208,60 @@ void CSE_HeuristicRL::Softmax(ArrayStack& choices) } } +//------------------------------------------------------------------------ +// DumpChoices: dump out information on current choices +// +// Arguments: +// choices - array of choices +// highlight - highlight this choice +// +void CSE_HeuristicRL::DumpChoices(ArrayStack& choices, int highlight) +{ + for (int i = 0; i < choices.Height(); i++) + { + Choice& choice = choices.TopRef(i); + CSEdsc* const cse = choice.m_dsc; + const char* msg = i == highlight ? "=>" : " "; + if (cse != nullptr) + { + printf("%s%2d: " FMT_CSE " preference %10.7f likelihood %10.7f\n", msg, i, cse->csdIndex, + choice.m_preference, choice.m_softmax); + } + else + { + printf("%s%2d: QUIT preference %10.7f likelihood %10.7f\n", msg, i, choice.m_preference, + choice.m_softmax); + } + } +} + +//------------------------------------------------------------------------ +// DumpChoices: dump out information on current choices +// +// Arguments: +// choices - array of choices +// highlight - highlight this choice +// +void CSE_HeuristicRL::DumpChoices(ArrayStack& choices, CSEdsc* highlight) +{ + for (int i = 0; i < choices.Height(); i++) + { + Choice& choice = choices.TopRef(i); + CSEdsc* const cse = choice.m_dsc; + const char* msg = cse == highlight ? "=>" : " "; + if (cse != nullptr) + { + printf("%s%2d: " FMT_CSE " preference %10.7f likelihood %10.7f\n", msg, i, cse->csdIndex, + choice.m_preference, choice.m_softmax); + } + else + { + printf("%s%2d: QUIT preference %10.7f likelihood %10.7f\n", msg, i, choice.m_preference, + choice.m_softmax); + } + } +} + //------------------------------------------------------------------------ // UpdateParameters: Replay an existing CSE sequence with known reward, // and update the model parameters via the policy gradient. @@ -4475,62 +4316,50 @@ bool CSE_HeuristicCommon::IsCompatibleType(var_types cseLclVarTyp, var_types exp return false; } -//------------------------------------------------------------------------ -// PerformCSE: takes a successful candidate and performs the appropriate replacements -// -// Arguments: -// successfulCandidate - cse candidate to perform +// PerformCSE() takes a successful candidate and performs the appropriate replacements: // // It will replace all of the CSE defs with assignments to a new "cse0" LclVar // and will replace all of the CSE uses with reads of the "cse0" LclVar // // It will also put cse0 into SSA if there is just one def. -// void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) { AdjustHeuristic(successfulCandidate); - CSEdsc* const dsc = successfulCandidate->CseDsc(); #ifdef DEBUG // Setup the message arg for lvaGrabTemp() // - const char* heuristicTempMessage = ""; + const char* grabTempMessage = "CSE - unknown"; if (successfulCandidate->IsAggressive()) { - heuristicTempMessage = ": aggressive"; + grabTempMessage = "CSE - aggressive"; } else if (successfulCandidate->IsModerate()) { - heuristicTempMessage = ": moderate"; + grabTempMessage = "CSE - moderate"; } else if (successfulCandidate->IsConservative()) { - heuristicTempMessage = ": conservative"; + grabTempMessage = "CSE - conservative"; } else if (successfulCandidate->IsStressCSE()) { - heuristicTempMessage = ": stress"; + grabTempMessage = "CSE - stress mode"; } else if (successfulCandidate->IsRandom()) { - heuristicTempMessage = ": random"; + grabTempMessage = "CSE - random"; } - - const char* const grabTempMessage = m_pCompiler->printfAlloc(FMT_CSE "%s", dsc->csdIndex, heuristicTempMessage); - - // Add this candidate to the CSE sequence - // - m_sequence->push_back(dsc->csdIndex); - #endif // DEBUG - // Allocate a CSE temp - // + /* Introduce a new temp for the CSE */ + + // we will create a long lifetime temp for the new CSE LclVar unsigned cseLclVarNum = m_pCompiler->lvaGrabTemp(false DEBUGARG(grabTempMessage)); var_types cseLclVarTyp = genActualType(successfulCandidate->Expr()->TypeGet()); - LclVarDsc* const lclDsc = m_pCompiler->lvaGetDesc(cseLclVarNum); + LclVarDsc* lclDsc = m_pCompiler->lvaGetDesc(cseLclVarNum); if (cseLclVarTyp == TYP_STRUCT) { m_pCompiler->lvaSetStruct(cseLclVarNum, successfulCandidate->Expr()->GetLayout(m_pCompiler), false); @@ -4539,10 +4368,8 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) lclDsc->lvIsCSE = true; // Record that we created a new LclVar for use as a CSE temp - // m_addCSEcount++; m_pCompiler->optCSEcount++; - m_pCompiler->Metrics.CseCount++; // Walk all references to this CSE, adding an assignment // to the CSE temp to all defs and changing all refs to @@ -4550,9 +4377,11 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) // // Later we will unmark any nested CSE's for the CSE uses. // + CSEdsc* dsc = successfulCandidate->CseDsc(); + INDEBUG(m_sequence->push_back(dsc->csdIndex)); + // If there's just a single def for the CSE, we'll put this // CSE into SSA form on the fly. We won't need any PHIs. - // unsigned cseSsaNum = SsaConfig::RESERVED_SSA_NUM; LclSsaVarDsc* ssaVarDsc = nullptr; @@ -5197,10 +5026,12 @@ CSE_HeuristicCommon* Compiler::optGetCSEheuristic() if (JitConfig.JitRandomCSE() > 0) { + JITDUMP("Using Random CSE heuristic (JitRandomCSE)\n"); useRandomHeuristic = true; } else if (compStressCompile(Compiler::STRESS_MAKE_CSE, MAX_STRESS_WEIGHT)) { + JITDUMP("Using Random CSE heuristic (stress)\n"); useRandomHeuristic = true; } @@ -5224,24 +5055,12 @@ CSE_HeuristicCommon* Compiler::optGetCSEheuristic() #endif - // Parameterized (greedy) RL-based heuristic - // - if (optCSEheuristic == nullptr) - { - bool useGreedyHeuristic = (JitConfig.JitRLCSEGreedy() > 0); - - if (useGreedyHeuristic) - { - optCSEheuristic = new (this, CMK_CSE) CSE_HeuristicParameterized(this); - } - } - if (optCSEheuristic == nullptr) { + JITDUMP("Using standard CSE heuristic\n"); optCSEheuristic = new (this, CMK_CSE) CSE_Heuristic(this); } - INDEBUG(optCSEheuristic->Announce()); return optCSEheuristic; } @@ -5264,7 +5083,6 @@ PhaseStatus Compiler::optOptimizeValnumCSEs() // Determine which heuristic to use... // CSE_HeuristicCommon* const heuristic = optGetCSEheuristic(); - INDEBUG(heuristic->Announce()); optValnumCSE_phase = true; optCSEweight = -1.0f; diff --git a/src/coreclr/jit/optcse.h b/src/coreclr/jit/optcse.h index 550f754f6a8b6e..86ff3b742d3041 100644 --- a/src/coreclr/jit/optcse.h +++ b/src/coreclr/jit/optcse.h @@ -142,23 +142,22 @@ class CSE_HeuristicReplay : public CSE_HeuristicCommon #endif }; -#endif // DEBUG - -// Parameterized Policy - -class CSE_HeuristicParameterized : public CSE_HeuristicCommon +// Reinforcement Learning CSE heuristic +// +// Uses a "linear" feature model with +// softmax policy. +// +class CSE_HeuristicRL : public CSE_HeuristicCommon { -protected: +private: struct Choice { - Choice(CSEdsc* dsc, double preference) : m_dsc(dsc), m_preference(preference), m_softmax(0), m_performed(false) + Choice(CSEdsc* dsc, double preference) : m_dsc(dsc), m_preference(preference), m_softmax(0) { } - CSEdsc* m_dsc; double m_preference; double m_softmax; - bool m_performed; }; enum @@ -168,70 +167,32 @@ class CSE_HeuristicParameterized : public CSE_HeuristicCommon maxSteps = 65, // MAX_CSE_CNT + 1 (for stopping) }; - static double s_defaultParameters[numParameters]; double m_parameters[numParameters]; + double m_alpha; + double m_rewards[maxSteps]; + CLRRandom m_cseRNG; + bool m_updateParameters; + bool m_greedy; + bool m_verbose; unsigned m_registerPressure; jitstd::vector* m_localWeights; - bool m_verbose; -public: - CSE_HeuristicParameterized(Compiler*); - void ConsiderCandidates(); - bool ConsiderTree(GenTree* tree, bool isReturn); void CaptureLocalWeights(); - void GreedyPolicy(); - void GetFeatures(CSEdsc* dsc, double* features); double Preference(CSEdsc* dsc); void GetStoppingFeatures(double* features); double StoppingPreference(); - void BuildChoices(ArrayStack& choices); - - Choice& ChooseGreedy(ArrayStack& choices, bool recompute); - - virtual const char* Name() const - { - return "Parameterized CSE Heuristic"; - } - -#ifdef DEBUG void DumpFeatures(CSEdsc* dsc, double* features); - void DumpChoices(ArrayStack& choices, int higlight = -1); - void DumpChoices(ArrayStack& choices, CSEdsc* higlight); - void DumpMetrics(); - void Announce(); - - // Likelihood of each choice made in the sequence - jitstd::vector* m_likelihoods; - // Likelihood of each action from starting state - jitstd::vector* m_baseLikelihoods; - // Features of each candidate - jitstd::vector* m_features; - -#endif -}; - -#ifdef DEBUG - -// Reinforcement Learning CSE heuristic -// -// Uses a "linear" feature model with -// softmax policy. -// -class CSE_HeuristicRL : public CSE_HeuristicParameterized -{ -private: - double m_alpha; - double m_rewards[maxSteps]; - CLRRandom m_cseRNG; - bool m_updateParameters; - bool m_greedy; - Choice& ChooseSoftmax(ArrayStack& choices); + Choice& ChooseGreedy(ArrayStack& choices); + void BuildChoices(ArrayStack& choices); void Softmax(ArrayStack& choices); + void DumpChoices(ArrayStack& choices, int higlight = -1); + void DumpChoices(ArrayStack& choices, CSEdsc* higlight); + void UpdateParameters(); + void GreedyPolicy(); void SoftmaxPolicy(); void UpdateParametersStep(CSEdsc* dsc, ArrayStack& choices, double reward, double* delta); - void UpdateParameters(); Choice* FindChoice(CSEdsc* dsc, ArrayStack& choices); const char* Name() const; @@ -242,6 +203,11 @@ class CSE_HeuristicRL : public CSE_HeuristicParameterized #ifdef DEBUG virtual void DumpMetrics(); virtual void Announce(); + // Likelihood of each choice made in the sequence + jitstd::vector* m_likelihoods; + // Likelihood of each action from starting state + jitstd::vector* m_baseLikelihoods; + jitstd::vector* m_features; #endif }; diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index 4363e946af51d4..f9616636681b5c 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -46,7 +46,7 @@ class OptBoolsDsc private: BasicBlock* m_b1; // The first basic block with the BBJ_COND conditional jump type BasicBlock* m_b2; // The next basic block of m_b1. Either BBJ_COND or BBJ_RETURN type - BasicBlock* m_b3; // m_b1's target block. Null if m_b2 is not a return block. + BasicBlock* m_b3; // m_b1->bbTarget. Null if m_b2 is not a return block. Compiler* m_comp; // The pointer to the Compiler instance @@ -89,7 +89,7 @@ class OptBoolsDsc // Notes: // m_b1 and m_b2 are set on entry. // -// Case 1: if b1->TargetIs(b2->GetTarget()), it transforms +// Case 1: if b1.bbTarget == b2.bbTarget, it transforms // B1 : brtrue(t1, Bx) // B2 : brtrue(t2, Bx) // B3 : @@ -107,7 +107,7 @@ class OptBoolsDsc // B3: GT_RETURN (BBJ_RETURN) // B4: GT_RETURN (BBJ_RETURN) // -// Case 2: if B2->FalseTargetIs(B1->GetTarget()), it transforms +// Case 2: if B2->FalseTargetIs(B1.bbTarget), it transforms // B1 : brtrue(t1, B3) // B2 : brtrue(t2, Bx) // B3 : @@ -123,7 +123,7 @@ bool OptBoolsDsc::optOptimizeBoolsCondBlock() m_t3 = nullptr; - // Check if m_b1 and m_b2 have the same target + // Check if m_b1 and m_b2 have the same bbTarget if (m_b1->TrueTargetIs(m_b2->GetTrueTarget())) { @@ -751,8 +751,6 @@ bool OptBoolsDsc::optOptimizeRangeTests() // BasicBlock* notInRangeBb = m_b1->GetTrueTarget(); BasicBlock* inRangeBb; - weight_t inRangeLikelihood = m_b1->GetFalseEdge()->getLikelihood(); - if (m_b2->TrueTargetIs(notInRangeBb)) { // Shape 1: both conditions jump to NotInRange @@ -766,7 +764,6 @@ bool OptBoolsDsc::optOptimizeRangeTests() // InRange: // ... inRangeBb = m_b2->GetFalseTarget(); - inRangeLikelihood *= m_b2->GetFalseEdge()->getLikelihood(); } else if (m_b2->FalseTargetIs(notInRangeBb)) { @@ -781,7 +778,6 @@ bool OptBoolsDsc::optOptimizeRangeTests() // NotInRange: // ... inRangeBb = m_b2->GetTrueTarget(); - inRangeLikelihood *= m_b2->GetTrueEdge()->getLikelihood(); } else { @@ -812,35 +808,22 @@ bool OptBoolsDsc::optOptimizeRangeTests() } // Re-direct firstBlock to jump to inRangeBb - FlowEdge* const newEdge = m_comp->fgAddRefPred(inRangeBb, m_b1); - FlowEdge* const oldFalseEdge = m_b1->GetFalseEdge(); - FlowEdge* const oldTrueEdge = m_b1->GetTrueEdge(); - + m_comp->fgAddRefPred(inRangeBb, m_b1); if (!cmp2IsReversed) { - m_b1->SetFalseEdge(oldTrueEdge); - m_b1->SetTrueEdge(newEdge); - assert(m_b1->TrueTargetIs(inRangeBb)); - assert(m_b1->FalseTargetIs(notInRangeBb)); - - newEdge->setLikelihood(inRangeLikelihood); - oldTrueEdge->setLikelihood(1.0 - inRangeLikelihood); + m_b1->SetTrueTarget(inRangeBb); + m_b1->SetFalseTarget(notInRangeBb); } else { - m_b1->SetFalseEdge(newEdge); - assert(m_b1->TrueTargetIs(notInRangeBb)); - assert(m_b1->FalseTargetIs(inRangeBb)); - - oldTrueEdge->setLikelihood(inRangeLikelihood); - newEdge->setLikelihood(1.0 - inRangeLikelihood); + m_b1->SetFalseTarget(inRangeBb); } // Remove the 2nd condition block as we no longer need it - m_comp->fgRemoveRefPred(oldFalseEdge); + m_comp->fgRemoveRefPred(m_b2, m_b1); m_comp->fgRemoveBlock(m_b2, true); - Statement* const stmt = m_b1->lastStmt(); + Statement* stmt = m_b1->lastStmt(); m_comp->gtSetStmtInfo(stmt); m_comp->fgSetStmtSeq(stmt); m_comp->gtUpdateStmtSideEffects(stmt); @@ -1029,8 +1012,8 @@ bool OptBoolsDsc::optOptimizeCompareChainCondBlock() m_comp->fgSetStmtSeq(s2); // Update the flow. - m_comp->fgRemoveRefPred(m_b1->GetTrueEdge()); - m_b1->SetKindAndTargetEdge(BBJ_ALWAYS, m_b1->GetFalseEdge()); + m_comp->fgRemoveRefPred(m_b1->GetTrueTarget(), m_b1); + m_b1->SetKindAndTarget(BBJ_ALWAYS, m_b1->GetFalseTarget()); m_b1->SetFlags(BBF_NONE_QUIRK); // Fixup flags. @@ -1279,73 +1262,71 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() m_comp->fgSetStmtSeq(m_testInfo1.testStmt); } - /* Modify the target of the conditional jump and update bbRefs and bbPreds */ - - if (optReturnBlock) - { - assert(m_b1->KindIs(BBJ_COND)); - assert(m_b2->KindIs(BBJ_RETURN)); - assert(m_b1->FalseTargetIs(m_b2)); - assert(m_b3 != nullptr); - m_b1->SetKindAndTargetEdge(BBJ_RETURN); - } - else + if (!optReturnBlock) { - // Modify b1, if necessary, so it has the same - // true target as b2. - // - FlowEdge* const origB1TrueEdge = m_b1->GetTrueEdge(); - FlowEdge* const origB2TrueEdge = m_b2->GetTrueEdge(); - FlowEdge* const origB2FalseEdge = m_b2->GetFalseEdge(); + // Update edges if m_b1: BBJ_COND and m_b2: BBJ_COND - weight_t const origB1TrueLikelihood = origB1TrueEdge->getLikelihood(); - weight_t newB1TrueLikelihood = 0; + FlowEdge* edge1 = m_comp->fgGetPredForBlock(m_b1->GetTrueTarget(), m_b1); + FlowEdge* edge2; if (m_sameTarget) { - // We originally reached B2's true target via - // B1 true OR B1 false B2 true. - // - newB1TrueLikelihood = origB1TrueLikelihood + (1.0 - origB1TrueLikelihood) * origB2TrueEdge->getLikelihood(); + edge2 = m_comp->fgGetPredForBlock(m_b2->GetTrueTarget(), m_b2); } else { - // We originally reached B2's true target via - // B1 false OR B1 true B2 false. - // - // We will now reach via B1 true. - // Modify flow for true side of B1 - // - m_comp->fgRedirectTrueEdge(m_b1, m_b2->GetTrueTarget()); + edge2 = m_comp->fgGetPredForBlock(m_b2->GetFalseTarget(), m_b2); + + m_comp->fgRemoveRefPred(m_b1->GetTrueTarget(), m_b1); - newB1TrueLikelihood = - (1.0 - origB1TrueLikelihood) + origB1TrueLikelihood * origB2FalseEdge->getLikelihood(); + m_b1->SetTrueTarget(m_b2->GetTrueTarget()); + + m_comp->fgAddRefPred(m_b2->GetTrueTarget(), m_b1); } - // Fix B1 true edge likelihood and min/max weights - // - origB1TrueEdge->setLikelihood(newB1TrueLikelihood); - weight_t const newB1TrueWeight = m_b1->bbWeight * newB1TrueLikelihood; - origB1TrueEdge->setEdgeWeights(newB1TrueWeight, newB1TrueWeight, m_b1->GetTrueTarget()); + assert(edge1 != nullptr); + assert(edge2 != nullptr); + weight_t edgeSumMin = edge1->edgeWeightMin() + edge2->edgeWeightMin(); + weight_t edgeSumMax = edge1->edgeWeightMax() + edge2->edgeWeightMax(); + if ((edgeSumMax >= edge1->edgeWeightMax()) && (edgeSumMax >= edge2->edgeWeightMax())) + { + edge1->setEdgeWeights(edgeSumMin, edgeSumMax, m_b1->GetTrueTarget()); + } + else + { + edge1->setEdgeWeights(BB_ZERO_WEIGHT, BB_MAX_WEIGHT, m_b1->GetTrueTarget()); + } + } + + /* Modify the target of the conditional jump and update bbRefs and bbPreds */ + + if (optReturnBlock) + { + assert(m_b1->KindIs(BBJ_COND)); + assert(m_b2->KindIs(BBJ_RETURN)); + assert(m_b1->FalseTargetIs(m_b2)); + assert(m_b3 != nullptr); + m_b1->SetKindAndTarget(BBJ_RETURN); + } + else + { assert(m_b1->KindIs(BBJ_COND)); assert(m_b2->KindIs(BBJ_COND)); assert(m_b1->TrueTargetIs(m_b2->GetTrueTarget())); assert(m_b1->FalseTargetIs(m_b2)); assert(!m_b2->IsLast()); + } - // We now reach B2's false target via B1 false. - // - m_comp->fgReplacePred(origB2FalseEdge, m_b1); - m_comp->fgRemoveRefPred(origB2TrueEdge); - FlowEdge* const newB1FalseEdge = origB2FalseEdge; - m_b1->SetFalseEdge(newB1FalseEdge); - - // Fix B1 false edge likelihood and min/max weights. + if (!optReturnBlock) + { + // Update bbRefs and bbPreds // - newB1FalseEdge->setLikelihood(1.0 - newB1TrueLikelihood); - weight_t const newB1FalseWeight = m_b1->bbWeight * (1.0 - newB1TrueLikelihood); - newB1FalseEdge->setEdgeWeights(newB1FalseWeight, newB1FalseWeight, m_b1->GetTrueTarget()); + // Replace pred 'm_b2' for 'm_b2->bbFalseTarget' with 'm_b1' + // Remove pred 'm_b2' for 'm_b2->bbTrueTarget' + m_comp->fgReplacePred(m_b2->GetFalseTarget(), m_b2, m_b1); + m_comp->fgRemoveRefPred(m_b2->GetTrueTarget(), m_b2); + m_b1->SetFalseTarget(m_b2->GetFalseTarget()); } // Get rid of the second block @@ -1380,7 +1361,7 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() // Notes: // m_b1, m_b2 and m_b3 of OptBoolsDsc are set on entry. // -// if B1->TargetIs(b3), it transforms +// if B1.bbTarget == b3, it transforms // B1 : brtrue(t1, B3) // B2 : ret(t2) // B3 : ret(0) diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 993ebe6cbb374c..e2f1d335e21277 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -21,7 +21,13 @@ void Compiler::optInit() { fgHasLoops = false; - optLoopsCanonical = false; + optLoopsCanonical = false; + optNumNaturalLoopsFound = 0; + +#ifdef DEBUG + loopAlignCandidates = 0; + loopsAligned = 0; +#endif /* Keep track of the number of calls and indirect calls made by this method */ optCallCount = 0; @@ -34,7 +40,6 @@ void Compiler::optInit() optCSECandidateCount = 0; optCSEattempt = 0; optCSEheuristic = nullptr; - optCSEunmarks = 0; } DataFlow::DataFlow(Compiler* pCompiler) : m_pCompiler(pCompiler) @@ -43,10 +48,8 @@ DataFlow::DataFlow(Compiler* pCompiler) : m_pCompiler(pCompiler) //------------------------------------------------------------------------ // optSetBlockWeights: adjust block weights, as follows: -// 1. Lexical block ranges where the bottom reaches the top are scaled as a loop. -// This is a more general definition of "loop" than natural loops. -// 2. A block that is not reachable from the entry block is marked "run rarely". -// 3. If we're not using profile weights, then any block with a non-zero weight +// 1. A block that is not reachable from the entry block is marked "run rarely". +// 2. If we're not using profile weights, then any block with a non-zero weight // that doesn't dominate all the return blocks has its weight dropped in half // (but only if the first block *does* dominate all the returns). // @@ -59,29 +62,13 @@ DataFlow::DataFlow(Compiler* pCompiler) : m_pCompiler(pCompiler) PhaseStatus Compiler::optSetBlockWeights() { noway_assert(opts.OptimizationEnabled()); - - assert(m_dfsTree != nullptr); - if (m_domTree == nullptr) - { - m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); - } - if (m_reachabilitySets == nullptr) - { - m_reachabilitySets = BlockReachabilitySets::Build(m_dfsTree); - } - - if (m_dfsTree->HasCycle()) - { - optMarkLoopHeads(); - optFindAndScaleGeneralLoopBlocks(); - } + assert(m_domTree != nullptr); + assert(fgReturnBlocksComputed); bool madeChanges = false; bool firstBBDominatesAllReturns = true; const bool usingProfileWeights = fgIsUsingProfileWeights(); - fgComputeReturnBlocks(); - // TODO-Quirk: Previously, this code ran on a dominator tree based only on // regular flow. This meant that all handlers were not considered to be // dominated by fgFirstBB. When those handlers could reach a return @@ -246,13 +233,6 @@ void Compiler::optScaleLoopBlocks(BasicBlock* begBlk, BasicBlock* endBlk) continue; } - // Don't change the block weight if it's unreachable. - if (!m_reachabilitySets->GetDfsTree()->Contains(curBlk)) - { - reportBlockWeight(curBlk, "; unchanged: unreachable"); - continue; - } - // For curBlk to be part of a loop that starts at begBlk, curBlk must be reachable from begBlk and // (since this is a loop) begBlk must likewise be reachable from curBlk. @@ -573,7 +553,7 @@ void Compiler::optCheckPreds() #endif // DEBUG //------------------------------------------------------------------------ -// optSetMappedBlockTargets: Initialize the branch successors of a block based on a block map. +// optRedirectBlock: Initialize the branch successors of a block based on a block map. // // Updates the successors of `newBlk`, a copy of `blk`: // If `blk2` is a branch successor of `blk`, and there is a mapping @@ -590,7 +570,7 @@ void Compiler::optCheckPreds() // Upon returning, `newBlk` should have all of its successors initialized. // `blk` must have its successors set upon entry; these won't be changed. // -void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, BlockToBlockMap* redirectMap) +void Compiler::optRedirectBlock(BasicBlock* blk, BasicBlock* newBlk, BlockToBlockMap* redirectMap) { // Caller should not have initialized newBlk's target yet assert(newBlk->KindIs(BBJ_ALWAYS)); @@ -610,24 +590,20 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo case BBJ_CALLFINALLY: case BBJ_CALLFINALLYRET: case BBJ_LEAVE: - { - FlowEdge* newEdge; - // Determine if newBlk should be redirected to a different target from blk's target if (redirectMap->Lookup(blk->GetTarget(), &newTarget)) { // newBlk needs to be redirected to a new target - newEdge = fgAddRefPred(newTarget, newBlk); + newBlk->SetKindAndTarget(blk->GetKind(), newTarget); } else { // newBlk uses the same target as blk - newEdge = fgAddRefPred(blk->GetTarget(), newBlk); + newBlk->SetKindAndTarget(blk->GetKind(), blk->GetTarget()); } - newBlk->SetKindAndTargetEdge(blk->GetKind(), newEdge); + fgAddRefPred(newBlk->GetTarget(), newBlk); break; - } case BBJ_COND: { @@ -656,11 +632,9 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo falseTarget = blk->GetFalseTarget(); } - FlowEdge* const oldTrueEdge = blk->GetTrueEdge(); - FlowEdge* const trueEdge = fgAddRefPred(trueTarget, newBlk, oldTrueEdge); - FlowEdge* const oldFalseEdge = blk->GetFalseEdge(); - FlowEdge* const falseEdge = fgAddRefPred(falseTarget, newBlk, oldFalseEdge); - newBlk->SetCond(trueEdge, falseEdge); + fgAddRefPred(trueTarget, newBlk); + fgAddRefPred(falseTarget, newBlk); + newBlk->SetCond(trueTarget, falseTarget); break; } @@ -698,33 +672,23 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo { BBswtDesc* currSwtDesc = blk->GetSwitchTargets(); BBswtDesc* newSwtDesc = new (this, CMK_BasicBlock) BBswtDesc(currSwtDesc); - newSwtDesc->bbsDstTab = new (this, CMK_FlowEdge) FlowEdge*[newSwtDesc->bbsCount]; + newSwtDesc->bbsDstTab = new (this, CMK_BasicBlock) BasicBlock*[newSwtDesc->bbsCount]; + BasicBlock** jumpPtr = newSwtDesc->bbsDstTab; - for (unsigned i = 0; i < newSwtDesc->bbsCount; i++) + for (BasicBlock* const switchTarget : blk->SwitchTargets()) { - FlowEdge* const inspiringEdge = currSwtDesc->bbsDstTab[i]; - BasicBlock* const switchTarget = inspiringEdge->getDestinationBlock(); - FlowEdge* newEdge; - // Determine if newBlk should target switchTarget, or be redirected if (redirectMap->Lookup(switchTarget, &newTarget)) { - newEdge = fgAddRefPred(newTarget, newBlk); + *jumpPtr = newTarget; } else { - newEdge = fgAddRefPred(switchTarget, newBlk); - } - - // Transfer likelihood... instead of doing this gradually - // for dup'd edges, we set it once, when we add the last dup. - // - if (newEdge->getDupCount() == inspiringEdge->getDupCount()) - { - newEdge->setLikelihood(inspiringEdge->getLikelihood()); + *jumpPtr = switchTarget; } - newSwtDesc->bbsDstTab[i] = newEdge; + fgAddRefPred(*jumpPtr, newBlk); + jumpPtr++; } newBlk->SetSwitch(newSwtDesc); @@ -733,18 +697,16 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: - { // newBlk's jump target should not need to be redirected assert(!redirectMap->Lookup(blk->GetTarget(), &newTarget)); - FlowEdge* newEdge = fgAddRefPred(newBlk->GetTarget(), newBlk); - newBlk->SetKindAndTargetEdge(blk->GetKind(), newEdge); + newBlk->SetKindAndTarget(blk->GetKind(), blk->GetTarget()); + fgAddRefPred(newBlk->GetTarget(), newBlk); break; - } default: // blk doesn't have a jump destination assert(blk->NumSucc() == 0); - newBlk->SetKindAndTargetEdge(blk->GetKind()); + newBlk->SetKindAndTarget(blk->GetKind()); break; } @@ -1338,8 +1300,6 @@ PhaseStatus Compiler::optUnrollLoops() { assert(anyIRchange); - Metrics.LoopsUnrolled += unrollCount; - #ifdef DEBUG if (verbose) { @@ -1749,10 +1709,12 @@ void Compiler::optRedirectPrevUnrollIteration(FlowGraphNaturalLoop* loop, BasicB testCopyStmt->SetRootNode(sideEffList); } + fgRemoveRefPred(prevTestBlock->GetTrueTarget(), prevTestBlock); + fgRemoveRefPred(prevTestBlock->GetFalseTarget(), prevTestBlock); + // Redirect exit edge from previous iteration to new entry. - fgRedirectTrueEdge(prevTestBlock, target); - fgRemoveRefPred(prevTestBlock->GetFalseEdge()); - prevTestBlock->SetKindAndTargetEdge(BBJ_ALWAYS, prevTestBlock->GetTrueEdge()); + prevTestBlock->SetKindAndTarget(BBJ_ALWAYS, target); + fgAddRefPred(target, prevTestBlock); JITDUMP("Redirecting previously created exiting " FMT_BB " -> " FMT_BB "\n", prevTestBlock->bbNum, target->bbNum); @@ -1892,36 +1854,32 @@ Compiler::OptInvertCountTreeInfoType Compiler::optInvertCountTreeInfo(GenTree* t // // Specifically, we're looking for the following case: // -// block: // ... // jmp test // `block` argument -// top: +// loop: // ... // ... // test: // ..stmts.. // cond -// jtrue top +// jtrue loop // // If we find this, and the condition is simple enough, we change // the loop to the following: // -// block: // ... -// jmp bNewCond -// bNewCond: // ..stmts.. // duplicated cond block statements // cond // duplicated cond -// jfalse join +// jfalse done // // else fall-through -// top: +// loop: // ... // ... // test: // ..stmts.. // cond -// jtrue top -// join: +// jtrue loop +// done: // // Makes no changes if the flow pattern match fails. // @@ -1965,7 +1923,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) return false; } - // Since bTest is a BBJ_COND it will have a false target + // Since bTest is a BBJ_COND it will have a bbFalseTarget // BasicBlock* const bJoin = bTest->GetFalseTarget(); noway_assert(bJoin != nullptr); @@ -1987,7 +1945,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) } // It has to be a forward jump. Defer this check until after all the cheap checks - // are done, since it iterates forward in the block list looking for block's target. + // are done, since it iterates forward in the block list looking for bbTarget. // TODO-CQ: Check if we can also optimize the backwards jump as well. // if (!fgIsForwardBranch(block, block->GetTarget())) @@ -2177,8 +2135,10 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) bool foundCondTree = false; // Create a new block after `block` to put the copied condition code. - // - BasicBlock* const bNewCond = fgNewBBafter(BBJ_COND, block, /*extendRegion*/ true); + BasicBlock* bNewCond = fgNewBBafter(BBJ_COND, block, /*extendRegion*/ true, bJoin); + block->SetKindAndTarget(BBJ_ALWAYS, bNewCond); + block->SetFlags(BBF_NONE_QUIRK); + assert(block->JumpsToNext()); // Clone each statement in bTest and append to bNewCond. for (Statement* const stmt : bTest->Statements()) @@ -2237,27 +2197,12 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) // Update pred info // - // For now we set the likelihood of the newCond branch to match - // the likelihood of the test branch (though swapped, since we're - // currently reversing the condition). This may or may not match - // the block weight adjustments we're making. All this becomes - // easier to reconcile once we rely on edge likelihoods more and - // have synthesis running (so block weights ==> frequencies). - // - // Until then we won't worry that edges and blocks are potentially - // out of sync. - // - FlowEdge* const testTopEdge = bTest->GetTrueEdge(); - FlowEdge* const testJoinEdge = bTest->GetFalseEdge(); - FlowEdge* const newCondJoinEdge = fgAddRefPred(bJoin, bNewCond, testJoinEdge); - FlowEdge* const newCondTopEdge = fgAddRefPred(bTop, bNewCond, testTopEdge); + bNewCond->SetFalseTarget(bTop); + fgAddRefPred(bJoin, bNewCond); + fgAddRefPred(bTop, bNewCond); - bNewCond->SetTrueEdge(newCondJoinEdge); - bNewCond->SetFalseEdge(newCondTopEdge); - - fgRedirectTargetEdge(block, bNewCond); - block->SetFlags(BBF_NONE_QUIRK); - assert(block->JumpsToNext()); + fgAddRefPred(bNewCond, block); + fgRemoveRefPred(bTest, block); // Move all predecessor edges that look like loop entry edges to point to the new cloned condition // block, not the existing condition block. The idea is that if we only move `block` to point to @@ -2268,9 +2213,10 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) // is maintained no matter which condition block we point to, but we'll lose optimization potential // (and create spaghetti code) if we get it wrong. // + unsigned const loopFirstNum = bTop->bbNum; unsigned const loopBottomNum = bTest->bbNum; - for (BasicBlock* const predBlock : bTest->PredBlocksEditing()) + for (BasicBlock* const predBlock : bTest->PredBlocks()) { unsigned const bNum = predBlock->bbNum; if ((loopFirstNum <= bNum) && (bNum <= loopBottomNum)) @@ -2335,8 +2281,8 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) weight_t const testToNextWeight = weightTop * testToNextLikelihood; weight_t const testToAfterWeight = weightTop * testToAfterLikelihood; - FlowEdge* const edgeTestToNext = bTest->GetTrueEdge(); - FlowEdge* const edgeTestToAfter = bTest->GetFalseEdge(); + FlowEdge* const edgeTestToNext = fgGetPredForBlock(bTop, bTest); + FlowEdge* const edgeTestToAfter = fgGetPredForBlock(bTest->GetFalseTarget(), bTest); JITDUMP("Setting weight of " FMT_BB " -> " FMT_BB " to " FMT_WT " (iterate loop)\n", bTest->bbNum, bTop->bbNum, testToNextWeight); @@ -2356,8 +2302,8 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) weight_t const blockToNextWeight = weightBlock * blockToNextLikelihood; weight_t const blockToAfterWeight = weightBlock * blockToAfterLikelihood; - FlowEdge* const edgeBlockToNext = bNewCond->GetFalseEdge(); - FlowEdge* const edgeBlockToAfter = bNewCond->GetTrueEdge(); + FlowEdge* const edgeBlockToNext = fgGetPredForBlock(bNewCond->GetFalseTarget(), bNewCond); + FlowEdge* const edgeBlockToAfter = fgGetPredForBlock(bNewCond->GetTrueTarget(), bNewCond); JITDUMP("Setting weight of " FMT_BB " -> " FMT_BB " to " FMT_WT " (enter loop)\n", bNewCond->bbNum, bNewCond->GetFalseTarget()->bbNum, blockToNextWeight); @@ -2368,21 +2314,18 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) edgeBlockToAfter->setEdgeWeights(blockToAfterWeight, blockToAfterWeight, bNewCond->GetTrueTarget()); #ifdef DEBUG - // If we're checking profile data, see if profile for the two target blocks is consistent. + // If we're checkig profile data, see if profile for the two target blocks is consistent. // if ((activePhaseChecks & PhaseChecks::CHECK_PROFILE) == PhaseChecks::CHECK_PROFILE) { - if (JitConfig.JitProfileChecks() > 0) - { - const ProfileChecks checks = (ProfileChecks)JitConfig.JitProfileChecks(); - const bool nextProfileOk = fgDebugCheckIncomingProfileData(bNewCond->GetFalseTarget(), checks); - const bool jumpProfileOk = fgDebugCheckIncomingProfileData(bNewCond->GetTrueTarget(), checks); + const ProfileChecks checks = (ProfileChecks)JitConfig.JitProfileChecks(); + const bool nextProfileOk = fgDebugCheckIncomingProfileData(bNewCond->GetFalseTarget(), checks); + const bool jumpProfileOk = fgDebugCheckIncomingProfileData(bNewCond->GetTrueTarget(), checks); - if (hasFlag(checks, ProfileChecks::RAISE_ASSERT)) - { - assert(nextProfileOk); - assert(jumpProfileOk); - } + if (hasFlag(checks, ProfileChecks::RAISE_ASSERT)) + { + assert(nextProfileOk); + assert(jumpProfileOk); } } #endif // DEBUG @@ -2530,13 +2473,13 @@ void Compiler::optMarkLoopHeads() { printf("*************** In optMarkLoopHeads()\n"); } + + assert(m_reachabilitySets != nullptr); fgDebugCheckBBNumIncreasing(); int loopHeadsMarked = 0; #endif - assert((m_dfsTree != nullptr) && (m_reachabilitySets != nullptr)); - bool hasLoops = false; for (BasicBlock* const block : Blocks()) @@ -2612,7 +2555,15 @@ void Compiler::optFindAndScaleGeneralLoopBlocks() // This code depends on block number ordering. INDEBUG(fgDebugCheckBBNumIncreasing()); - assert((m_dfsTree != nullptr) && (m_domTree != nullptr) && (m_reachabilitySets != nullptr)); + assert(m_dfsTree != nullptr); + if (m_reachabilitySets == nullptr) + { + m_reachabilitySets = BlockReachabilitySets::Build(m_dfsTree); + } + if (m_domTree == nullptr) + { + m_domTree = FlowGraphDominatorTree::Build(m_dfsTree); + } unsigned generalLoopCount = 0; @@ -2688,7 +2639,7 @@ void Compiler::optFindAndScaleGeneralLoopBlocks() } //----------------------------------------------------------------------------- -// optFindLoopsPhase: find loops in the function. +// optFindLoops: find loops in the function. // // The JIT recognizes two types of loops in a function: natural loops and "general" (or "unnatural") loops. // Natural loops are those which get added to Compiler::m_loops. Most downstream optimizations require @@ -2709,12 +2660,17 @@ PhaseStatus Compiler::optFindLoopsPhase() } #endif - fgRenumberBlocks(); + optMarkLoopHeads(); assert(m_dfsTree != nullptr); optFindLoops(); - Metrics.LoopsFoundDuringOpts = (int)m_loops->NumLoops(); + if (fgHasLoops) + { + optFindAndScaleGeneralLoopBlocks(); + } + + optNumNaturalLoopsFound = (unsigned)m_loops->NumLoops(); return PhaseStatus::MODIFIED_EVERYTHING; } @@ -3020,7 +2976,7 @@ bool Compiler::optCreatePreheader(FlowGraphNaturalLoop* loop) insertBefore = header; } - BasicBlock* preheader = fgNewBBbefore(BBJ_ALWAYS, insertBefore, false); + BasicBlock* preheader = fgNewBBbefore(BBJ_ALWAYS, insertBefore, false, header); preheader->SetFlags(BBF_INTERNAL); fgSetEHRegionForNewPreheaderOrExit(preheader); @@ -3033,8 +2989,7 @@ bool Compiler::optCreatePreheader(FlowGraphNaturalLoop* loop) JITDUMP("Created new preheader " FMT_BB " for " FMT_LP "\n", preheader->bbNum, loop->GetIndex()); - FlowEdge* const newEdge = fgAddRefPred(header, preheader); - preheader->SetTargetEdge(newEdge); + fgAddRefPred(header, preheader); for (FlowEdge* enterEdge : loop->EntryEdges()) { @@ -3125,9 +3080,6 @@ bool Compiler::optCanonicalizeExit(FlowGraphNaturalLoop* loop, BasicBlock* exit) BasicBlock* newExit; - JITDUMP("Canonicalize exit " FMT_BB " for " FMT_LP " to have only loop predecessors\n", exit->bbNum, - loop->GetIndex()); - #if FEATURE_EH_CALLFINALLY_THUNKS if (exit->KindIs(BBJ_CALLFINALLY)) { @@ -3140,31 +3092,33 @@ bool Compiler::optCanonicalizeExit(FlowGraphNaturalLoop* loop, BasicBlock* exit) BasicBlock* bottom = loop->GetLexicallyBottomMostBlock(); if (bottom->hasTryIndex() && (bottom->getTryIndex() == finallyBlock->getHndIndex()) && !bottom->hasHndIndex()) { - newExit = fgNewBBafter(BBJ_ALWAYS, bottom, true); + newExit = fgNewBBafter(BBJ_ALWAYS, bottom, true, exit); } else { // Otherwise just do the heavy-handed thing and insert it anywhere in the right region. - newExit = fgNewBBinRegion(BBJ_ALWAYS, finallyBlock->bbHndIndex, 0, nullptr, /* putInFilter */ false, + newExit = fgNewBBinRegion(BBJ_ALWAYS, finallyBlock->bbHndIndex, 0, nullptr, exit, /* putInFilter */ false, /* runRarely */ false, /* insertAtEnd */ true); } } else -#endif // FEATURE_EH_CALLFINALLY_THUNKS +#endif { - newExit = fgNewBBbefore(BBJ_ALWAYS, exit, false); + newExit = fgNewBBbefore(BBJ_ALWAYS, exit, false, exit); newExit->SetFlags(BBF_NONE_QUIRK); fgSetEHRegionForNewPreheaderOrExit(newExit); } newExit->SetFlags(BBF_INTERNAL); - FlowEdge* const newEdge = fgAddRefPred(exit, newExit); - newExit->SetTargetEdge(newEdge); + fgAddRefPred(exit, newExit); newExit->bbCodeOffs = exit->bbCodeOffs; - for (BasicBlock* pred : exit->PredBlocksEditing()) + JITDUMP("Created new exit " FMT_BB " to replace " FMT_BB " for " FMT_LP "\n", newExit->bbNum, exit->bbNum, + loop->GetIndex()); + + for (BasicBlock* pred : exit->PredBlocks()) { if (loop->ContainsBlock(pred)) { @@ -3173,9 +3127,6 @@ bool Compiler::optCanonicalizeExit(FlowGraphNaturalLoop* loop, BasicBlock* exit) } optSetWeightForPreheaderOrExit(loop, newExit); - - JITDUMP("Created new exit " FMT_BB " to replace " FMT_BB " exit for " FMT_LP "\n", newExit->bbNum, exit->bbNum, - loop->GetIndex()); return true; } @@ -3317,7 +3268,7 @@ void Compiler::optSetWeightForPreheaderOrExit(FlowGraphNaturalLoop* loop, BasicB } // Normalize block -> target weight - FlowEdge* const edgeFromBlock = block->GetTargetEdge(); + FlowEdge* const edgeFromBlock = fgGetPredForBlock(block->GetTarget(), block); assert(edgeFromBlock != nullptr); edgeFromBlock->setEdgeWeights(block->bbWeight, block->bbWeight, block->GetTarget()); } @@ -5196,8 +5147,6 @@ void Compiler::optHoistCandidate(GenTree* tree, // Record the hoisted expression in hoistCtxt hoistCtxt->GetHoistedInCurLoop(this)->Set(tree->gtVNPair.GetLiberal(), true); - - Metrics.HoistedExpressions++; } bool Compiler::optVNIsLoopInvariant(ValueNum vn, FlowGraphNaturalLoop* loop, VNSet* loopVnInvariantCache) @@ -5664,6 +5613,7 @@ void Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk, FlowGraphNatura case GT_XCHG: case GT_CMPXCHG: case GT_MEMORYBARRIER: + case GT_STORE_DYN_BLK: { memoryHavoc |= memoryKindSet(GcHeap, ByrefExposed); } diff --git a/src/coreclr/jit/patchpoint.cpp b/src/coreclr/jit/patchpoint.cpp index 158154583f7798..27b94470962ef0 100644 --- a/src/coreclr/jit/patchpoint.cpp +++ b/src/coreclr/jit/patchpoint.cpp @@ -101,12 +101,13 @@ class PatchpointTransformer // Arguments: // jumpKind - jump kind for the new basic block // insertAfter - basic block, after which compiler has to insert the new one. + // jumpDest - jump target for the new basic block. Defaults to nullptr. // // Return Value: // new basic block. - BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter) + BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter, BasicBlock* jumpDest = nullptr) { - BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true); + BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true, jumpDest); block->SetFlags(BBF_IMPORTED); return block; } @@ -142,21 +143,21 @@ class PatchpointTransformer // Current block now becomes the test block BasicBlock* remainderBlock = compiler->fgSplitBlockAtBeginning(block); - BasicBlock* helperBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, block); + BasicBlock* helperBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, block, block->Next()); // Update flow and flags + block->SetCond(remainderBlock, helperBlock); block->SetFlags(BBF_INTERNAL); + helperBlock->SetFlags(BBF_BACKWARD_JUMP | BBF_NONE_QUIRK); - assert(block->TargetIs(remainderBlock)); FlowEdge* const falseEdge = compiler->fgAddRefPred(helperBlock, block); - FlowEdge* const trueEdge = block->GetTargetEdge(); + FlowEdge* const trueEdge = compiler->fgGetPredForBlock(remainderBlock, block); trueEdge->setLikelihood(HIGH_PROBABILITY / 100.0); falseEdge->setLikelihood((100 - HIGH_PROBABILITY) / 100.0); - block->SetCond(trueEdge, falseEdge); FlowEdge* const newEdge = compiler->fgAddRefPred(remainderBlock, helperBlock); - helperBlock->SetTargetEdge(newEdge); + newEdge->setLikelihood(1.0); // Update weights remainderBlock->inheritWeight(block); @@ -237,7 +238,7 @@ class PatchpointTransformer } // Update flow - block->SetKindAndTargetEdge(BBJ_THROW); + block->SetKindAndTarget(BBJ_THROW); // Add helper call // diff --git a/src/coreclr/jit/phase.cpp b/src/coreclr/jit/phase.cpp index 78bf4eec09c752..717d0a7d270d5e 100644 --- a/src/coreclr/jit/phase.cpp +++ b/src/coreclr/jit/phase.cpp @@ -171,7 +171,10 @@ void Phase::PostPhase(PhaseStatus status) comp->fgDebugCheckLinkedLocals(); } - comp->fgDebugCheckFlowGraphAnnotations(); + if (comp->m_dfsTree != nullptr) + { + comp->fgDebugCheckDfsTree(); + } } #endif // DEBUG } diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 2f7b1e0b31372c..f66748633a69e3 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -1205,8 +1205,6 @@ class LocalsUseVisitor : public GenTreeVisitor } } - m_compiler->Metrics.PhysicallyPromotedFields += totalNumPromotions; - if (totalNumPromotions <= 0) { return false; diff --git a/src/coreclr/jit/redundantbranchopts.cpp b/src/coreclr/jit/redundantbranchopts.cpp index e8b346faccc376..3ea6142de4cd3f 100644 --- a/src/coreclr/jit/redundantbranchopts.cpp +++ b/src/coreclr/jit/redundantbranchopts.cpp @@ -927,7 +927,6 @@ bool Compiler::optRedundantBranch(BasicBlock* const block) JITDUMP("\nRedundant branch opt in " FMT_BB ":\n", block->bbNum); fgMorphBlockStmt(block, stmt DEBUGARG(__FUNCTION__)); - Metrics.RedundantBranchesEliminated++; return true; } @@ -1600,7 +1599,7 @@ bool Compiler::optJumpThreadCore(JumpThreadInfo& jti) // If this pred is in the set that will reuse block, do nothing. // Else revise pred to branch directly to the appropriate successor of block. // - for (BasicBlock* const predBlock : jti.m_block->PredBlocksEditing()) + for (BasicBlock* const predBlock : jti.m_block->PredBlocks()) { // If this was an ambiguous pred, skip. // @@ -1684,7 +1683,6 @@ bool Compiler::optJumpThreadCore(JumpThreadInfo& jti) // We optimized. // - Metrics.JumpThreadingsPerformed++; fgModified = true; return true; } diff --git a/src/coreclr/jit/scev.cpp b/src/coreclr/jit/scev.cpp deleted file mode 100644 index 5819b56bdfd3a1..00000000000000 --- a/src/coreclr/jit/scev.cpp +++ /dev/null @@ -1,1006 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// This file contains code to analyze how the value of induction variables -// evolve (scalar evolution analysis), and to turn them into the SCEV IR -// defined in scev.h. The analysis is inspired by "Michael Wolfe. 1992. Beyond -// induction variables." and also by LLVM's scalar evolution analysis. -// -// The main idea of scalar evolution nalysis is to give a closed form -// describing the value of tree nodes inside loops even when taking into -// account that they are changing on each loop iteration. This is useful for -// optimizations that want to reason about values of IR nodes inside loops, -// such as IV widening or strength reduction. -// -// To represent the possibility of evolution the SCEV IR includes the concept -// of an add recurrence , which describes a value that -// starts at "start" and changes by adding "step" at each iteration. The IR -// nodes that change in this way (or depend on something that changes in this -// way) are generally called induction variables. -// -// An add recurrence arises only when a local exists in the loop that is -// mutated in each iteration. Such a local will naturally end up with a phi -// node in the loop header. These locals are called primary (or basic) -// induction variables. The non-primary IVs (which always must depend on the -// primary IVs) are sometimes called secondary IVs. -// -// The job of the analysis is to go from a tree node to a SCEV node that -// describes its value (possibly taking its evolution into account). Note that -// SCEV nodes are immutable and the values they represent are _not_ -// flow-dependent; that is, they don't exist at a specific location inside the -// loop, even though some particular tree node gave rise to that SCEV node. The -// analysis itself _is_ flow-dependent and guarantees that the Scev* returned -// describes the value that corresponds to what the tree node computes at its -// specific location. However, it would be perfectly legal for two trees at -// different locations in the loop to analyze to the same SCEV node (even -// potentially returning the same pointer). For example, in theory "i" and "j" -// in the following loop would both be represented by the same add recurrence -// , and the analysis could even return the same Scev* for both of -// them, even if it does not today: -// -// int i = 0; -// while (true) -// { -// i++; -// ... -// int j = i - 1; -// } -// -// Actually materializing the value of a SCEV node back into tree IR is not -// implemented yet, but generally would depend on the availability of tree -// nodes that compute the dependent values at the point where the IR is to be -// materialized. -// -// Besides the add recurrences the analysis itself is generally a -// straightforward translation from JIT IR into the SCEV IR. Creating the add -// recurrences requires paying attention to the structure of PHIs, and -// disambiguating the values coming from outside the loop and the values coming -// from the backedges. -// - -#include "jitpch.h" - -//------------------------------------------------------------------------ -// GetConstantValue: If this SSA use refers to a constant, then fetch that -// constant. -// -// Parameters: -// comp - Compiler instance -// cns - [out] Constant value; only valid if this function returns true. -// -// Returns: -// True if this SSA use refers to a constant; otherwise false, -// -bool ScevLocal::GetConstantValue(Compiler* comp, int64_t* cns) -{ - LclVarDsc* dsc = comp->lvaGetDesc(LclNum); - LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(SsaNum); - GenTreeLclVarCommon* defNode = ssaDsc->GetDefNode(); - if ((defNode != nullptr) && defNode->Data()->OperIs(GT_CNS_INT, GT_CNS_LNG)) - { - *cns = defNode->Data()->AsIntConCommon()->IntegralValue(); - return true; - } - - return false; -} - -//------------------------------------------------------------------------ -// Scev::GetConstantValue: If this SCEV is always a constant (i.e. either an -// inline constant or an SSA use referring to a constant) then obtain that -// constant. -// -// Parameters: -// comp - Compiler instance -// cns - [out] Constant value; only valid if this function returns true. -// -// Returns: -// True if a constant could be extracted. -// -bool Scev::GetConstantValue(Compiler* comp, int64_t* cns) -{ - if (OperIs(ScevOper::Constant)) - { - *cns = ((ScevConstant*)this)->Value; - return true; - } - - if (OperIs(ScevOper::Local)) - { - return ((ScevLocal*)this)->GetConstantValue(comp, cns); - } - - return false; -} - -#ifdef DEBUG -//------------------------------------------------------------------------ -// Dump: Print this scev node to stdout. -// -// Parameters: -// comp - Compiler instance -// -void Scev::Dump(Compiler* comp) -{ - switch (Oper) - { - case ScevOper::Constant: - { - ScevConstant* cns = (ScevConstant*)this; - printf("%zd", (ssize_t)cns->Value); - break; - } - case ScevOper::Local: - { - ScevLocal* invariantLocal = (ScevLocal*)this; - printf("V%02u.%u", invariantLocal->LclNum, invariantLocal->SsaNum); - - int64_t cns; - if (invariantLocal->GetConstantValue(comp, &cns)) - { - printf(" (%lld)", (long long)cns); - } - break; - } - case ScevOper::ZeroExtend: - case ScevOper::SignExtend: - { - ScevUnop* unop = (ScevUnop*)this; - printf("%cext<%d>(", unop->Oper == ScevOper::ZeroExtend ? 'z' : 's', genTypeSize(unop->Type) * 8); - unop->Op1->Dump(comp); - printf(")"); - break; - } - case ScevOper::Add: - case ScevOper::Mul: - case ScevOper::Lsh: - { - ScevBinop* binop = (ScevBinop*)this; - printf("("); - binop->Op1->Dump(comp); - const char* op; - switch (binop->Oper) - { - case ScevOper::Add: - op = "+"; - break; - case ScevOper::Mul: - op = "*"; - break; - case ScevOper::Lsh: - op = "<<"; - break; - default: - unreached(); - } - printf(" %s ", op); - binop->Op2->Dump(comp); - printf(")"); - break; - } - case ScevOper::AddRec: - { - ScevAddRec* addRec = (ScevAddRec*)this; - printf("<" FMT_LP, addRec->Loop->GetIndex()); - printf(", "); - addRec->Start->Dump(comp); - printf(", "); - addRec->Step->Dump(comp); - printf(">"); - break; - } - default: - unreached(); - } -} -#endif - -//------------------------------------------------------------------------ -// ScalarEvolutionContext: Construct an instance of a context to do scalar evolution in. -// -// Parameters: -// comp - Compiler instance -// -// Remarks: -// After construction the context should be reset for a new loop by calling -// ResetForLoop. -// -ScalarEvolutionContext::ScalarEvolutionContext(Compiler* comp) - : m_comp(comp), m_cache(comp->getAllocator(CMK_LoopIVOpts)), m_ephemeralCache(comp->getAllocator(CMK_LoopIVOpts)) -{ -} - -//------------------------------------------------------------------------ -// ResetForLoop: Reset the internal cache in preparation of scalar -// evolution analysis inside a new loop. -// -// Parameters: -// loop - The loop. -// -void ScalarEvolutionContext::ResetForLoop(FlowGraphNaturalLoop* loop) -{ - m_loop = loop; - m_cache.RemoveAll(); -} - -//------------------------------------------------------------------------ -// NewConstant: Create a SCEV node that represents a constant. -// -// Returns: -// The new node. -// -ScevConstant* ScalarEvolutionContext::NewConstant(var_types type, int64_t value) -{ - ScevConstant* constant = new (m_comp, CMK_LoopIVOpts) ScevConstant(type, value); - return constant; -} - -//------------------------------------------------------------------------ -// NewLocal: Create a SCEV node that represents an invariant local (i.e. a -// use of an SSA def from outside the loop). -// -// Parameters: -// lclNum - The local -// ssaNum - The SSA number of the def outside the loop that is being used. -// -// Returns: -// The new node. -// -ScevLocal* ScalarEvolutionContext::NewLocal(unsigned lclNum, unsigned ssaNum) -{ - var_types type = genActualType(m_comp->lvaGetDesc(lclNum)); - ScevLocal* invariantLocal = new (m_comp, CMK_LoopIVOpts) ScevLocal(type, lclNum, ssaNum); - return invariantLocal; -} - -//------------------------------------------------------------------------ -// NewExtension: Create a SCEV node that represents a zero or sign extension. -// -// Parameters: -// oper - The operation (ScevOper::ZeroExtend or ScevOper::SignExtend) -// targetType - The target type of the extension -// op - The operand being extended. -// -// Returns: -// The new node. -// -ScevUnop* ScalarEvolutionContext::NewExtension(ScevOper oper, var_types targetType, Scev* op) -{ - assert(op != nullptr); - ScevUnop* ext = new (m_comp, CMK_LoopIVOpts) ScevUnop(oper, targetType, op); - return ext; -} - -//------------------------------------------------------------------------ -// NewBinop: Create a SCEV node that represents a binary operation. -// -// Parameters: -// oper - The operation -// op1 - First operand -// op2 - Second operand -// -// Returns: -// The new node. -// -ScevBinop* ScalarEvolutionContext::NewBinop(ScevOper oper, Scev* op1, Scev* op2) -{ - assert((op1 != nullptr) && (op2 != nullptr)); - ScevBinop* binop = new (m_comp, CMK_LoopIVOpts) ScevBinop(oper, op1->Type, op1, op2); - return binop; -} - -//------------------------------------------------------------------------ -// NewAddRec: Create a SCEV node that represents a new add recurrence. -// -// Parameters: -// loop - The loop where this add recurrence is evolving -// start - Value of the recurrence at the first iteration -// step - Step value of the recurrence -// -// Returns: -// The new node. -// -ScevAddRec* ScalarEvolutionContext::NewAddRec(Scev* start, Scev* step) -{ - assert((start != nullptr) && (step != nullptr)); - ScevAddRec* addRec = new (m_comp, CMK_LoopIVOpts) ScevAddRec(start->Type, start, step DEBUGARG(m_loop)); - return addRec; -} - -//------------------------------------------------------------------------ -// CreateSimpleInvariantScev: Create a "simple invariant" SCEV node for a tree: -// either an invariant local use or a constant. -// -// Parameters: -// tree - The tree -// -// Returns: -// SCEV node or nullptr if the tree is not a simple invariant. -// -Scev* ScalarEvolutionContext::CreateSimpleInvariantScev(GenTree* tree) -{ - if (tree->OperIs(GT_CNS_INT, GT_CNS_LNG)) - { - return CreateScevForConstant(tree->AsIntConCommon()); - } - - if (tree->OperIs(GT_LCL_VAR) && tree->AsLclVarCommon()->HasSsaName()) - { - LclVarDsc* dsc = m_comp->lvaGetDesc(tree->AsLclVarCommon()); - LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(tree->AsLclVarCommon()->GetSsaNum()); - - if ((ssaDsc->GetBlock() == nullptr) || !m_loop->ContainsBlock(ssaDsc->GetBlock())) - { - return NewLocal(tree->AsLclVarCommon()->GetLclNum(), tree->AsLclVarCommon()->GetSsaNum()); - } - } - - return nullptr; -} - -//------------------------------------------------------------------------ -// CreateScevForConstant: Given an integer constant, create a SCEV node for it. -// -// Parameters: -// tree - The integer constant -// -// Returns: -// SCEV node or nullptr if the integer constant is not representable (e.g. a handle). -// -Scev* ScalarEvolutionContext::CreateScevForConstant(GenTreeIntConCommon* tree) -{ - if (tree->IsIconHandle() || !tree->TypeIs(TYP_INT, TYP_LONG)) - { - return nullptr; - } - - return NewConstant(tree->TypeGet(), tree->AsIntConCommon()->IntegralValue()); -} - -//------------------------------------------------------------------------ -// AnalyzeNew: Analyze the specified tree in the specified block, without going -// through the cache. -// -// Parameters: -// block - Block containing the tree -// tree - Tree node -// depth - Current analysis depth -// -// Returns: -// SCEV node if the tree was analyzable; otherwise nullptr if the value is -// cannot be described. -// -Scev* ScalarEvolutionContext::AnalyzeNew(BasicBlock* block, GenTree* tree, int depth) -{ - switch (tree->OperGet()) - { - case GT_CNS_INT: - case GT_CNS_LNG: - { - return CreateScevForConstant(tree->AsIntConCommon()); - } - case GT_LCL_VAR: - case GT_PHI_ARG: - { - if (!tree->AsLclVarCommon()->HasSsaName()) - { - return nullptr; - } - - assert(m_comp->lvaInSsa(tree->AsLclVarCommon()->GetLclNum())); - LclVarDsc* dsc = m_comp->lvaGetDesc(tree->AsLclVarCommon()); - LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(tree->AsLclVarCommon()->GetSsaNum()); - - if ((ssaDsc->GetBlock() == nullptr) || !m_loop->ContainsBlock(ssaDsc->GetBlock())) - { - return NewLocal(tree->AsLclVarCommon()->GetLclNum(), tree->AsLclVarCommon()->GetSsaNum()); - } - - if (ssaDsc->GetDefNode() == nullptr) - { - // GT_CALL retbuf def? - return nullptr; - } - - if (ssaDsc->GetDefNode()->GetLclNum() != tree->AsLclVarCommon()->GetLclNum()) - { - // Should be a def of the parent - assert(dsc->lvIsStructField && (ssaDsc->GetDefNode()->GetLclNum() == dsc->lvParentLcl)); - return nullptr; - } - - return Analyze(ssaDsc->GetBlock(), ssaDsc->GetDefNode(), depth + 1); - } - case GT_STORE_LCL_VAR: - { - GenTreeLclVarCommon* store = tree->AsLclVarCommon(); - GenTree* data = store->Data(); - if (!data->OperIs(GT_PHI)) - { - return Analyze(block, data, depth + 1); - } - - if (block != m_loop->GetHeader()) - { - return nullptr; - } - - // We have a phi def for the current loop. Look for a primary - // induction variable. - GenTreePhi* phi = data->AsPhi(); - GenTreePhiArg* enterSsa = nullptr; - GenTreePhiArg* backedgeSsa = nullptr; - - for (GenTreePhi::Use& use : phi->Uses()) - { - GenTreePhiArg* phiArg = use.GetNode()->AsPhiArg(); - GenTreePhiArg*& ssaArg = m_loop->ContainsBlock(phiArg->gtPredBB) ? backedgeSsa : enterSsa; - if ((ssaArg == nullptr) || (ssaArg->GetSsaNum() == phiArg->GetSsaNum())) - { - ssaArg = phiArg; - } - else - { - return nullptr; - } - } - - if ((enterSsa == nullptr) || (backedgeSsa == nullptr)) - { - return nullptr; - } - - ScevLocal* enterScev = NewLocal(enterSsa->GetLclNum(), enterSsa->GetSsaNum()); - - LclVarDsc* dsc = m_comp->lvaGetDesc(store); - LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(backedgeSsa->GetSsaNum()); - - if (ssaDsc->GetDefNode() == nullptr) - { - // GT_CALL retbuf def - return nullptr; - } - - if (ssaDsc->GetDefNode()->GetLclNum() != store->GetLclNum()) - { - assert(dsc->lvIsStructField && ssaDsc->GetDefNode()->GetLclNum() == dsc->lvParentLcl); - return nullptr; - } - - assert(ssaDsc->GetBlock() != nullptr); - - // Try simple but most common case first, where we have a direct - // add recurrence like i = i + 1. - Scev* simpleAddRec = CreateSimpleAddRec(store, enterScev, ssaDsc->GetBlock(), ssaDsc->GetDefNode()->Data()); - if (simpleAddRec != nullptr) - { - return simpleAddRec; - } - - // Otherwise try a more powerful approach; we create a symbolic - // node representing the recurrence and then invoke the analysis - // recursively. This handles for example cases like - // - // int i = start; - // while (i < n) - // { - // int j = i + 1; - // ... - // i = j; - // } - // => - // - // where we need to follow SSA defs. In this case the analysis will result in - // + 1. The symbolic node represents a recurrence, - // so this corresponds to the infinite sequence [start, start + 1, - // start + 1 + 1, ...] which can be represented by . - // - // This approach also generalizes to handle chains of recurrences. - // For example: - // - // int i = 0; - // int j = 0; - // while (i < n) - // { - // j++; - // i += j; - // } - // => > - // - // Here `i` will analyze to + . - // Like before this corresponds to an infinite sequence - // [start, start + , start + 2 * , ...] - // which again can be represented as >. - // - // More generally, as long as we have only additions and only a - // single operand is the recurrence, we can represent it as an add - // recurrence. See MakeAddRecFromRecursiveScev for the details. - // - ScevConstant* symbolicAddRec = NewConstant(data->TypeGet(), 0xdeadbeef); - m_ephemeralCache.Emplace(store, symbolicAddRec); - - Scev* result; - if (m_usingEphemeralCache) - { - result = Analyze(ssaDsc->GetBlock(), ssaDsc->GetDefNode()->Data(), depth + 1); - } - else - { - m_usingEphemeralCache = true; - result = Analyze(ssaDsc->GetBlock(), ssaDsc->GetDefNode()->Data(), depth + 1); - m_usingEphemeralCache = false; - m_ephemeralCache.RemoveAll(); - } - - if (result == nullptr) - { - return nullptr; - } - - return MakeAddRecFromRecursiveScev(enterScev, result, symbolicAddRec); - } - case GT_CAST: - { - GenTreeCast* cast = tree->AsCast(); - if (cast->gtCastType != TYP_LONG) - { - return nullptr; - } - - Scev* op = Analyze(block, cast->CastOp(), depth + 1); - if (op == nullptr) - { - return nullptr; - } - - return NewExtension(cast->IsUnsigned() ? ScevOper::ZeroExtend : ScevOper::SignExtend, TYP_LONG, op); - } - case GT_ADD: - case GT_SUB: - case GT_MUL: - case GT_LSH: - { - Scev* op1 = Analyze(block, tree->gtGetOp1(), depth + 1); - if (op1 == nullptr) - return nullptr; - - Scev* op2 = Analyze(block, tree->gtGetOp2(), depth + 1); - if (op2 == nullptr) - return nullptr; - - ScevOper oper; - switch (tree->OperGet()) - { - case GT_ADD: - oper = ScevOper::Add; - break; - case GT_SUB: - oper = ScevOper::Add; - op2 = NewBinop(ScevOper::Mul, op2, NewConstant(op2->Type, -1)); - break; - case GT_MUL: - oper = ScevOper::Mul; - break; - case GT_LSH: - oper = ScevOper::Lsh; - break; - default: - unreached(); - } - - return NewBinop(oper, op1, op2); - } - case GT_COMMA: - { - return Analyze(block, tree->gtGetOp2(), depth + 1); - } - case GT_ARR_ADDR: - { - return Analyze(block, tree->AsArrAddr()->Addr(), depth + 1); - } - default: - return nullptr; - } -} - -//------------------------------------------------------------------------ -// CreateSimpleAddRec: Create a "simple" add-recurrence. This handles the most -// common patterns for primary induction variables where we see a store like -// "i = i + 1". -// -// Parameters: -// headerStore - Phi definition of the candidate primary induction variable -// enterScev - SCEV describing start value of the primary induction variable -// stepDefBlock - Block containing the def of the step value -// stepDefData - Value of the def of the step value -// -// Returns: -// SCEV node if this is a simple addrec shape. Otherwise nullptr. -// -Scev* ScalarEvolutionContext::CreateSimpleAddRec(GenTreeLclVarCommon* headerStore, - ScevLocal* enterScev, - BasicBlock* stepDefBlock, - GenTree* stepDefData) -{ - if (!stepDefData->OperIs(GT_ADD)) - { - return nullptr; - } - - GenTree* stepTree; - GenTree* op1 = stepDefData->gtGetOp1(); - GenTree* op2 = stepDefData->gtGetOp2(); - if (op1->OperIs(GT_LCL_VAR) && (op1->AsLclVar()->GetLclNum() == headerStore->GetLclNum()) && - (op1->AsLclVar()->GetSsaNum() == headerStore->GetSsaNum())) - { - stepTree = op2; - } - else if (op2->OperIs(GT_LCL_VAR) && (op2->AsLclVar()->GetLclNum() == headerStore->GetLclNum()) && - (op2->AsLclVar()->GetSsaNum() == headerStore->GetSsaNum())) - { - stepTree = op1; - } - else - { - // Not a simple IV shape (i.e. more complex than "i = i + k") - return nullptr; - } - - Scev* stepScev = CreateSimpleInvariantScev(stepTree); - if (stepScev == nullptr) - { - return nullptr; - } - - return NewAddRec(enterScev, stepScev); -} - -//------------------------------------------------------------------------ -// ExtractAddOperands: Extract all operands of potentially nested add -// operations. -// -// Parameters: -// binop - The binop representing an add -// operands - Array stack to add the operands to -// -void ScalarEvolutionContext::ExtractAddOperands(ScevBinop* binop, ArrayStack& operands) -{ - assert(binop->OperIs(ScevOper::Add)); - - if (binop->Op1->OperIs(ScevOper::Add)) - { - ExtractAddOperands(static_cast(binop->Op1), operands); - } - else - { - operands.Push(binop->Op1); - } - - if (binop->Op2->OperIs(ScevOper::Add)) - { - ExtractAddOperands(static_cast(binop->Op2), operands); - } - else - { - operands.Push(binop->Op2); - } -} - -//------------------------------------------------------------------------ -// MakeAddRecFromRecursiveScev: Given a recursive SCEV and a symbolic SCEV -// whose appearances represent an occurrence of the full SCEV, create a -// non-recursive add-rec from it. -// -// Parameters: -// startScev - The start value of the addrec -// scev - The scev -// recursiveScev - A symbolic node whose appearance represents the value of "scev" -// -// Returns: -// A non-recursive addrec, or nullptr if the recursive SCEV is not -// representable as an add recurrence. -// -Scev* ScalarEvolutionContext::MakeAddRecFromRecursiveScev(Scev* startScev, Scev* scev, Scev* recursiveScev) -{ - if (!scev->OperIs(ScevOper::Add)) - { - return nullptr; - } - - ArrayStack addOperands(m_comp->getAllocator(CMK_LoopIVOpts)); - ExtractAddOperands(static_cast(scev), addOperands); - - assert(addOperands.Height() >= 2); - - int numAppearances = 0; - for (int i = 0; i < addOperands.Height(); i++) - { - Scev* addOperand = addOperands.Bottom(i); - if (addOperand == recursiveScev) - { - numAppearances++; - } - else - { - ScevVisit result = addOperand->Visit([=](Scev* node) { - if (node == recursiveScev) - { - return ScevVisit::Abort; - } - - return ScevVisit::Continue; - }); - - if (result == ScevVisit::Abort) - { - // We do not handle nested occurrences. Some of these may be representable, some won't. - return nullptr; - } - } - } - - if (numAppearances == 0) - { - // TODO-CQ: We currently cannot handle cases like - // i = arr.Length; - // j = i - 1; - // i = j; - // while (true) { ...; j = i - 1; i = j; } - // - // These cases can arise from loop structures like "for (int i = - // arr.Length; --i >= 0;)" when Roslyn emits a "sub; dup; stloc" - // sequence, and local prop + loop inversion converts the duplicated - // local into a fully fledged IV. - // In this case we see that i = , but for - // j we will see + (-1) in this function - // as the value coming around the backedge, and we cannot reconcile - // this. - // - return nullptr; - } - - if (numAppearances > 1) - { - // Multiple occurrences -- cannot be represented as an addrec - // (corresponds to a geometric progression). - return nullptr; - } - - Scev* step = nullptr; - for (int i = 0; i < addOperands.Height(); i++) - { - Scev* addOperand = addOperands.Bottom(i); - if (addOperand == recursiveScev) - { - continue; - } - - if (step == nullptr) - { - step = addOperand; - } - else - { - step = NewBinop(ScevOper::Add, step, addOperand); - } - } - - return NewAddRec(startScev, step); -} - -//------------------------------------------------------------------------ -// Analyze: Analyze the specified tree in the specified block. -// -// Parameters: -// block - Block containing the tree -// tree - Tree node -// -// Returns: -// SCEV node if the tree was analyzable; otherwise nullptr if the value is -// cannot be described. -// -Scev* ScalarEvolutionContext::Analyze(BasicBlock* block, GenTree* tree) -{ - return Analyze(block, tree, 0); -} - -// Since the analysis follows SSA defs we have no upper bound on the potential -// depth of the analysis performed. We put an artificial limit on this for two -// reasons: -// 1. The analysis is recursive, and we should not stack overflow regardless of -// the input program. -// 2. If we produced arbitrarily deep SCEV trees then all algorithms over their -// structure would similarly be at risk of stack overflows if they were -// recursive. However, these algorithms are generally much more elegant when -// they make use of recursion. -const int SCALAR_EVOLUTION_ANALYSIS_MAX_DEPTH = 64; - -//------------------------------------------------------------------------ -// Analyze: Analyze the specified tree in the specified block. -// -// Parameters: -// block - Block containing the tree -// tree - Tree node -// depth - Current analysis depth -// -// Returns: -// SCEV node if the tree was analyzable; otherwise nullptr if the value is -// cannot be described. -// -Scev* ScalarEvolutionContext::Analyze(BasicBlock* block, GenTree* tree, int depth) -{ - Scev* result; - if (!m_cache.Lookup(tree, &result) && (!m_usingEphemeralCache || !m_ephemeralCache.Lookup(tree, &result))) - { - if (depth >= SCALAR_EVOLUTION_ANALYSIS_MAX_DEPTH) - { - return nullptr; - } - - result = AnalyzeNew(block, tree, depth); - - if (m_usingEphemeralCache) - { - m_ephemeralCache.Set(tree, result, ScalarEvolutionMap::Overwrite); - } - else - { - m_cache.Set(tree, result); - } - } - - return result; -} - -//------------------------------------------------------------------------ -// FoldBinop: Fold simple binops. -// -// Type parameters: -// T - Type that the binop is being evaluated in -// -// Parameters: -// oper - Binary operation -// op1 - First operand -// op2 - Second operand -// -// Returns: -// Folded value. -// -template -static T FoldBinop(ScevOper oper, T op1, T op2) -{ - switch (oper) - { - case ScevOper::Add: - return op1 + op2; - case ScevOper::Mul: - return op1 * op2; - case ScevOper::Lsh: - return op1 << op2; - default: - unreached(); - } -} - -//------------------------------------------------------------------------ -// Simplify: Try to simplify a SCEV node by folding and canonicalization. -// -// Parameters: -// scev - The node -// -// Returns: -// Simplified node. -// -// Remarks: -// Canonicalization is done for binops; constants are moved to the right and -// addrecs are moved to the left. -// -// Simple unops/binops on constants are folded. Operands are distributed into -// add recs whenever possible. -// -Scev* ScalarEvolutionContext::Simplify(Scev* scev) -{ - switch (scev->Oper) - { - case ScevOper::Constant: - case ScevOper::Local: - { - return scev; - } - case ScevOper::ZeroExtend: - case ScevOper::SignExtend: - { - ScevUnop* unop = (ScevUnop*)scev; - assert(genTypeSize(unop->Type) >= genTypeSize(unop->Op1->Type)); - - Scev* op1 = Simplify(unop->Op1); - - if (unop->Type == op1->Type) - { - return op1; - } - - assert((unop->Type == TYP_LONG) && (op1->Type == TYP_INT)); - - if (op1->OperIs(ScevOper::Constant)) - { - ScevConstant* cns = (ScevConstant*)op1; - return NewConstant(unop->Type, unop->OperIs(ScevOper::ZeroExtend) ? (uint64_t)(int32_t)cns->Value - : (int64_t)(int32_t)cns->Value); - } - - if (op1->OperIs(ScevOper::AddRec)) - { - // TODO-Cleanup: This requires some proof that it is ok, but - // currently we do not rely on this. - return op1; - } - - return (op1 == unop->Op1) ? unop : NewExtension(unop->Oper, unop->Type, op1); - } - case ScevOper::Add: - case ScevOper::Mul: - case ScevOper::Lsh: - { - ScevBinop* binop = (ScevBinop*)scev; - Scev* op1 = Simplify(binop->Op1); - Scev* op2 = Simplify(binop->Op2); - - if (binop->OperIs(ScevOper::Add, ScevOper::Mul)) - { - // Normalize addrecs to the left - if (op2->OperIs(ScevOper::AddRec) && !op1->OperIs(ScevOper::AddRec)) - { - std::swap(op1, op2); - } - // Normalize constants to the right - if (op1->OperIs(ScevOper::Constant) && !op2->OperIs(ScevOper::Constant)) - { - std::swap(op1, op2); - } - } - - if (op1->OperIs(ScevOper::AddRec)) - { - // + x => - // * x => - ScevAddRec* addRec = (ScevAddRec*)op1; - Scev* newStart = Simplify(NewBinop(binop->Oper, addRec->Start, op2)); - Scev* newStep = scev->OperIs(ScevOper::Mul, ScevOper::Lsh) - ? Simplify(NewBinop(binop->Oper, addRec->Step, op2)) - : addRec->Step; - return NewAddRec(newStart, newStep); - } - - if (op1->OperIs(ScevOper::Constant) && op2->OperIs(ScevOper::Constant)) - { - ScevConstant* cns1 = (ScevConstant*)op1; - ScevConstant* cns2 = (ScevConstant*)op2; - int64_t newValue; - if (binop->TypeIs(TYP_INT)) - { - newValue = FoldBinop(binop->Oper, static_cast(cns1->Value), - static_cast(cns2->Value)); - } - else - { - assert(binop->TypeIs(TYP_LONG)); - newValue = FoldBinop(binop->Oper, cns1->Value, cns2->Value); - } - - return NewConstant(binop->Type, newValue); - } - - return (op1 == binop->Op1) && (op2 == binop->Op2) ? binop : NewBinop(binop->Oper, op1, op2); - } - case ScevOper::AddRec: - { - ScevAddRec* addRec = (ScevAddRec*)scev; - Scev* start = Simplify(addRec->Start); - Scev* step = Simplify(addRec->Step); - return (start == addRec->Start) && (step == addRec->Step) ? addRec : NewAddRec(start, step); - } - default: - unreached(); - } -} diff --git a/src/coreclr/jit/scev.h b/src/coreclr/jit/scev.h deleted file mode 100644 index 0800be905503a9..00000000000000 --- a/src/coreclr/jit/scev.h +++ /dev/null @@ -1,222 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#pragma once - -// This file contains the definition of the scalar evolution IR. This IR allows -// representing the values of IR nodes inside loops in a closed form, taking -// into account that they are changing on each loop iteration. The IR is based -// around the following possible operations. At the core is ScevOper::AddRec, -// which represents a value that evolves by an add recurrence. In dumps it is -// described by where "loop" is the loop the value is -// evolving in, "start" is the initial value and "step" is the step by which -// the value evolves in every iteration. -// -// See scev.cpp for further documentation. -// -enum class ScevOper -{ - Constant, - Local, - ZeroExtend, - SignExtend, - Add, - Mul, - Lsh, - AddRec, -}; - -static bool ScevOperIs(ScevOper oper, ScevOper otherOper) -{ - return oper == otherOper; -} - -template -static bool ScevOperIs(ScevOper oper, ScevOper operFirst, Args... operTail) -{ - return oper == operFirst || ScevOperIs(oper, operTail...); -} - -enum class ScevVisit -{ - Abort, - Continue, -}; - -struct Scev -{ - const ScevOper Oper; - const var_types Type; - - Scev(ScevOper oper, var_types type) : Oper(oper), Type(type) - { - } - - template - bool OperIs(Args... opers) - { - return ScevOperIs(Oper, opers...); - } - - bool TypeIs(var_types type) - { - return Type == type; - } - - bool GetConstantValue(Compiler* comp, int64_t* cns); - -#ifdef DEBUG - void Dump(Compiler* comp); -#endif - template - ScevVisit Visit(TVisitor visitor); -}; - -struct ScevConstant : Scev -{ - ScevConstant(var_types type, int64_t value) : Scev(ScevOper::Constant, type), Value(value) - { - } - - int64_t Value; -}; - -struct ScevLocal : Scev -{ - ScevLocal(var_types type, unsigned lclNum, unsigned ssaNum) - : Scev(ScevOper::Local, type), LclNum(lclNum), SsaNum(ssaNum) - { - } - - const unsigned LclNum; - const unsigned SsaNum; - - bool GetConstantValue(Compiler* comp, int64_t* cns); -}; - -struct ScevUnop : Scev -{ - ScevUnop(ScevOper oper, var_types type, Scev* op1) : Scev(oper, type), Op1(op1) - { - } - - Scev* const Op1; -}; - -struct ScevBinop : ScevUnop -{ - ScevBinop(ScevOper oper, var_types type, Scev* op1, Scev* op2) : ScevUnop(oper, type, op1), Op2(op2) - { - } - - Scev* const Op2; -}; - -// Represents a value that evolves by an add recurrence. -// The value at iteration N is Start + N * Step. -// "Start" and "Step" are guaranteed to be invariant in "Loop". -struct ScevAddRec : Scev -{ - ScevAddRec(var_types type, Scev* start, Scev* step DEBUGARG(FlowGraphNaturalLoop* loop)) - : Scev(ScevOper::AddRec, type), Start(start), Step(step) DEBUGARG(Loop(loop)) - { - } - - Scev* const Start; - Scev* const Step; - INDEBUG(FlowGraphNaturalLoop* const Loop); -}; - -//------------------------------------------------------------------------ -// Scev::Visit: Recursively visit all SCEV nodes in the SCEV tree. -// -// Parameters: -// visitor - Callback with signature Scev* -> ScevVisit. -// -// Returns: -// ScevVisit::Abort if "visitor" aborted, otherwise ScevVisit::Continue. -// -// Remarks: -// The visit is done in preorder. -// -template -ScevVisit Scev::Visit(TVisitor visitor) -{ - if (visitor(this) == ScevVisit::Abort) - return ScevVisit::Abort; - - switch (Oper) - { - case ScevOper::Constant: - case ScevOper::Local: - break; - case ScevOper::ZeroExtend: - case ScevOper::SignExtend: - return static_cast(this)->Op1->Visit(visitor); - case ScevOper::Add: - case ScevOper::Mul: - case ScevOper::Lsh: - { - ScevBinop* binop = static_cast(this); - if (binop->Op1->Visit(visitor) == ScevVisit::Abort) - return ScevVisit::Abort; - - return binop->Op2->Visit(visitor); - } - case ScevOper::AddRec: - { - ScevAddRec* addrec = static_cast(this); - if (addrec->Start->Visit(visitor) == ScevVisit::Abort) - return ScevVisit::Abort; - - return addrec->Step->Visit(visitor); - } - default: - unreached(); - } - - return ScevVisit::Continue; -} - -typedef JitHashTable, Scev*> ScalarEvolutionMap; - -// Scalar evolution is analyzed in the context of a single loop, and are -// computed on-demand by the use of the "Analyze" method on this class, which -// also maintains a cache. -class ScalarEvolutionContext -{ - Compiler* m_comp; - FlowGraphNaturalLoop* m_loop = nullptr; - ScalarEvolutionMap m_cache; - - // During analysis of PHIs we insert a symbolic node representing the - // "recurrence"; we use this cache to be able to invalidate things that end - // up depending on the symbolic node quickly. - ScalarEvolutionMap m_ephemeralCache; - bool m_usingEphemeralCache = false; - - Scev* Analyze(BasicBlock* block, GenTree* tree, int depth); - Scev* AnalyzeNew(BasicBlock* block, GenTree* tree, int depth); - Scev* CreateSimpleAddRec(GenTreeLclVarCommon* headerStore, - ScevLocal* start, - BasicBlock* stepDefBlock, - GenTree* stepDefData); - Scev* MakeAddRecFromRecursiveScev(Scev* start, Scev* scev, Scev* recursiveScev); - Scev* CreateSimpleInvariantScev(GenTree* tree); - Scev* CreateScevForConstant(GenTreeIntConCommon* tree); - void ExtractAddOperands(ScevBinop* add, ArrayStack& operands); - -public: - ScalarEvolutionContext(Compiler* comp); - - void ResetForLoop(FlowGraphNaturalLoop* loop); - - ScevConstant* NewConstant(var_types type, int64_t value); - ScevLocal* NewLocal(unsigned lclNum, unsigned ssaNum); - ScevUnop* NewExtension(ScevOper oper, var_types targetType, Scev* op); - ScevBinop* NewBinop(ScevOper oper, Scev* op1, Scev* op2); - ScevAddRec* NewAddRec(Scev* start, Scev* step); - - Scev* Analyze(BasicBlock* block, GenTree* tree); - Scev* Simplify(Scev* scev); -}; diff --git a/src/coreclr/jit/sideeffects.cpp b/src/coreclr/jit/sideeffects.cpp index a2dd47c994ef49..d2c1de6c749a5b 100644 --- a/src/coreclr/jit/sideeffects.cpp +++ b/src/coreclr/jit/sideeffects.cpp @@ -174,7 +174,7 @@ AliasSet::NodeInfo::NodeInfo(Compiler* compiler, GenTree* node) // Is the operation a write? If so, set `node` to the location that is being written to. bool isWrite = false; - if (node->OperIsStore() || node->OperIs(GT_MEMORYBARRIER)) + if (node->OperIsStore() || node->OperIs(GT_STORE_DYN_BLK, GT_MEMORYBARRIER)) { isWrite = true; } diff --git a/src/coreclr/jit/simd.cpp b/src/coreclr/jit/simd.cpp index 48c23eb646411f..d69730ad520ed3 100644 --- a/src/coreclr/jit/simd.cpp +++ b/src/coreclr/jit/simd.cpp @@ -549,6 +549,8 @@ bool areFieldAddressesTheSame(GenTreeFieldAddr* op1, GenTreeFieldAddr* op2) bool Compiler::areFieldsContiguous(GenTreeIndir* op1, GenTreeIndir* op2) { assert(op1->isIndir() && op2->isIndir()); + // TODO-1stClassStructs: delete once IND nodes are no more. + assert(!op1->TypeIs(TYP_STRUCT) && !op2->TypeIs(TYP_STRUCT)); var_types op1Type = op1->TypeGet(); var_types op2Type = op2->TypeGet(); diff --git a/src/coreclr/jit/simd.h b/src/coreclr/jit/simd.h index aec72eaab542e8..9e8781714dbc41 100644 --- a/src/coreclr/jit/simd.h +++ b/src/coreclr/jit/simd.h @@ -277,55 +277,6 @@ struct simd64_t }; static_assert_no_msg(sizeof(simd64_t) == 64); -struct simdmask_t -{ - union { - int8_t i8[8]; - int16_t i16[4]; - int32_t i32[2]; - int64_t i64[1]; - uint8_t u8[8]; - uint16_t u16[4]; - uint32_t u32[2]; - uint64_t u64[1]; - }; - - bool operator==(const simdmask_t& other) const - { - return (u64[0] == other.u64[0]); - } - - bool operator!=(const simdmask_t& other) const - { - return !(*this == other); - } - - static simdmask_t AllBitsSet() - { - simdmask_t result; - - result.u64[0] = 0xFFFFFFFFFFFFFFFF; - - return result; - } - - bool IsAllBitsSet() const - { - return *this == AllBitsSet(); - } - - bool IsZero() const - { - return *this == Zero(); - } - - static simdmask_t Zero() - { - return {}; - } -}; -static_assert_no_msg(sizeof(simdmask_t) == 8); - typedef simd64_t simd_t; #else typedef simd16_t simd_t; diff --git a/src/coreclr/jit/ssabuilder.cpp b/src/coreclr/jit/ssabuilder.cpp index fb8b33aa313294..2072591bcd5a9a 100644 --- a/src/coreclr/jit/ssabuilder.cpp +++ b/src/coreclr/jit/ssabuilder.cpp @@ -1181,7 +1181,6 @@ void SsaBuilder::RenameVariables() { JITDUMP("*************** In SsaBuilder::RenameVariables()\n"); - m_pCompiler->Metrics.VarsInSsa = 0; // The first thing we do is treat parameters and must-init variables as if they have a // virtual definition before entry -- they start out at SSA name 1. for (unsigned lclNum = 0; lclNum < m_pCompiler->lvaCount; lclNum++) @@ -1191,8 +1190,6 @@ void SsaBuilder::RenameVariables() continue; } - m_pCompiler->Metrics.VarsInSsa++; - LclVarDsc* varDsc = m_pCompiler->lvaGetDesc(lclNum); assert(varDsc->lvTracked); diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 3f975416f93a99..fa6abd0f23e8bb 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -51,8 +51,8 @@ PhaseStatus Compiler::optSwitchRecognition() // // Arguments: // block - The block to check -// trueEdge - [out] The successor edge taken if X == CNS -// falseEdge - [out] The successor edge taken if X != CNS +// blockIfTrue - [out] The block that will be jumped to if X == CNS +// blockIfFalse - [out] The block that will be jumped to if X != CNS // isReversed - [out] True if the condition is reversed (GT_NE) // variableNode - [out] The variable node (X in the example above) // cns - [out] The constant value (CNS in the example above) @@ -61,8 +61,8 @@ PhaseStatus Compiler::optSwitchRecognition() // True if the block represents a constant test, false otherwise // bool IsConstantTestCondBlock(const BasicBlock* block, - BasicBlock** trueTarget, - BasicBlock** falseTarget, + BasicBlock** blockIfTrue, + BasicBlock** blockIfFalse, bool* isReversed, GenTree** variableNode = nullptr, ssize_t* cns = nullptr) @@ -94,9 +94,9 @@ bool IsConstantTestCondBlock(const BasicBlock* block, return false; } - *isReversed = rootNode->gtGetOp1()->OperIs(GT_NE); - *trueTarget = *isReversed ? block->GetFalseTarget() : block->GetTrueTarget(); - *falseTarget = *isReversed ? block->GetTrueTarget() : block->GetFalseTarget(); + *isReversed = rootNode->gtGetOp1()->OperIs(GT_NE); + *blockIfTrue = *isReversed ? block->GetFalseTarget() : block->GetTrueTarget(); + *blockIfFalse = *isReversed ? block->GetTrueTarget() : block->GetFalseTarget(); if (block->FalseTargetIs(block) || block->TrueTargetIs(block)) { @@ -141,14 +141,14 @@ bool Compiler::optSwitchDetectAndConvert(BasicBlock* firstBlock) GenTree* variableNode = nullptr; ssize_t cns = 0; - BasicBlock* trueTarget = nullptr; - BasicBlock* falseTarget = nullptr; + BasicBlock* blockIfTrue = nullptr; + BasicBlock* blockIfFalse = nullptr; // The algorithm is simple - we check that the given block is a constant test block // and then try to accumulate as many constant test blocks as possible. Once we hit // a block that doesn't match the pattern, we start processing the accumulated blocks. bool isReversed = false; - if (IsConstantTestCondBlock(firstBlock, &trueTarget, &falseTarget, &isReversed, &variableNode, &cns)) + if (IsConstantTestCondBlock(firstBlock, &blockIfTrue, &blockIfFalse, &isReversed, &variableNode, &cns)) { if (isReversed) { @@ -161,71 +161,65 @@ bool Compiler::optSwitchDetectAndConvert(BasicBlock* firstBlock) // No more than SWITCH_MAX_TABLE_SIZE blocks are allowed (arbitrary limit in this context) int testValueIndex = 0; ssize_t testValues[SWITCH_MAX_DISTANCE] = {}; - testValues[testValueIndex] = cns; - testValueIndex++; + testValues[testValueIndex++] = cns; - // Track likelihood of reaching the false block - // - weight_t falseLikelihood = firstBlock->GetFalseEdge()->getLikelihood(); - const BasicBlock* prevBlock = firstBlock; + const BasicBlock* prevBlock = firstBlock; // Now walk the next blocks and see if they are basically the same type of test for (const BasicBlock* currBb = firstBlock->Next(); currBb != nullptr; currBb = currBb->Next()) { GenTree* currVariableNode = nullptr; ssize_t currCns = 0; - BasicBlock* currTrueTarget = nullptr; - BasicBlock* currFalseTarget = nullptr; + BasicBlock* currBlockIfTrue = nullptr; + BasicBlock* currBlockIfFalse = nullptr; if (!currBb->hasSingleStmt()) { // Only the first conditional block can have multiple statements. // Stop searching and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } // Inspect secondary blocks - if (IsConstantTestCondBlock(currBb, &currTrueTarget, &currFalseTarget, &isReversed, &currVariableNode, + if (IsConstantTestCondBlock(currBb, &currBlockIfTrue, &currBlockIfFalse, &isReversed, &currVariableNode, &currCns)) { - if (currTrueTarget != trueTarget) + if (currBlockIfTrue != blockIfTrue) { // This blocks jumps to a different target, stop searching and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } if (!GenTree::Compare(currVariableNode, variableNode)) { // A different variable node is used, stop searching and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } if (currBb->GetUniquePred(this) != prevBlock) { // Multiple preds in a secondary block, stop searching and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } if (!BasicBlock::sameEHRegion(prevBlock, currBb)) { // Current block is in a different EH region, stop searching and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } // Ok we can work with that, add the test value to the list testValues[testValueIndex++] = currCns; - falseLikelihood *= currBb->GetFalseEdge()->getLikelihood(); - if (testValueIndex == SWITCH_MAX_DISTANCE) { // Too many suitable tests found - stop and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } if (isReversed) { // We only support reversed test (GT_NE) for the last block. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } prevBlock = currBb; @@ -233,7 +227,7 @@ bool Compiler::optSwitchDetectAndConvert(BasicBlock* firstBlock) else { // Current block is not a suitable test, stop searching and process what we already have. - return optSwitchConvert(firstBlock, testValueIndex, testValues, falseLikelihood, variableNode); + return optSwitchConvert(firstBlock, testValueIndex, testValues, variableNode); } } } @@ -251,14 +245,12 @@ bool Compiler::optSwitchDetectAndConvert(BasicBlock* firstBlock) // firstBlock - First conditional block in the chain // testsCount - Number of conditional blocks in the chain // testValues - Array of constants that are tested against the variable -// falseLikelihood - Likelihood of control flow reaching the false block // nodeToTest - Variable node that is tested against the constants // // Return Value: // True if the conversion was successful, false otherwise // -bool Compiler::optSwitchConvert( - BasicBlock* firstBlock, int testsCount, ssize_t* testValues, weight_t falseLikelihood, GenTree* nodeToTest) +bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* testValues, GenTree* nodeToTest) { assert(firstBlock->KindIs(BBJ_COND)); assert(!varTypeIsSmall(nodeToTest)); @@ -327,10 +319,6 @@ bool Compiler::optSwitchConvert( const bool isTest = IsConstantTestCondBlock(lastBlock, &blockIfTrue, &blockIfFalse, &isReversed); assert(isTest); - assert(firstBlock->TrueTargetIs(blockIfTrue)); - FlowEdge* const trueEdge = firstBlock->GetTrueEdge(); - FlowEdge* const falseEdge = firstBlock->GetFalseEdge(); - // Convert firstBlock to a switch block firstBlock->SetSwitch(new (this, CMK_BasicBlock) BBswtDesc); firstBlock->bbCodeOffsEnd = lastBlock->bbCodeOffsEnd; @@ -350,17 +338,16 @@ bool Compiler::optSwitchConvert( gtUpdateStmtSideEffects(firstBlock->lastStmt()); // Unlink and remove the whole chain of conditional blocks - fgRemoveRefPred(falseEdge); - BasicBlock* blockToRemove = falseEdge->getDestinationBlock(); - assert(firstBlock->NextIs(blockToRemove)); + BasicBlock* blockToRemove = firstBlock->Next(); + fgRemoveRefPred(blockToRemove, firstBlock); while (!lastBlock->NextIs(blockToRemove)) { blockToRemove = fgRemoveBlock(blockToRemove, true); } - const unsigned jumpCount = static_cast(maxValue - minValue + 1); + const auto jumpCount = static_cast(maxValue - minValue + 1); assert((jumpCount > 0) && (jumpCount <= SWITCH_MAX_DISTANCE + 1)); - FlowEdge** jmpTab = new (this, CMK_FlowEdge) FlowEdge*[jumpCount + 1 /*default case*/]; + const auto jmpTab = new (this, CMK_BasicBlock) BasicBlock*[jumpCount + 1 /*default case*/]; // Quirk: lastBlock's false target may have diverged from bbNext. If the false target is behind firstBlock, // we may create a cycle in the BasicBlock list by setting firstBlock->bbNext to it. @@ -374,20 +361,16 @@ bool Compiler::optSwitchConvert( if (isReversed) { assert(lastBlock->FalseTargetIs(blockIfTrue)); - fgRemoveRefPred(trueEdge); - BasicBlock* targetBlock = blockIfTrue; - blockIfTrue = fgNewBBafter(BBJ_ALWAYS, firstBlock, true); - FlowEdge* const newEdge = fgAddRefPred(targetBlock, blockIfTrue); - skipPredRemoval = true; - blockIfTrue->SetTargetEdge(newEdge); + fgRemoveRefPred(blockIfTrue, firstBlock); + blockIfTrue = fgNewBBafter(BBJ_ALWAYS, firstBlock, true, blockIfTrue); + fgAddRefPred(blockIfTrue->GetTarget(), blockIfTrue); + skipPredRemoval = true; } else { assert(lastBlock->FalseTargetIs(blockIfFalse)); - BasicBlock* targetBlock = blockIfFalse; - blockIfFalse = fgNewBBafter(BBJ_ALWAYS, firstBlock, true); - FlowEdge* const newEdge = fgAddRefPred(targetBlock, blockIfFalse); - blockIfFalse->SetTargetEdge(newEdge); + blockIfFalse = fgNewBBafter(BBJ_ALWAYS, firstBlock, true, blockIfFalse); + fgAddRefPred(blockIfFalse->GetTarget(), blockIfFalse); } } @@ -412,32 +395,21 @@ bool Compiler::optSwitchConvert( // Unlink blockIfTrue from firstBlock, we're going to link it again in the loop below. if (!skipPredRemoval) { - fgRemoveRefPred(trueEdge); + fgRemoveRefPred(blockIfTrue, firstBlock); } - FlowEdge* switchTrueEdge = nullptr; - for (unsigned i = 0; i < jumpCount; i++) { // value exists in the testValues array (via bitVector) - 'true' case. const bool isTrue = (bitVector & static_cast(1ULL << i)) != 0; + jmpTab[i] = isTrue ? blockIfTrue : blockIfFalse; - FlowEdge* const newEdge = fgAddRefPred((isTrue ? blockIfTrue : blockIfFalse), firstBlock); - jmpTab[i] = newEdge; - - if ((switchTrueEdge == nullptr) && isTrue) - { - switchTrueEdge = newEdge; - } + fgAddRefPred(jmpTab[i], firstBlock); } // Link the 'default' case - FlowEdge* const switchDefaultEdge = fgAddRefPred(blockIfFalse, firstBlock); - jmpTab[jumpCount] = switchDefaultEdge; - - // Fix likelihoods - switchDefaultEdge->setLikelihood(falseLikelihood); - switchTrueEdge->setLikelihood(1.0 - falseLikelihood); + jmpTab[jumpCount] = blockIfFalse; + fgAddRefPred(blockIfFalse, firstBlock); return true; } diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 4a7033732e7a13..4abe71984b57cf 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -41,9 +41,9 @@ #define MAX_PASS_MULTIREG_BYTES 32 // Maximum size of a struct that could be passed in more than one register (Max is two SIMD16s) #define MAX_RET_MULTIREG_BYTES 32 // Maximum size of a struct that could be returned in more than one register (Max is two SIMD16s) #define MAX_ARG_REG_COUNT 2 // Maximum registers used to pass a single argument in multiple registers. - #define MAX_RET_REG_COUNT 4 // Maximum registers used to return a value. + #define MAX_RET_REG_COUNT 2 // Maximum registers used to return a value. - #define MAX_MULTIREG_COUNT 4 // Maximum number of registers defined by a single instruction (including calls). + #define MAX_MULTIREG_COUNT 2 // Maximum number of registers defined by a single instruction (including calls). // This is also the maximum number of registers for a MultiReg node. #else // !UNIX_AMD64_ABI #define WINDOWS_AMD64_ABI // Uses the Windows ABI for AMD64 @@ -206,11 +206,11 @@ // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_REF and CORINFO_HELP_CHECKED_ASSIGN_REF. #define RBM_CALLEE_GCTRASH_WRITEBARRIER RBM_CALLEE_TRASH_NOGC - // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_RAX | RBM_RCX) - // Registers killed by CORINFO_HELP_ASSIGN_BYREF. - #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_RSI | RBM_RDI | RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF) + #define RBM_CALLEE_TRASH_WRITEBARRIER_BYREF (RBM_RSI | RBM_RDI | RBM_CALLEE_TRASH_NOGC) + + // Registers no longer containing GC pointers after CORINFO_HELP_ASSIGN_BYREF. + #define RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF (RBM_CALLEE_TRASH_NOGC & ~(RBM_RDI | RBM_RSI)) // We have two register classifications // * callee trash: aka volatile or caller saved @@ -563,15 +563,4 @@ #define RBM_STACK_PROBE_HELPER_TRASH RBM_RAX #endif // !UNIX_AMD64_ABI -#ifdef UNIX_AMD64_ABI - #define SWIFT_SUPPORT - #define REG_SWIFT_ERROR REG_R12 - #define RBM_SWIFT_ERROR RBM_R12 - #define REG_SWIFT_SELF REG_R13 - - #define REG_SWIFT_INTRET_ORDER REG_RAX,REG_RDX,REG_RCX,REG_R8 - #define REG_SWIFT_FLOATRET_ORDER REG_XMM0,REG_XMM1,REG_XMM2,REG_XMM3 - #define REG_SWIFT_ARG_RET_BUFF REG_RAX -#endif - // clang-format on diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 4a99ca2a79f989..3646ecb4407bf7 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -140,14 +140,6 @@ #define REG_JUMP_THUNK_PARAM REG_R12 #define RBM_JUMP_THUNK_PARAM RBM_R12 - #define RBM_LOWMASK (RBM_P0 | RBM_P1 | RBM_P2 | RBM_P3 | RBM_P4 | RBM_P5 | RBM_P6 | RBM_P7) - #define RBM_HIGHMASK (RBM_P8 | RBM_P9 | RBM_P10 | RBM_P11 | RBM_P12 | RBM_P13 | RBM_P14 | RBM_P15) - #define RBM_ALLMASK (RBM_LOWMASK | RBM_HIGHMASK) - - // TODO-SVE: Fix when adding predicate register allocation - #define RBM_MSK_CALLEE_SAVED (0) - #define RBM_MSK_CALLEE_TRASH (0) - // ARM64 write barrier ABI (see vm\arm64\asmhelpers.asm, vm\arm64\asmhelpers.S): // CORINFO_HELP_ASSIGN_REF (JIT_WriteBarrier), CORINFO_HELP_CHECKED_ASSIGN_REF (JIT_CheckedWriteBarrier): // On entry: @@ -378,11 +370,4 @@ #define REG_ZERO_INIT_FRAME_REG2 REG_R10 #define REG_ZERO_INIT_FRAME_SIMD REG_V16 - #define SWIFT_SUPPORT - #define REG_SWIFT_ERROR REG_R21 - #define RBM_SWIFT_ERROR RBM_R21 - #define REG_SWIFT_SELF REG_R20 - #define REG_SWIFT_INTRET_ORDER REG_R0,REG_R1,REG_R2,REG_R3 - #define REG_SWIFT_FLOATRET_ORDER REG_V0,REG_V1,REG_V2,REG_V3 - // clang-format on diff --git a/src/coreclr/jit/targetriscv64.h b/src/coreclr/jit/targetriscv64.h index 5ac82fa9a00974..9cf0185a569351 100644 --- a/src/coreclr/jit/targetriscv64.h +++ b/src/coreclr/jit/targetriscv64.h @@ -12,6 +12,8 @@ #define ROUND_FLOAT 0 // Do not round intermed float expression results #define CPU_HAS_BYTE_REGS 0 + #define CPBLK_UNROLL_LIMIT 64 // Upper bound to let the code generator to loop unroll CpBlk + #define INITBLK_UNROLL_LIMIT 64 // Upper bound to let the code generator to loop unroll InitBlk #ifdef FEATURE_SIMD #pragma error("SIMD Unimplemented yet RISCV64") diff --git a/src/coreclr/jit/typelist.h b/src/coreclr/jit/typelist.h index 1a9a8c4072f6bf..8b8da6db011f5b 100644 --- a/src/coreclr/jit/typelist.h +++ b/src/coreclr/jit/typelist.h @@ -63,10 +63,8 @@ DEF_TP(SIMD16 ,"simd16" , TYP_SIMD16, 16,16, 16, 4,16, VTR_FLOAT, available #if defined(TARGET_XARCH) DEF_TP(SIMD32 ,"simd32" , TYP_SIMD32, 32,32, 32, 8,16, VTR_FLOAT, availableDoubleRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_S|VTF_VEC) DEF_TP(SIMD64 ,"simd64" , TYP_SIMD64, 64,64, 64, 16,16, VTR_FLOAT, availableDoubleRegs, RBM_FLT_CALLEE_SAVED, RBM_FLT_CALLEE_TRASH, VTF_S|VTF_VEC) -#endif // TARGET_XARCH -#if defined(TARGET_XARCH) || defined(TARGET_ARM64) DEF_TP(MASK ,"mask" , TYP_MASK, 8, 8, 8, 2, 8, VTR_MASK, availableMaskRegs, RBM_MSK_CALLEE_SAVED, RBM_MSK_CALLEE_TRASH, VTF_S) -#endif // TARGET_XARCH || TARGET_ARM64 +#endif // TARGET_XARCH #endif // FEATURE_SIMD DEF_TP(UNKNOWN ,"unknown" ,TYP_UNKNOWN, 0, 0, 0, 0, 0, VTR_INT, availableIntRegs, RBM_INT_CALLEE_SAVED, RBM_INT_CALLEE_TRASH, VTF_ANY) diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 4922e1f3da0a6c..5b2af35004cb6e 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -437,7 +437,6 @@ ValueNumStore::ValueNumStore(Compiler* comp, CompAllocator alloc) #if defined(TARGET_XARCH) , m_simd32CnsMap(nullptr) , m_simd64CnsMap(nullptr) - , m_simdMaskCnsMap(nullptr) #endif // TARGET_XARCH #endif // FEATURE_SIMD , m_VNFunc0Map(nullptr) @@ -1707,12 +1706,6 @@ ValueNumStore::Chunk::Chunk(CompAllocator alloc, ValueNum* pNextBaseVN, var_type m_defs = new (alloc) Alloc::Type[ChunkSize]; break; } - - case TYP_MASK: - { - m_defs = new (alloc) Alloc::Type[ChunkSize]; - break; - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -1877,11 +1870,6 @@ ValueNum ValueNumStore::VNForSimd64Con(simd64_t cnsVal) { return VnForConst(cnsVal, GetSimd64CnsMap(), TYP_SIMD64); } - -ValueNum ValueNumStore::VNForSimdMaskCon(simdmask_t cnsVal) -{ - return VnForConst(cnsVal, GetSimdMaskCnsMap(), TYP_MASK); -} #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -1983,11 +1971,6 @@ ValueNum ValueNumStore::VNForGenericCon(var_types typ, uint8_t* cnsVal) READ_VALUE(simd64_t); return VNForSimd64Con(val); } - case TYP_MASK: - { - READ_VALUE(simdmask_t); - return VNForSimdMaskCon(val); - } #endif // TARGET_XARCH #endif // FEATURE_SIMD default: @@ -2102,11 +2085,6 @@ ValueNum ValueNumStore::VNZeroForType(var_types typ) { return VNForSimd64Con(simd64_t::Zero()); } - - case TYP_MASK: - { - return VNForSimdMaskCon(simdmask_t::Zero()); - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -2197,11 +2175,6 @@ ValueNum ValueNumStore::VNAllBitsForType(var_types typ) { return VNForSimd64Con(simd64_t::AllBitsSet()); } - - case TYP_MASK: - { - return VNForSimdMaskCon(simdmask_t::AllBitsSet()); - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -2323,13 +2296,6 @@ ValueNum ValueNumStore::VNOneForSimdType(var_types simdType, var_types simdBaseT memcpy(&simd64Val, &simdVal, sizeof(simd64_t)); return VNForSimd64Con(simd64Val); } - - case TYP_MASK: - { - // '1' doesn't make sense for TYP_MASK? - // Or should it be AllBitsSet? - unreached(); - } #endif // TARGET_XARCH default: @@ -2498,87 +2464,6 @@ ValueNum ValueNumStore::VNForFunc(var_types typ, VNFunc func, ValueNum arg0VN) return resultVN; } -//---------------------------------------------------------------------------------------- -// VNForCast: Returns VN associated with castclass/isinst -// -// Arguments: -// func - Either VNF_CastClass or VNF_IsInstanceOf -// castToVN - VN of the "Cast to" argument -// objVN - VN of the "Cast object" argument -// -// Return Value: -// ValueNum associated with castclass/isinst -// -ValueNum ValueNumStore::VNForCast(VNFunc func, ValueNum castToVN, ValueNum objVN) -{ - assert((func == VNF_CastClass) || (func == VNF_IsInstanceOf)); - - if (objVN == VNForNull()) - { - // CastClass(cls, null) -> null - // IsInstanceOf(cls, null) -> null - // - return VNForNull(); - } - - // - // Fold "CAST(IsInstanceOf(obj, cls), cls)" to "IsInstanceOf(obj, cls)" - // where CAST is either ISINST or CASTCLASS. - // - VNFuncApp funcApp; - if (GetVNFunc(objVN, &funcApp) && (funcApp.m_func == VNF_IsInstanceOf) && (funcApp.m_args[0] == castToVN)) - { - // The outer cast is redundant, remove it and preserve its side effects - // We do ignoreRoot here because the actual cast node never throws any exceptions. - return objVN; - } - - // Check if we can fold the cast based on the runtime types of the arguments. - // - if (IsVNTypeHandle(castToVN)) - { - bool isExact; - bool isNonNull; - CORINFO_CLASS_HANDLE castFrom = GetObjectType(objVN, &isExact, &isNonNull); - CORINFO_CLASS_HANDLE castTo; - if ((castFrom != NO_CLASS_HANDLE) && - EmbeddedHandleMapLookup(ConstantValue(castToVN), (ssize_t*)&castTo)) - { - TypeCompareState castResult = m_pComp->info.compCompHnd->compareTypesForCast(castFrom, castTo); - if (castResult == TypeCompareState::Must) - { - // IsInstanceOf/CastClass is guaranteed to succeed (we don't need to check for isExact here) - return objVN; - } - - if ((castResult == TypeCompareState::MustNot) && isExact && (func == VNF_IsInstanceOf)) - { - // IsInstanceOf is guaranteed to fail -> return null (we need to check for isExact here) - return VNForNull(); - } - } - } - - if (func == VNF_CastClass) - { - // CastClass(cls, obj) -> obj (may throw InvalidCastException) - // - ValueNum vnExcSet = VNExcSetSingleton(VNForFuncNoFolding(TYP_REF, VNF_InvalidCastExc, objVN, castToVN)); - return VNWithExc(objVN, vnExcSet); - } - - // IsInstanceOf(cls, obj) -> either obj or null - we don't know - // - assert(func == VNF_IsInstanceOf); - Chunk* const c = GetAllocChunk(TYP_REF, CEA_Func2); - unsigned const offsetWithinChunk = c->AllocVN(); - VNDefFuncAppFlexible* fapp = c->PointerToFuncApp(offsetWithinChunk, 2); - fapp->m_func = VNF_IsInstanceOf; - fapp->m_args[0] = castToVN; - fapp->m_args[1] = objVN; - return c->m_baseVN + offsetWithinChunk; -} - //---------------------------------------------------------------------------------------- // VNForFunc - Returns the ValueNum associated with 'func'('arg0VN','arg1VN') // There is a one-to-one relationship between the ValueNum @@ -2642,9 +2527,24 @@ ValueNum ValueNumStore::VNForFunc(var_types typ, VNFunc func, ValueNum arg0VN, V } else { - if ((func == VNF_CastClass) || (func == VNF_IsInstanceOf)) + if (func == VNF_CastClass) { - resultVN = VNForCast(func, arg0VN, arg1VN); + if (arg1VN == VNForNull()) + { + // CastClass(cls, null) -> null + resultVN = VNForNull(); + } + else + { + // CastClass(cls, obj) -> obj (may throw InvalidCastException) + ValueNum vnExcSet = VNExcSetSingleton(VNForFuncNoFolding(TYP_REF, VNF_InvalidCastExc, arg1VN, arg0VN)); + resultVN = VNWithExc(arg1VN, vnExcSet); + } + } + else if ((func == VNF_IsInstanceOf) && (arg1VN == VNForNull())) + { + // IsInstanceOf(cls, null) -> null + resultVN = VNForNull(); } else { @@ -3775,7 +3675,7 @@ simd32_t ValueNumStore::GetConstantSimd32(ValueNum argVN) return ConstantValue(argVN); } -// Given a simd64 constant value number return its value as a simd64. +// Given a simd64 constant value number return its value as a simd32. // simd64_t ValueNumStore::GetConstantSimd64(ValueNum argVN) { @@ -3784,16 +3684,6 @@ simd64_t ValueNumStore::GetConstantSimd64(ValueNum argVN) return ConstantValue(argVN); } - -// Given a simdmask constant value number return its value as a simdmask. -// -simdmask_t ValueNumStore::GetConstantSimdMask(ValueNum argVN) -{ - assert(IsVNConstant(argVN)); - assert(TypeOfVN(argVN) == TYP_MASK); - - return ConstantValue(argVN); -} #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -9126,22 +9016,19 @@ void ValueNumStore::vnDump(Compiler* comp, ValueNum vn, bool isPtr) ssize_t val = ConstantValue(vn); const GenTreeFlags handleFlags = GetHandleFlags(vn); printf("Hnd const: 0x%p %s", dspPtr(val), GenTree::gtGetHandleKindString(handleFlags)); - if (!comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && !comp->opts.IsReadyToRun()) + switch (handleFlags & GTF_ICON_HDL_MASK) { - switch (handleFlags & GTF_ICON_HDL_MASK) - { - case GTF_ICON_CLASS_HDL: - printf(" %s", comp->eeGetClassName((CORINFO_CLASS_HANDLE)val)); - break; - case GTF_ICON_METHOD_HDL: - printf(" %s", comp->eeGetMethodFullName((CORINFO_METHOD_HANDLE)val)); - break; - case GTF_ICON_FIELD_HDL: - printf(" %s", comp->eeGetFieldName((CORINFO_FIELD_HANDLE)val, true)); - break; - default: - break; - } + case GTF_ICON_CLASS_HDL: + printf(" %s", comp->eeGetClassName((CORINFO_CLASS_HANDLE)val)); + break; + case GTF_ICON_METHOD_HDL: + printf(" %s", comp->eeGetMethodFullName((CORINFO_METHOD_HANDLE)val)); + break; + case GTF_ICON_FIELD_HDL: + printf(" %s", comp->eeGetFieldName((CORINFO_FIELD_HANDLE)val, true)); + break; + default: + break; } } else if (IsVNConstant(vn)) @@ -9265,13 +9152,6 @@ void ValueNumStore::vnDump(Compiler* comp, ValueNum vn, bool isPtr) cnsVal.u64[6], cnsVal.u64[7]); break; } - - case TYP_MASK: - { - simdmask_t cnsVal = GetConstantSimdMask(vn); - printf("SimdMaskCns[0x%08x, 0x%08x]", cnsVal.u32[0], cnsVal.u32[1]); - break; - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -9624,8 +9504,8 @@ const uint8_t ValueNumStore::s_vnfOpAttribs[VNF_COUNT] = { static genTreeOps genTreeOpsIllegalAsVNFunc[] = {GT_IND, // When we do heap memory. GT_NULLCHECK, GT_QMARK, GT_COLON, GT_LOCKADD, GT_XADD, GT_XCHG, - GT_CMPXCHG, GT_LCLHEAP, GT_BOX, GT_XORR, GT_XAND, GT_STORE_LCL_VAR, - GT_STORE_LCL_FLD, GT_STOREIND, GT_STORE_BLK, + GT_CMPXCHG, GT_LCLHEAP, GT_BOX, GT_XORR, GT_XAND, GT_STORE_DYN_BLK, + GT_STORE_LCL_VAR, GT_STORE_LCL_FLD, GT_STOREIND, GT_STORE_BLK, // These need special semantics: GT_COMMA, // == second argument (but with exception(s) from first). GT_ARR_ADDR, GT_BOUNDS_CHECK, @@ -9925,7 +9805,7 @@ class ValueNumberState return false; } - if (!predBlock->KindIs(BBJ_COND) || predBlock->TrueEdgeIs(predBlock->GetFalseEdge())) + if (!predBlock->KindIs(BBJ_COND) || predBlock->TrueTargetIs(predBlock->GetFalseTarget())) { return true; } @@ -10712,15 +10592,6 @@ void Compiler::fgValueNumberTreeConst(GenTree* tree) tree->gtVNPair.SetBoth(vnStore->VNForSimd64Con(simd64Val)); break; } - - case TYP_MASK: - { - simdmask_t simdmaskVal; - memcpy(&simdmaskVal, &tree->AsVecCon()->gtSimdVal, sizeof(simdmask_t)); - - tree->gtVNPair.SetBoth(vnStore->VNForSimdMaskCon(simdmaskVal)); - break; - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -11384,9 +11255,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) break; case GT_CATCH_ARG: - case GT_SWIFT_ERROR: // We know nothing about the value of a caught expression. - // We also know nothing about the error register's value post-Swift call. tree->gtVNPair.SetBoth(vnStore->VNForExpr(compCurBB, tree->TypeGet())); break; @@ -11548,7 +11417,12 @@ void Compiler::fgValueNumberTree(GenTree* tree) unsigned loadSize = tree->AsIndir()->Size(); VNFuncApp funcApp{VNF_COUNT}; - if (fgValueNumberConstLoad(tree->AsIndir())) + // TODO-1stClassStructs: delete layout-less "IND(struct)" nodes and the "loadSize == 0" condition. + if (loadSize == 0) + { + tree->gtVNPair.SetBoth(vnStore->VNForExpr(compCurBB, loadType)); + } + else if (fgValueNumberConstLoad(tree->AsIndir())) { // VN is assigned inside fgValueNumberConstLoad } @@ -11815,6 +11689,30 @@ void Compiler::fgValueNumberTree(GenTree* tree) break; #endif // FEATURE_HW_INTRINSICS + case GT_STORE_DYN_BLK: + { + // Conservatively, mutate the heaps - we don't analyze these rare stores. + // Likewise, any locals possibly defined by them we mark as address-exposed. + fgMutateGcHeap(tree DEBUGARG("dynamic block store")); + + GenTreeStoreDynBlk* store = tree->AsStoreDynBlk(); + ValueNumPair vnpExcSet = ValueNumStore::VNPForEmptyExcSet(); + + // Propagate the exceptions... + vnpExcSet = vnStore->VNPUnionExcSet(store->Addr()->gtVNPair, vnpExcSet); + vnpExcSet = vnStore->VNPUnionExcSet(store->Data()->gtVNPair, vnpExcSet); + vnpExcSet = vnStore->VNPUnionExcSet(store->gtDynamicSize->gtVNPair, vnpExcSet); + + // This is a store, it produces no value. Thus we use VNPForVoid(). + store->gtVNPair = vnStore->VNPWithExc(vnStore->VNPForVoid(), vnpExcSet); + + // Note that we are only adding the exception for the destination address. + // Currently, "Data()" is an explicit indirection in case this is a "cpblk". + assert(store->Data()->gtEffectiveVal()->OperIsIndir() || store->OperIsInitBlkOp()); + fgValueNumberAddExceptionSetForIndirection(store, store->Addr()); + break; + } + case GT_CMPXCHG: // Specialop { // For CMPXCHG and other intrinsics add an arbitrary side effect on GcHeap/ByrefExposed. @@ -14012,68 +13910,3 @@ bool ValueNumPair::BothDefined() const { return (m_liberal != ValueNumStore::NoVN) && (m_conservative != ValueNumStore::NoVN); } - -//-------------------------------------------------------------------------------- -// GetObjectType: Try to get a class handle (hopefully, exact) for given object via VN -// -// Arguments: -// vn - Value number of the object -// pIsExact - [out] set to true if the class handle is exact -// pIsNonNull - [out] set to true if the object is known to be non-null -// -// Return Value: -// Class handle for the object, or NO_CLASS_HANDLE if not available -// -CORINFO_CLASS_HANDLE ValueNumStore::GetObjectType(ValueNum vn, bool* pIsExact, bool* pIsNonNull) -{ - *pIsNonNull = false; - *pIsExact = false; - - if (TypeOfVN(vn) != TYP_REF) - { - // Not an object - return NO_CLASS_HANDLE; - } - - if (IsVNObjHandle(vn)) - { - // We know exact type for nongc objects, and they can never be null - *pIsNonNull = true; - *pIsExact = true; - size_t handle = CoercedConstantValue(vn); - return m_pComp->info.compCompHnd->getObjectType((CORINFO_OBJECT_HANDLE)handle); - } - - VNFuncApp funcApp; - if (!GetVNFunc(vn, &funcApp)) - { - // We can't make any assumptions about the object - return NO_CLASS_HANDLE; - } - - // CastClass/IsInstanceOf/JitNew all have the class handle as the first argument - const VNFunc func = funcApp.m_func; - if ((func == VNF_CastClass) || (func == VNF_IsInstanceOf) || (func == VNF_JitNew)) - { - ssize_t clsHandle; - ValueNum clsVN = funcApp.m_args[0]; - if (IsVNTypeHandle(clsVN) && EmbeddedHandleMapLookup(ConstantValue(clsVN), &clsHandle)) - { - // JitNew returns an exact and non-null obj, castclass and isinst do not have this guarantee. - *pIsNonNull = func == VNF_JitNew; - *pIsExact = func == VNF_JitNew; - return (CORINFO_CLASS_HANDLE)clsHandle; - } - } - - // obj.GetType() is guaranteed to return a non-null RuntimeType object - if (func == VNF_ObjGetType) - { - *pIsNonNull = true; - // Let's not assume whether RuntimeType is exact or not here (it was not in the past for NAOT) - // Callers usually call isExact anyway. - return m_pComp->info.compCompHnd->getBuiltinClass(CLASSID_RUNTIME_TYPE); - } - - return NO_CLASS_HANDLE; -} diff --git a/src/coreclr/jit/valuenum.h b/src/coreclr/jit/valuenum.h index ef8bd48229a15e..554c0ce6dc9a31 100644 --- a/src/coreclr/jit/valuenum.h +++ b/src/coreclr/jit/valuenum.h @@ -374,7 +374,6 @@ class ValueNumStore #if defined(TARGET_XARCH) simd32_t GetConstantSimd32(ValueNum argVN); simd64_t GetConstantSimd64(ValueNum argVN); - simdmask_t GetConstantSimdMask(ValueNum argVN); #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -460,7 +459,6 @@ class ValueNumStore #if defined(TARGET_XARCH) ValueNum VNForSimd32Con(simd32_t cnsVal); ValueNum VNForSimd64Con(simd64_t cnsVal); - ValueNum VNForSimdMaskCon(simdmask_t cnsVal); #endif // TARGET_XARCH #endif // FEATURE_SIMD ValueNum VNForGenericCon(var_types typ, uint8_t* cnsVal); @@ -513,8 +511,6 @@ class ValueNumStore return nullptr; } - CORINFO_CLASS_HANDLE GetObjectType(ValueNum vn, bool* pIsExact, bool* pIsNonNull); - // And the single constant for an object reference type. static ValueNum VNForNull() { @@ -692,8 +688,6 @@ class ValueNumStore // Skip all folding checks. ValueNum VNForFuncNoFolding(var_types typ, VNFunc func, ValueNum op1VNwx, ValueNum op2VNwx); - ValueNum VNForCast(VNFunc func, ValueNum castToVN, ValueNum objVN); - ValueNum VNForMapSelect(ValueNumKind vnk, var_types type, ValueNum map, ValueNum index); ValueNum VNForMapPhysicalSelect(ValueNumKind vnk, var_types type, ValueNum map, unsigned offset, unsigned size); @@ -1762,35 +1756,6 @@ class ValueNumStore } return m_simd64CnsMap; } - - struct SimdMaskPrimitiveKeyFuncs : public JitKeyFuncsDefEquals - { - static bool Equals(simdmask_t x, simdmask_t y) - { - return x == y; - } - - static unsigned GetHashCode(const simdmask_t val) - { - unsigned hash = 0; - - hash = static_cast(hash ^ val.u32[0]); - hash = static_cast(hash ^ val.u32[1]); - - return hash; - } - }; - - typedef VNMap SimdMaskToValueNumMap; - SimdMaskToValueNumMap* m_simdMaskCnsMap; - SimdMaskToValueNumMap* GetSimdMaskCnsMap() - { - if (m_simdMaskCnsMap == nullptr) - { - m_simdMaskCnsMap = new (m_alloc) SimdMaskToValueNumMap(m_alloc); - } - return m_simdMaskCnsMap; - } #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -1973,13 +1938,6 @@ struct ValueNumStore::VarTypConv typedef simd64_t Type; typedef simd64_t Lang; }; - -template <> -struct ValueNumStore::VarTypConv -{ - typedef simdmask_t Type; - typedef simdmask_t Lang; -}; #endif // TARGET_XARCH #endif // FEATURE_SIMD @@ -2056,13 +2014,6 @@ FORCEINLINE simd64_t ValueNumStore::SafeGetConstantValue(Chunk* c, uns assert(c->m_typ == TYP_SIMD64); return reinterpret_cast::Lang*>(c->m_defs)[offset]; } - -template <> -FORCEINLINE simdmask_t ValueNumStore::SafeGetConstantValue(Chunk* c, unsigned offset) -{ - assert(c->m_typ == TYP_MASK); - return reinterpret_cast::Lang*>(c->m_defs)[offset]; -} #endif // TARGET_XARCH template <> @@ -2135,20 +2086,6 @@ FORCEINLINE simd64_t ValueNumStore::ConstantValueInternal(ValueNum vn return SafeGetConstantValue(c, offset); } - -template <> -FORCEINLINE simdmask_t ValueNumStore::ConstantValueInternal(ValueNum vn DEBUGARG(bool coerce)) -{ - Chunk* c = m_chunks.GetNoExpand(GetChunkNum(vn)); - assert(c->m_attribs == CEA_Const); - - unsigned offset = ChunkOffset(vn); - - assert(c->m_typ == TYP_MASK); - assert(!coerce); - - return SafeGetConstantValue(c, offset); -} #endif // TARGET_XARCH #endif // FEATURE_SIMD diff --git a/src/coreclr/jit/vartype.h b/src/coreclr/jit/vartype.h index ed57a76b6e7ad8..27dd5b3329574f 100644 --- a/src/coreclr/jit/vartype.h +++ b/src/coreclr/jit/vartype.h @@ -321,7 +321,7 @@ inline bool varTypeUsesMaskReg(T vt) // However, we only have one type that uses VTR_MASK today // and so its quite a bit cheaper to just check that directly -#if defined(FEATURE_SIMD) && (defined(TARGET_XARCH) || defined(TARGET_ARM64)) +#if defined(FEATURE_SIMD) && defined(TARGET_XARCH) assert((TypeGet(vt) == TYP_MASK) || (varTypeRegister[TypeGet(vt)] != VTR_MASK)); return TypeGet(vt) == TYP_MASK; #else diff --git a/src/coreclr/nativeaot/Bootstrap/main.cpp b/src/coreclr/nativeaot/Bootstrap/main.cpp index 16ffe8f9dbcb06..ed343d16ed8c2b 100644 --- a/src/coreclr/nativeaot/Bootstrap/main.cpp +++ b/src/coreclr/nativeaot/Bootstrap/main.cpp @@ -103,54 +103,39 @@ extern "C" bool RhRegisterOSModule(void * pModule, extern "C" void* PalGetModuleHandleFromPointer(void* pointer); -#if defined(HOST_X86) && defined(HOST_WINDOWS) -#define STRINGIFY(s) #s -#define MANAGED_RUNTIME_EXPORT_ALTNAME(_method) STRINGIFY(/alternatename:_##_method=_method) -#define MANAGED_RUNTIME_EXPORT(_name) \ - __pragma(comment (linker, MANAGED_RUNTIME_EXPORT_ALTNAME(_name))) \ - extern "C" void __cdecl _name(); -#define MANAGED_RUNTIME_EXPORT_NAME(_name) _name -#define CDECL __cdecl -#else -#define MANAGED_RUNTIME_EXPORT(_name) \ - extern "C" void _name(); -#define MANAGED_RUNTIME_EXPORT_NAME(_name) _name -#define CDECL -#endif - -MANAGED_RUNTIME_EXPORT(GetRuntimeException) -MANAGED_RUNTIME_EXPORT(RuntimeFailFast) -MANAGED_RUNTIME_EXPORT(AppendExceptionStackFrame) -MANAGED_RUNTIME_EXPORT(GetSystemArrayEEType) -MANAGED_RUNTIME_EXPORT(OnFirstChanceException) -MANAGED_RUNTIME_EXPORT(OnUnhandledException) -MANAGED_RUNTIME_EXPORT(IDynamicCastableIsInterfaceImplemented) -MANAGED_RUNTIME_EXPORT(IDynamicCastableGetInterfaceImplementation) +extern "C" void GetRuntimeException(); +extern "C" void RuntimeFailFast(); +extern "C" void AppendExceptionStackFrame(); +extern "C" void GetSystemArrayEEType(); +extern "C" void OnFirstChanceException(); +extern "C" void OnUnhandledException(); +extern "C" void IDynamicCastableIsInterfaceImplemented(); +extern "C" void IDynamicCastableGetInterfaceImplementation(); #ifdef FEATURE_OBJCMARSHAL -MANAGED_RUNTIME_EXPORT(ObjectiveCMarshalTryGetTaggedMemory) -MANAGED_RUNTIME_EXPORT(ObjectiveCMarshalGetIsTrackedReferenceCallback) -MANAGED_RUNTIME_EXPORT(ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback) -MANAGED_RUNTIME_EXPORT(ObjectiveCMarshalGetUnhandledExceptionPropagationHandler) +extern "C" void ObjectiveCMarshalTryGetTaggedMemory(); +extern "C" void ObjectiveCMarshalGetIsTrackedReferenceCallback(); +extern "C" void ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback(); +extern "C" void ObjectiveCMarshalGetUnhandledExceptionPropagationHandler(); #endif -typedef void (CDECL *pfn)(); +typedef void(*pfn)(); static const pfn c_classlibFunctions[] = { - &MANAGED_RUNTIME_EXPORT_NAME(GetRuntimeException), - &MANAGED_RUNTIME_EXPORT_NAME(RuntimeFailFast), + &GetRuntimeException, + &RuntimeFailFast, nullptr, // &UnhandledExceptionHandler, - &MANAGED_RUNTIME_EXPORT_NAME(AppendExceptionStackFrame), + &AppendExceptionStackFrame, nullptr, // &CheckStaticClassConstruction, - &MANAGED_RUNTIME_EXPORT_NAME(GetSystemArrayEEType), - &MANAGED_RUNTIME_EXPORT_NAME(OnFirstChanceException), - &MANAGED_RUNTIME_EXPORT_NAME(OnUnhandledException), - &MANAGED_RUNTIME_EXPORT_NAME(IDynamicCastableIsInterfaceImplemented), - &MANAGED_RUNTIME_EXPORT_NAME(IDynamicCastableGetInterfaceImplementation), + &GetSystemArrayEEType, + &OnFirstChanceException, + &OnUnhandledException, + &IDynamicCastableIsInterfaceImplemented, + &IDynamicCastableGetInterfaceImplementation, #ifdef FEATURE_OBJCMARSHAL - &MANAGED_RUNTIME_EXPORT_NAME(ObjectiveCMarshalTryGetTaggedMemory), - &MANAGED_RUNTIME_EXPORT_NAME(ObjectiveCMarshalGetIsTrackedReferenceCallback), - &MANAGED_RUNTIME_EXPORT_NAME(ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback), - &MANAGED_RUNTIME_EXPORT_NAME(ObjectiveCMarshalGetUnhandledExceptionPropagationHandler), + &ObjectiveCMarshalTryGetTaggedMemory, + &ObjectiveCMarshalGetIsTrackedReferenceCallback, + &ObjectiveCMarshalGetOnEnteredFinalizerQueueCallback, + &ObjectiveCMarshalGetUnhandledExceptionPropagationHandler, #else nullptr, nullptr, @@ -213,7 +198,7 @@ static int InitializeRuntime() #ifndef NATIVEAOT_DLL #if defined(_WIN32) -int CDECL wmain(int argc, wchar_t* argv[]) +int __cdecl wmain(int argc, wchar_t* argv[]) #else int main(int argc, char* argv[]) #endif diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets index 62f159c5dc5268..3a9c4d0153a7b1 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.DotNet.ILCompiler.SingleEntry.targets @@ -4,11 +4,7 @@ <_hostOS>$(NETCoreSdkPortableRuntimeIdentifier.SubString(0, $(NETCoreSdkPortableRuntimeIdentifier.LastIndexOf('-')))) - <_targetsNonPortableSdkRid>false - <_targetsNonPortableSdkRid Condition="'$(RuntimeIdentifier)' == '$(NETCoreSdkRuntimeIdentifier)' and '$(RuntimeIdentifier)' != '$(NETCoreSdkPortableRuntimeIdentifier)'">true - <_originalTargetOS>$(RuntimeIdentifier.SubString(0, $(RuntimeIdentifier.LastIndexOf('-')))) - <_originalTargetOS Condition="'$(_targetsNonPortableSdkRid)' == 'true'">$(NETCoreSdkPortableRuntimeIdentifier.SubString(0, $(NETCoreSdkPortableRuntimeIdentifier.LastIndexOf('-')))) <_originalTargetOS Condition="$(_originalTargetOS.Contains('.'))">$(_originalTargetOS.SubString(0, $(_originalTargetOS.IndexOf('.')))) <_originalTargetOS Condition="$(_originalTargetOS.StartsWith('win'))">win @@ -21,9 +17,7 @@ <_targetArchitecture>$(RuntimeIdentifier.SubString($([MSBuild]::Add($(RuntimeIdentifier.LastIndexOf('-')), 1)))) <_hostPackageName>runtime.$(_hostOS)-$(_hostArchitecture).Microsoft.DotNet.ILCompiler - <_hostPackageName Condition="'$(_targetsNonPortableSdkRid)' == 'true'">runtime.$(RuntimeIdentifier).Microsoft.DotNet.ILCompiler <_targetPackageName>runtime.$(_originalTargetOS)-$(_targetArchitecture).Microsoft.DotNet.ILCompiler - <_targetPackageName Condition="'$(_targetsNonPortableSdkRid)' == 'true'">runtime.$(RuntimeIdentifier).Microsoft.DotNet.ILCompiler <_targetOS>$(_originalTargetOS) diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Publish.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Publish.targets index f57c573057cec8..e9a4424e819a23 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Publish.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Publish.targets @@ -55,8 +55,6 @@ - - diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets index 9f710609a2644e..32a946dd5839ad 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets @@ -230,19 +230,6 @@ The .NET Foundation licenses this file to you under the MIT license. - - - - - - - <_XcodeVersion>$([System.Text.RegularExpressions.Regex]::Match($(_XcodeVersionString), '[1-9]\d*')) - - - - - - <_CommandProbe>command -v <_CommandProbe Condition="$([MSBuild]::IsOSPlatform('Windows'))">where /Q diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index 75f579fc4a8617..c2039ec502d1f8 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -168,7 +168,7 @@ The .NET Foundation licenses this file to you under the MIT license. - + @@ -360,6 +360,7 @@ The .NET Foundation licenses this file to you under the MIT license. <_IgnoreLinkerWarnings>false <_IgnoreLinkerWarnings Condition="'$(_IsApplePlatform)' == 'true'">true + <_StripFlag Condition="'$(_IsApplePlatform)' == 'true' and '$(IlcExportUnmanagedEntrypoints)' == 'true'">-x @@ -387,7 +388,7 @@ The .NET Foundation licenses this file to you under the MIT license. + strip -no_code_signature_warning $(_StripFlag) "$(NativeBinary)"" /> NUL diff --git a/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs b/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs index 6d9c043416f7cc..42e677737be96b 100644 --- a/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs +++ b/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs @@ -27,7 +27,7 @@ internal static partial class StartupCodeHelpers /// private static IntPtr s_moduleGCStaticsSpines; - [UnmanagedCallersOnly(EntryPoint = "InitializeModules")] + [UnmanagedCallersOnly(EntryPoint = "InitializeModules", CallConvs = new Type[] { typeof(CallConvCdecl) })] internal static unsafe void InitializeModules(IntPtr osModule, IntPtr* pModuleHeaders, int count, IntPtr* pClasslibFunctions, int nClasslibFunctions) { RuntimeImports.RhpRegisterOsModule(osModule); @@ -206,12 +206,11 @@ private static unsafe object[] InitializeStatics(IntPtr gcStaticRegionStart, int nint blockAddr = MethodTable.SupportsRelativePointers ? (nint)ReadRelPtr32(pBlock) : *pBlock; if ((blockAddr & GCStaticRegionConstants.Uninitialized) == GCStaticRegionConstants.Uninitialized) { -#pragma warning disable CS8500 // takes address of managed type object? obj = null; RuntimeImports.RhAllocateNewObject( new IntPtr(blockAddr & ~GCStaticRegionConstants.Mask), (uint)GC_ALLOC_FLAGS.GC_ALLOC_PINNED_OBJECT_HEAP, - &obj); + Unsafe.AsPointer(ref obj)); if (obj == null) { RuntimeExceptionHelpers.FailFast("Failed allocating GC static bases"); @@ -233,8 +232,7 @@ private static unsafe object[] InitializeStatics(IntPtr gcStaticRegionStart, int Unsafe.Add(ref rawSpineData, currentBase) = obj; // Update the base pointer to point to the pinned object - *pBlock = *(IntPtr*)&obj; -#pragma warning restore CS8500 + *pBlock = *(IntPtr*)Unsafe.AsPointer(ref obj); } currentBase++; @@ -292,7 +290,12 @@ private static unsafe void RehydrateData(IntPtr dehydratedData, int length) { // At the time of writing this, 90% of DehydratedDataCommand.Copy cases // would fall into the above specialized cases. 10% fall back to memmove. - Unsafe.CopyBlock(pDest, pCurrent, (uint)payload); + memmove(pDest, pCurrent, (nuint)payload); + + // Not a DllImport - we don't need a GC transition since this is early startup + [MethodImplAttribute(MethodImplOptions.InternalCall)] + [RuntimeImport("*", "memmove")] + static extern unsafe void* memmove(byte* dmem, byte* smem, nuint size); } pDest += payload; diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs index 341cd832604715..136872edd4a472 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs @@ -119,25 +119,5 @@ public static T ReadUnaligned(void* source) { throw new PlatformNotSupportedException(); } - - /// - /// Copies bytes from the source address to the destination address. - /// - [Intrinsic] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void CopyBlock(void* destination, void* source, uint byteCount) - { - throw new PlatformNotSupportedException(); - } - - /// - /// Copies bytes from the source address to the destination address. - /// - [Intrinsic] - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void CopyBlock(ref byte destination, ref readonly byte source, uint byteCount) - { - throw new PlatformNotSupportedException(); - } } } diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs index 7ce239610a7ab6..c8d9a0a74c2645 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs @@ -68,14 +68,6 @@ private struct EHEnum private IntPtr _dummy; // For alignment } - internal struct MethodRegionInfo - { - internal byte* _hotStartAddress; - internal nuint _hotSize; - internal byte* _coldStartAddress; - internal nuint _coldSize; - } - #pragma warning disable IDE0060 // This is a fail-fast function used by the runtime as a last resort that will terminate the process with // as little effort as possible. No guarantee is made about the semantics of this fail-fast. @@ -771,13 +763,7 @@ private static void DispatchEx(scoped ref StackFrameIterator frameIter, ref ExIn DebugScanCallFrame(exInfo._passNumber, frameIter.ControlPC, frameIter.SP); -#if !NATIVEAOT - // Don't add frames at collided unwind - if (startIdx == MaxTryRegionIdx) -#endif - { - UpdateStackTrace(exceptionObj, exInfo._frameIter.FramePointer, (IntPtr)frameIter.OriginalControlPC, frameIter.SP, ref isFirstRethrowFrame, ref prevFramePtr, ref isFirstFrame, ref exInfo); - } + UpdateStackTrace(exceptionObj, exInfo._frameIter.FramePointer, (IntPtr)frameIter.OriginalControlPC, frameIter.SP, ref isFirstRethrowFrame, ref prevFramePtr, ref isFirstFrame, ref exInfo); byte* pHandler; if (FindFirstPassHandler(exceptionObj, startIdx, ref frameIter, @@ -946,20 +932,6 @@ private static void DebugVerifyHandlingFrame(UIntPtr handlingFrameSP) "Handling frame must have a valid stack frame pointer"); } - // Caclulate the code offset from the start of the method as if the hot and cold regions were - // stored sequentially in memory. - private static uint CalculateCodeOffset(byte* pbControlPC, in MethodRegionInfo methodRegionInfo) - { - uint codeOffset = (uint)(pbControlPC - methodRegionInfo._hotStartAddress); - // If the PC is in the cold region, adjust the offset to be relative to the start of the method. - if ((methodRegionInfo._coldSize != 0) && (codeOffset >= methodRegionInfo._hotSize)) - { - codeOffset = (uint)(methodRegionInfo._hotSize + (nuint)(pbControlPC - methodRegionInfo._coldStartAddress)); - } - - return codeOffset; - } - private static void UpdateStackTrace(object exceptionObj, UIntPtr curFramePtr, IntPtr ip, UIntPtr sp, ref bool isFirstRethrowFrame, ref UIntPtr prevFramePtr, ref bool isFirstFrame, ref ExInfo exInfo) { @@ -986,13 +958,13 @@ private static bool FindFirstPassHandler(object exception, uint idxStart, tryRegionIdx = MaxTryRegionIdx; EHEnum ehEnum; - MethodRegionInfo methodRegionInfo; - if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref frameIter, out methodRegionInfo, &ehEnum)) + byte* pbMethodStartAddress; + if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref frameIter, &pbMethodStartAddress, &ehEnum)) return false; byte* pbControlPC = frameIter.ControlPC; - uint codeOffset = CalculateCodeOffset(pbControlPC, in methodRegionInfo); + uint codeOffset = (uint)(pbControlPC - pbMethodStartAddress); uint lastTryStart = 0, lastTryEnd = 0; @@ -1139,14 +1111,13 @@ private static void InvokeSecondPass(ref ExInfo exInfo, uint idxStart) private static void InvokeSecondPass(ref ExInfo exInfo, uint idxStart, uint idxLimit) { EHEnum ehEnum; - MethodRegionInfo methodRegionInfo; - - if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref exInfo._frameIter, out methodRegionInfo, &ehEnum)) + byte* pbMethodStartAddress; + if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref exInfo._frameIter, &pbMethodStartAddress, &ehEnum)) return; byte* pbControlPC = exInfo._frameIter.ControlPC; - uint codeOffset = CalculateCodeOffset(pbControlPC, in methodRegionInfo); + uint codeOffset = (uint)(pbControlPC - pbMethodStartAddress); uint lastTryStart = 0, lastTryEnd = 0; @@ -1219,7 +1190,7 @@ private static void InvokeSecondPass(ref ExInfo exInfo, uint idxStart, uint idxL #if NATIVEAOT #pragma warning disable IDE0060 - [UnmanagedCallersOnly(EntryPoint = "RhpFailFastForPInvokeExceptionPreemp")] + [UnmanagedCallersOnly(EntryPoint = "RhpFailFastForPInvokeExceptionPreemp", CallConvs = new Type[] { typeof(CallConvCdecl) })] public static void RhpFailFastForPInvokeExceptionPreemp(IntPtr PInvokeCallsiteReturnAddr, void* pExceptionRecord, void* pContextRecord) { FailFastViaClasslib(RhFailFastReason.UnhandledExceptionFromPInvoke, null, PInvokeCallsiteReturnAddr); diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs index 8f5098ab31adcd..806208187e92fc 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs @@ -66,6 +66,7 @@ internal static void RhCollect(int generation, InternalGCCollectionMode mode, bo } [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] private static extern void RhpCollect(int generation, InternalGCCollectionMode mode, Interop.BOOL lowMemoryP); [RuntimeExport("RhGetGcTotalMemory")] @@ -75,6 +76,7 @@ internal static long RhGetGcTotalMemory() } [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] private static extern long RhpGetGcTotalMemory(); [RuntimeExport("RhStartNoGCRegion")] @@ -156,6 +158,10 @@ internal static int RhEndNoGCRegion() [RuntimeImport(Redhawk.BaseName, "RhpGcSafeZeroMemory")] internal static extern unsafe ref byte RhpGcSafeZeroMemory(ref byte dmem, nuint size); + [MethodImplAttribute(MethodImplOptions.InternalCall)] + [RuntimeImport(Redhawk.BaseName, "memmove")] + internal static extern unsafe void* memmove(byte* dmem, byte* smem, nuint size); + [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(Redhawk.BaseName, "RhBulkMoveWithWriteBarrier")] internal static extern unsafe void RhBulkMoveWithWriteBarrier(ref byte dmem, ref byte smem, nuint size); @@ -171,7 +177,7 @@ internal static int RhEndNoGCRegion() [RuntimeImport(Redhawk.BaseName, "RhpEHEnumInitFromStackFrameIterator")] [MethodImpl(MethodImplOptions.InternalCall)] - internal static extern unsafe bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, out EH.MethodRegionInfo pMethodRegionInfo, void* pEHEnum); + internal static extern unsafe bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, byte** pMethodStartAddress, void* pEHEnum); [RuntimeImport(Redhawk.BaseName, "RhpEHEnumNext")] [MethodImpl(MethodImplOptions.InternalCall)] @@ -272,23 +278,28 @@ internal static extern unsafe IntPtr RhpCallPropagateExceptionCallback( // Block the current thread until at least one object needs to be finalized (returns true) or // memory is low (returns false and the finalizer thread should initiate a garbage collection). [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static extern uint RhpWaitForFinalizerRequest(); // Indicate that the current round of finalizations is complete. [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static extern void RhpSignalFinalizationComplete(uint fCount); [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static extern ulong RhpGetTickCount64(); // Enters a no GC region, possibly doing a blocking GC if there is not enough // memory available to satisfy the caller's request. [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static extern int RhpStartNoGCRegion(long totalSize, Interop.BOOL hasLohSize, long lohSize, Interop.BOOL disallowFullBlockingGC); // Exits a no GC region, possibly doing a GC to clean up the garbage that // the caller allocated. [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static extern int RhpEndNoGCRegion(); } } diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeExports.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeExports.cs index 378fabf56836f0..8c83e3cc3c836f 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeExports.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/RuntimeExports.cs @@ -114,7 +114,9 @@ public static unsafe object RhBox(MethodTable* pEEType, ref byte data) } else { - Unsafe.CopyBlock(ref result.GetRawData(), ref dataAdjustedForNullable, pEEType->ValueTypeSize); + fixed (byte* pFields = &result.GetRawData()) + fixed (byte* pData = &dataAdjustedForNullable) + InternalCalls.memmove(pFields, pData, pEEType->ValueTypeSize); } return result; @@ -269,7 +271,9 @@ public static unsafe void RhUnbox(object? obj, ref byte data, MethodTable* pUnbo else { // Copy the boxed fields into the new location. - Unsafe.CopyBlock(ref data, ref fields, pEEType->ValueTypeSize); + fixed (byte *pData = &data) + fixed (byte* pFields = &fields) + InternalCalls.memmove(pData, pFields, pEEType->ValueTypeSize); } } @@ -283,6 +287,7 @@ public static unsafe int RhGetCurrentThreadStackTrace(IntPtr[] outputBuffer) #pragma warning disable SYSLIB1054 // Use DllImport here instead of LibraryImport because this file is used by Test.CoreLib. [DllImport(Redhawk.BaseName)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] private static extern unsafe int RhpGetCurrentThreadStackTrace(IntPtr* pOutputBuffer, uint outputBufferLength, UIntPtr addressInCurrentFrame); #pragma warning restore SYSLIB1054 @@ -298,7 +303,7 @@ public static unsafe int RhGetCurrentThreadStackTrace(IntPtr[] outputBuffer) // NOTE: We don't want to allocate the array on behalf of the caller because we don't know which class // library's objects the caller understands (we support multiple class libraries with multiple root // System.Object types). - [UnmanagedCallersOnly(EntryPoint = "RhpCalculateStackTraceWorker")] + [UnmanagedCallersOnly(EntryPoint = "RhpCalculateStackTraceWorker", CallConvs = new Type[] { typeof(CallConvCdecl) })] private static unsafe int RhpCalculateStackTraceWorker(IntPtr* pOutputBuffer, uint outputBufferLength, UIntPtr addressInCurrentFrame) { uint nFrames = 0; diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/__Finalizer.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/__Finalizer.cs index a55b7fc040d25a..b88c8033eb8f83 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/__Finalizer.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/__Finalizer.cs @@ -16,7 +16,7 @@ namespace System.Runtime // We choose this name to avoid clashing with any future public class with the name Finalizer. internal static class __Finalizer { - [UnmanagedCallersOnly(EntryPoint = "ProcessFinalizers")] + [UnmanagedCallersOnly(EntryPoint = "ProcessFinalizers", CallConvs = new Type[] { typeof(CallConvCdecl) })] public static void ProcessFinalizers() { #if INPLACE_RUNTIME diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index cb9a63c1973777..7ef6f223fdb580 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -263,15 +263,13 @@ add_definitions(-D_LIB) # there is a problem with undefined symbols when this is set # add_definitions(-DSTRESS_HEAP) -if(CLR_CMAKE_TARGET_WIN32) +if(WIN32) set(FEATURE_ETW 1) add_definitions(-DFEATURE_ETW) add_definitions(-DFEATURE_SUSPEND_REDIRECTION) - if (CLR_CMAKE_TARGET_ARCH_AMD64) - add_definitions(-DFEATURE_SPECIAL_USER_MODE_APC) - endif() + add_definitions(-DFEATURE_SPECIAL_USER_MODE_APC) else() - if(NOT CLR_CMAKE_TARGET_APPLE) + if(NOT CLR_CMAKE_TARGET_MACCATALYST AND NOT CLR_CMAKE_TARGET_IOS AND NOT CLR_CMAKE_TARGET_TVOS) add_definitions(-DFEATURE_READONLY_GS_COOKIE) endif() include(unix/configure.cmake) diff --git a/src/coreclr/nativeaot/Runtime/CachedInterfaceDispatch.cpp b/src/coreclr/nativeaot/Runtime/CachedInterfaceDispatch.cpp index 2938ee70974073..eaf8bce80c236a 100644 --- a/src/coreclr/nativeaot/Runtime/CachedInterfaceDispatch.cpp +++ b/src/coreclr/nativeaot/Runtime/CachedInterfaceDispatch.cpp @@ -445,7 +445,7 @@ bool InitializeInterfaceDispatch() return true; } -FCIMPL4(PCODE, RhpUpdateDispatchCellCache, InterfaceDispatchCell * pCell, PCODE pTargetCode, MethodTable* pInstanceType, DispatchCellInfo *pNewCellInfo) +COOP_PINVOKE_HELPER(PCODE, RhpUpdateDispatchCellCache, (InterfaceDispatchCell * pCell, PCODE pTargetCode, MethodTable* pInstanceType, DispatchCellInfo *pNewCellInfo)) { // Attempt to update the cache with this new mapping (if we have any cache at all, the initial state // is none). @@ -515,9 +515,8 @@ FCIMPL4(PCODE, RhpUpdateDispatchCellCache, InterfaceDispatchCell * pCell, PCODE return (PCODE)pTargetCode; } -FCIMPLEND -FCIMPL2(PCODE, RhpSearchDispatchCellCache, InterfaceDispatchCell * pCell, MethodTable* pInstanceType) +COOP_PINVOKE_HELPER(PCODE, RhpSearchDispatchCellCache, (InterfaceDispatchCell * pCell, MethodTable* pInstanceType)) { // This function must be implemented in native code so that we do not take a GC while walking the cache InterfaceDispatchCache * pCache = (InterfaceDispatchCache*)pCell->GetCache(); @@ -531,15 +530,13 @@ FCIMPL2(PCODE, RhpSearchDispatchCellCache, InterfaceDispatchCell * pCell, Method return (PCODE)nullptr; } -FCIMPLEND // Given a dispatch cell, get the type and slot associated with it. This function MUST be implemented // in cooperative native code, as the m_pCache field on the cell is unsafe to access from managed // code due to its use of the GC state as a lock, and as lifetime control -FCIMPL2(void, RhpGetDispatchCellInfo, InterfaceDispatchCell * pCell, DispatchCellInfo* pDispatchCellInfo) +COOP_PINVOKE_HELPER(void, RhpGetDispatchCellInfo, (InterfaceDispatchCell * pCell, DispatchCellInfo* pDispatchCellInfo)) { *pDispatchCellInfo = pCell->GetDispatchCellInfo(); } -FCIMPLEND #endif // FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/nativeaot/Runtime/CommonMacros.h b/src/coreclr/nativeaot/Runtime/CommonMacros.h index e9e62638813ba9..9c762216dd7f17 100644 --- a/src/coreclr/nativeaot/Runtime/CommonMacros.h +++ b/src/coreclr/nativeaot/Runtime/CommonMacros.h @@ -17,8 +17,8 @@ #define STDCALL #endif -#define F_CALL_CONV FASTCALL -#define QCALLTYPE +#define NATIVEAOT_API +#define REDHAWK_CALLCONV FASTCALL #ifdef _MSC_VER @@ -177,145 +177,14 @@ typedef uint8_t CODE_LOCATION; // Define an unmanaged function called from managed code that needs to execute in co-operative GC mode. (There // should be very few of these, most such functions will be simply p/invoked). // - -#define FCALL_METHOD_NAME(name, ...) name -#define FCALL_METHOD_NAME_(tuple) FCALL_METHOD_NAME tuple - -#if defined(HOST_X86) && defined(HOST_WINDOWS) - -// x86 is special case. It supports multiple calling conventions (fastcall, stdcall, cdecl) -// and mangles the method names according to the calling convention (eg. @fastcall@4, _stdcall@4, -// _cdecl). -// -// The managed code uses its own calling convention that is different from the native call -// conventions. It's similar to fastcall but pushes the arguments to stack in reverse order. -// Additionally, for the sake of simplicity we don't decorate the symbol names. -// -// In order to bridge the managed calling convention we use two tricks: -// - The FCIMPL and FCDECL macros reorder parameters for any method with 4 or more arguments. -// - A linker comment is used to pass the "/alternatename:foo=@foo@4" switch to allow the -// symbols to be resolved to the fastcall decorated name. - -#define FCALL_ARGHELPER_NAME(_0, _1, _2, _3, _4, _5, NAME, ...) NAME -#define FCALL_ARGHELPER_NAME_(tuple) FCALL_ARGHELPER_NAME tuple - -#define FCALL_ARGHELPER0(dummy) () -#define FCALL_ARGHELPER1(dummy, a) (a) -#define FCALL_ARGHELPER2(dummy, a, b) (a, b) -#define FCALL_ARGHELPER3(dummy, a, b, c) (a, b, c) -#define FCALL_ARGHELPER4(dummy, a, b, c, d) (a, b, d, c) -#define FCALL_ARGHELPER5(dummy, a, b, c, d, e) (a, b, e, d, c) - -#define FCALL_STRINGIFY(s) #s -#define FCALL_XSTRINGIFY(s) FCALL_STRINGIFY(s) - -#define FCALL_METHOD_ARGS(...) FCALL_ARGHELPER_NAME_((__VA_ARGS__, FCALL_ARGHELPER5, FCALL_ARGHELPER4, FCALL_ARGHELPER3, FCALL_ARGHELPER2, FCALL_ARGHELPER1, FCALL_ARGHELPER0)) (__VA_ARGS__) -#define FCALL_METHOD_ARGS_(tuple) FCALL_METHOD_ARGS tuple - -#define FCALL_ARGHELPER_STACKSIZE(...) FCALL_ARGHELPER_NAME_((__VA_ARGS__, 20, 16, 12, 8, 4, 0)) -#define FCALL_IMPL_ALTNAME(_method, _argSize) FCALL_XSTRINGIFY(/alternatename:_method=@_method@_argSize) -#define FCALL_DECL_ALTNAME(_method, _argSize) FCALL_XSTRINGIFY(/alternatename:@_method@_argSize=_method) -#define FCDECL_RENAME(_rettype, ...) \ - _Pragma(FCALL_XSTRINGIFY(comment (linker, FCALL_DECL_ALTNAME(FCALL_METHOD_NAME_((__VA_ARGS__)), FCALL_ARGHELPER_STACKSIZE(__VA_ARGS__))))) -#define FCIMPL_RENAME(_rettype, ...) \ - _Pragma(FCALL_XSTRINGIFY(comment (linker, FCALL_IMPL_ALTNAME(FCALL_METHOD_NAME_((__VA_ARGS__)), FCALL_ARGHELPER_STACKSIZE(__VA_ARGS__))))) -#define FCIMPL_RENAME_ARGSIZE(_rettype, _method, _argSize) \ - _Pragma(FCALL_XSTRINGIFY(comment (linker, FCALL_XSTRINGIFY(/alternatename:_method=@_method##_FCall@_argSize)))) - -#define FCIMPL1_F(_rettype, _method, a) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 4) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (a) \ - { -#define FCIMPL1_D(_rettype, _method, a) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 8) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (a) \ - { -#define FCIMPL1_L FCIMPL1_D -#define FCIMPL2_FF(_rettype, _method, a, b) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 8) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (b, a) \ - { -#define FCIMPL2_DD(_rettype, _method, a, b) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 16) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (b, a) \ - { -#define FCIMPL2_FI(_rettype, _method, a, b) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 8) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (a, b) \ - { -#define FCIMPL2_DI(_rettype, _method, a, b) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 12) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (a, b) \ - { -#define FCIMPL3_FFF(_rettype, _method, a, b, c) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 12) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (c, b, a) \ - { -#define FCIMPL3_DDD(_rettype, _method, a, b, c) \ - FCIMPL_RENAME_ARGSIZE(_rettype, _method, 24) \ - EXTERN_C _rettype F_CALL_CONV _method##_FCall (c, b, a) \ - { - +#define COOP_PINVOKE_HELPER(_rettype, _method, _args) EXTERN_C NATIVEAOT_API _rettype REDHAWK_CALLCONV _method _args +#ifdef HOST_X86 +// We have helpers that act like memcpy and memset from the CRT, so they need to be __cdecl. +#define COOP_PINVOKE_CDECL_HELPER(_rettype, _method, _args) EXTERN_C NATIVEAOT_API _rettype __cdecl _method _args #else - -#define FCDECL_RENAME(_rettype, ...) -#define FCIMPL_RENAME(_rettype, ...) - -#define FCALL_METHOD_ARGS(dummy, ...) (__VA_ARGS__) -#define FCALL_METHOD_ARGS_(tuple) FCALL_METHOD_ARGS tuple - -#define FCIMPL1_F(_rettype, _method, a) \ - EXTERN_C _rettype F_CALL_CONV _method (a) \ - { -#define FCIMPL1_D(_rettype, _method, a) \ - EXTERN_C _rettype F_CALL_CONV _method (a) \ - { -#define FCIMPL1_L FCIMPL1_D -#define FCIMPL2_FF(_rettype, _method, a, b) \ - EXTERN_C _rettype F_CALL_CONV _method (a, b) \ - { -#define FCIMPL2_DD(_rettype, _method, a, b) \ - EXTERN_C _rettype F_CALL_CONV _method (a, b) \ - { -#define FCIMPL2_FI(_rettype, _method, a, b) \ - EXTERN_C _rettype F_CALL_CONV _method (a, b) \ - { -#define FCIMPL2_DI(_rettype, _method, a, b) \ - EXTERN_C _rettype F_CALL_CONV _method (a, b) \ - { -#define FCIMPL3_FFF(_rettype, _method, a, b, c) \ - EXTERN_C _rettype F_CALL_CONV _method (a, b, c) \ - { -#define FCIMPL3_DDD(_rettype, _method, a, b, c) \ - EXTERN_C _rettype F_CALL_CONV _method (a, b, c) \ - { - +#define COOP_PINVOKE_CDECL_HELPER COOP_PINVOKE_HELPER #endif -#define FCDECL_(_rettype, ...) \ - FCDECL_RENAME(_rettype, __VA_ARGS__) \ - EXTERN_C _rettype F_CALL_CONV FCALL_METHOD_NAME_((__VA_ARGS__)) FCALL_METHOD_ARGS_((__VA_ARGS__)) -#define FCDECL0(_rettype, _method) FCDECL_(_rettype, _method) -#define FCDECL1(_rettype, _method, a) FCDECL_(_rettype, _method, a) -#define FCDECL2(_rettype, _method, a, b) FCDECL_(_rettype, _method, a, b) -#define FCDECL3(_rettype, _method, a, b, c) FCDECL_(_rettype, _method, a, b, c) -#define FCDECL4(_rettype, _method, a, b, c, d) FCDECL_(_rettype, _method, a, b, c, d) -#define FCDECL5(_rettype, _method, a, b, c, d, e) FCDECL_(_rettype, _method, a, b, c, d, e) - -#define FCIMPL_(_rettype, ...) \ - FCIMPL_RENAME(_rettype, __VA_ARGS__) \ - EXTERN_C _rettype F_CALL_CONV FCALL_METHOD_NAME_((__VA_ARGS__)) FCALL_METHOD_ARGS_((__VA_ARGS__)) \ - { -#define FCIMPL0(_rettype, _method) FCIMPL_(_rettype, _method) -#define FCIMPL1(_rettype, _method, a) FCIMPL_(_rettype, _method, a) -#define FCIMPL2(_rettype, _method, a, b) FCIMPL_(_rettype, _method, a, b) -#define FCIMPL3(_rettype, _method, a, b, c) FCIMPL_(_rettype, _method, a, b, c) -#define FCIMPL4(_rettype, _method, a, b, c, d) FCIMPL_(_rettype, _method, a, b, c, d) -#define FCIMPL5(_rettype, _method, a, b, c, d, e) FCIMPL_(_rettype, _method, a, b, c, d, e) - -#define FCIMPLEND \ - } - typedef bool CLR_BOOL; #if defined(TARGET_X86) || defined(TARGET_AMD64) diff --git a/src/coreclr/nativeaot/Runtime/EHHelpers.cpp b/src/coreclr/nativeaot/Runtime/EHHelpers.cpp index c440aced22dd95..28cb9e617f09bb 100644 --- a/src/coreclr/nativeaot/Runtime/EHHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/EHHelpers.cpp @@ -29,60 +29,43 @@ #include "MethodTable.inl" #include "CommonMacros.inl" -struct MethodRegionInfo -{ - void* hotStartAddress; - size_t hotSize; - void* coldStartAddress; - size_t coldSize; -}; - -FCIMPL3(FC_BOOL_RET, RhpEHEnumInitFromStackFrameIterator, - StackFrameIterator* pFrameIter, MethodRegionInfo* pMethodRegionInfoOut, EHEnum* pEHEnum) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhpEHEnumInitFromStackFrameIterator, ( + StackFrameIterator* pFrameIter, void ** pMethodStartAddressOut, EHEnum* pEHEnum)) { ICodeManager * pCodeManager = pFrameIter->GetCodeManager(); pEHEnum->m_pCodeManager = pCodeManager; - pMethodRegionInfoOut->hotSize = 0; // unknown - pMethodRegionInfoOut->coldStartAddress = nullptr; - pMethodRegionInfoOut->coldSize = 0; - - FC_RETURN_BOOL(pCodeManager->EHEnumInit(pFrameIter->GetMethodInfo(), &pMethodRegionInfoOut->hotStartAddress, &pEHEnum->m_state)); + FC_RETURN_BOOL(pCodeManager->EHEnumInit(pFrameIter->GetMethodInfo(), pMethodStartAddressOut, &pEHEnum->m_state)); } -FCIMPLEND -FCIMPL2(FC_BOOL_RET, RhpEHEnumNext, EHEnum* pEHEnum, EHClause* pEHClause) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhpEHEnumNext, (EHEnum* pEHEnum, EHClause* pEHClause)) { FC_RETURN_BOOL(pEHEnum->m_pCodeManager->EHEnumNext(&pEHEnum->m_state, pEHClause)); } -FCIMPLEND // Unmanaged helper to locate one of two classlib-provided functions that the runtime needs to // implement throwing of exceptions out of Rtm, and fail-fast. This may return NULL if the classlib // found via the provided address does not have the necessary exports. -FCIMPL2(void *, RhpGetClasslibFunctionFromCodeAddress, void * address, ClasslibFunctionId functionId) +COOP_PINVOKE_HELPER(void *, RhpGetClasslibFunctionFromCodeAddress, (void * address, ClasslibFunctionId functionId)) { return GetRuntimeInstance()->GetClasslibFunctionFromCodeAddress(address, functionId); } -FCIMPLEND // Unmanaged helper to locate one of two classlib-provided functions that the runtime needs to // implement throwing of exceptions out of Rtm, and fail-fast. This may return NULL if the classlib // found via the provided address does not have the necessary exports. -FCIMPL2(void *, RhpGetClasslibFunctionFromEEType, MethodTable * pEEType, ClasslibFunctionId functionId) +COOP_PINVOKE_HELPER(void *, RhpGetClasslibFunctionFromEEType, (MethodTable * pEEType, ClasslibFunctionId functionId)) { return pEEType->GetTypeManagerPtr()->AsTypeManager()->GetClasslibFunction(functionId); } -FCIMPLEND -FCIMPL0(void, RhpValidateExInfoStack) +COOP_PINVOKE_HELPER(void, RhpValidateExInfoStack, ()) { Thread * pThisThread = ThreadStore::GetCurrentThread(); pThisThread->ValidateExInfoStack(); } -FCIMPLEND -FCIMPL0(void, RhpClearThreadDoNotTriggerGC) +COOP_PINVOKE_HELPER(void, RhpClearThreadDoNotTriggerGC, ()) { Thread * pThisThread = ThreadStore::GetCurrentThread(); @@ -91,9 +74,8 @@ FCIMPL0(void, RhpClearThreadDoNotTriggerGC) pThisThread->ClearDoNotTriggerGc(); } -FCIMPLEND -FCIMPL0(void, RhpSetThreadDoNotTriggerGC) +COOP_PINVOKE_HELPER(void, RhpSetThreadDoNotTriggerGC, ()) { Thread * pThisThread = ThreadStore::GetCurrentThread(); @@ -102,15 +84,13 @@ FCIMPL0(void, RhpSetThreadDoNotTriggerGC) pThisThread->SetDoNotTriggerGc(); } -FCIMPLEND -FCIMPL2(int32_t, RhGetModuleFileName, HANDLE moduleHandle, _Out_ const TCHAR** pModuleNameOut) +COOP_PINVOKE_HELPER(int32_t, RhGetModuleFileName, (HANDLE moduleHandle, _Out_ const TCHAR** pModuleNameOut)) { return PalGetModuleFileName(pModuleNameOut, moduleHandle); } -FCIMPLEND -FCIMPL3(void, RhpCopyContextFromExInfo, void * pOSContext, int32_t cbOSContext, PAL_LIMITED_CONTEXT * pPalContext) +COOP_PINVOKE_HELPER(void, RhpCopyContextFromExInfo, (void * pOSContext, int32_t cbOSContext, PAL_LIMITED_CONTEXT * pPalContext)) { ASSERT((size_t)cbOSContext >= sizeof(CONTEXT)); CONTEXT* pContext = (CONTEXT *)pOSContext; @@ -193,7 +173,6 @@ FCIMPL3(void, RhpCopyContextFromExInfo, void * pOSContext, int32_t cbOSContext, #error Not Implemented for this architecture -- RhpCopyContextFromExInfo #endif } -FCIMPLEND #if defined(HOST_AMD64) || defined(HOST_ARM) || defined(HOST_X86) || defined(HOST_ARM64) struct DISPATCHER_CONTEXT @@ -210,10 +189,10 @@ struct EXCEPTION_REGISTRATION_RECORD }; #endif // HOST_X86 -EXTERN_C void QCALLTYPE RhpFailFastForPInvokeExceptionPreemp(intptr_t PInvokeCallsiteReturnAddr, - void* pExceptionRecord, void* pContextRecord); -FCDECL3(void, RhpFailFastForPInvokeExceptionCoop, intptr_t PInvokeCallsiteReturnAddr, - void* pExceptionRecord, void* pContextRecord); +EXTERN_C void __cdecl RhpFailFastForPInvokeExceptionPreemp(intptr_t PInvokeCallsiteReturnAddr, + void* pExceptionRecord, void* pContextRecord); +EXTERN_C void REDHAWK_CALLCONV RhpFailFastForPInvokeExceptionCoop(intptr_t PInvokeCallsiteReturnAddr, + void* pExceptionRecord, void* pContextRecord); int32_t __stdcall RhpVectoredExceptionHandler(PEXCEPTION_POINTERS pExPtrs); EXTERN_C int32_t __stdcall RhpPInvokeExceptionGuard(PEXCEPTION_RECORD pExceptionRecord, @@ -267,38 +246,32 @@ EXTERN_C int32_t RhpPInvokeExceptionGuard() #endif #if defined(HOST_AMD64) || defined(HOST_ARM) || defined(HOST_X86) || defined(HOST_ARM64) || defined(HOST_WASM) -FCDECL2(void, RhpThrowHwEx, int exceptionCode, TADDR faultingIP); +EXTERN_C NATIVEAOT_API void REDHAWK_CALLCONV RhpThrowHwEx(); #else -FCIMPL0(void, RhpThrowHwEx) +COOP_PINVOKE_HELPER(void, RhpThrowHwEx, ()) { ASSERT_UNCONDITIONALLY("RhpThrowHwEx NYI for this architecture!"); } -FCIMPLEND -FCIMPL0(void, RhpThrowEx) +COOP_PINVOKE_HELPER(void, RhpThrowEx, ()) { ASSERT_UNCONDITIONALLY("RhpThrowEx NYI for this architecture!"); } -FCIMPLEND -FCIMPL0(void, RhpCallCatchFunclet) +COOP_PINVOKE_HELPER(void, RhpCallCatchFunclet, ()) { ASSERT_UNCONDITIONALLY("RhpCallCatchFunclet NYI for this architecture!"); } -FCIMPLEND -FCIMPL0(void, RhpCallFinallyFunclet) +COOP_PINVOKE_HELPER(void, RhpCallFinallyFunclet, ()) { ASSERT_UNCONDITIONALLY("RhpCallFinallyFunclet NYI for this architecture!"); } -FCIMPLEND -FCIMPL0(void, RhpCallFilterFunclet) +COOP_PINVOKE_HELPER(void, RhpCallFilterFunclet, ()) { ASSERT_UNCONDITIONALLY("RhpCallFilterFunclet NYI for this architecture!"); } -FCIMPLEND -FCIMPL0(void, RhpRethrow) +COOP_PINVOKE_HELPER(void, RhpRethrow, ()) { ASSERT_UNCONDITIONALLY("RhpRethrow NYI for this architecture!"); } -FCIMPLEND EXTERN_C void* RhpCallCatchFunclet2 = NULL; EXTERN_C void* RhpCallFinallyFunclet2 = NULL; @@ -309,23 +282,7 @@ EXTERN_C void* RhpRethrow2 = NULL; #endif EXTERN_C CODE_LOCATION RhpAssignRefAVLocation; -#if defined(HOST_X86) -EXTERN_C CODE_LOCATION RhpAssignRefEAXAVLocation; -EXTERN_C CODE_LOCATION RhpAssignRefECXAVLocation; -EXTERN_C CODE_LOCATION RhpAssignRefEBXAVLocation; -EXTERN_C CODE_LOCATION RhpAssignRefESIAVLocation; -EXTERN_C CODE_LOCATION RhpAssignRefEDIAVLocation; -EXTERN_C CODE_LOCATION RhpAssignRefEBPAVLocation; -#endif EXTERN_C CODE_LOCATION RhpCheckedAssignRefAVLocation; -#if defined(HOST_X86) -EXTERN_C CODE_LOCATION RhpCheckedAssignRefEAXAVLocation; -EXTERN_C CODE_LOCATION RhpCheckedAssignRefECXAVLocation; -EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBXAVLocation; -EXTERN_C CODE_LOCATION RhpCheckedAssignRefESIAVLocation; -EXTERN_C CODE_LOCATION RhpCheckedAssignRefEDIAVLocation; -EXTERN_C CODE_LOCATION RhpCheckedAssignRefEBPAVLocation; -#endif EXTERN_C CODE_LOCATION RhpCheckedLockCmpXchgAVLocation; EXTERN_C CODE_LOCATION RhpCheckedXchgAVLocation; #if !defined(HOST_AMD64) && !defined(HOST_ARM64) @@ -351,31 +308,13 @@ static bool InWriteBarrierHelper(uintptr_t faultingIP) static uintptr_t writeBarrierAVLocations[] = { (uintptr_t)&RhpAssignRefAVLocation, -#if defined(HOST_X86) - (uintptr_t)&RhpAssignRefEAXAVLocation, - (uintptr_t)&RhpAssignRefECXAVLocation, - (uintptr_t)&RhpAssignRefEBXAVLocation, - (uintptr_t)&RhpAssignRefESIAVLocation, - (uintptr_t)&RhpAssignRefEDIAVLocation, - (uintptr_t)&RhpAssignRefEBPAVLocation, -#endif (uintptr_t)&RhpCheckedAssignRefAVLocation, -#if defined(HOST_X86) - (uintptr_t)&RhpCheckedAssignRefEAXAVLocation, - (uintptr_t)&RhpCheckedAssignRefECXAVLocation, - (uintptr_t)&RhpCheckedAssignRefEBXAVLocation, - (uintptr_t)&RhpCheckedAssignRefESIAVLocation, - (uintptr_t)&RhpCheckedAssignRefEDIAVLocation, - (uintptr_t)&RhpCheckedAssignRefEBPAVLocation, -#endif (uintptr_t)&RhpCheckedLockCmpXchgAVLocation, (uintptr_t)&RhpCheckedXchgAVLocation, #if !defined(HOST_AMD64) && !defined(HOST_ARM64) -#if !defined(HOST_X86) (uintptr_t)&RhpLockCmpXchg8AVLocation, (uintptr_t)&RhpLockCmpXchg16AVLocation, (uintptr_t)&RhpLockCmpXchg32AVLocation, -#endif (uintptr_t)&RhpLockCmpXchg64AVLocation, #endif (uintptr_t)&RhpByRefAssignRefAVLocation1, @@ -656,10 +595,9 @@ int32_t __stdcall RhpVectoredExceptionHandler(PEXCEPTION_POINTERS pExPtrs) #endif // TARGET_UNIX -FCIMPL0(void, RhpFallbackFailFast) +COOP_PINVOKE_HELPER(void, RhpFallbackFailFast, ()) { RhFailFast(); } -FCIMPLEND #endif // !DACCESS_COMPILE diff --git a/src/coreclr/nativeaot/Runtime/FinalizerHelpers.cpp b/src/coreclr/nativeaot/Runtime/FinalizerHelpers.cpp index 7a4d8a853a8b74..24a456a5f5a7ed 100644 --- a/src/coreclr/nativeaot/Runtime/FinalizerHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/FinalizerHelpers.cpp @@ -27,7 +27,7 @@ GPTR_DECL(Thread, g_pFinalizerThread); CLREventStatic g_FinalizerEvent; CLREventStatic g_FinalizerDoneEvent; -EXTERN_C void QCALLTYPE ProcessFinalizers(); +extern "C" void __cdecl ProcessFinalizers(); // Unmanaged front-end to the finalizer thread. We require this because at the point the GC creates the // finalizer thread we can't run managed code. Instead this method waits @@ -89,12 +89,12 @@ void RhEnableFinalization() g_FinalizerEvent.Set(); } -EXTERN_C void QCALLTYPE RhInitializeFinalizerThread() +EXTERN_C NATIVEAOT_API void __cdecl RhInitializeFinalizerThread() { g_FinalizerEvent.Set(); } -EXTERN_C void QCALLTYPE RhWaitForPendingFinalizers(UInt32_BOOL allowReentrantWait) +EXTERN_C NATIVEAOT_API void __cdecl RhWaitForPendingFinalizers(UInt32_BOOL allowReentrantWait) { // This must be called via p/invoke rather than RuntimeImport since it blocks and could starve the GC if // called in cooperative mode. @@ -115,7 +115,7 @@ EXTERN_C void QCALLTYPE RhWaitForPendingFinalizers(UInt32_BOOL allowReentrantWai // Block the current thread until at least one object needs to be finalized (returns true) or memory is low // (returns false and the finalizer thread should initiate a garbage collection). -EXTERN_C UInt32_BOOL QCALLTYPE RhpWaitForFinalizerRequest() +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhpWaitForFinalizerRequest() { // We can wait for two events; finalization queue has been populated and low memory resource notification. // But if the latter is signalled we shouldn't wait on it again immediately -- if the garbage collection @@ -182,7 +182,7 @@ EXTERN_C UInt32_BOOL QCALLTYPE RhpWaitForFinalizerRequest() } // Indicate that the current round of finalizations is complete. -EXTERN_C void QCALLTYPE RhpSignalFinalizationComplete(uint32_t fcount) +EXTERN_C NATIVEAOT_API void __cdecl RhpSignalFinalizationComplete(uint32_t fcount) { FireEtwGCFinalizersEnd_V1(fcount, GetClrInstanceId()); g_FinalizerDoneEvent.Set(); @@ -194,7 +194,7 @@ EXTERN_C void QCALLTYPE RhpSignalFinalizationComplete(uint32_t fcount) // // Fetch next object which needs finalization or return null if we've reached the end of the list. -FCIMPL0(OBJECTREF, RhpGetNextFinalizableObject) +COOP_PINVOKE_HELPER(OBJECTREF, RhpGetNextFinalizableObject, ()) { while (true) { @@ -216,4 +216,3 @@ FCIMPL0(OBJECTREF, RhpGetNextFinalizableObject) return refNext; } } -FCIMPLEND diff --git a/src/coreclr/nativeaot/Runtime/GCHelpers.cpp b/src/coreclr/nativeaot/Runtime/GCHelpers.cpp index 7f6b1a08627faa..2f2a088d507347 100644 --- a/src/coreclr/nativeaot/Runtime/GCHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/GCHelpers.cpp @@ -95,7 +95,7 @@ void MethodTable::InitializeAsGcFreeType() m_uBaseSize = sizeof(Array) + SYNC_BLOCK_SKEW; } -EXTERN_C void QCALLTYPE RhpCollect(uint32_t uGeneration, uint32_t uMode, UInt32_BOOL lowMemoryP) +EXTERN_C NATIVEAOT_API void __cdecl RhpCollect(uint32_t uGeneration, uint32_t uMode, UInt32_BOOL lowMemoryP) { // This must be called via p/invoke rather than RuntimeImport to make the stack crawlable. @@ -110,7 +110,7 @@ EXTERN_C void QCALLTYPE RhpCollect(uint32_t uGeneration, uint32_t uMode, UInt32_ pCurThread->EnablePreemptiveMode(); } -EXTERN_C int64_t QCALLTYPE RhpGetGcTotalMemory() +EXTERN_C NATIVEAOT_API int64_t __cdecl RhpGetGcTotalMemory() { // This must be called via p/invoke rather than RuntimeImport to make the stack crawlable. @@ -126,7 +126,7 @@ EXTERN_C int64_t QCALLTYPE RhpGetGcTotalMemory() return ret; } -EXTERN_C int32_t QCALLTYPE RhpStartNoGCRegion(int64_t totalSize, UInt32_BOOL hasLohSize, int64_t lohSize, UInt32_BOOL disallowFullBlockingGC) +EXTERN_C NATIVEAOT_API int32_t __cdecl RhpStartNoGCRegion(int64_t totalSize, UInt32_BOOL hasLohSize, int64_t lohSize, UInt32_BOOL disallowFullBlockingGC) { Thread *pCurThread = ThreadStore::GetCurrentThread(); ASSERT(!pCurThread->IsCurrentThreadInCooperativeMode()); @@ -141,154 +141,132 @@ EXTERN_C int32_t QCALLTYPE RhpStartNoGCRegion(int64_t totalSize, UInt32_BOOL has return result; } -EXTERN_C int32_t QCALLTYPE RhpEndNoGCRegion() +EXTERN_C NATIVEAOT_API int32_t __cdecl RhpEndNoGCRegion() { ASSERT(!ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode()); return GCHeapUtilities::GetGCHeap()->EndNoGCRegion(); } -FCIMPL1(void, RhSuppressFinalize, OBJECTREF refObj) +COOP_PINVOKE_HELPER(void, RhSuppressFinalize, (OBJECTREF refObj)) { if (!refObj->GetMethodTable()->HasFinalizer()) return; GCHeapUtilities::GetGCHeap()->SetFinalizationRun(refObj); } -FCIMPLEND -FCIMPL1(FC_BOOL_RET, RhReRegisterForFinalize, OBJECTREF refObj) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhReRegisterForFinalize, (OBJECTREF refObj)) { if (!refObj->GetMethodTable()->HasFinalizer()) FC_RETURN_BOOL(true); FC_RETURN_BOOL(GCHeapUtilities::GetGCHeap()->RegisterForFinalization(-1, refObj)); } -FCIMPLEND -FCIMPL0(int32_t, RhGetMaxGcGeneration) +COOP_PINVOKE_HELPER(int32_t, RhGetMaxGcGeneration, ()) { return GCHeapUtilities::GetGCHeap()->GetMaxGeneration(); } -FCIMPLEND -FCIMPL2(int32_t, RhGetGcCollectionCount, int32_t generation, CLR_BOOL getSpecialGCCount) +COOP_PINVOKE_HELPER(int32_t, RhGetGcCollectionCount, (int32_t generation, CLR_BOOL getSpecialGCCount)) { return GCHeapUtilities::GetGCHeap()->CollectionCount(generation, getSpecialGCCount); } -FCIMPLEND -FCIMPL1(int32_t, RhGetGeneration, OBJECTREF obj) +COOP_PINVOKE_HELPER(int32_t, RhGetGeneration, (OBJECTREF obj)) { return GCHeapUtilities::GetGCHeap()->WhichGeneration(obj); } -FCIMPLEND -FCIMPL1(int64_t, RhGetGenerationSize, int32_t gen) +COOP_PINVOKE_HELPER(int64_t, RhGetGenerationSize, (int32_t gen)) { return (int64_t)(GCHeapUtilities::GetGCHeap()->GetLastGCGenerationSize(gen)); } -FCIMPLEND -FCIMPL0(int64_t, RhGetLastGCPercentTimeInGC) +COOP_PINVOKE_HELPER(int64_t, RhGetLastGCPercentTimeInGC, ()) { return GCHeapUtilities::GetGCHeap()->GetLastGCPercentTimeInGC(); } -FCIMPLEND -FCIMPL0(int32_t, RhGetGcLatencyMode) +COOP_PINVOKE_HELPER(int32_t, RhGetGcLatencyMode, ()) { return GCHeapUtilities::GetGCHeap()->GetGcLatencyMode(); } -FCIMPLEND -FCIMPL1(int32_t, RhSetGcLatencyMode, int32_t newLatencyMode) +COOP_PINVOKE_HELPER(int32_t, RhSetGcLatencyMode, (int32_t newLatencyMode)) { return GCHeapUtilities::GetGCHeap()->SetGcLatencyMode(newLatencyMode); } -FCIMPLEND -FCIMPL1(FC_BOOL_RET, RhIsPromoted, OBJECTREF obj) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhIsPromoted, (OBJECTREF obj)) { FC_RETURN_BOOL(GCHeapUtilities::GetGCHeap()->IsPromoted(obj)); } -FCIMPLEND -FCIMPL0(FC_BOOL_RET, RhIsServerGc) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhIsServerGc, ()) { FC_RETURN_BOOL(GCHeapUtilities::IsServerHeap()); } -FCIMPLEND -FCIMPL2(FC_BOOL_RET, RhRegisterGcCallout, GcRestrictedCalloutKind eKind, void * pCallout) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhRegisterGcCallout, (GcRestrictedCalloutKind eKind, void * pCallout)) { FC_RETURN_BOOL(RestrictedCallouts::RegisterGcCallout(eKind, pCallout)); } -FCIMPLEND -FCIMPL2(void, RhUnregisterGcCallout, GcRestrictedCalloutKind eKind, void * pCallout) +COOP_PINVOKE_HELPER(void, RhUnregisterGcCallout, (GcRestrictedCalloutKind eKind, void * pCallout)) { RestrictedCallouts::UnregisterGcCallout(eKind, pCallout); } -FCIMPLEND #ifdef FEATURE_OBJCMARSHAL -FCIMPL1(FC_BOOL_RET, RhRegisterObjectiveCMarshalBeginEndCallback, void * pCallback) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhRegisterObjectiveCMarshalBeginEndCallback, (void * pCallback)) { FC_RETURN_BOOL(ObjCMarshalNative::RegisterBeginEndCallback(pCallback)); } -FCIMPLEND #endif -FCIMPL0(int32_t, RhGetLohCompactionMode) +COOP_PINVOKE_HELPER(int32_t, RhGetLohCompactionMode, ()) { return GCHeapUtilities::GetGCHeap()->GetLOHCompactionMode(); } -FCIMPLEND -FCIMPL1(void, RhSetLohCompactionMode, int32_t newLohCompactionMode) +COOP_PINVOKE_HELPER(void, RhSetLohCompactionMode, (int32_t newLohCompactionMode)) { GCHeapUtilities::GetGCHeap()->SetLOHCompactionMode(newLohCompactionMode); } -FCIMPLEND -FCIMPL0(int64_t, RhGetCurrentObjSize) +COOP_PINVOKE_HELPER(int64_t, RhGetCurrentObjSize, ()) { return GCHeapUtilities::GetGCHeap()->GetCurrentObjSize(); } -FCIMPLEND -FCIMPL0(int64_t, RhGetGCNow) +COOP_PINVOKE_HELPER(int64_t, RhGetGCNow, ()) { return GCHeapUtilities::GetGCHeap()->GetNow(); } -FCIMPLEND -FCIMPL1(int64_t, RhGetLastGCStartTime, int32_t generation) +COOP_PINVOKE_HELPER(int64_t, RhGetLastGCStartTime, (int32_t generation)) { return GCHeapUtilities::GetGCHeap()->GetLastGCStartTime(generation); } -FCIMPLEND -FCIMPL1(int64_t, RhGetLastGCDuration, int32_t generation) +COOP_PINVOKE_HELPER(int64_t, RhGetLastGCDuration, (int32_t generation)) { return GCHeapUtilities::GetGCHeap()->GetLastGCDuration(generation); } -FCIMPLEND -FCIMPL2(FC_BOOL_RET, RhRegisterForFullGCNotification, int32_t maxGenerationThreshold, int32_t largeObjectHeapThreshold) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhRegisterForFullGCNotification, (int32_t maxGenerationThreshold, int32_t largeObjectHeapThreshold)) { ASSERT(maxGenerationThreshold >= 1 && maxGenerationThreshold <= 99); ASSERT(largeObjectHeapThreshold >= 1 && largeObjectHeapThreshold <= 99); FC_RETURN_BOOL(GCHeapUtilities::GetGCHeap()->RegisterForFullGCNotification(maxGenerationThreshold, largeObjectHeapThreshold)); } -FCIMPLEND -FCIMPL0(FC_BOOL_RET, RhCancelFullGCNotification) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhCancelFullGCNotification, ()) { FC_RETURN_BOOL(GCHeapUtilities::GetGCHeap()->CancelFullGCNotification()); } -FCIMPLEND -FCIMPL1(int32_t, RhWaitForFullGCApproach, int32_t millisecondsTimeout) +COOP_PINVOKE_HELPER(int32_t, RhWaitForFullGCApproach, (int32_t millisecondsTimeout)) { ASSERT(millisecondsTimeout >= -1); ASSERT(ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode()); @@ -296,9 +274,8 @@ FCIMPL1(int32_t, RhWaitForFullGCApproach, int32_t millisecondsTimeout) int timeout = millisecondsTimeout == -1 ? INFINITE : millisecondsTimeout; return GCHeapUtilities::GetGCHeap()->WaitForFullGCApproach(millisecondsTimeout); } -FCIMPLEND -FCIMPL1(int32_t, RhWaitForFullGCComplete, int32_t millisecondsTimeout) +COOP_PINVOKE_HELPER(int32_t, RhWaitForFullGCComplete, (int32_t millisecondsTimeout)) { ASSERT(millisecondsTimeout >= -1); ASSERT(ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode()); @@ -306,25 +283,22 @@ FCIMPL1(int32_t, RhWaitForFullGCComplete, int32_t millisecondsTimeout) int timeout = millisecondsTimeout == -1 ? INFINITE : millisecondsTimeout; return GCHeapUtilities::GetGCHeap()->WaitForFullGCComplete(millisecondsTimeout); } -FCIMPLEND -FCIMPL0(int64_t, RhGetGCSegmentSize) +COOP_PINVOKE_HELPER(int64_t, RhGetGCSegmentSize, ()) { size_t first = GCHeapUtilities::GetGCHeap()->GetValidSegmentSize(true); size_t second = GCHeapUtilities::GetGCHeap()->GetValidSegmentSize(false); return (first > second) ? first : second; } -FCIMPLEND -FCIMPL0(int64_t, RhGetAllocatedBytesForCurrentThread) +COOP_PINVOKE_HELPER(int64_t, RhGetAllocatedBytesForCurrentThread, ()) { Thread *pThread = ThreadStore::GetCurrentThread(); gc_alloc_context *ac = pThread->GetAllocContext(); int64_t currentAllocated = ac->alloc_bytes + ac->alloc_bytes_uoh - (ac->alloc_limit - ac->alloc_ptr); return currentAllocated; } -FCIMPLEND struct RH_GC_GENERATION_INFO { @@ -360,7 +334,7 @@ struct RH_GH_MEMORY_INFO uint64_t pauseDuration1; }; -FCIMPL2(void, RhGetMemoryInfo, RH_GH_MEMORY_INFO* pData, int kind) +COOP_PINVOKE_HELPER(void, RhGetMemoryInfo, (RH_GH_MEMORY_INFO* pData, int kind)) { uint64_t* genInfoRaw = (uint64_t*)&(pData->generationInfo0); uint64_t* pauseInfoRaw = (uint64_t*)&(pData->pauseDuration0); @@ -384,7 +358,6 @@ FCIMPL2(void, RhGetMemoryInfo, RH_GH_MEMORY_INFO* pData, int kind) pauseInfoRaw, kind); } -FCIMPLEND // The MethodTable is remembered in some slow-path allocation paths. This value is used in event tracing. @@ -397,7 +370,7 @@ MethodTable* GetLastAllocEEType() return tls_pLastAllocationEEType; } -FCIMPL0(int64_t, RhGetTotalAllocatedBytes) +COOP_PINVOKE_HELPER(int64_t, RhGetTotalAllocatedBytes, ()) { uint64_t allocated_bytes = GCHeapUtilities::GetGCHeap()->GetTotalAllocatedBytes() - Thread::GetDeadThreadsNonAllocBytes(); @@ -416,9 +389,10 @@ FCIMPL0(int64_t, RhGetTotalAllocatedBytes) return current_high; } -FCIMPLEND -EXTERN_C void QCALLTYPE RhEnumerateConfigurationValues(void* configurationContext, ConfigurationValueFunc callback) +using EnumerateConfigurationValuesCallback = void (*)(void* context, void* name, void* publicKey, GCConfigurationType type, int64_t data); + +EXTERN_C NATIVEAOT_API void __cdecl RhEnumerateConfigurationValues(void* configurationContext, EnumerateConfigurationValuesCallback callback) { IGCHeap* pHeap = GCHeapUtilities::GetGCHeap(); pHeap->EnumerateConfigurationValues(configurationContext, callback); @@ -427,28 +401,27 @@ EXTERN_C void QCALLTYPE RhEnumerateConfigurationValues(void* configurationContex GCHeapHardLimitInfo g_gcHeapHardLimitInfo; bool g_gcHeapHardLimitInfoSpecified = false; -FCIMPL1(void, RhRefreshMemoryLimit, GCHeapHardLimitInfo heapHardLimitInfo) +EXTERN_C NATIVEAOT_API void __cdecl RhRefreshMemoryLimit(GCHeapHardLimitInfo heapHardLimitInfo) { IGCHeap* pHeap = GCHeapUtilities::GetGCHeap(); g_gcHeapHardLimitInfo = heapHardLimitInfo; g_gcHeapHardLimitInfoSpecified = true; pHeap->RefreshMemoryLimit(); } -FCIMPLEND -EXTERN_C uint64_t QCALLTYPE RhGetGenerationBudget(int generation) +EXTERN_C NATIVEAOT_API uint64_t __cdecl RhGetGenerationBudget(int generation) { IGCHeap* pHeap = GCHeapUtilities::GetGCHeap(); return pHeap->GetGenerationBudget(generation); } -EXTERN_C void QCALLTYPE RhEnableNoGCRegionCallback(NoGCRegionCallbackFinalizerWorkItem* pCallback, int64_t totalSize) +EXTERN_C NATIVEAOT_API void __cdecl RhEnableNoGCRegionCallback(NoGCRegionCallbackFinalizerWorkItem* pCallback, int64_t totalSize) { IGCHeap* pHeap = GCHeapUtilities::GetGCHeap(); pHeap->EnableNoGCRegionCallback(pCallback, totalSize); } -EXTERN_C int64_t QCALLTYPE RhGetTotalAllocatedBytesPrecise() +EXTERN_C NATIVEAOT_API int64_t __cdecl RhGetTotalAllocatedBytesPrecise() { int64_t allocated; @@ -562,7 +535,8 @@ static Object* GcAllocInternal(MethodTable* pEEType, uint32_t uFlags, uintptr_t // numElements - number of array elements // pTransitionFrame- transition frame to make stack crawlable // Returns a pointer to the object allocated or NULL on failure. -EXTERN_C void* F_CALL_CONV RhpGcAlloc(MethodTable* pEEType, uint32_t uFlags, uintptr_t numElements, PInvokeTransitionFrame* pTransitionFrame) + +COOP_PINVOKE_HELPER(void*, RhpGcAlloc, (MethodTable* pEEType, uint32_t uFlags, uintptr_t numElements, PInvokeTransitionFrame* pTransitionFrame)) { Thread* pThread = ThreadStore::GetCurrentThread(); @@ -597,7 +571,7 @@ EXTERN_C void* F_CALL_CONV RhpGcAlloc(MethodTable* pEEType, uint32_t uFlags, uin return GcAllocInternal(pEEType, uFlags, numElements, pThread); } -EXTERN_C void QCALLTYPE RhAllocateNewArray(MethodTable* pArrayEEType, uint32_t numElements, uint32_t flags, Array** pResult) +EXTERN_C NATIVEAOT_API void RhAllocateNewArray(MethodTable* pArrayEEType, uint32_t numElements, uint32_t flags, Array** pResult) { Thread* pThread = ThreadStore::GetCurrentThread(); @@ -611,7 +585,7 @@ EXTERN_C void QCALLTYPE RhAllocateNewArray(MethodTable* pArrayEEType, uint32_t n pThread->EnablePreemptiveMode(); } -EXTERN_C void QCALLTYPE RhAllocateNewObject(MethodTable* pEEType, uint32_t flags, Object** pResult) +EXTERN_C NATIVEAOT_API void RhAllocateNewObject(MethodTable* pEEType, uint32_t flags, Object** pResult) { Thread* pThread = ThreadStore::GetCurrentThread(); @@ -625,13 +599,12 @@ EXTERN_C void QCALLTYPE RhAllocateNewObject(MethodTable* pEEType, uint32_t flags pThread->EnablePreemptiveMode(); } -FCIMPL0(int64_t, RhGetTotalPauseDuration) +COOP_PINVOKE_HELPER(int64_t, RhGetTotalPauseDuration, ()) { return GCHeapUtilities::GetGCHeap()->GetTotalPauseDuration(); } -FCIMPLEND -FCIMPL1(void, RhRegisterForGCReporting, GCFrameRegistration* pRegistration) +COOP_PINVOKE_HELPER(void, RhRegisterForGCReporting, (GCFrameRegistration* pRegistration)) { Thread* pThread = ThreadStore::GetCurrentThread(); @@ -640,9 +613,8 @@ FCIMPL1(void, RhRegisterForGCReporting, GCFrameRegistration* pRegistration) pThread->PushGCFrameRegistration(pRegistration); } -FCIMPLEND -FCIMPL1(void, RhUnregisterForGCReporting, GCFrameRegistration* pRegistration) +COOP_PINVOKE_HELPER(void, RhUnregisterForGCReporting, (GCFrameRegistration* pRegistration)) { Thread* pThread = pRegistration->m_pThread; if (pThread == NULL) @@ -651,9 +623,8 @@ FCIMPL1(void, RhUnregisterForGCReporting, GCFrameRegistration* pRegistration) ASSERT(pThread == ThreadStore::GetCurrentThread()); pThread->PopGCFrameRegistration(pRegistration); } -FCIMPLEND -EXTERN_C void* QCALLTYPE RhRegisterFrozenSegment(void* pSection, size_t allocSize, size_t commitSize, size_t reservedSize) +EXTERN_C NATIVEAOT_API void* __cdecl RhRegisterFrozenSegment(void* pSection, size_t allocSize, size_t commitSize, size_t reservedSize) { ASSERT(allocSize <= commitSize); ASSERT(commitSize <= reservedSize); @@ -673,26 +644,25 @@ EXTERN_C void* QCALLTYPE RhRegisterFrozenSegment(void* pSection, size_t allocSiz #endif // FEATURE_BASICFREEZE } -EXTERN_C void QCALLTYPE RhUpdateFrozenSegment(void* pSegmentHandle, uint8_t* allocated, uint8_t* committed) +EXTERN_C NATIVEAOT_API void __cdecl RhUpdateFrozenSegment(void* pSegmentHandle, uint8_t* allocated, uint8_t* committed) { ASSERT(allocated <= committed); GCHeapUtilities::GetGCHeap()->UpdateFrozenSegment((segment_handle)pSegmentHandle, allocated, committed); } -EXTERN_C void QCALLTYPE RhUnregisterFrozenSegment(void* pSegmentHandle) +EXTERN_C NATIVEAOT_API void __cdecl RhUnregisterFrozenSegment(void* pSegmentHandle) { GCHeapUtilities::GetGCHeap()->UnregisterFrozenSegment((segment_handle)pSegmentHandle); } -FCIMPL1(uint32_t, RhGetGCDescSize, MethodTable* pMT) +COOP_PINVOKE_HELPER(uint32_t, RhGetGCDescSize, (MethodTable* pMT)) { if (!pMT->ContainsPointersOrCollectible()) return 0; return (uint32_t)CGCDesc::GetCGCDescFromMT(pMT)->GetSize(); } -FCIMPLEND #ifdef FEATURE_GC_STRESS @@ -701,7 +671,7 @@ EXTERN_C UInt32_BOOL g_fGcStressStarted; UInt32_BOOL g_fGcStressStarted = UInt32_FALSE; // UInt32_BOOL because asm code reads it // static -EXTERN_C void F_CALL_CONV RhpStressGc() +EXTERN_C void RhpStressGc() { // The GarbageCollect operation below may trash the last win32 error. We save the error here so that it can be // restored after the GC operation; @@ -716,9 +686,8 @@ EXTERN_C void F_CALL_CONV RhpStressGc() PalSetLastError(lastErrorOnEntry); } -FCIMPL0(void, RhpInitializeGcStress) +COOP_PINVOKE_HELPER(void, RhpInitializeGcStress, ()) { g_fGcStressStarted = UInt32_TRUE; } -FCIMPLEND #endif // FEATURE_GC_STRESS diff --git a/src/coreclr/nativeaot/Runtime/GCMemoryHelpers.cpp b/src/coreclr/nativeaot/Runtime/GCMemoryHelpers.cpp index 0154bd1fde5077..0f40aadbf05988 100644 --- a/src/coreclr/nativeaot/Runtime/GCMemoryHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/GCMemoryHelpers.cpp @@ -18,7 +18,7 @@ // in a read on another thread getting incorrect data. // Unaligned memory at the beginning and remaining bytes at the end are written bytewise. // USAGE: The caller is responsible for null-checking the reference. -FCIMPL2(void *, RhpGcSafeZeroMemory, void * mem, size_t size) +COOP_PINVOKE_CDECL_HELPER(void *, RhpGcSafeZeroMemory, (void * mem, size_t size)) { // The caller must do the null-check because we cannot take an AV in the runtime and translate it to managed. ASSERT(mem != nullptr); @@ -28,7 +28,6 @@ FCIMPL2(void *, RhpGcSafeZeroMemory, void * mem, size_t size) // memset returns the destination buffer return mem; } -FCIMPLEND #if defined(TARGET_X86) || defined(TARGET_AMD64) // @@ -42,7 +41,7 @@ FCIMPLEND // Move memory, in a way that is compatible with a move onto the heap, but // does not require the destination pointer to be on the heap. -FCIMPL3(void, RhBulkMoveWithWriteBarrier, uint8_t* pDest, uint8_t* pSrc, size_t cbDest) +COOP_PINVOKE_HELPER(void, RhBulkMoveWithWriteBarrier, (uint8_t* pDest, uint8_t* pSrc, size_t cbDest)) { // It is possible that the bulk write is publishing object references accessible so far only // by the current thread to shared memory. @@ -57,4 +56,3 @@ FCIMPL3(void, RhBulkMoveWithWriteBarrier, uint8_t* pDest, uint8_t* pSrc, size_t InlinedBulkWriteBarrier(pDest, cbDest); } -FCIMPLEND diff --git a/src/coreclr/nativeaot/Runtime/HandleTableHelpers.cpp b/src/coreclr/nativeaot/Runtime/HandleTableHelpers.cpp index 23e985357d343a..3f4b88e527159a 100644 --- a/src/coreclr/nativeaot/Runtime/HandleTableHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/HandleTableHelpers.cpp @@ -15,58 +15,49 @@ #include "gchandleutilities.h" -FCIMPL2(OBJECTHANDLE, RhpHandleAlloc, Object *pObject, int type) +COOP_PINVOKE_HELPER(OBJECTHANDLE, RhpHandleAlloc, (Object *pObject, int type)) { return GCHandleUtilities::GetGCHandleManager()->GetGlobalHandleStore()->CreateHandleOfType(pObject, (HandleType)type); } -FCIMPLEND -FCIMPL2(OBJECTHANDLE, RhpHandleAllocDependent, Object *pPrimary, Object *pSecondary) +COOP_PINVOKE_HELPER(OBJECTHANDLE, RhpHandleAllocDependent, (Object *pPrimary, Object *pSecondary)) { return GCHandleUtilities::GetGCHandleManager()->GetGlobalHandleStore()->CreateDependentHandle(pPrimary, pSecondary); } -FCIMPLEND -FCIMPL1(void, RhHandleFree, OBJECTHANDLE handle) +COOP_PINVOKE_HELPER(void, RhHandleFree, (OBJECTHANDLE handle)) { GCHandleUtilities::GetGCHandleManager()->DestroyHandleOfUnknownType(handle); } -FCIMPLEND -FCIMPL1(Object *, RhHandleGet, OBJECTHANDLE handle) +COOP_PINVOKE_HELPER(Object *, RhHandleGet, (OBJECTHANDLE handle)) { return ObjectFromHandle(handle); } -FCIMPLEND -FCIMPL2(Object *, RhHandleGetDependent, OBJECTHANDLE handle, Object **ppSecondary) +COOP_PINVOKE_HELPER(Object *, RhHandleGetDependent, (OBJECTHANDLE handle, Object **ppSecondary)) { Object *pPrimary = ObjectFromHandle(handle); *ppSecondary = (pPrimary != NULL) ? GetDependentHandleSecondary(handle) : NULL; return pPrimary; } -FCIMPLEND -FCIMPL2(void, RhHandleSetDependentSecondary, OBJECTHANDLE handle, Object *pSecondary) +COOP_PINVOKE_HELPER(void, RhHandleSetDependentSecondary, (OBJECTHANDLE handle, Object *pSecondary)) { SetDependentHandleSecondary(handle, pSecondary); } -FCIMPLEND -FCIMPL2(void, RhHandleSet, OBJECTHANDLE handle, Object *pObject) +COOP_PINVOKE_HELPER(void, RhHandleSet, (OBJECTHANDLE handle, Object *pObject)) { GCHandleUtilities::GetGCHandleManager()->StoreObjectInHandle(handle, pObject); } -FCIMPLEND -FCIMPL2(FC_BOOL_RET, RhRegisterRefCountedHandleCallback, void * pCallout, MethodTable * pTypeFilter) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhRegisterRefCountedHandleCallback, (void * pCallout, MethodTable * pTypeFilter)) { FC_RETURN_BOOL(RestrictedCallouts::RegisterRefCountedHandleCallback(pCallout, pTypeFilter)); } -FCIMPLEND -FCIMPL2(void, RhUnregisterRefCountedHandleCallback, void * pCallout, MethodTable * pTypeFilter) +COOP_PINVOKE_HELPER(void, RhUnregisterRefCountedHandleCallback, (void * pCallout, MethodTable * pTypeFilter)) { RestrictedCallouts::UnregisterRefCountedHandleCallback(pCallout, pTypeFilter); } -FCIMPLEND diff --git a/src/coreclr/nativeaot/Runtime/MathHelpers.cpp b/src/coreclr/nativeaot/Runtime/MathHelpers.cpp index 9c4220ef5a7eed..1d73305679168a 100644 --- a/src/coreclr/nativeaot/Runtime/MathHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/MathHelpers.cpp @@ -9,7 +9,7 @@ // Floating point and 64-bit integer math helpers. // -FCIMPL1_D(uint64_t, RhpDbl2ULng, double val) +EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpDbl2ULng(double val) { const double two63 = 2147483648.0 * 4294967296.0; uint64_t ret; @@ -24,13 +24,12 @@ FCIMPL1_D(uint64_t, RhpDbl2ULng, double val) } return ret; } -FCIMPLEND #undef min #undef max #include -FCIMPL2_FF(float, RhpFltRem, float dividend, float divisor) +EXTERN_C NATIVEAOT_API float REDHAWK_CALLCONV RhpFltRem(float dividend, float divisor) { // // From the ECMA standard: @@ -54,9 +53,8 @@ FCIMPL2_FF(float, RhpFltRem, float dividend, float divisor) // else... return fmodf(dividend,divisor); } -FCIMPLEND -FCIMPL2_DD(double, RhpDblRem, double dividend, double divisor) +EXTERN_C NATIVEAOT_API double REDHAWK_CALLCONV RhpDblRem(double dividend, double divisor) { // // From the ECMA standard: @@ -79,304 +77,104 @@ FCIMPL2_DD(double, RhpDblRem, double dividend, double divisor) // else... return(fmod(dividend,divisor)); } -FCIMPLEND -#ifndef HOST_64BIT -EXTERN_C int64_t QCALLTYPE RhpLDiv(int64_t i, int64_t j) +#ifdef HOST_ARM +EXTERN_C NATIVEAOT_API int32_t REDHAWK_CALLCONV RhpIDiv(int32_t i, int32_t j) { ASSERT(j && "Divide by zero!"); return i / j; } -EXTERN_C uint64_t QCALLTYPE RhpULDiv(uint64_t i, uint64_t j) +EXTERN_C NATIVEAOT_API uint32_t REDHAWK_CALLCONV RhpUDiv(uint32_t i, uint32_t j) { ASSERT(j && "Divide by zero!"); return i / j; } -EXTERN_C int64_t QCALLTYPE RhpLMod(int64_t i, int64_t j) +EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpLDiv(int64_t i, int64_t j) { ASSERT(j && "Divide by zero!"); - return i % j; + return i / j; } -EXTERN_C uint64_t QCALLTYPE RhpULMod(uint64_t i, uint64_t j) +EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpULDiv(uint64_t i, uint64_t j) { ASSERT(j && "Divide by zero!"); - return i % j; + return i / j; } -FCIMPL1_D(int64_t, RhpDbl2Lng, double val) +EXTERN_C NATIVEAOT_API int32_t REDHAWK_CALLCONV RhpIMod(int32_t i, int32_t j) { - return (int64_t)val; + ASSERT(j && "Divide by zero!"); + return i % j; } -FCIMPLEND -FCIMPL1_D(int32_t, RhpDbl2Int, double val) +EXTERN_C NATIVEAOT_API uint32_t REDHAWK_CALLCONV RhpUMod(uint32_t i, uint32_t j) { - return (int32_t)val; + ASSERT(j && "Divide by zero!"); + return i % j; } -FCIMPLEND -FCIMPL1_D(uint32_t, RhpDbl2UInt, double val) +EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpLMod(int64_t i, int64_t j) { - return (uint32_t)val; + ASSERT(j && "Divide by zero!"); + return i % j; } -FCIMPLEND -FCIMPL1_L(double, RhpLng2Dbl, int64_t val) +EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpULMod(uint64_t i, uint64_t j) { - return (double)val; + ASSERT(j && "Divide by zero!"); + return i % j; } -FCIMPLEND -FCIMPL1_L(double, RhpULng2Dbl, uint64_t val) +EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpLMul(int64_t i, int64_t j) { - return (double)val; + return i * j; } -FCIMPLEND -#endif +EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpULMul(uint64_t i, uint64_t j) +{ + return i * j; +} -#ifdef HOST_ARM -EXTERN_C int32_t F_CALL_CONV RhpIDiv(int32_t i, int32_t j) +EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpLRsz(uint64_t i, int32_t j) { - ASSERT(j && "Divide by zero!"); - return i / j; + return i >> (j & 0x3f); } -EXTERN_C uint32_t F_CALL_CONV RhpUDiv(uint32_t i, uint32_t j) +EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpLRsh(int64_t i, int32_t j) { - ASSERT(j && "Divide by zero!"); - return i / j; + return i >> (j & 0x3f); } -EXTERN_C int32_t F_CALL_CONV RhpIMod(int32_t i, int32_t j) +EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpLLsh(int64_t i, int32_t j) { - ASSERT(j && "Divide by zero!"); - return i % j; + return i << (j & 0x3f); } -EXTERN_C uint32_t F_CALL_CONV RhpUMod(uint32_t i, uint32_t j) +EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpDbl2Lng(double val) { - ASSERT(j && "Divide by zero!"); - return i % j; + return (int64_t)val; } -EXTERN_C int64_t F_CALL_CONV RhpLMul(int64_t i, int64_t j) +EXTERN_C NATIVEAOT_API int32_t REDHAWK_CALLCONV RhpDbl2Int(double val) { - return i * j; + return (int32_t)val; } -EXTERN_C uint64_t F_CALL_CONV RhpLRsz(uint64_t i, int32_t j) +EXTERN_C NATIVEAOT_API uint32_t REDHAWK_CALLCONV RhpDbl2UInt(double val) { - return i >> (j & 0x3f); + return (uint32_t)val; } -EXTERN_C int64_t F_CALL_CONV RhpLRsh(int64_t i, int32_t j) +EXTERN_C NATIVEAOT_API double REDHAWK_CALLCONV RhpLng2Dbl(int64_t val) { - return i >> (j & 0x3f); + return (double)val; } -EXTERN_C int64_t F_CALL_CONV RhpLLsh(int64_t i, int32_t j) +EXTERN_C NATIVEAOT_API double REDHAWK_CALLCONV RhpULng2Dbl(uint64_t val) { - return i << (j & 0x3f); + return (double)val; } #endif // HOST_ARM - -#ifdef HOST_X86 - -FCIMPL1_D(double, acos, double x) - return std::acos(x); -FCIMPLEND - -FCIMPL1_F(float, acosf, float x) - return std::acosf(x); -FCIMPLEND - -FCIMPL1_D(double, acosh, double x) - return std::acosh(x); -FCIMPLEND - -FCIMPL1_F(float, acoshf, float x) - return std::acoshf(x); -FCIMPLEND - -FCIMPL1_D(double, asin, double x) - return std::asin(x); -FCIMPLEND - -FCIMPL1_F(float, asinf, float x) - return std::asinf(x); -FCIMPLEND - -FCIMPL1_D(double, asinh, double x) - return std::asinh(x); -FCIMPLEND - -FCIMPL1_F(float, asinhf, float x) - return std::asinhf(x); -FCIMPLEND - -FCIMPL1_D(double, atan, double x) - return std::atan(x); -FCIMPLEND - -FCIMPL1_F(float, atanf, float x) - return std::atanf(x); -FCIMPLEND - -FCIMPL2_DD(double, atan2, double x, double y) - return std::atan2(x, y); -FCIMPLEND - -FCIMPL2_FF(float, atan2f, float x, float y) - return std::atan2f(x, y); -FCIMPLEND - -FCIMPL1_D(double, atanh, double x) - return std::atanh(x); -FCIMPLEND - -FCIMPL1_F(float, atanhf, float x) - return std::atanhf(x); -FCIMPLEND - -FCIMPL1_D(double, cbrt, double x) - return std::cbrt(x); -FCIMPLEND - -FCIMPL1_F(float, cbrtf, float x) - return std::cbrtf(x); -FCIMPLEND - -FCIMPL1_D(double, ceil, double x) - return std::ceil(x); -FCIMPLEND - -FCIMPL1_F(float, ceilf, float x) - return std::ceilf(x); -FCIMPLEND - -FCIMPL1_D(double, cos, double x) - return std::cos(x); -FCIMPLEND - -FCIMPL1_F(float, cosf, float x) - return std::cosf(x); -FCIMPLEND - -FCIMPL1_D(double, cosh, double x) - return std::cosh(x); -FCIMPLEND - -FCIMPL1_F(float, coshf, float x) - return std::coshf(x); -FCIMPLEND - -FCIMPL1_D(double, exp, double x) - return std::exp(x); -FCIMPLEND - -FCIMPL1_F(float, expf, float x) - return std::expf(x); -FCIMPLEND - -FCIMPL1_D(double, floor, double x) - return std::floor(x); -FCIMPLEND - -FCIMPL1_F(float, floorf, float x) - return std::floorf(x); -FCIMPLEND - -FCIMPL1_D(double, log, double x) - return std::log(x); -FCIMPLEND - -FCIMPL1_F(float, logf, float x) - return std::logf(x); -FCIMPLEND - -FCIMPL1_D(double, log2, double x) - return std::log2(x); -FCIMPLEND - -FCIMPL1_F(float, log2f, float x) - return std::log2f(x); -FCIMPLEND - -FCIMPL1_D(double, log10, double x) - return std::log10(x); -FCIMPLEND - -FCIMPL1_F(float, log10f, float x) - return std::log10f(x); -FCIMPLEND - -FCIMPL2_DD(double, pow, double x, double y) - return std::pow(x, y); -FCIMPLEND - -FCIMPL2_FF(float, powf, float x, float y) - return std::powf(x, y); -FCIMPLEND - -FCIMPL1_D(double, sin, double x) - return std::sin(x); -FCIMPLEND - -FCIMPL1_F(float, sinf, float x) - return std::sinf(x); -FCIMPLEND - -FCIMPL1_D(double, sinh, double x) - return std::sinh(x); -FCIMPLEND - -FCIMPL1_F(float, sinhf, float x) - return std::sinhf(x); -FCIMPLEND - -FCIMPL1_D(double, sqrt, double x) - return std::sqrt(x); -FCIMPLEND - -FCIMPL1_F(float, sqrtf, float x) - return std::sqrtf(x); -FCIMPLEND - -FCIMPL1_D(double, tan, double x) - return std::tan(x); -FCIMPLEND - -FCIMPL1_F(float, tanf, float x) - return std::tanf(x); -FCIMPLEND - -FCIMPL1_D(double, tanh, double x) - return std::tanh(x); -FCIMPLEND - -FCIMPL1_F(float, tanhf, float x) - return std::tanhf(x); -FCIMPLEND - -FCIMPL3_DDD(double, fma, double x, double y, double z) - return std::fma(x, y, z); -FCIMPLEND - -FCIMPL3_FFF(float, fmaf, float x, float y, float z) - return std::fmaf(x, y, z); -FCIMPLEND - -FCIMPL2_DI(double, modf, double x, double* intptr) - return std::modf(x, intptr); -FCIMPLEND - -FCIMPL2_FI(float, modff, float x, float* intptr) - return std::modff(x, intptr); -FCIMPLEND - -#endif diff --git a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp index 6189b606880105..eacc90297a69b4 100644 --- a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp @@ -37,14 +37,13 @@ #include "RhConfig.h" #include -FCIMPL0(void, RhDebugBreak) +COOP_PINVOKE_HELPER(void, RhDebugBreak, ()) { PalDebugBreak(); } -FCIMPLEND // Busy spin for the given number of iterations. -EXTERN_C void QCALLTYPE RhSpinWait(int32_t iterations) +EXTERN_C NATIVEAOT_API void __cdecl RhSpinWait(int32_t iterations) { ASSERT(iterations > 0); @@ -57,7 +56,7 @@ EXTERN_C void QCALLTYPE RhSpinWait(int32_t iterations) } // Yield the cpu to another thread ready to process, if one is available. -EXTERN_C UInt32_BOOL QCALLTYPE RhYield() +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhYield() { // This must be called via p/invoke -- it's a wait operation and we don't want to block thread suspension on this. ASSERT_MSG(!ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode(), @@ -66,7 +65,7 @@ EXTERN_C UInt32_BOOL QCALLTYPE RhYield() return PalSwitchToThread(); } -EXTERN_C void QCALLTYPE RhFlushProcessWriteBuffers() +EXTERN_C NATIVEAOT_API void __cdecl RhFlushProcessWriteBuffers() { // This must be called via p/invoke -- it's a wait operation and we don't want to block thread suspension on this. ASSERT_MSG(!ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode(), @@ -82,7 +81,7 @@ EXTERN_C void QCALLTYPE RhFlushProcessWriteBuffers() // modules are available based on the return count. It is also possible to call this method without an array, // in which case just the module count is returned (note that it's still possible for the module count to // increase between calls to this method). -FCIMPL1(uint32_t, RhGetLoadedOSModules, Array * pResultArray) +COOP_PINVOKE_HELPER(uint32_t, RhGetLoadedOSModules, (Array * pResultArray)) { // Note that we depend on the fact that this is a COOP helper to make writing into an unpinned array safe. @@ -108,9 +107,8 @@ FCIMPL1(uint32_t, RhGetLoadedOSModules, Array * pResultArray) return cModules; } -FCIMPLEND -FCIMPL1(HANDLE, RhGetOSModuleFromPointer, PTR_VOID pPointerVal) +COOP_PINVOKE_HELPER(HANDLE, RhGetOSModuleFromPointer, (PTR_VOID pPointerVal)) { ICodeManager * pCodeManager = GetRuntimeInstance()->GetCodeManagerForAddress(pPointerVal); @@ -119,9 +117,8 @@ FCIMPL1(HANDLE, RhGetOSModuleFromPointer, PTR_VOID pPointerVal) return NULL; } -FCIMPLEND -FCIMPL4(FC_BOOL_RET, RhFindBlob, TypeManagerHandle *pTypeManagerHandle, uint32_t blobId, uint8_t ** ppbBlob, uint32_t * pcbBlob) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhFindBlob, (TypeManagerHandle *pTypeManagerHandle, uint32_t blobId, uint8_t ** ppbBlob, uint32_t * pcbBlob)) { TypeManagerHandle typeManagerHandle = *pTypeManagerHandle; @@ -140,13 +137,11 @@ FCIMPL4(FC_BOOL_RET, RhFindBlob, TypeManagerHandle *pTypeManagerHandle, uint32_t FC_RETURN_BOOL(pBlob != NULL); } -FCIMPLEND -FCIMPL1(void *, RhGetTargetOfUnboxingAndInstantiatingStub, void * pUnboxStub) +COOP_PINVOKE_HELPER(void *, RhGetTargetOfUnboxingAndInstantiatingStub, (void * pUnboxStub)) { return GetRuntimeInstance()->GetTargetOfUnboxingAndInstantiatingStub(pUnboxStub); } -FCIMPLEND #if TARGET_ARM //***************************************************************************** @@ -198,7 +193,7 @@ inline int32_t GetThumb2BlRel24(uint16_t * p) // Given a pointer to code, find out if this points to an import stub // or unboxing stub, and if so, return the address that stub jumps to -FCIMPL1(uint8_t *, RhGetCodeTarget, uint8_t * pCodeOrg) +COOP_PINVOKE_HELPER(uint8_t *, RhGetCodeTarget, (uint8_t * pCodeOrg)) { bool unboxingStub = false; @@ -341,16 +336,15 @@ FCIMPL1(uint8_t *, RhGetCodeTarget, uint8_t * pCodeOrg) return pCodeOrg; } -FCIMPLEND -EXTERN_C uint64_t QCALLTYPE RhpGetTickCount64() +EXTERN_C NATIVEAOT_API uint64_t __cdecl RhpGetTickCount64() { return PalGetTickCount64(); } -EXTERN_C int32_t QCALLTYPE RhpCalculateStackTraceWorker(void* pOutputBuffer, uint32_t outputBufferLength, void* pAddressInCurrentFrame); +EXTERN_C int32_t __cdecl RhpCalculateStackTraceWorker(void* pOutputBuffer, uint32_t outputBufferLength, void* pAddressInCurrentFrame); -EXTERN_C int32_t QCALLTYPE RhpGetCurrentThreadStackTrace(void* pOutputBuffer, uint32_t outputBufferLength, void* pAddressInCurrentFrame) +EXTERN_C NATIVEAOT_API int32_t __cdecl RhpGetCurrentThreadStackTrace(void* pOutputBuffer, uint32_t outputBufferLength, void* pAddressInCurrentFrame) { // This must be called via p/invoke rather than RuntimeImport to make the stack crawlable. @@ -359,7 +353,7 @@ EXTERN_C int32_t QCALLTYPE RhpGetCurrentThreadStackTrace(void* pOutputBuffer, ui return RhpCalculateStackTraceWorker(pOutputBuffer, outputBufferLength, pAddressInCurrentFrame); } -FCIMPL2(FC_BOOL_RET, RhCompareObjectContentsAndPadding, Object* pObj1, Object* pObj2) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhCompareObjectContentsAndPadding, (Object* pObj1, Object* pObj2)) { ASSERT(pObj1->GetMethodTable() == pObj2->GetMethodTable()); ASSERT(pObj1->GetMethodTable()->IsValueType()); @@ -373,45 +367,39 @@ FCIMPL2(FC_BOOL_RET, RhCompareObjectContentsAndPadding, Object* pObj1, Object* p // memcmp is ok in this COOP method as we are comparing structs which are typically small. FC_RETURN_BOOL(memcmp(pbFields1, pbFields2, cbFields) == 0); } -FCIMPLEND -FCIMPL3(void*, RhpGetModuleSection, TypeManagerHandle *pModule, int32_t headerId, int32_t* length) +COOP_PINVOKE_HELPER(void*, RhpGetModuleSection, (TypeManagerHandle *pModule, int32_t headerId, int32_t* length)) { return pModule->AsTypeManager()->GetModuleSection((ReadyToRunSectionType)headerId, length); } -FCIMPLEND -FCIMPL2(void, RhGetCurrentThreadStackBounds, PTR_VOID * ppStackLow, PTR_VOID * ppStackHigh) +COOP_PINVOKE_HELPER(void, RhGetCurrentThreadStackBounds, (PTR_VOID * ppStackLow, PTR_VOID * ppStackHigh)) { ThreadStore::GetCurrentThread()->GetStackBounds(ppStackLow, ppStackHigh); } -FCIMPLEND // Function to call when a thread is detached from the runtime ThreadExitCallback g_threadExitCallback; -FCIMPL1(void, RhSetThreadExitCallback, void * pCallback) +COOP_PINVOKE_HELPER(void, RhSetThreadExitCallback, (void * pCallback)) { g_threadExitCallback = (ThreadExitCallback)pCallback; } -FCIMPLEND -FCIMPL0(int32_t, RhGetProcessCpuCount) +COOP_PINVOKE_HELPER(int32_t, RhGetProcessCpuCount, ()) { return PalGetProcessCpuCount(); } -FCIMPLEND -FCIMPL2(uint32_t, RhGetKnobValues, char *** pResultKeys, char *** pResultValues) +COOP_PINVOKE_HELPER(uint32_t, RhGetKnobValues, (char *** pResultKeys, char *** pResultValues)) { *pResultKeys = g_pRhConfig->GetKnobNames(); *pResultValues = g_pRhConfig->GetKnobValues(); return g_pRhConfig->GetKnobCount(); } -FCIMPLEND #if defined(TARGET_X86) || defined(TARGET_AMD64) -EXTERN_C void QCALLTYPE RhCpuIdEx(int* cpuInfo, int functionId, int subFunctionId) +EXTERN_C NATIVEAOT_API void __cdecl RhCpuIdEx(int* cpuInfo, int functionId, int subFunctionId) { __cpuidex(cpuInfo, functionId, subFunctionId); } diff --git a/src/coreclr/nativeaot/Runtime/RestrictedCallouts.h b/src/coreclr/nativeaot/Runtime/RestrictedCallouts.h index 2c1a2e61e0951b..40eaf8395baed6 100644 --- a/src/coreclr/nativeaot/Runtime/RestrictedCallouts.h +++ b/src/coreclr/nativeaot/Runtime/RestrictedCallouts.h @@ -97,6 +97,6 @@ class RestrictedCallouts static CrstStatic s_sLock; // Prototypes for the callouts. - typedef void (F_CALL_CONV * GcRestrictedCallbackFunction)(uint32_t uiCondemnedGeneration); - typedef CLR_BOOL (* HandleTableRestrictedCallbackFunction)(Object * pObject); + typedef void (REDHAWK_CALLCONV * GcRestrictedCallbackFunction)(uint32_t uiCondemnedGeneration); + typedef CLR_BOOL (REDHAWK_CALLCONV * HandleTableRestrictedCallbackFunction)(Object * pObject); }; diff --git a/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp b/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp index 708356c2e3a909..fedc3989eba761 100644 --- a/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp +++ b/src/coreclr/nativeaot/Runtime/RuntimeInstance.cpp @@ -43,30 +43,27 @@ ThreadStore * RuntimeInstance::GetThreadStore() return m_pThreadStore; } -FCIMPL1(uint8_t *, RhGetCrashInfoBuffer, int32_t* pcbMaxSize) +COOP_PINVOKE_HELPER(uint8_t *, RhGetCrashInfoBuffer, (int32_t* pcbMaxSize)) { *pcbMaxSize = MAX_CRASHINFOBUFFER_SIZE; return g_CrashInfoBuffer; } -FCIMPLEND #if TARGET_UNIX #include "PalCreateDump.h" -FCIMPL2(void, RhCreateCrashDumpIfEnabled, PEXCEPTION_RECORD pExceptionRecord, PCONTEXT pExContext) +COOP_PINVOKE_HELPER(void, RhCreateCrashDumpIfEnabled, (PEXCEPTION_RECORD pExceptionRecord, PCONTEXT pExContext)) { PalCreateCrashDumpIfEnabled(pExceptionRecord, pExContext); } -FCIMPLEND #endif -FCIMPL1(uint8_t *, RhGetRuntimeVersion, int32_t* pcbLength) +COOP_PINVOKE_HELPER(uint8_t *, RhGetRuntimeVersion, (int32_t* pcbLength)) { *pcbLength = sizeof(CLR_PRODUCT_VERSION) - 1; // don't include the terminating null return (uint8_t*)&CLR_PRODUCT_VERSION; } -FCIMPLEND -FCIMPL1(uint8_t *, RhFindMethodStartAddress, void * codeAddr) +COOP_PINVOKE_HELPER(uint8_t *, RhFindMethodStartAddress, (void * codeAddr)) { uint8_t *startAddress = dac_cast(GetRuntimeInstance()->FindMethodStartAddress(dac_cast(codeAddr))); #if TARGET_ARM @@ -75,7 +72,6 @@ FCIMPL1(uint8_t *, RhFindMethodStartAddress, void * codeAddr) return startAddress; #endif } -FCIMPLEND PTR_uint8_t RuntimeInstance::FindMethodStartAddress(PTR_VOID ControlPC) { @@ -273,16 +269,15 @@ bool RuntimeInstance::RegisterTypeManager(TypeManager * pTypeManager) return true; } -FCIMPL4(TypeManagerHandle, RhpCreateTypeManager, HANDLE osModule, void* pModuleHeader, PTR_PTR_VOID pClasslibFunctions, uint32_t nClasslibFunctions) +COOP_PINVOKE_HELPER(TypeManagerHandle, RhpCreateTypeManager, (HANDLE osModule, void* pModuleHeader, PTR_PTR_VOID pClasslibFunctions, uint32_t nClasslibFunctions)) { TypeManager * typeManager = TypeManager::Create(osModule, pModuleHeader, pClasslibFunctions, nClasslibFunctions); GetRuntimeInstance()->RegisterTypeManager(typeManager); return TypeManagerHandle::Create(typeManager); } -FCIMPLEND -FCIMPL1(void*, RhpRegisterOsModule, HANDLE hOsModule) +COOP_PINVOKE_HELPER(void*, RhpRegisterOsModule, (HANDLE hOsModule)) { RuntimeInstance::OsModuleEntry * pEntry = new (nothrow) RuntimeInstance::OsModuleEntry(); if (NULL == pEntry) @@ -294,7 +289,6 @@ FCIMPL1(void*, RhpRegisterOsModule, HANDLE hOsModule) return hOsModule; // Return non-null on success } -FCIMPLEND RuntimeInstance::TypeManagerList& RuntimeInstance::GetTypeManagerList() { @@ -350,9 +344,9 @@ bool RuntimeInstance::ShouldHijackCallsiteForGcStress(uintptr_t CallsiteIP) } #ifdef FEATURE_CACHED_INTERFACE_DISPATCH -EXTERN_C void F_CALL_CONV RhpInitialDynamicInterfaceDispatch(); +EXTERN_C void RhpInitialDynamicInterfaceDispatch(); -FCIMPL2(void *, RhNewInterfaceDispatchCell, MethodTable * pInterface, int32_t slotNumber) +COOP_PINVOKE_HELPER(void *, RhNewInterfaceDispatchCell, (MethodTable * pInterface, int32_t slotNumber)) { InterfaceDispatchCell * pCell = new (nothrow) InterfaceDispatchCell[2]; if (pCell == NULL) @@ -370,7 +364,6 @@ FCIMPL2(void *, RhNewInterfaceDispatchCell, MethodTable * pInterface, int32_t sl return pCell; } -FCIMPLEND #endif // FEATURE_CACHED_INTERFACE_DISPATCH #endif // DACCESS_COMPILE diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp index f6d560fa295111..63da57ba78c3c7 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp @@ -41,6 +41,9 @@ EXTERN_C CODE_LOCATION ReturnFromUniversalTransition; EXTERN_C CODE_LOCATION ReturnFromUniversalTransition_DebugStepTailCall; #endif +#ifdef TARGET_X86 +EXTERN_C CODE_LOCATION RhpCallFunclet2; +#endif EXTERN_C CODE_LOCATION RhpCallCatchFunclet2; EXTERN_C CODE_LOCATION RhpCallFinallyFunclet2; EXTERN_C CODE_LOCATION RhpCallFilterFunclet2; @@ -776,6 +779,20 @@ void StackFrameIterator::UnwindFuncletInvokeThunk() PTR_uintptr_t SP; +#ifdef TARGET_X86 + // First, unwind RhpCallFunclet + SP = (PTR_uintptr_t)(m_RegDisplay.SP + 0x4); // skip the saved assembly-routine-EBP + m_RegDisplay.SetIP(*SP++); + m_RegDisplay.SetSP((uintptr_t)dac_cast(SP)); + SetControlPC(dac_cast(m_RegDisplay.GetIP())); + + ASSERT( + EQUALS_RETURN_ADDRESS(m_ControlPC, RhpCallCatchFunclet2) || + EQUALS_RETURN_ADDRESS(m_ControlPC, RhpCallFinallyFunclet2) || + EQUALS_RETURN_ADDRESS(m_ControlPC, RhpCallFilterFunclet2) + ); +#endif + bool isFilterInvoke = EQUALS_RETURN_ADDRESS(m_ControlPC, RhpCallFilterFunclet2); #if defined(UNIX_AMD64_ABI) @@ -859,7 +876,7 @@ void StackFrameIterator::UnwindFuncletInvokeThunk() m_RegDisplay.pR15 = SP++; #elif defined(TARGET_X86) - SP = (PTR_uintptr_t)(m_RegDisplay.SP + 0x4); // skip the saved assembly-routine-EBP + SP = (PTR_uintptr_t)(m_RegDisplay.SP); if (!isFilterInvoke) { @@ -1792,6 +1809,25 @@ StackFrameIterator::ReturnAddressCategory StackFrameIterator::CategorizeUnadjust return InThrowSiteThunk; } +#ifdef TARGET_X86 + if (EQUALS_RETURN_ADDRESS(returnAddress, RhpCallFunclet2)) + { + PORTABILITY_ASSERT("CategorizeUnadjustedReturnAddress"); +#if 0 + // See if it is a filter funclet based on the caller of RhpCallFunclet + PTR_uintptr_t SP = (PTR_uintptr_t)(m_RegDisplay.SP + 0x4); // skip the saved assembly-routine-EBP + PTR_uintptr_t ControlPC = *SP++; + if (EQUALS_RETURN_ADDRESS(ControlPC, RhpCallFilterFunclet2)) + { + return InFilterFuncletInvokeThunk; + } + else +#endif + { + return InFuncletInvokeThunk; + } + } +#else // TARGET_X86 if (EQUALS_RETURN_ADDRESS(returnAddress, RhpCallCatchFunclet2) || EQUALS_RETURN_ADDRESS(returnAddress, RhpCallFinallyFunclet2)) { @@ -1802,13 +1838,14 @@ StackFrameIterator::ReturnAddressCategory StackFrameIterator::CategorizeUnadjust { return InFilterFuncletInvokeThunk; } +#endif // TARGET_X86 return InManagedCode; #endif // defined(USE_PORTABLE_HELPERS) } #ifndef DACCESS_COMPILE -bool StackFrameIterator::Init(PAL_LIMITED_CONTEXT* pStackwalkCtx, bool instructionFault) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhpSfiInit, (StackFrameIterator* pThis, PAL_LIMITED_CONTEXT* pStackwalkCtx, CLR_BOOL instructionFault, CLR_BOOL* pfIsExceptionIntercepted)) { Thread * pCurThread = ThreadStore::GetCurrentThread(); @@ -1820,38 +1857,41 @@ bool StackFrameIterator::Init(PAL_LIMITED_CONTEXT* pStackwalkCtx, bool instructi // Passing NULL is a special-case to request a standard managed stack trace for the current thread. if (pStackwalkCtx == NULL) - InternalInitForStackTrace(); + pThis->InternalInitForStackTrace(); else - InternalInitForEH(pCurThread, pStackwalkCtx, instructionFault); + pThis->InternalInitForEH(pCurThread, pStackwalkCtx, instructionFault); - bool isValid = IsValid(); + bool isValid = pThis->IsValid(); if (isValid) - CalculateCurrentMethodState(); + pThis->CalculateCurrentMethodState(); - return isValid; + if (pfIsExceptionIntercepted) + { + *pfIsExceptionIntercepted = false; + } + + FC_RETURN_BOOL(isValid); } -bool StackFrameIterator::Next(uint32_t* puExCollideClauseIdx, bool* pfUnwoundReversePInvoke) +COOP_PINVOKE_HELPER(FC_BOOL_RET, RhpSfiNext, (StackFrameIterator* pThis, uint32_t* puExCollideClauseIdx, CLR_BOOL* pfUnwoundReversePInvoke, CLR_BOOL* pfIsExceptionIntercepted)) { - Thread * pCurThread = ThreadStore::GetCurrentThread(); - // The stackwalker is intolerant to hijacked threads, as it is largely expecting to be called from C++ // where the hijack state of the thread is invariant. Because we've exposed the iterator out to C#, we // need to unhijack every time we callback into C++ because the thread could have been hijacked during our // time executing C#. - pCurThread->Unhijack(); + ThreadStore::GetCurrentThread()->Unhijack(); const uint32_t MaxTryRegionIdx = 0xFFFFFFFF; - ExInfo * pCurExInfo = m_pNextExInfo; - Next(); - bool isValid = IsValid(); + ExInfo * pCurExInfo = pThis->m_pNextExInfo; + pThis->Next(); + bool isValid = pThis->IsValid(); if (isValid) - CalculateCurrentMethodState(); + pThis->CalculateCurrentMethodState(); if (puExCollideClauseIdx != NULL) { - if (m_dwFlags & StackFrameIterator::ExCollide) + if (pThis->m_dwFlags & StackFrameIterator::ExCollide) { ASSERT(pCurExInfo->m_idxCurClause != MaxTryRegionIdx); *puExCollideClauseIdx = pCurExInfo->m_idxCurClause; @@ -1865,29 +1905,9 @@ bool StackFrameIterator::Next(uint32_t* puExCollideClauseIdx, bool* pfUnwoundRev if (pfUnwoundReversePInvoke != NULL) { - *pfUnwoundReversePInvoke = (m_dwFlags & StackFrameIterator::UnwoundReversePInvoke) != 0; + *pfUnwoundReversePInvoke = (pThis->m_dwFlags & StackFrameIterator::UnwoundReversePInvoke) != 0; } - return isValid; -} - -FCIMPL4(FC_BOOL_RET, RhpSfiInit, StackFrameIterator* pThis, PAL_LIMITED_CONTEXT* pStackwalkCtx, CLR_BOOL instructionFault, CLR_BOOL* pfIsExceptionIntercepted) -{ - bool isValid = pThis->Init(pStackwalkCtx, instructionFault); - - if (pfIsExceptionIntercepted) - { - *pfIsExceptionIntercepted = false; - } - - FC_RETURN_BOOL(isValid); -} -FCIMPLEND - -FCIMPL4(FC_BOOL_RET, RhpSfiNext, StackFrameIterator* pThis, uint32_t* puExCollideClauseIdx, CLR_BOOL* pfUnwoundReversePInvoke, CLR_BOOL* pfIsExceptionIntercepted) -{ - bool isValid = pThis->Next(puExCollideClauseIdx, pfUnwoundReversePInvoke); - if (pfIsExceptionIntercepted) { *pfIsExceptionIntercepted = false; @@ -1895,6 +1915,5 @@ FCIMPL4(FC_BOOL_RET, RhpSfiNext, StackFrameIterator* pThis, uint32_t* puExCollid FC_RETURN_BOOL(isValid); } -FCIMPLEND #endif // !DACCESS_COMPILE diff --git a/src/coreclr/nativeaot/Runtime/StackFrameIterator.h b/src/coreclr/nativeaot/Runtime/StackFrameIterator.h index 10b4c2bd65bc51..d102cda234d8f8 100644 --- a/src/coreclr/nativeaot/Runtime/StackFrameIterator.h +++ b/src/coreclr/nativeaot/Runtime/StackFrameIterator.h @@ -28,6 +28,8 @@ struct EHEnum }; class StackFrameIterator; +EXTERN_C FC_BOOL_RET FASTCALL RhpSfiInit(StackFrameIterator* pThis, PAL_LIMITED_CONTEXT* pStackwalkCtx, CLR_BOOL instructionFault, CLR_BOOL* pfIsExceptionIntercepted); +EXTERN_C FC_BOOL_RET FASTCALL RhpSfiNext(StackFrameIterator* pThis, uint32_t* puExCollideClauseIdx, CLR_BOOL* pfUnwoundReversePInvoke, CLR_BOOL* pfIsExceptionIntercepted); struct PInvokeTransitionFrame; typedef DPTR(PInvokeTransitionFrame) PTR_PInvokeTransitionFrame; @@ -36,6 +38,8 @@ typedef DPTR(PAL_LIMITED_CONTEXT) PTR_PAL_LIMITED_CONTEXT; class StackFrameIterator { friend class AsmOffsets; + friend FC_BOOL_RET FASTCALL RhpSfiInit(StackFrameIterator* pThis, PAL_LIMITED_CONTEXT* pStackwalkCtx, CLR_BOOL instructionFault, CLR_BOOL* pfIsExceptionIntercepted); + friend FC_BOOL_RET FASTCALL RhpSfiNext(StackFrameIterator* pThis, uint32_t* puExCollideClauseIdx, CLR_BOOL* pfUnwoundReversePInvoke, CLR_BOOL* pfIsExceptionIntercepted); public: StackFrameIterator() {} @@ -65,10 +69,6 @@ class StackFrameIterator bool HasStackRangeToReportConservatively(); void GetStackRangeToReportConservatively(PTR_OBJECTREF * ppLowerBound, PTR_OBJECTREF * ppUpperBound); - // Implementations of RhpSfiInit and RhpSfiNext called from managed code - bool Init(PAL_LIMITED_CONTEXT* pStackwalkCtx, bool instructionFault); - bool Next(uint32_t* puExCollideClauseIdx, bool* pfUnwoundReversePInvoke); - private: // The invoke of a funclet is a bit special and requires an assembly thunk, but we don't want to break the // stackwalk due to this. So this routine will unwind through the assembly thunks used to invoke funclets. diff --git a/src/coreclr/nativeaot/Runtime/ThunksMapping.cpp b/src/coreclr/nativeaot/Runtime/ThunksMapping.cpp index d22f30e19d9e0c..025a20ab917619 100644 --- a/src/coreclr/nativeaot/Runtime/ThunksMapping.cpp +++ b/src/coreclr/nativeaot/Runtime/ThunksMapping.cpp @@ -55,48 +55,42 @@ void EncodeThumb2Mov32(uint16_t * pCode, uint32_t value, uint8_t rDestination) } #endif -FCIMPL0(int, RhpGetNumThunkBlocksPerMapping) +COOP_PINVOKE_HELPER(int, RhpGetNumThunkBlocksPerMapping, ()) { ASSERT_MSG((THUNKS_MAP_SIZE % OS_PAGE_SIZE) == 0, "Thunks map size should be in multiples of pages"); return THUNKS_MAP_SIZE / OS_PAGE_SIZE; } -FCIMPLEND -FCIMPL0(int, RhpGetNumThunksPerBlock) +COOP_PINVOKE_HELPER(int, RhpGetNumThunksPerBlock, ()) { return min( OS_PAGE_SIZE / THUNK_SIZE, // Number of thunks that can fit in a page (OS_PAGE_SIZE - POINTER_SIZE) / (POINTER_SIZE * 2) // Number of pointer pairs, minus the jump stub cell, that can fit in a page ); } -FCIMPLEND -FCIMPL0(int, RhpGetThunkSize) +COOP_PINVOKE_HELPER(int, RhpGetThunkSize, ()) { return THUNK_SIZE; } -FCIMPLEND -FCIMPL1(void*, RhpGetThunkDataBlockAddress, void* pThunkStubAddress) +COOP_PINVOKE_HELPER(void*, RhpGetThunkDataBlockAddress, (void* pThunkStubAddress)) { return (void*)(((uintptr_t)pThunkStubAddress & ~(OS_PAGE_SIZE - 1)) + THUNKS_MAP_SIZE); } -FCIMPLEND -FCIMPL1(void*, RhpGetThunkStubsBlockAddress, void* pThunkDataAddress) +COOP_PINVOKE_HELPER(void*, RhpGetThunkStubsBlockAddress, (void* pThunkDataAddress)) { return (void*)(((uintptr_t)pThunkDataAddress & ~(OS_PAGE_SIZE - 1)) - THUNKS_MAP_SIZE); } -FCIMPLEND -FCIMPL0(int, RhpGetThunkBlockSize) +COOP_PINVOKE_HELPER(int, RhpGetThunkBlockSize, ()) { return OS_PAGE_SIZE; } -FCIMPLEND -EXTERN_C void* QCALLTYPE RhAllocateThunksMapping() +EXTERN_C NATIVEAOT_API void* __cdecl RhAllocateThunksMapping() { #ifdef WIN32 @@ -266,13 +260,13 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping() extern "C" uintptr_t g_pThunkStubData; uintptr_t g_pThunkStubData = NULL; -FCDECL0(int, RhpGetThunkBlockCount); -FCDECL0(int, RhpGetNumThunkBlocksPerMapping); -FCDECL0(int, RhpGetThunkBlockSize); -FCDECL1(void*, RhpGetThunkDataBlockAddress, void* addr); -FCDECL1(void*, RhpGetThunkStubsBlockAddress, void* addr); +COOP_PINVOKE_HELPER(int, RhpGetThunkBlockCount, ()); +COOP_PINVOKE_HELPER(int, RhpGetNumThunkBlocksPerMapping, ()); +COOP_PINVOKE_HELPER(int, RhpGetThunkBlockSize, ()); +COOP_PINVOKE_HELPER(void*, RhpGetThunkDataBlockAddress, (void* addr)); +COOP_PINVOKE_HELPER(void*, RhpGetThunkStubsBlockAddress, (void* addr)); -EXTERN_C void* QCALLTYPE RhAllocateThunksMapping() +EXTERN_C NATIVEAOT_API void* __cdecl RhAllocateThunksMapping() { static int nextThunkDataMapping = 0; @@ -319,13 +313,13 @@ EXTERN_C void* QCALLTYPE RhAllocateThunksMapping() #else // FEATURE_FIXED_POOL_THUNKS -FCDECL0(void*, RhpGetThunksBase); -FCDECL0(int, RhpGetNumThunkBlocksPerMapping); -FCDECL0(int, RhpGetNumThunksPerBlock); -FCDECL0(int, RhpGetThunkSize); -FCDECL0(int, RhpGetThunkBlockSize); +COOP_PINVOKE_HELPER(void*, RhpGetThunksBase, ()); +COOP_PINVOKE_HELPER(int, RhpGetNumThunkBlocksPerMapping, ()); +COOP_PINVOKE_HELPER(int, RhpGetNumThunksPerBlock, ()); +COOP_PINVOKE_HELPER(int, RhpGetThunkSize, ()); +COOP_PINVOKE_HELPER(int, RhpGetThunkBlockSize, ()); -EXTERN_C void* QCALLTYPE RhAllocateThunksMapping() +EXTERN_C NATIVEAOT_API void* __cdecl RhAllocateThunksMapping() { static void* pThunksTemplateAddress = NULL; diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S index 8078be2a9a80d3..cc740a9c0601f7 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S +++ b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.S @@ -258,10 +258,7 @@ LEAF_END RhpCheckedXchg, _TEXT // // On exit: // rdi, rsi are incremented by 8, -// rcx, rax: trashed -// -// NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -// if you add more trashed registers. +// rcx, r10, r11: trashed // // WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: // - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 @@ -283,15 +280,16 @@ ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 UPDATE_GC_SHADOW BASENAME, rcx, rdi #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - cmp qword ptr [C_VAR(g_write_watch_table)], 0x0 + mov r11, [C_VAR(g_write_watch_table)] + cmp r11, 0x0 je LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable) - mov rax, rdi - shr rax, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, [C_VAR(g_write_watch_table)] - cmp byte ptr [rax], 0x0 + mov r10, rdi + shr r10, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift + add r10, r11 + cmp byte ptr [r10], 0x0 jne LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable) - mov byte ptr [rax], 0xFF + mov byte ptr [r10], 0xFF #endif LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable): @@ -311,12 +309,12 @@ LOCAL_LABEL(RhpByRefAssignRef_CheckCardTable): // an entire byte in the card table since it's quicker than messing around with bitmasks and we only write // the byte if it hasn't already been done since writes are expensive and impact scaling. shr rcx, 0x0B - mov rax, [C_VAR(g_card_table)] - cmp byte ptr [rcx + rax], 0x0FF + mov r10, [C_VAR(g_card_table)] + cmp byte ptr [rcx + r10], 0x0FF je LOCAL_LABEL(RhpByRefAssignRef_NoBarrierRequired) // We get here if it's necessary to update the card table. - mov byte ptr [rcx + rax], 0xFF + mov byte ptr [rcx + r10], 0xFF #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES // Shift rcx by 0x0A more to get the card bundle byte (we shifted by 0x0B already) diff --git a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm index 7fdf76f0fd8320..148aa7d1301b1c 100644 --- a/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm +++ b/src/coreclr/nativeaot/Runtime/amd64/WriteBarriers.asm @@ -274,10 +274,7 @@ LEAF_END RhpCheckedXchg, _TEXT ;; ;; On exit: ;; rdi, rsi are incremented by 8, -;; rcx, rax: trashed -;; -;; NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -;; if you add more trashed registers. +;; rcx, r10, r11: trashed ;; ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1/2 @@ -299,15 +296,16 @@ ALTERNATE_ENTRY RhpByRefAssignRefAVLocation2 UPDATE_GC_SHADOW BASENAME, rcx, rdi ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP - cmp [g_write_watch_table], 0 + mov r11, [g_write_watch_table] + cmp r11, 0 je RhpByRefAssignRef_CheckCardTable - mov rax, rdi - shr rax, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, [g_write_watch_table] - cmp byte ptr [rax], 0 + mov r10, rdi + shr r10, 0Ch ;; SoftwareWriteWatch::AddressToTableByteIndexShift + add r10, r11 + cmp byte ptr [r10], 0 jne RhpByRefAssignRef_CheckCardTable - mov byte ptr [rax], 0FFh + mov byte ptr [r10], 0FFh endif RhpByRefAssignRef_CheckCardTable: @@ -327,12 +325,12 @@ RhpByRefAssignRef_CheckCardTable: ;; an entire byte in the card table since it's quicker than messing around with bitmasks and we only write ;; the byte if it hasn't already been done since writes are expensive and impact scaling. shr rcx, 0Bh - mov rax, [g_card_table] - cmp byte ptr [rcx + rax], 0FFh + mov r10, [g_card_table] + cmp byte ptr [rcx + r10], 0FFh je RhpByRefAssignRef_NoBarrierRequired ;; We get here if it's necessary to update the card table. - mov byte ptr [rcx + rax], 0FFh + mov byte ptr [rcx + r10], 0FFh ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES ;; Shift rcx by 0Ah more to get the card bundle byte (we shifted by 0Bh already) diff --git a/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S b/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S index 966b052a2b9f9e..79ffed2b05210d 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S +++ b/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S @@ -90,11 +90,9 @@ LOCAL_LABEL(RhpNewFast_RarePath): // Set the new objects MethodTable pointer on success. cbz x0, LOCAL_LABEL(NewOutOfMemory) - .cfi_remember_state POP_COOP_PINVOKE_FRAME EPILOG_RETURN - .cfi_restore_state LOCAL_LABEL(NewOutOfMemory): // This is the OOM failure path. We are going to tail-call to a managed helper that will throw // an out of memory exception that the caller of this allocator understands. @@ -264,11 +262,9 @@ LOCAL_LABEL(RhpNewArray_Rare): // Set the new objects MethodTable pointer and length on success. cbz x0, LOCAL_LABEL(ArrayOutOfMemory) - .cfi_remember_state POP_COOP_PINVOKE_FRAME EPILOG_RETURN - .cfi_restore_state LOCAL_LABEL(ArrayOutOfMemory): // This is the OOM failure path. We are going to tail-call to a managed helper that will throw // an out of memory exception that the caller of this allocator understands. diff --git a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S index 8075335ea0b2bc..abe7555b761134 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S +++ b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S @@ -146,11 +146,8 @@ NESTED_ENTRY RhpWaitForGC, _TEXT, NoHandler ldr x2, [sp, #OFFSETOF__PInvokeTransitionFrame__m_Flags] tbnz x2, #PTFF_THREAD_ABORT_BIT, LOCAL_LABEL(ThrowThreadAbort) - .cfi_remember_state POP_PROBE_FRAME EPILOG_RETURN - - .cfi_restore_state LOCAL_LABEL(ThrowThreadAbort): POP_PROBE_FRAME mov w0, #STATUS_REDHAWK_THREAD_ABORT diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S index 275ae9401dca72..835466c3b9e7e4 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S +++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S @@ -192,9 +192,6 @@ // x15 : trashed // x12, x17 : trashed // -// NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -// if you add more trashed registers. -// // WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: // - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1 // - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm index 1389921eff5440..26a8ef30387c5e 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm +++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.asm @@ -206,9 +206,6 @@ INVALIDGCVALUE EQU 0xCCCCCCCD ;; x15 : trashed ;; x12, x17 : trashed ;; -;; NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -;; if you add more trashed registers. -;; ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at RhpByRefAssignRefAVLocation1 ;; - Function "UnwindSimpleHelperToCaller" assumes no registers were pushed and LR contains the return address diff --git a/src/coreclr/nativeaot/Runtime/disabledeventpipeinternal.cpp b/src/coreclr/nativeaot/Runtime/disabledeventpipeinternal.cpp index 6c48533b05c1ea..4f34808b1c569e 100644 --- a/src/coreclr/nativeaot/Runtime/disabledeventpipeinternal.cpp +++ b/src/coreclr/nativeaot/Runtime/disabledeventpipeinternal.cpp @@ -13,7 +13,7 @@ struct EventPipeEventInstanceData; struct EventPipeSessionInfo; -EXTERN_C uint64_t QCALLTYPE EventPipeInternal_Enable( +EXTERN_C NATIVEAOT_API uint64_t __cdecl RhEventPipeInternal_Enable( const WCHAR* outputFile, EventPipeSerializationFormat format, uint32_t circularBufferSizeInMB, @@ -23,11 +23,11 @@ EXTERN_C uint64_t QCALLTYPE EventPipeInternal_Enable( return 0; } -EXTERN_C void QCALLTYPE EventPipeInternal_Disable(uint64_t sessionID) +EXTERN_C NATIVEAOT_API void __cdecl RhEventPipeInternal_Disable(uint64_t sessionID) { } -EXTERN_C intptr_t QCALLTYPE EventPipeInternal_CreateProvider( +EXTERN_C NATIVEAOT_API intptr_t __cdecl RhEventPipeInternal_CreateProvider( const WCHAR* providerName, EventPipeCallback pCallbackFunc, void* pCallbackContext) @@ -35,7 +35,7 @@ EXTERN_C intptr_t QCALLTYPE EventPipeInternal_CreateProvider( return 0; } -EXTERN_C intptr_t QCALLTYPE EventPipeInternal_DefineEvent( +EXTERN_C NATIVEAOT_API intptr_t __cdecl RhEventPipeInternal_DefineEvent( intptr_t provHandle, uint32_t eventID, int64_t keywords, @@ -47,21 +47,21 @@ EXTERN_C intptr_t QCALLTYPE EventPipeInternal_DefineEvent( return 0; } -EXTERN_C intptr_t QCALLTYPE EventPipeInternal_GetProvider(const WCHAR* providerName) +EXTERN_C NATIVEAOT_API intptr_t __cdecl RhEventPipeInternal_GetProvider(const WCHAR* providerName) { return 0; } -EXTERN_C void QCALLTYPE EventPipeInternal_DeleteProvider(intptr_t provHandle) +EXTERN_C NATIVEAOT_API void __cdecl RhEventPipeInternal_DeleteProvider(intptr_t provHandle) { } -EXTERN_C int QCALLTYPE EventPipeInternal_EventActivityIdControl(uint32_t controlCode, GUID *pActivityId) +EXTERN_C NATIVEAOT_API int __cdecl RhEventPipeInternal_EventActivityIdControl(uint32_t controlCode, GUID *pActivityId) { return 0; } -EXTERN_C void QCALLTYPE EventPipeInternal_WriteEventData( +EXTERN_C NATIVEAOT_API void __cdecl RhEventPipeInternal_WriteEventData( intptr_t eventHandle, EventData *pEventData, uint32_t eventDataCount, @@ -70,22 +70,22 @@ EXTERN_C void QCALLTYPE EventPipeInternal_WriteEventData( { } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_GetSessionInfo(uint64_t sessionID, EventPipeSessionInfo *pSessionInfo) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_GetSessionInfo(uint64_t sessionID, EventPipeSessionInfo *pSessionInfo) { return FALSE; } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_GetNextEvent(uint64_t sessionID, EventPipeEventInstanceData *pInstance) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_GetNextEvent(uint64_t sessionID, EventPipeEventInstanceData *pInstance) { return FALSE; } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_SignalSession(uint64_t sessionID) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_SignalSession(uint64_t sessionID) { return FALSE; } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_WaitForSessionSignal(uint64_t sessionID, int32_t timeoutMs) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_WaitForSessionSignal(uint64_t sessionID, int32_t timeoutMs) { return FALSE; } diff --git a/src/coreclr/nativeaot/Runtime/disabledruntimeeventinternal.cpp b/src/coreclr/nativeaot/Runtime/disabledruntimeeventinternal.cpp index f590021e6ff924..17a8010ec1cde8 100644 --- a/src/coreclr/nativeaot/Runtime/disabledruntimeeventinternal.cpp +++ b/src/coreclr/nativeaot/Runtime/disabledruntimeeventinternal.cpp @@ -9,43 +9,43 @@ // We will do a no-op for events in the disabled EventPipe This is similar to the way eventpipe checks if the provider and an event is enabled before firting the event, and no-op otherwise. -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogContentionLockCreated(intptr_t LockID, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogContentionLockCreated(intptr_t LockID, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogContentionStart(uint8_t ContentionFlags, uint16_t ClrInstanceID, intptr_t LockID, intptr_t AssociatedObjectID, uint64_t LockOwnerThreadID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogContentionStart(uint8_t ContentionFlags, uint16_t ClrInstanceID, intptr_t LockID, intptr_t AssociatedObjectID, uint64_t LockOwnerThreadID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(uint32_t activeWorkerThreadCount, uint32_t retiredWorkerThreadCount, uint16_t clrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(uint32_t activeWorkerThreadCount, uint32_t retiredWorkerThreadCount, uint16_t clrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(uint16_t MinWorkerThreads, uint16_t MaxWorkerThreads, uint16_t MinIOCompletionThreads, uint16_t MaxIOCompletionThreads, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(uint16_t MinWorkerThreads, uint16_t MaxWorkerThreads, uint16_t MinIOCompletionThreads, uint16_t MaxIOCompletionThreads, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint32_t NewWorkerThreadCount, uint32_t Reason, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint32_t NewWorkerThreadCount, uint32_t Reason, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats( +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats( double Duration, double Throughput, double ThreadPoolWorkerThreadWait, @@ -60,7 +60,7 @@ EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjust { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue( +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolIOEnqueue( void * NativeOverlapped, void * Overlapped, bool MultiDequeues, @@ -68,27 +68,27 @@ EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue( { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIODequeue(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolIODequeue(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(uint32_t Count, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(uint32_t Count, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOPack(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolIOPack(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogExceptionThrown(const WCHAR* exceptionTypeName, const WCHAR* exceptionMessage, void* faultingIP, HRESULT hresult) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogExceptionThrown(const WCHAR* exceptionTypeName, const WCHAR* exceptionMessage, void* faultingIP, HRESULT hresult) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogWaitHandleWaitStart(uint8_t WaitSource, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogWaitHandleWaitStart(uint8_t WaitSource, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) { } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogWaitHandleWaitStop(uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogWaitHandleWaitStop(uint16_t ClrInstanceID) { } diff --git a/src/coreclr/nativeaot/Runtime/eventpipeinternal.cpp b/src/coreclr/nativeaot/Runtime/eventpipeinternal.cpp index 3febb68b73e785..0e4f52a80b379f 100644 --- a/src/coreclr/nativeaot/Runtime/eventpipeinternal.cpp +++ b/src/coreclr/nativeaot/Runtime/eventpipeinternal.cpp @@ -43,7 +43,7 @@ struct EventPipeProviderConfigurationNative WCHAR *pFilterData; }; -EXTERN_C uint64_t QCALLTYPE EventPipeInternal_Enable( +EXTERN_C NATIVEAOT_API uint64_t __cdecl RhEventPipeInternal_Enable( const WCHAR* outputFile, EventPipeSerializationFormat format, uint32_t circularBufferSizeInMB, @@ -104,12 +104,12 @@ EXTERN_C uint64_t QCALLTYPE EventPipeInternal_Enable( return result; } -EXTERN_C void QCALLTYPE EventPipeInternal_Disable(uint64_t sessionID) +EXTERN_C NATIVEAOT_API void __cdecl RhEventPipeInternal_Disable(uint64_t sessionID) { ep_disable(sessionID); } -EXTERN_C intptr_t QCALLTYPE EventPipeInternal_CreateProvider( +EXTERN_C NATIVEAOT_API intptr_t __cdecl RhEventPipeInternal_CreateProvider( const WCHAR* providerName, EventPipeCallback pCallbackFunc, void* pCallbackContext) @@ -120,7 +120,7 @@ EXTERN_C intptr_t QCALLTYPE EventPipeInternal_CreateProvider( return reinterpret_cast(pProvider); } -EXTERN_C intptr_t QCALLTYPE EventPipeInternal_DefineEvent( +EXTERN_C NATIVEAOT_API intptr_t __cdecl RhEventPipeInternal_DefineEvent( intptr_t provHandle, uint32_t eventID, int64_t keywords, @@ -139,7 +139,7 @@ EXTERN_C intptr_t QCALLTYPE EventPipeInternal_DefineEvent( return reinterpret_cast(pEvent); } -EXTERN_C intptr_t QCALLTYPE EventPipeInternal_GetProvider(const WCHAR* providerName) +EXTERN_C NATIVEAOT_API intptr_t __cdecl RhEventPipeInternal_GetProvider(const WCHAR* providerName) { EventPipeProvider * provider = NULL; if (providerName) @@ -152,7 +152,7 @@ EXTERN_C intptr_t QCALLTYPE EventPipeInternal_GetProvider(const WCHAR* providerN return reinterpret_cast(provider); } -EXTERN_C void QCALLTYPE EventPipeInternal_DeleteProvider(intptr_t provHandle) +EXTERN_C NATIVEAOT_API void __cdecl RhEventPipeInternal_DeleteProvider(intptr_t provHandle) { if (provHandle != 0) { @@ -172,7 +172,7 @@ enum class ActivityControlCode EVENT_ACTIVITY_CONTROL_CREATE_SET_ID = 5 }; -EXTERN_C int QCALLTYPE EventPipeInternal_EventActivityIdControl(uint32_t controlCode, GUID *pActivityId) +EXTERN_C NATIVEAOT_API int __cdecl RhEventPipeInternal_EventActivityIdControl(uint32_t controlCode, GUID *pActivityId) { int retVal = 0; ep_rt_thread_activity_id_handle_t activityIdHandle = ep_thread_get_activity_id_handle (); @@ -224,7 +224,7 @@ EXTERN_C int QCALLTYPE EventPipeInternal_EventActivityIdControl(uint32_t control return retVal; } -EXTERN_C void QCALLTYPE EventPipeInternal_WriteEventData( +EXTERN_C NATIVEAOT_API void __cdecl RhEventPipeInternal_WriteEventData( intptr_t eventHandle, EventData *pEventData, uint32_t eventDataCount, @@ -236,7 +236,7 @@ EXTERN_C void QCALLTYPE EventPipeInternal_WriteEventData( ep_write_event_2(pEvent, pEventData, eventDataCount, reinterpret_cast(pActivityId), reinterpret_cast(pRelatedActivityId)); } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_GetSessionInfo(uint64_t sessionID, EventPipeSessionInfo *pSessionInfo) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_GetSessionInfo(uint64_t sessionID, EventPipeSessionInfo *pSessionInfo) { bool retVal = false; if (pSessionInfo != NULL) @@ -253,7 +253,7 @@ EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_GetSessionInfo(uint64_t session return retVal; } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_GetNextEvent(uint64_t sessionID, EventPipeEventInstanceData *pInstance) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_GetNextEvent(uint64_t sessionID, EventPipeEventInstanceData *pInstance) { EventPipeEventInstance *pNextInstance = NULL; _ASSERTE(pInstance != NULL); @@ -274,7 +274,7 @@ EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_GetNextEvent(uint64_t sessionID return pNextInstance != NULL; } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_SignalSession(uint64_t sessionID) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_SignalSession(uint64_t sessionID) { EventPipeSession *const session = ep_get_session (sessionID); if (!session) @@ -283,7 +283,7 @@ EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_SignalSession(uint64_t sessionI return ep_rt_wait_event_set (ep_session_get_wait_event (session)); } -EXTERN_C UInt32_BOOL QCALLTYPE EventPipeInternal_WaitForSessionSignal(uint64_t sessionID, int32_t timeoutMs) +EXTERN_C NATIVEAOT_API UInt32_BOOL __cdecl RhEventPipeInternal_WaitForSessionSignal(uint64_t sessionID, int32_t timeoutMs) { EventPipeSession *const session = ep_get_session (sessionID); if (!session) diff --git a/src/coreclr/nativeaot/Runtime/gcheaputilities.cpp b/src/coreclr/nativeaot/Runtime/gcheaputilities.cpp index 2678b12c1aea0b..42f7928cd9e252 100644 --- a/src/coreclr/nativeaot/Runtime/gcheaputilities.cpp +++ b/src/coreclr/nativeaot/Runtime/gcheaputilities.cpp @@ -38,7 +38,7 @@ GPTR_IMPL(GcDacVars, g_gcDacGlobals); // GC entrypoints for the linked-in GC. These symbols are invoked // directly if we are not using a standalone GC. -extern "C" HRESULT LOCALGC_CALLCONV GC_Initialize( +extern "C" HRESULT GC_Initialize( /* In */ IGCToCLR* clrToGC, /* Out */ IGCHeap** gcHeap, /* Out */ IGCHandleManager** gcHandleManager, diff --git a/src/coreclr/nativeaot/Runtime/i386/AsmMacros.inc b/src/coreclr/nativeaot/Runtime/i386/AsmMacros.inc index 896bf8e67dab53..b7f6554993cd19 100644 --- a/src/coreclr/nativeaot/Runtime/i386/AsmMacros.inc +++ b/src/coreclr/nativeaot/Runtime/i386/AsmMacros.inc @@ -10,9 +10,6 @@ include AsmOffsets.inc ; generated by the build from AsmOffsets.cpp FASTCALL_FUNC macro FuncName,cbArgs FuncNameReal EQU @&FuncName&@&cbArgs FuncNameReal proc public - FuncName label proc - PUBLIC FuncName - endm FASTCALL_ENDFUNC macro @@ -21,8 +18,18 @@ endm ALTERNATE_ENTRY macro Name -Name label proc -PUBLIC Name +decoratedName TEXTEQU @CatStr( _, Name ) ) + +decoratedName label proc +PUBLIC decoratedName + endm + +LABELED_RETURN_ADDRESS macro Name + +decoratedName TEXTEQU @CatStr( _, Name ) ) + +decoratedName label proc +PUBLIC decoratedName endm __tls_array equ 2Ch ;; offsetof(TEB, ThreadLocalStoragePointer) @@ -127,7 +134,7 @@ PTFF_SAVE_RAX equ 00000100h ;; RAX is saved if it contains a GC ref PTFF_SAVE_ALL_SCRATCH equ 00000700h PTFF_RAX_IS_GCREF equ 00010000h ;; iff PTFF_SAVE_RAX: set -> eax is Object, clear -> eax is scalar PTFF_RAX_IS_BYREF equ 00020000h ;; iff PTFF_SAVE_RAX: set -> eax is ByRef, clear -> eax is Object or scalar -PTFF_THREAD_ABORT equ 00100000h ;; indicates that ThreadAbortException should be thrown when returning from the transition +PTFF_THREAD_ABORT equ 00040000h ;; indicates that ThreadAbortException should be thrown when returning from the transition ;; These must match the TrapThreadsFlags enum TrapThreadsFlags_None equ 0 @@ -156,11 +163,6 @@ G_EPHEMERAL_HIGH equ _g_ephemeral_high G_CARD_TABLE equ _g_card_table RhpWaitForGC2 equ @RhpWaitForGC2@4 RhpTrapThreads equ _RhpTrapThreads -RhpStressGc equ @RhpStressGc@0 -RhpGcPoll2 equ @RhpGcPoll2@4 -RhHandleGet equ @RhHandleGet@4 -RhpGcSafeZeroMemory equ @RhpGcSafeZeroMemory@8 -RhpGetNumThunkBlocksPerMapping equ @RhpGetNumThunkBlocksPerMapping@0 ifdef FEATURE_GC_STRESS THREAD__HIJACKFORGCSTRESS equ ?HijackForGcStress@Thread@@SGXPAUPAL_LIMITED_CONTEXT@@@Z @@ -176,17 +178,6 @@ EXTERN RhExceptionHandling_FailedAllocation : PROC EXTERN RhThrowHwEx : PROC EXTERN RhThrowEx : PROC EXTERN RhRethrow : PROC -EXTERN RhpGcPoll2 : PROC - -;; The following imports are not used in the assembly helpers. Due to the -;; way the FCall mangling is handled in the C sources through linker directives -;; (see FCIMPL macro definitions in CommonMacros.h), we need to add dummy -;; references to some methods in few object files (HandleTableHelpers and -;; GCMemoryHelpers, ThunkMapping) to get the linker to see the #pragma -;; comment(linker, ...) directives embedded in those files. -EXTERN RhHandleGet : PROC -EXTERN RhpGcSafeZeroMemory : PROC -EXTERN RhpGetNumThunkBlocksPerMapping : PROC ifdef FEATURE_GC_STRESS EXTERN THREAD__HIJACKFORGCSTRESS : PROC diff --git a/src/coreclr/nativeaot/Runtime/i386/AsmOffsetsCpu.h b/src/coreclr/nativeaot/Runtime/i386/AsmOffsetsCpu.h index ad428db1250cef..7e19e77b7e885c 100644 --- a/src/coreclr/nativeaot/Runtime/i386/AsmOffsetsCpu.h +++ b/src/coreclr/nativeaot/Runtime/i386/AsmOffsetsCpu.h @@ -7,7 +7,7 @@ // // NOTE: the offsets MUST be in hex notation WITHOUT the 0x prefix -PLAT_ASM_SIZEOF(c0, ExInfo) +PLAT_ASM_SIZEOF(bc, ExInfo) PLAT_ASM_OFFSET(0, ExInfo, m_pPrevExInfo) PLAT_ASM_OFFSET(4, ExInfo, m_pExContext) PLAT_ASM_OFFSET(8, ExInfo, m_exception) @@ -15,7 +15,7 @@ PLAT_ASM_OFFSET(0c, ExInfo, m_kind) PLAT_ASM_OFFSET(0d, ExInfo, m_passNumber) PLAT_ASM_OFFSET(10, ExInfo, m_idxCurClause) PLAT_ASM_OFFSET(14, ExInfo, m_frameIter) -PLAT_ASM_OFFSET(bc, ExInfo, m_notifyDebuggerSP) +PLAT_ASM_OFFSET(b8, ExInfo, m_notifyDebuggerSP) PLAT_ASM_OFFSET(0, PInvokeTransitionFrame, m_RIP) PLAT_ASM_OFFSET(4, PInvokeTransitionFrame, m_FramePointer) @@ -23,12 +23,12 @@ PLAT_ASM_OFFSET(8, PInvokeTransitionFrame, m_pThread) PLAT_ASM_OFFSET(0c, PInvokeTransitionFrame, m_Flags) PLAT_ASM_OFFSET(10, PInvokeTransitionFrame, m_PreservedRegs) -PLAT_ASM_SIZEOF(a8, StackFrameIterator) +PLAT_ASM_SIZEOF(a4, StackFrameIterator) PLAT_ASM_OFFSET(08, StackFrameIterator, m_FramePointer) PLAT_ASM_OFFSET(0c, StackFrameIterator, m_ControlPC) PLAT_ASM_OFFSET(10, StackFrameIterator, m_RegDisplay) -PLAT_ASM_OFFSET(a0, StackFrameIterator, m_OriginalControlPC) -PLAT_ASM_OFFSET(a4, StackFrameIterator, m_pPreviousTransitionFrame) +PLAT_ASM_OFFSET(9c, StackFrameIterator, m_OriginalControlPC) +PLAT_ASM_OFFSET(a0, StackFrameIterator, m_pPreviousTransitionFrame) PLAT_ASM_SIZEOF(1c, PAL_LIMITED_CONTEXT) PLAT_ASM_OFFSET(0, PAL_LIMITED_CONTEXT, IP) @@ -40,7 +40,7 @@ PLAT_ASM_OFFSET(10, PAL_LIMITED_CONTEXT, Rsi) PLAT_ASM_OFFSET(14, PAL_LIMITED_CONTEXT, Rax) PLAT_ASM_OFFSET(18, PAL_LIMITED_CONTEXT, Rbx) -PLAT_ASM_SIZEOF(28, REGDISPLAY) +PLAT_ASM_SIZEOF(24, REGDISPLAY) PLAT_ASM_OFFSET(1c, REGDISPLAY, SP) PLAT_ASM_OFFSET(0c, REGDISPLAY, pRbx) diff --git a/src/coreclr/nativeaot/Runtime/i386/ExceptionHandling.asm b/src/coreclr/nativeaot/Runtime/i386/ExceptionHandling.asm index 2172d3982d1821..127c1b617b8f81 100644 --- a/src/coreclr/nativeaot/Runtime/i386/ExceptionHandling.asm +++ b/src/coreclr/nativeaot/Runtime/i386/ExceptionHandling.asm @@ -10,18 +10,10 @@ include AsmMacros.inc -;; input: ECX: possible exception object -;; EDX: funclet IP -;; EAX: funclet EBP -CALL_FUNCLET macro SUFFIX - push ebp - mov ebp, eax - mov eax, ecx - call edx -ALTERNATE_ENTRY _RhpCall&SUFFIX&Funclet2 - pop ebp -endm +RhpCallFunclet equ @RhpCallFunclet@0 +RhpThrowHwEx equ @RhpThrowHwEx@0 +extern RhpCallFunclet : proc ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; @@ -33,7 +25,7 @@ endm ;; OUTPUT: ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -FASTCALL_FUNC RhpThrowHwEx, 8 +FASTCALL_FUNC RhpThrowHwEx, 0 esp_offsetof_ExInfo textequ %0 esp_offsetof_Context textequ %SIZEOF__ExInfo @@ -82,7 +74,7 @@ FASTCALL_FUNC RhpThrowHwEx, 8 ;; edx contains the address of the ExInfo call RhThrowHwEx -ALTERNATE_ENTRY _RhpThrowHwEx2 +ALTERNATE_ENTRY RhpThrowHwEx2 ;; no return int 3 @@ -98,7 +90,7 @@ FASTCALL_ENDFUNC ;; OUTPUT: ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -FASTCALL_FUNC RhpThrowEx, 4 +FASTCALL_FUNC RhpThrowEx, 0 esp_offsetof_ExInfo textequ %0 esp_offsetof_Context textequ %SIZEOF__ExInfo @@ -159,7 +151,7 @@ FASTCALL_FUNC RhpThrowEx, 4 ;; edx contains the address of the ExInfo call RhThrowEx -ALTERNATE_ENTRY _RhpThrowEx2 +ALTERNATE_ENTRY RhpThrowEx2 ;; no return int 3 @@ -179,6 +171,7 @@ FASTCALL_ENDFUNC ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; FASTCALL_FUNC RhpRethrow, 0 + esp_offsetof_ExInfo textequ %0 esp_offsetof_Context textequ %SIZEOF__ExInfo @@ -273,14 +266,13 @@ endm ;; ;; INPUT: ECX: exception object ;; EDX: handler funclet address -;; [ESP + 4]: ExInfo* -;; [ESP + 8]: REGDISPLAY* -;; (CLR calling convention switches the last two parameters!) +;; [ESP + 4]: REGDISPLAY* +;; [ESP + 8]: ExInfo* ;; ;; OUTPUT: ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -FASTCALL_FUNC RhpCallCatchFunclet, 16 +FASTCALL_FUNC RhpCallCatchFunclet, 0 FUNCLET_CALL_PROLOGUE 2 @@ -291,8 +283,8 @@ FASTCALL_FUNC RhpCallCatchFunclet, 16 ;; [esp + 10h]: ebx save esp_offsetof_PrevEBP textequ %14h ;; [esp + 14h]: prev ebp esp_offsetof_RetAddr textequ %18h ;; [esp + 18h]: return address - esp_offsetof_RegDisplay textequ %20h ;; [esp + 20h]: REGDISPLAY* - esp_offsetof_ExInfo textequ %1ch ;; [esp + 1ch]: ExInfo* + esp_offsetof_RegDisplay textequ %1ch ;; [esp + 1Ch]: REGDISPLAY* + esp_offsetof_ExInfo textequ %20h ;; [esp + 20h]: ExInfo* ;; Clear the DoNotTriggerGc state before calling out to our managed catch funclet. INLINE_GETTHREAD eax, ebx ;; eax <- Thread*, ebx is trashed @@ -321,7 +313,9 @@ FASTCALL_FUNC RhpCallCatchFunclet, 16 ;; ECX still contains the exception object ;; EDX: funclet IP ;; EAX: funclet EBP - CALL_FUNCLET Catch + call RhpCallFunclet + +ALTERNATE_ENTRY RhpCallCatchFunclet2 ;; eax: resume IP mov [esp + esp_offsetof_ResumeIP], eax ;; save for later @@ -385,7 +379,7 @@ FASTCALL_ENDFUNC ;; OUTPUT: ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -FASTCALL_FUNC RhpCallFinallyFunclet, 8 +FASTCALL_FUNC RhpCallFinallyFunclet, 0 FUNCLET_CALL_PROLOGUE 0 @@ -415,7 +409,9 @@ FASTCALL_FUNC RhpCallFinallyFunclet, 8 ;; ECX: not used ;; EDX: funclet IP ;; EAX: funclet EBP - CALL_FUNCLET Finally + call RhpCallFunclet + +ALTERNATE_ENTRY RhpCallFinallyFunclet2 pop edx ;; restore REGDISPLAY* @@ -450,7 +446,7 @@ FASTCALL_ENDFUNC ;; OUTPUT: ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -FASTCALL_FUNC RhpCallFilterFunclet, 12 +FASTCALL_FUNC RhpCallFilterFunclet, 0 FUNCLET_CALL_PROLOGUE 0 @@ -467,7 +463,9 @@ FASTCALL_FUNC RhpCallFilterFunclet, 12 ;; EAX contains the funclet EBP value mov edx, [esp + 0] ;; reload filter funclet address - CALL_FUNCLET Filter + call RhpCallFunclet + +ALTERNATE_ENTRY RhpCallFilterFunclet2 ;; EAX contains the result of the filter execution mov edx, [ebp + 8] diff --git a/src/coreclr/nativeaot/Runtime/i386/GcProbe.asm b/src/coreclr/nativeaot/Runtime/i386/GcProbe.asm index 7e2715d3dd7685..b5876f059f6a40 100644 --- a/src/coreclr/nativeaot/Runtime/i386/GcProbe.asm +++ b/src/coreclr/nativeaot/Runtime/i386/GcProbe.asm @@ -11,6 +11,8 @@ include AsmMacros.inc DEFAULT_PROBE_SAVE_FLAGS equ PTFF_SAVE_ALL_PRESERVED + PTFF_SAVE_RSP +PROBE_SAVE_FLAGS_EVERYTHING equ DEFAULT_PROBE_SAVE_FLAGS + PTFF_SAVE_ALL_SCRATCH +PROBE_SAVE_FLAGS_RAX_IS_GCREF equ DEFAULT_PROBE_SAVE_FLAGS + PTFF_SAVE_RAX + PTFF_RAX_IS_GCREF ;; ;; The prolog for all GC suspension hijackes (normal and stress). Sets up an EBP frame, @@ -23,7 +25,7 @@ DEFAULT_PROBE_SAVE_FLAGS equ PTFF_SAVE_ALL_PRESERVED + PTFF_SAVE_RSP ;; EAX: not trashed or saved ;; EBP: new EBP frame with correct return address ;; ESP: points to saved scratch registers (ECX & EDX) -;; ECX: return value flags +;; ECX: trashed ;; EDX: thread pointer ;; HijackFixupProlog macro @@ -42,15 +44,11 @@ HijackFixupProlog macro mov ecx, [edx + OFFSETOF__Thread__m_pvHijackedReturnAddress] mov [ebp + 4], ecx - ;; Fetch the return address flags - mov ecx, [edx + OFFSETOF__Thread__m_uHijackedReturnValueFlags] - ;; ;; Clear hijack state ;; mov dword ptr [edx + OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation], 0 mov dword ptr [edx + OFFSETOF__Thread__m_pvHijackedReturnAddress], 0 - mov dword ptr [edx + OFFSETOF__Thread__m_uHijackedReturnValueFlags], 0 endm @@ -138,7 +136,7 @@ PopProbeFrame macro pop eax endm -RhpThrowHwEx equ @RhpThrowHwEx@8 +RhpThrowHwEx equ @RhpThrowHwEx@0 extern RhpThrowHwEx : proc ;; @@ -181,25 +179,6 @@ Abort: RhpWaitForGC endp -RhpGcPoll proc - cmp [RhpTrapThreads], TrapThreadsFlags_None - jne @F ; forward branch - predicted not taken - ret -@@: - jmp RhpGcPollRare - -RhpGcPoll endp - -RhpGcPollRare proc - push ebp - mov ebp, esp - PUSH_COOP_PINVOKE_FRAME ecx - call RhpGcPoll2 - POP_COOP_PINVOKE_FRAME - pop ebp - ret -RhpGcPollRare endp - ifdef FEATURE_GC_STRESS ;; ;; Set the Thread state and invoke RhpStressGC(). @@ -258,7 +237,7 @@ FASTCALL_FUNC RhpGcProbeHijack, 0 HijackFixupEpilog WaitForGC: - or ecx, DEFAULT_PROBE_SAVE_FLAGS + PTFF_SAVE_RAX + mov ecx, DEFAULT_PROBE_SAVE_FLAGS + PTFF_SAVE_RAX jmp RhpWaitForGC FASTCALL_ENDFUNC @@ -267,7 +246,7 @@ ifdef FEATURE_GC_STRESS FASTCALL_FUNC RhpGcStressHijack, 0 HijackFixupProlog - or ecx, DEFAULT_PROBE_SAVE_FLAGS + PTFF_SAVE_RAX + mov ecx, DEFAULT_PROBE_SAVE_FLAGS + PTFF_SAVE_RAX jmp RhpGcStressProbe FASTCALL_ENDFUNC diff --git a/src/coreclr/nativeaot/Runtime/i386/Interlocked.asm b/src/coreclr/nativeaot/Runtime/i386/Interlocked.asm index 9bce4da5c223dc..f9599b1b8666e3 100644 --- a/src/coreclr/nativeaot/Runtime/i386/Interlocked.asm +++ b/src/coreclr/nativeaot/Runtime/i386/Interlocked.asm @@ -1,32 +1,3 @@ -;; Licensed to the .NET Foundation under one or more agreements. -;; The .NET Foundation licenses this file to you under the MIT license. - - .586 - .xmm - .model flat - option casemap:none - .code - -include AsmMacros.inc - -FASTCALL_FUNC RhpLockCmpXchg64, 20 - -_value$ = 16 -_comparand$ = 8 - - mov eax, DWORD PTR _comparand$[esp-4] - mov edx, DWORD PTR _comparand$[esp] - push ebx - mov ebx, DWORD PTR _value$[esp] - push esi - mov esi, ecx - mov ecx, DWORD PTR _value$[esp+8] -ALTERNATE_ENTRY _RhpLockCmpXchg64AVLocation - lock cmpxchg8b QWORD PTR [esi] - pop esi - pop ebx - ret 16 - -FASTCALL_ENDFUNC +;; TODO: Implement end diff --git a/src/coreclr/nativeaot/Runtime/i386/InteropThunksHelpers.asm b/src/coreclr/nativeaot/Runtime/i386/InteropThunksHelpers.asm index f786fa592987d9..f47ff5eb3c1664 100644 --- a/src/coreclr/nativeaot/Runtime/i386/InteropThunksHelpers.asm +++ b/src/coreclr/nativeaot/Runtime/i386/InteropThunksHelpers.asm @@ -7,8 +7,6 @@ option casemap:none .code -include AsmMacros.inc - ;; ----------------------------------------------------------------------------------------------------------- ;; standard macros ;; ----------------------------------------------------------------------------------------------------------- @@ -68,8 +66,7 @@ LEAF_ENTRY RhCommonStub, _TEXT ;; store thunk address in thread static mov edx, [eax] mov eax, [eax + POINTER_SIZE] ;; eax <- target slot data - add ecx, SECTIONREL ThunkParamSlot - mov [ecx], edx ;; ThunkParamSlot <- context slot data + mov [ecx + OFFSET ThunkParamSlot], edx ;; ThunkParamSlot <- context slot data ;; restore the regs we used pop edx @@ -83,23 +80,22 @@ LEAF_END RhCommonStub, _TEXT ;; ;; IntPtr RhGetCommonStubAddress() ;; -FASTCALL_FUNC RhGetCommonStubAddress, 0 +LEAF_ENTRY RhGetCommonStubAddress, _TEXT lea eax, [RhCommonStub] ret -FASTCALL_ENDFUNC +LEAF_END RhGetCommonStubAddress, _TEXT ;; ;; IntPtr RhGetCurrentThunkContext() ;; -FASTCALL_FUNC RhGetCurrentThunkContext, 0 +LEAF_ENTRY RhGetCurrentThunkContext, _TEXT mov ecx, [__tls_index] mov edx, fs:[__tls_array] mov ecx, [edx + ecx * POINTER_SIZE] - add ecx, SECTIONREL ThunkParamSlot - mov eax, [ecx] ;; eax <- ThunkParamSlot + mov eax, [ecx + OFFSET ThunkParamSlot] ;; eax <- ThunkParamSlot ret -FASTCALL_ENDFUNC +LEAF_END RhGetCurrentThunkContext, _TEXT end diff --git a/src/coreclr/nativeaot/Runtime/i386/MiscStubs.asm b/src/coreclr/nativeaot/Runtime/i386/MiscStubs.asm index ac4b9c511cef14..7c1329d6f66b30 100644 --- a/src/coreclr/nativeaot/Runtime/i386/MiscStubs.asm +++ b/src/coreclr/nativeaot/Runtime/i386/MiscStubs.asm @@ -17,7 +17,7 @@ include AsmMacros.inc ; NOTE: this helper will modify a value of esp and must establish the frame pointer. PROBE_STEP equ 1000h -RhpStackProbe PROC public +_RhpStackProbe PROC public ; On entry: ; eax - the lowest address of the stack frame being allocated (i.e. [InitialSp - FrameSize]) ; @@ -37,158 +37,6 @@ ProbeLoop: pop ebp ret -RhpStackProbe ENDP - -;; *********************************************************************/ -;; LLsh - long shift left -;; -;; Purpose: -;; Does a Long Shift Left (signed and unsigned are identical) -;; Shifts a long left any number of bits. -;; -;; Entry: -;; EDX:EAX - long value to be shifted -;; ECX - number of bits to shift by -;; -;; Exit: -;; EDX:EAX - shifted value -;; -;; NOTE: Adapted from JIT_LLsh in CoreCLR -;; -RhpLLsh PROC public - ;; Reduce shift amount mod 64 - and ecx, 63 - - cmp ecx, 32 - jae LLshMORE32 - - ;; Handle shifts of between bits 0 and 31 - shld edx, eax, cl - shl eax, cl - ret - -LLshMORE32: - ;; Handle shifts of between bits 32 and 63 - ;; The x86 shift instructions only use the lower 5 bits. - mov edx, eax - xor eax, eax - shl edx, cl - ret -RhpLLsh ENDP - -;; *********************************************************************/ -;; LRsh - long shift right -;; -;; Purpose: -;; Does a signed Long Shift Right -;; Shifts a long right any number of bits. -;; -;; Entry: -;; EDX:EAX - long value to be shifted -;; ECX - number of bits to shift by -;; -;; Exit: -;; EDX:EAX - shifted value -;; -;; NOTE: Adapted from JIT_LRsh in CoreCLR -;; -RhpLRsh PROC public - ;; Reduce shift amount mod 64 - and ecx, 63 - - cmp ecx, 32 - jae LRshMORE32 - - ;; Handle shifts of between bits 0 and 31 - shrd eax, edx, cl - sar edx, cl - ret - -LRshMORE32: - ;; Handle shifts of between bits 32 and 63 - ;; The x86 shift instructions only use the lower 5 bits. - mov eax, edx - sar edx, 31 - sar eax, cl - ret -RhpLRsh ENDP - -;; *********************************************************************/ -;; LRsz -;; -;; Purpose: -;; Does a unsigned Long Shift Right -;; Shifts a long right any number of bits. -;; -;; Entry: -;; EDX:EAX - long value to be shifted -;; ECX - number of bits to shift by -;; -;; Exit: -;; EDX:EAX - shifted value -;; -;; NOTE: Adapted from JIT_LRsz in CoreCLR -;; -RhpLRsz PROC public - ;; Reduce shift amount mod 64 - and ecx, 63 - - cmp ecx, 32 - jae LRszMORE32 - - ;; Handle shifts of between bits 0 and 31 - shrd eax, edx, cl - shr edx, cl - ret - -LRszMORE32: - ;; Handle shifts of between bits 32 and 63 - ;; The x86 shift instructions only use the lower 5 bits. - mov eax, edx - xor edx, edx - shr eax, cl - ret -RhpLRsz ENDP - -;; *********************************************************************/ -;; LMul -;; -;; Purpose: -;; Does a long multiply (same for signed/unsigned) -;; -;; Entry: -;; Parameters are passed on the stack: -;; 1st pushed: multiplier (QWORD) -;; 2nd pushed: multiplicand (QWORD) -;; -;; Exit: -;; EDX:EAX - product of multiplier and multiplicand -;; -;; NOTE: Adapted from JIT_LMul in CoreCLR -;; -RhpLMul PROC public - mov eax, dword ptr [esp + 8] ; AHI - mov ecx, dword ptr [esp + 16] ; BHI - or ecx, eax ; test for both hiwords zero. - mov ecx, dword ptr [esp + 12] ; BLO - jnz LMul_hard ; both are zero, just mult ALO and BLO - - mov eax, dword ptr [esp + 4] - mul ecx - ret 16 - -LMul_hard: - push ebx - mul ecx ; eax has AHI, ecx has BLO, so AHI * BLO - mov ebx, eax ; save result - mov eax, dword ptr [esp + 8] ; ALO - mul dword ptr [esp + 20] ; ALO * BHI - add ebx, eax ; ebx = ((ALO * BHI) + (AHI * BLO)) - mov eax, dword ptr [esp + 8] ; ALO ;ecx = BLO - mul ecx ; so edx:eax = ALO*BLO - add edx, ebx ; now edx has all the LO*HI stuff - pop ebx - ret 16 -RhpLMul ENDP +_RhpStackProbe ENDP end diff --git a/src/coreclr/nativeaot/Runtime/i386/PInvoke.asm b/src/coreclr/nativeaot/Runtime/i386/PInvoke.asm index 7e03d12c580818..90f0d083a842a2 100644 --- a/src/coreclr/nativeaot/Runtime/i386/PInvoke.asm +++ b/src/coreclr/nativeaot/Runtime/i386/PInvoke.asm @@ -6,34 +6,7 @@ option casemap:none .code -include AsmMacros.inc - -FASTCALL_FUNC RhpPInvoke, 4 - INLINE_GETTHREAD eax, edx - - mov edx, [esp] ; edx <- return address - mov dword ptr [ecx + OFFSETOF__PInvokeTransitionFrame__m_pThread], eax - mov dword ptr [ecx + OFFSETOF__PInvokeTransitionFrame__m_FramePointer], ebp - mov dword ptr [ecx + OFFSETOF__PInvokeTransitionFrame__m_RIP], edx - - lea edx, [esp + 4] ; edx <- caller SP - mov dword ptr [ecx + OFFSETOF__PInvokeTransitionFrame__m_Flags], PTFF_SAVE_RSP - mov dword ptr [ecx + OFFSETOF__PInvokeTransitionFrame__m_PreservedRegs], edx - mov dword ptr [eax + OFFSETOF__Thread__m_pTransitionFrame], ecx - - ret -FASTCALL_ENDFUNC - -FASTCALL_FUNC RhpPInvokeReturn, 4 - mov edx, [ecx + OFFSETOF__PInvokeTransitionFrame__m_pThread] - mov dword ptr [edx + OFFSETOF__Thread__m_pTransitionFrame], 0 - cmp [RhpTrapThreads], TrapThreadsFlags_None - jne @F ; forward branch - predicted not taken - ret -@@: - ; passing transition frame pointer in rcx - jmp RhpWaitForGC2 -FASTCALL_ENDFUNC +include AsmMacros.inc end diff --git a/src/coreclr/nativeaot/Runtime/i386/StubDispatch.asm b/src/coreclr/nativeaot/Runtime/i386/StubDispatch.asm index 86dd2807fbc8aa..b1101fac53778e 100644 --- a/src/coreclr/nativeaot/Runtime/i386/StubDispatch.asm +++ b/src/coreclr/nativeaot/Runtime/i386/StubDispatch.asm @@ -30,18 +30,17 @@ endm ;; Macro that generates a stub consuming a cache with the given number of entries. DEFINE_INTERFACE_DISPATCH_STUB macro entries -StubName textequ @CatStr( _RhpInterfaceDispatch, entries, <@0> ) -StubAVLocation textequ @CatStr( _RhpInterfaceDispatchAVLocation, entries ) +StubName textequ @CatStr( _RhpInterfaceDispatch, entries ) StubName proc public ;; Check the instance here to catch null references. We're going to touch it again below (to cache ;; the MethodTable pointer), but that's after we've pushed ebx below, and taking an A/V there will - ;; mess up the stack trace. We also don't have a spare scratch register (eax holds the cache pointer - ;; and the push of ebx below is precisely so we can access a second register to hold the MethodTable - ;; pointer). - ALTERNATE_ENTRY StubAVLocation - cmp dword ptr [ecx], ecx + ;; mess up the stack trace for debugging. We also don't have a spare scratch register (eax holds + ;; the cache pointer and the push of ebx below is precisely so we can access a second register + ;; to hold the MethodTable pointer). + test ecx, ecx + je RhpInterfaceDispatchNullReference ;; eax currently contains the indirection cell address. We need to update it to point to the cache ;; block instead. @@ -95,8 +94,17 @@ RhpInterfaceDispatchSlow proc jmp _RhpUniversalTransition_DebugStepTailCall@0 RhpInterfaceDispatchSlow endp +;; Out of line helper used when we try to interface dispatch on a null pointer. Sets up the stack so the +;; debugger gives a reasonable stack trace. +RhpInterfaceDispatchNullReference proc public + push ebp + mov ebp, esp + mov ebx, [ecx] ;; This should A/V + int 3 +RhpInterfaceDispatchNullReference endp + ;; Stub dispatch routine for dispatch to a vtable slot -_RhpVTableOffsetDispatch@0 proc public +_RhpVTableOffsetDispatch proc public ;; eax currently contains the indirection cell address. We need to update it to point to the vtable offset (which is in the m_pCache field) mov eax, [eax + OFFSETOF__InterfaceDispatchCell__m_pCache] @@ -108,20 +116,17 @@ _RhpVTableOffsetDispatch@0 proc public ;; tail-jump to the target jmp eax -_RhpVTableOffsetDispatch@0 endp +_RhpVTableOffsetDispatch endp ;; Initial dispatch on an interface when we don't have a cache yet. -FASTCALL_FUNC RhpInitialDynamicInterfaceDispatch, 0 -ALTERNATE_ENTRY _RhpInitialInterfaceDispatch - ;; Trigger an AV if we're dispatching on a null this. - ;; The exception handling infrastructure is aware of the fact that this is the first - ;; instruction of RhpInitialInterfaceDispatch and uses it to translate an AV here - ;; to a NullReferenceException at the callsite. - cmp dword ptr [ecx], ecx +_RhpInitialInterfaceDispatch proc public + ALTERNATE_ENTRY RhpInitialDynamicInterfaceDispatch jmp RhpInterfaceDispatchSlow -FASTCALL_ENDFUNC + +_RhpInitialInterfaceDispatch endp + endif ;; FEATURE_CACHED_INTERFACE_DISPATCH diff --git a/src/coreclr/nativeaot/Runtime/i386/UniversalTransition.asm b/src/coreclr/nativeaot/Runtime/i386/UniversalTransition.asm index f5abc17e985d71..d204066863184f 100644 --- a/src/coreclr/nativeaot/Runtime/i386/UniversalTransition.asm +++ b/src/coreclr/nativeaot/Runtime/i386/UniversalTransition.asm @@ -62,7 +62,7 @@ FASTCALL_FUNC Rhp&FunctionName&_FAKE_ENTRY, 0 mov ebp, esp push eax push eax -ALTERNATE_ENTRY _Rhp&FunctionName&@0 +ALTERNATE_ENTRY Rhp&FunctionName&@0 push ecx push edx @@ -74,7 +74,7 @@ ALTERNATE_ENTRY _Rhp&FunctionName&@0 lea ecx, [ebp-10h] ; Get pointer to edx value pushed above call eax -ALTERNATE_ENTRY _ReturnFrom&FunctionName +ALTERNATE_ENTRY ReturnFrom&FunctionName ; We cannot make the label public as that tricks DIA stackwalker into thinking ; it's the beginning of a method. For this reason we export an auxiliary variable diff --git a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm b/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm index 953ea11b74de6d..246f4297900646 100644 --- a/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm +++ b/src/coreclr/nativeaot/Runtime/i386/WriteBarriers.asm @@ -99,16 +99,15 @@ DEFINE_WRITE_BARRIER macro DESTREG, REFREG ;; Define a helper with a name of the form RhpAssignRefEAX etc. (along with suitable calling standard ;; decoration). The location to be updated is in DESTREG. The object reference that will be assigned into that ;; location is in one of the other general registers determined by the value of REFREG. -FASTCALL_FUNC RhpAssignRef&REFREG&, 8 +FASTCALL_FUNC RhpAssignRef&REFREG&, 0 ;; Export the canonical write barrier under unqualified name as well ifidni , - ALTERNATE_ENTRY RhpAssignRef - ALTERNATE_ENTRY _RhpAssignRefAVLocation + @RhpAssignRef@0 label proc + PUBLIC @RhpAssignRef@0 + ALTERNATE_ENTRY RhpAssignRefAVLocation endif - ALTERNATE_ENTRY _RhpAssignRef&REFREG&AVLocation - ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here ;; and the card table update we may perform below. mov dword ptr [DESTREG], REFREG @@ -197,16 +196,15 @@ DEFINE_CHECKED_WRITE_BARRIER macro DESTREG, REFREG ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen on the first instruction ;; - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address -FASTCALL_FUNC RhpCheckedAssignRef&REFREG&, 8 +FASTCALL_FUNC RhpCheckedAssignRef&REFREG&, 0 ;; Export the canonical write barrier under unqualified name as well ifidni , - ALTERNATE_ENTRY RhpCheckedAssignRef - ALTERNATE_ENTRY _RhpCheckedAssignRefAVLocation + @RhpCheckedAssignRef@0 label proc + PUBLIC @RhpCheckedAssignRef@0 + ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation endif - ALTERNATE_ENTRY _RhpCheckedAssignRef&REFREG&AVLocation - ;; Write the reference into the location. Note that we rely on the fact that no GC can occur between here ;; and the card table update we may perform below. mov dword ptr [DESTREG], REFREG @@ -240,76 +238,29 @@ DEFINE_CHECKED_WRITE_BARRIER EDX, EBP ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at @RhpCheckedLockCmpXchgAVLocation@0 ;; - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address -FASTCALL_FUNC RhpCheckedLockCmpXchg, 12 - mov eax, [esp+4] -ALTERNATE_ENTRY _RhpCheckedLockCmpXchgAVLocation +;; pass third argument in EAX +FASTCALL_FUNC RhpCheckedLockCmpXchg +ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation lock cmpxchg [ecx], edx - jne RhpCheckedLockCmpXchg_NoBarrierRequired_ECX_EDX + jne RhpCheckedLockCmpXchg_NoBarrierRequired_ECX_EDX - DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, ECX, EDX, ret 4 + DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedLockCmpXchg, ECX, EDX, ret FASTCALL_ENDFUNC ;; WARNING: Code in EHHelpers.cpp makes assumptions about write barrier code, in particular: ;; - Function "InWriteBarrierHelper" assumes an AV due to passed in null pointer will happen at @RhpCheckedXchgAVLocation@0 ;; - Function "UnwindSimpleHelperToCaller" assumes the stack contains just the pushed return address -FASTCALL_FUNC RhpCheckedXchg, 8 +FASTCALL_FUNC RhpCheckedXchg, 0 ;; Setup eax with the new object for the exchange, that way it will automatically hold the correct result ;; afterwards and we can leave edx unaltered ready for the GC write barrier below. mov eax, edx -ALTERNATE_ENTRY _RhpCheckedXchgAVLocation +ALTERNATE_ENTRY RhpCheckedXchgAVLocation xchg [ecx], eax DEFINE_CHECKED_WRITE_BARRIER_CORE RhpCheckedXchg, ECX, EDX, ret FASTCALL_ENDFUNC -;; -;; RhpByRefAssignRef simulates movs instruction for object references. -;; -;; On entry: -;; edi: address of ref-field (assigned to) -;; esi: address of the data (source) -;; -;; On exit: -;; edi, esi are incremented by 4, -;; ecx: trashed -;; -FASTCALL_FUNC RhpByRefAssignRef, 8 -ALTERNATE_ENTRY _RhpByRefAssignRefAVLocation1 - mov ecx, [esi] -ALTERNATE_ENTRY _RhpByRefAssignRefAVLocation2 - mov [edi], ecx - - ;; Check whether the writes were even into the heap. If not there's no card update required. - cmp edi, [G_LOWEST_ADDRESS] - jb RhpByRefAssignRef_NoBarrierRequired - cmp edi, [G_HIGHEST_ADDRESS] - jae RhpByRefAssignRef_NoBarrierRequired - - UPDATE_GC_SHADOW BASENAME, ecx, edi - - ;; If the reference is to an object that's not in an ephemeral generation we have no need to track it - ;; (since the object won't be collected or moved by an ephemeral collection). - cmp ecx, [G_EPHEMERAL_LOW] - jb RhpByRefAssignRef_NoBarrierRequired - cmp ecx, [G_EPHEMERAL_HIGH] - jae RhpByRefAssignRef_NoBarrierRequired - - mov ecx, edi - shr ecx, 10 - add ecx, [G_CARD_TABLE] - cmp byte ptr [ecx], 0FFh - je RhpByRefAssignRef_NoBarrierRequired - - mov byte ptr [ecx], 0FFh - -RhpByRefAssignRef_NoBarrierRequired: - ;; Increment the pointers before leaving - add esi,4 - add edi,4 - ret -FASTCALL_ENDFUNC - end diff --git a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h index 6a3b24a3944870..750faccc828383 100644 --- a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h +++ b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h @@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants static const uint32_t Signature = 0x00525452; // 'RTR' static const uint32_t CurrentMajorVersion = 9; - static const uint32_t CurrentMinorVersion = 2; + static const uint32_t CurrentMinorVersion = 1; }; struct ReadyToRunHeader diff --git a/src/coreclr/nativeaot/Runtime/interoplibinterface.h b/src/coreclr/nativeaot/Runtime/interoplibinterface.h index fce04b81f90249..da57618b3f7896 100644 --- a/src/coreclr/nativeaot/Runtime/interoplibinterface.h +++ b/src/coreclr/nativeaot/Runtime/interoplibinterface.h @@ -6,11 +6,11 @@ class ObjCMarshalNative { public: - using TryGetCallback = void*(F_CALL_CONV *)(void); - using TryGetTaggedMemoryCallback = CLR_BOOL(F_CALL_CONV *)(_In_ Object *, _Out_ void **); - using BeginEndCallback = void(F_CALL_CONV *)(void); - using IsReferencedCallback = int(F_CALL_CONV *)(_In_ void*); - using EnteredFinalizationCallback = void(F_CALL_CONV *)(_In_ void*); + using TryGetCallback = void*(REDHAWK_CALLCONV *)(void); + using TryGetTaggedMemoryCallback = CLR_BOOL(REDHAWK_CALLCONV *)(_In_ Object *, _Out_ void **); + using BeginEndCallback = void(REDHAWK_CALLCONV *)(void); + using IsReferencedCallback = int(REDHAWK_CALLCONV *)(_In_ void*); + using EnteredFinalizationCallback = void(REDHAWK_CALLCONV *)(_In_ void*); public: // Instance inspection static bool IsTrackedReference(_In_ Object * pObject, _Out_ bool* isReferenced); diff --git a/src/coreclr/nativeaot/Runtime/portable.cpp b/src/coreclr/nativeaot/Runtime/portable.cpp index d797ad14687ba0..7ddb06baee5a87 100644 --- a/src/coreclr/nativeaot/Runtime/portable.cpp +++ b/src/coreclr/nativeaot/Runtime/portable.cpp @@ -33,7 +33,7 @@ #include "GCMemoryHelpers.inl" #if defined(USE_PORTABLE_HELPERS) -EXTERN_C void* F_CALL_CONV RhpGcAlloc(MethodTable *pEEType, uint32_t uFlags, uintptr_t numElements, void * pTransitionFrame); +EXTERN_C NATIVEAOT_API void* REDHAWK_CALLCONV RhpGcAlloc(MethodTable *pEEType, uint32_t uFlags, uintptr_t numElements, void * pTransitionFrame); static Object* AllocateObject(MethodTable* pEEType, uint32_t uFlags, uintptr_t numElements) { @@ -55,7 +55,7 @@ struct gc_alloc_context // // Allocations // -FCIMPL1(Object *, RhpNewFast, MethodTable* pEEType) +COOP_PINVOKE_HELPER(Object *, RhpNewFast, (MethodTable* pEEType)) { ASSERT(!pEEType->HasFinalizer()); @@ -75,20 +75,18 @@ FCIMPL1(Object *, RhpNewFast, MethodTable* pEEType) return AllocateObject(pEEType, 0, 0); } -FCIMPLEND #define GC_ALLOC_FINALIZE 0x1 // TODO: Defined in gc.h #define GC_ALLOC_ALIGN8_BIAS 0x4 // TODO: Defined in gc.h #define GC_ALLOC_ALIGN8 0x8 // TODO: Defined in gc.h -FCIMPL1(Object *, RhpNewFinalizable, MethodTable* pEEType) +COOP_PINVOKE_HELPER(Object *, RhpNewFinalizable, (MethodTable* pEEType)) { ASSERT(pEEType->HasFinalizer()); return AllocateObject(pEEType, GC_ALLOC_FINALIZE, 0); } -FCIMPLEND -FCIMPL2(Array *, RhpNewArray, MethodTable * pArrayEEType, int numElements) +COOP_PINVOKE_HELPER(Array *, RhpNewArray, (MethodTable * pArrayEEType, int numElements)) { Thread * pCurThread = ThreadStore::GetCurrentThread(); gc_alloc_context * acontext = pCurThread->GetAllocContext(); @@ -124,15 +122,13 @@ FCIMPL2(Array *, RhpNewArray, MethodTable * pArrayEEType, int numElements) return (Array*)AllocateObject(pArrayEEType, 0, numElements); } -FCIMPLEND -FCIMPL2(String *, RhNewString, MethodTable * pArrayEEType, int numElements) +COOP_PINVOKE_HELPER(String *, RhNewString, (MethodTable * pArrayEEType, int numElements)) { // TODO: Implement. We tail call to RhpNewArray for now since there's a bunch of TODOs in the places // that matter anyway. return (String*)RhpNewArray(pArrayEEType, numElements); } -FCIMPLEND #endif #if defined(USE_PORTABLE_HELPERS) @@ -140,14 +136,13 @@ FCIMPLEND GPTR_DECL(MethodTable, g_pFreeObjectEEType); -FCIMPL1(Object *, RhpNewFinalizableAlign8, MethodTable* pEEType) +COOP_PINVOKE_HELPER(Object *, RhpNewFinalizableAlign8, (MethodTable* pEEType)) { return AllocateObject(pEEType, GC_ALLOC_FINALIZE | GC_ALLOC_ALIGN8, 0); } -FCIMPLEND #ifndef HOST_64BIT -FCIMPL1(Object*, RhpNewFastAlign8, MethodTable* pEEType) +COOP_PINVOKE_HELPER(Object*, RhpNewFastAlign8, (MethodTable* pEEType)) { ASSERT(!pEEType->HasFinalizer()); @@ -182,9 +177,8 @@ FCIMPL1(Object*, RhpNewFastAlign8, MethodTable* pEEType) return AllocateObject(pEEType, GC_ALLOC_ALIGN8, 0); } -FCIMPLEND -FCIMPL1(Object*, RhpNewFastMisalign, MethodTable* pEEType) +COOP_PINVOKE_HELPER(Object*, RhpNewFastMisalign, (MethodTable* pEEType)) { Thread* pCurThread = ThreadStore::GetCurrentThread(); gc_alloc_context* acontext = pCurThread->GetAllocContext(); @@ -216,9 +210,8 @@ FCIMPL1(Object*, RhpNewFastMisalign, MethodTable* pEEType) return AllocateObject(pEEType, GC_ALLOC_ALIGN8 | GC_ALLOC_ALIGN8_BIAS, 0); } -FCIMPLEND -FCIMPL2(Array*, RhpNewArrayAlign8, MethodTable* pArrayEEType, int numElements) +COOP_PINVOKE_HELPER(Array*, RhpNewArrayAlign8, (MethodTable* pArrayEEType, int numElements)) { Thread* pCurThread = ThreadStore::GetCurrentThread(); gc_alloc_context* acontext = pCurThread->GetAllocContext(); @@ -266,63 +259,53 @@ FCIMPL2(Array*, RhpNewArrayAlign8, MethodTable* pArrayEEType, int numElements) return (Array*)AllocateObject(pArrayEEType, GC_ALLOC_ALIGN8, numElements); } -FCIMPLEND #endif // !HOST_64BIT #endif // defined(HOST_ARM) || defined(HOST_WASM) -FCIMPL0(void, RhpInitialDynamicInterfaceDispatch) +COOP_PINVOKE_HELPER(void, RhpInitialDynamicInterfaceDispatch, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch1) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch1, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch2) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch2, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch4) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch4, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch8) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch8, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch16) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch16, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch32) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch32, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpInterfaceDispatch64) +COOP_PINVOKE_HELPER(void, RhpInterfaceDispatch64, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpVTableOffsetDispatch) +COOP_PINVOKE_HELPER(void, RhpVTableOffsetDispatch, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND // @TODO Implement UniversalTransition EXTERN_C void * ReturnFromUniversalTransition; @@ -338,158 +321,137 @@ void * ReturnFromUniversalTransition_DebugStepTailCall; // // Return address hijacking // -FCIMPL0(void, RhpGcStressHijack) +COOP_PINVOKE_HELPER(void, RhpGcStressHijack, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND -FCIMPL0(void, RhpGcProbeHijack) +COOP_PINVOKE_HELPER(void, RhpGcProbeHijack, ()) { ASSERT_UNCONDITIONALLY("NYI"); } -FCIMPLEND #endif // defined(USE_PORTABLE_HELPERS) || defined(TARGET_UNIX) #if defined(USE_PORTABLE_HELPERS) #if !defined (HOST_ARM64) -FCIMPL2(void, RhpAssignRef, Object ** dst, Object * ref) +COOP_PINVOKE_HELPER(void, RhpAssignRef, (Object ** dst, Object * ref)) { // @TODO: USE_PORTABLE_HELPERS - Null check *dst = ref; InlineWriteBarrier(dst, ref); } -FCIMPLEND -FCIMPL2(void, RhpCheckedAssignRef, Object ** dst, Object * ref) +COOP_PINVOKE_HELPER(void, RhpCheckedAssignRef, (Object ** dst, Object * ref)) { // @TODO: USE_PORTABLE_HELPERS - Null check *dst = ref; InlineCheckedWriteBarrier(dst, ref); } -FCIMPLEND #endif -FCIMPL3(Object *, RhpCheckedLockCmpXchg, Object ** location, Object * value, Object * comparand) +COOP_PINVOKE_HELPER(Object *, RhpCheckedLockCmpXchg, (Object ** location, Object * value, Object * comparand)) { // @TODO: USE_PORTABLE_HELPERS - Null check Object * ret = (Object *)PalInterlockedCompareExchangePointer((void * volatile *)location, value, comparand); InlineCheckedWriteBarrier(location, value); return ret; } -FCIMPLEND -FCIMPL2(Object *, RhpCheckedXchg, Object ** location, Object * value) +COOP_PINVOKE_HELPER(Object *, RhpCheckedXchg, (Object ** location, Object * value)) { // @TODO: USE_PORTABLE_HELPERS - Null check Object * ret = (Object *)PalInterlockedExchangePointer((void * volatile *)location, value); InlineCheckedWriteBarrier(location, value); return ret; } -FCIMPLEND -FCIMPL3(uint8_t, RhpLockCmpXchg8, uint8_t * location, uint8_t value, uint8_t comparand) +COOP_PINVOKE_HELPER(uint8_t, RhpLockCmpXchg8, (uint8_t * location, uint8_t value, uint8_t comparand)) { ASSERT_UNCONDITIONALLY("NYI"); return 0; } -FCIMPLEND -FCIMPL3(int16_t, RhpLockCmpXchg16, int16_t * location, int16_t value, int16_t comparand) +COOP_PINVOKE_HELPER(int16_t, RhpLockCmpXchg16, (int16_t * location, int16_t value, int16_t comparand)) { ASSERT_UNCONDITIONALLY("NYI"); return 0; } -FCIMPLEND -FCIMPL3(int32_t, RhpLockCmpXchg32, int32_t * location, int32_t value, int32_t comparand) +COOP_PINVOKE_HELPER(int32_t, RhpLockCmpXchg32, (int32_t * location, int32_t value, int32_t comparand)) { // @TODO: USE_PORTABLE_HELPERS - Null check return PalInterlockedCompareExchange(location, value, comparand); } -FCIMPLEND -FCIMPL3(int64_t, RhpLockCmpXchg64, int64_t * location, int64_t value, int64_t comparand) +COOP_PINVOKE_HELPER(int64_t, RhpLockCmpXchg64, (int64_t * location, int64_t value, int64_t comparand)) { // @TODO: USE_PORTABLE_HELPERS - Null check return PalInterlockedCompareExchange64(location, value, comparand); } -FCIMPLEND -FCIMPL0(void*, RhAllocateThunksMapping) +EXTERN_C NATIVEAOT_API void* __cdecl RhAllocateThunksMapping() { return NULL; } -FCIMPLEND -FCIMPL0(void *, RhpGetThunksBase) +COOP_PINVOKE_HELPER(void *, RhpGetThunksBase, ()) { return NULL; } -FCIMPLEND -FCIMPL0(int, RhpGetNumThunkBlocksPerMapping) +COOP_PINVOKE_HELPER(int, RhpGetNumThunkBlocksPerMapping, ()) { ASSERT_UNCONDITIONALLY("NYI"); return 0; } -FCIMPLEND -FCIMPL0(int, RhpGetNumThunksPerBlock) +COOP_PINVOKE_HELPER(int, RhpGetNumThunksPerBlock, ()) { ASSERT_UNCONDITIONALLY("NYI"); return 0; } -FCIMPLEND -FCIMPL0(int, RhpGetThunkSize) +COOP_PINVOKE_HELPER(int, RhpGetThunkSize, ()) { ASSERT_UNCONDITIONALLY("NYI"); return 0; } -FCIMPLEND -FCIMPL1(void*, RhpGetThunkDataBlockAddress, void* pThunkStubAddress) +COOP_PINVOKE_HELPER(void*, RhpGetThunkDataBlockAddress, (void* pThunkStubAddress)) { ASSERT_UNCONDITIONALLY("NYI"); return NULL; } -FCIMPLEND -FCIMPL1(void*, RhpGetThunkStubsBlockAddress, void* pThunkDataAddress) +COOP_PINVOKE_HELPER(void*, RhpGetThunkStubsBlockAddress, (void* pThunkDataAddress)) { ASSERT_UNCONDITIONALLY("NYI"); return NULL; } -FCIMPLEND -FCIMPL0(int, RhpGetThunkBlockSize) +COOP_PINVOKE_HELPER(int, RhpGetThunkBlockSize, ()) { ASSERT_UNCONDITIONALLY("NYI"); return 0; } -FCIMPLEND -FCIMPL0(void *, RhGetCommonStubAddress) +COOP_PINVOKE_HELPER(void *, RhGetCommonStubAddress, ()) { ASSERT_UNCONDITIONALLY("NYI"); return NULL; } -FCIMPLEND -FCIMPL0(void *, RhGetCurrentThunkContext) +COOP_PINVOKE_HELPER(void *, RhGetCurrentThunkContext, ()) { ASSERT_UNCONDITIONALLY("NYI"); return NULL; } -FCIMPLEND -FCIMPL0(void, RhpGcPoll) +COOP_PINVOKE_HELPER(void, RhpGcPoll, ()) { // TODO: implement } -FCIMPLEND #endif diff --git a/src/coreclr/nativeaot/Runtime/regdisplay.h b/src/coreclr/nativeaot/Runtime/regdisplay.h index d5082fb9efbe74..b9c0175210578a 100644 --- a/src/coreclr/nativeaot/Runtime/regdisplay.h +++ b/src/coreclr/nativeaot/Runtime/regdisplay.h @@ -46,41 +46,8 @@ struct REGDISPLAY inline void SetIP(PCODE IP) { this->IP = IP; } inline void SetSP(uintptr_t SP) { this->SP = SP; } - -#ifdef TARGET_X86 - TADDR PCTAddr; - - inline unsigned long *GetEaxLocation() { return (unsigned long *)pRax; } - inline unsigned long *GetEcxLocation() { return (unsigned long *)pRcx; } - inline unsigned long *GetEdxLocation() { return (unsigned long *)pRdx; } - inline unsigned long *GetEbpLocation() { return (unsigned long *)pRbp; } - inline unsigned long *GetEbxLocation() { return (unsigned long *)pRbx; } - inline unsigned long *GetEsiLocation() { return (unsigned long *)pRsi; } - inline unsigned long *GetEdiLocation() { return (unsigned long *)pRdi; } - - inline void SetEaxLocation(unsigned long *loc) { pRax = (PTR_uintptr_t)loc; } - inline void SetEcxLocation(unsigned long *loc) { pRcx = (PTR_uintptr_t)loc; } - inline void SetEdxLocation(unsigned long *loc) { pRdx = (PTR_uintptr_t)loc; } - inline void SetEbxLocation(unsigned long *loc) { pRbx = (PTR_uintptr_t)loc; } - inline void SetEsiLocation(unsigned long *loc) { pRsi = (PTR_uintptr_t)loc; } - inline void SetEdiLocation(unsigned long *loc) { pRdi = (PTR_uintptr_t)loc; } - inline void SetEbpLocation(unsigned long *loc) { pRbp = (PTR_uintptr_t)loc; } -#endif }; -#ifdef TARGET_X86 -inline TADDR GetRegdisplayFP(REGDISPLAY *display) -{ - return (TADDR)*display->GetEbpLocation(); -} - -inline void SetRegdisplayPCTAddr(REGDISPLAY *display, TADDR addr) -{ - display->PCTAddr = addr; - display->SetIP(*PTR_PCODE(addr)); -} -#endif - #elif defined(TARGET_ARM) struct REGDISPLAY diff --git a/src/coreclr/nativeaot/Runtime/rhassert.h b/src/coreclr/nativeaot/Runtime/rhassert.h index 34403e216f5b16..ecf0297980ae19 100644 --- a/src/coreclr/nativeaot/Runtime/rhassert.h +++ b/src/coreclr/nativeaot/Runtime/rhassert.h @@ -44,10 +44,6 @@ void Assert(const char * expr, const char * file, unsigned int line_num, const c #define _ASSERTE(_expr) ASSERT(_expr) #endif -#ifndef _ASSERTE_ALL_BUILDS -#define _ASSERTE_ALL_BUILDS(_expr) ASSERT(_expr) -#endif - #define PORTABILITY_ASSERT(message) \ ASSERT_UNCONDITIONALLY(message); \ ASSUME(0); \ diff --git a/src/coreclr/nativeaot/Runtime/runtimeeventinternal.cpp b/src/coreclr/nativeaot/Runtime/runtimeeventinternal.cpp index 5de8fa373c781c..7b4afafd8671f3 100644 --- a/src/coreclr/nativeaot/Runtime/runtimeeventinternal.cpp +++ b/src/coreclr/nativeaot/Runtime/runtimeeventinternal.cpp @@ -5,52 +5,52 @@ #ifdef FEATURE_PERFTRACING -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogContentionLockCreated(intptr_t LockID, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogContentionLockCreated(intptr_t LockID, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) { FireEtwContentionLockCreated(reinterpret_cast(LockID), reinterpret_cast(AssociatedObjectID), ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogContentionStart(uint8_t ContentionFlags, uint16_t ClrInstanceID, intptr_t LockID, intptr_t AssociatedObjectID, uint64_t LockOwnerThreadID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogContentionStart(uint8_t ContentionFlags, uint16_t ClrInstanceID, intptr_t LockID, intptr_t AssociatedObjectID, uint64_t LockOwnerThreadID) { FireEtwContentionStart_V2((const unsigned char)(ContentionFlags), ClrInstanceID, reinterpret_cast(LockID), reinterpret_cast(AssociatedObjectID), LockOwnerThreadID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs) { FireEtwContentionStop_V1((const unsigned char)(ContentionFlags), ClrInstanceID, DurationNs); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(uint32_t activeWorkerThreadCount, uint32_t retiredWorkerThreadCount, uint16_t clrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(uint32_t activeWorkerThreadCount, uint32_t retiredWorkerThreadCount, uint16_t clrInstanceID) { FireEtwThreadPoolWorkerThreadStart(activeWorkerThreadCount, retiredWorkerThreadCount, clrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) { FireEtwThreadPoolWorkerThreadStop(ActiveWorkerThreadCount, RetiredWorkerThreadCount, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(uint32_t ActiveWorkerThreadCount, uint32_t RetiredWorkerThreadCount, uint16_t ClrInstanceID) { FireEtwThreadPoolWorkerThreadWait(ActiveWorkerThreadCount, RetiredWorkerThreadCount, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(uint16_t MinWorkerThreads, uint16_t MaxWorkerThreads, uint16_t MinIOCompletionThreads, uint16_t MaxIOCompletionThreads, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(uint16_t MinWorkerThreads, uint16_t MaxWorkerThreads, uint16_t MinIOCompletionThreads, uint16_t MaxIOCompletionThreads, uint16_t ClrInstanceID) { FireEtwThreadPoolMinMaxThreads(MinWorkerThreads, MaxWorkerThreads, MinIOCompletionThreads, MaxIOCompletionThreads, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, uint16_t ClrInstanceID) { FireEtwThreadPoolWorkerThreadAdjustmentSample(Throughput, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint32_t NewWorkerThreadCount, uint32_t Reason, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint32_t NewWorkerThreadCount, uint32_t Reason, uint16_t ClrInstanceID) { FireEtwThreadPoolWorkerThreadAdjustmentAdjustment(AverageThroughput, NewWorkerThreadCount, Reason, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats( +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats( double Duration, double Throughput, double ThreadPoolWorkerThreadWait, @@ -66,7 +66,7 @@ EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjust FireEtwThreadPoolWorkerThreadAdjustmentStats(Duration, Throughput, ThreadPoolWorkerThreadWait, ThroughputWave, ThroughputErrorEstimate, AverageThroughputErrorEstimate, ThroughputRatio, Confidence, NewControlSetting, NewThreadWaveMagnitude, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue( +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolIOEnqueue( void * NativeOverlapped, void * Overlapped, BOOL MultiDequeues, @@ -75,22 +75,22 @@ EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue( FireEtwThreadPoolIOEnqueue(NativeOverlapped, Overlapped, MultiDequeues, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIODequeue(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolIODequeue(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) { FireEtwThreadPoolIODequeue(NativeOverlapped, Overlapped, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(uint32_t Count, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(uint32_t Count, uint16_t ClrInstanceID) { FireEtwThreadPoolWorkingThreadCount(Count, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOPack(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogThreadPoolIOPack(void * NativeOverlapped, void * Overlapped, uint16_t ClrInstanceID) { FireEtwThreadPoolIOPack(NativeOverlapped, Overlapped, ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogExceptionThrown(const WCHAR* exceptionTypeName, const WCHAR* exceptionMessage, void* faultingIP, HRESULT hresult) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogExceptionThrown(const WCHAR* exceptionTypeName, const WCHAR* exceptionMessage, void* faultingIP, HRESULT hresult) { FireEtwExceptionThrown_V1(exceptionTypeName, exceptionMessage, @@ -100,12 +100,12 @@ EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogExceptionThrown(const WCHAR* GetClrInstanceId()); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogWaitHandleWaitStart(uint8_t WaitSource, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogWaitHandleWaitStart(uint8_t WaitSource, intptr_t AssociatedObjectID, uint16_t ClrInstanceID) { FireEtwWaitHandleWaitStart(WaitSource, reinterpret_cast(AssociatedObjectID), ClrInstanceID); } -EXTERN_C void QCALLTYPE NativeRuntimeEventSource_LogWaitHandleWaitStop(uint16_t ClrInstanceID) +EXTERN_C NATIVEAOT_API void __cdecl NativeRuntimeEventSource_LogWaitHandleWaitStop(uint16_t ClrInstanceID) { FireEtwWaitHandleWaitStop(ClrInstanceID); } diff --git a/src/coreclr/nativeaot/Runtime/startup.cpp b/src/coreclr/nativeaot/Runtime/startup.cpp index 4096c61f36ed12..f1da550fc01cc9 100644 --- a/src/coreclr/nativeaot/Runtime/startup.cpp +++ b/src/coreclr/nativeaot/Runtime/startup.cpp @@ -67,7 +67,12 @@ typedef size_t GSCookie; #ifdef FEATURE_READONLY_GS_COOKIE -#define READONLY_ATTR __attribute__((section(".rodata"))) +#ifdef __APPLE__ +#define READONLY_ATTR_ARGS section("__DATA,__const") +#else +#define READONLY_ATTR_ARGS section(".rodata") +#endif +#define READONLY_ATTR __attribute__((READONLY_ATTR_ARGS)) // const is so that it gets placed in the .text section (which is read-only) // volatile is so that accesses to it do not get optimized away because of the const diff --git a/src/coreclr/nativeaot/Runtime/thread.cpp b/src/coreclr/nativeaot/Runtime/thread.cpp index 3f30185600224f..e2be011a21c6aa 100644 --- a/src/coreclr/nativeaot/Runtime/thread.cpp +++ b/src/coreclr/nativeaot/Runtime/thread.cpp @@ -30,6 +30,10 @@ #ifndef DACCESS_COMPILE +EXTERN_C NATIVEAOT_API void* REDHAWK_CALLCONV RhpHandleAlloc(void* pObject, int type); +EXTERN_C NATIVEAOT_API void REDHAWK_CALLCONV RhHandleSet(void* handle, void* pObject); +EXTERN_C NATIVEAOT_API void REDHAWK_CALLCONV RhHandleFree(void* handle); + static int (*g_RuntimeInitializationCallback)(); static Thread* g_RuntimeInitializingThread; @@ -1061,11 +1065,10 @@ void Thread::ValidateExInfoPop(ExInfo * pExInfo, void * limitSP) #endif // _DEBUG } -FCIMPL3(void, RhpValidateExInfoPop, Thread * pThread, ExInfo * pExInfo, void * limitSP) +COOP_PINVOKE_HELPER(void, RhpValidateExInfoPop, (Thread * pThread, ExInfo * pExInfo, void * limitSP)) { pThread->ValidateExInfoPop(pExInfo, limitSP); } -FCIMPLEND void Thread::SetDoNotTriggerGc() { @@ -1129,7 +1132,7 @@ void Thread::ValidateExInfoStack() #ifndef DACCESS_COMPILE #ifndef TARGET_UNIX -EXTERN_C uint32_t QCALLTYPE RhCompatibleReentrantWaitAny(UInt32_BOOL alertable, uint32_t timeout, uint32_t count, HANDLE* pHandles) +EXTERN_C NATIVEAOT_API uint32_t __cdecl RhCompatibleReentrantWaitAny(UInt32_BOOL alertable, uint32_t timeout, uint32_t count, HANDLE* pHandles) { return PalCompatibleWaitAny(alertable, timeout, count, pHandles, /*allowReentrantWait:*/ TRUE); } @@ -1203,24 +1206,22 @@ void Thread::SetThreadAbortException(Object *exception) m_threadAbortException = exception; } -FCIMPL0(Object *, RhpGetThreadAbortException) +COOP_PINVOKE_HELPER(Object *, RhpGetThreadAbortException, ()) { Thread * pCurThread = ThreadStore::RawGetCurrentThread(); return pCurThread->GetThreadAbortException(); } -FCIMPLEND Object** Thread::GetThreadStaticStorage() { return &m_pThreadLocalStatics; } -FCIMPL0(Object**, RhGetThreadStaticStorage) +COOP_PINVOKE_HELPER(Object**, RhGetThreadStaticStorage, ()) { Thread* pCurrentThread = ThreadStore::RawGetCurrentThread(); return pCurrentThread->GetThreadStaticStorage(); } -FCIMPLEND InlinedThreadStaticRoot* Thread::GetInlinedThreadStaticList() { @@ -1236,15 +1237,14 @@ void Thread::RegisterInlinedThreadStaticRoot(InlinedThreadStaticRoot* newRoot, T newRoot->m_typeManager = typeManager; } -FCIMPL2(void, RhRegisterInlinedThreadStaticRoot, Object** root, TypeManager* typeManager) +COOP_PINVOKE_HELPER(void, RhRegisterInlinedThreadStaticRoot, (Object** root, TypeManager* typeManager)) { Thread* pCurrentThread = ThreadStore::RawGetCurrentThread(); pCurrentThread->RegisterInlinedThreadStaticRoot((InlinedThreadStaticRoot*)root, typeManager); } -FCIMPLEND // This is function is used to quickly query a value that can uniquely identify a thread -FCIMPL0(uint8_t*, RhCurrentNativeThreadId) +COOP_PINVOKE_HELPER(uint8_t*, RhCurrentNativeThreadId, ()) { #ifndef TARGET_UNIX return PalNtCurrentTeb(); @@ -1252,14 +1252,12 @@ FCIMPL0(uint8_t*, RhCurrentNativeThreadId) return (uint8_t*)ThreadStore::RawGetCurrentThread(); #endif // TARGET_UNIX } -FCIMPLEND // This function is used to get the OS thread identifier for the current thread. -FCIMPL0(uint64_t, RhCurrentOSThreadId) +COOP_PINVOKE_HELPER(uint64_t, RhCurrentOSThreadId, ()) { return PalGetCurrentOSThreadId(); } -FCIMPLEND // Standard calling convention variant and actual implementation for RhpReversePInvokeAttachOrTrapThread EXTERN_C NOINLINE void FASTCALL RhpReversePInvokeAttachOrTrapThread2(ReversePInvokeFrame* pFrame) @@ -1272,7 +1270,7 @@ EXTERN_C NOINLINE void FASTCALL RhpReversePInvokeAttachOrTrapThread2(ReversePInv // PInvoke // -FCIMPL1(void, RhpReversePInvoke, ReversePInvokeFrame * pFrame) +COOP_PINVOKE_HELPER(void, RhpReversePInvoke, (ReversePInvokeFrame * pFrame)) { Thread * pCurThread = ThreadStore::RawGetCurrentThread(); pFrame->m_savedThread = pCurThread; @@ -1281,29 +1279,25 @@ FCIMPL1(void, RhpReversePInvoke, ReversePInvokeFrame * pFrame) RhpReversePInvokeAttachOrTrapThread2(pFrame); } -FCIMPLEND -FCIMPL1(void, RhpReversePInvokeReturn, ReversePInvokeFrame * pFrame) +COOP_PINVOKE_HELPER(void, RhpReversePInvokeReturn, (ReversePInvokeFrame * pFrame)) { pFrame->m_savedThread->InlineReversePInvokeReturn(pFrame); } -FCIMPLEND #ifdef USE_PORTABLE_HELPERS -FCIMPL1(void, RhpPInvoke2, PInvokeTransitionFrame* pFrame) +COOP_PINVOKE_HELPER(void, RhpPInvoke2, (PInvokeTransitionFrame* pFrame)) { Thread * pCurThread = ThreadStore::RawGetCurrentThread(); pCurThread->InlinePInvoke(pFrame); } -FCIMPLEND -FCIMPL1(void, RhpPInvokeReturn2, PInvokeTransitionFrame* pFrame) +COOP_PINVOKE_HELPER(void, RhpPInvokeReturn2, (PInvokeTransitionFrame* pFrame)) { //reenter cooperative mode pFrame->m_pThread->InlinePInvokeReturn(pFrame); } -FCIMPLEND #endif //USE_PORTABLE_HELPERS diff --git a/src/coreclr/nativeaot/Runtime/threadstore.cpp b/src/coreclr/nativeaot/Runtime/threadstore.cpp index 259d07e7ab0bb7..6ec8a44ed2e7bd 100644 --- a/src/coreclr/nativeaot/Runtime/threadstore.cpp +++ b/src/coreclr/nativeaot/Runtime/threadstore.cpp @@ -410,22 +410,20 @@ void ThreadStore::CancelThreadAbort(Thread* targetThread) ResumeAllThreads(/* waitForGCEvent = */ false); } -EXTERN_C void* QCALLTYPE RhpGetCurrentThread() +COOP_PINVOKE_HELPER(void *, RhpGetCurrentThread, ()) { return ThreadStore::GetCurrentThread(); } -FCIMPL3(void, RhpInitiateThreadAbort, void* thread, Object * threadAbortException, CLR_BOOL doRudeAbort) +COOP_PINVOKE_HELPER(void, RhpInitiateThreadAbort, (void* thread, Object * threadAbortException, CLR_BOOL doRudeAbort)) { GetThreadStore()->InitiateThreadAbort((Thread*)thread, threadAbortException, doRudeAbort); } -FCIMPLEND -FCIMPL1(void, RhpCancelThreadAbort, void* thread) +COOP_PINVOKE_HELPER(void, RhpCancelThreadAbort, (void* thread)) { GetThreadStore()->CancelThreadAbort((Thread*)thread); } -FCIMPLEND C_ASSERT(sizeof(Thread) == sizeof(ThreadBuffer)); @@ -479,6 +477,7 @@ GVAL_IMPL(uint32_t, SECTIONREL__tls_CurrentThread); // // This routine supports the !Thread debugger extension routine // +typedef DPTR(TADDR) PTR_TADDR; // static PTR_Thread ThreadStore::GetThreadFromTEB(TADDR pTEB) { diff --git a/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp b/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp index c9e1adf6b5f759..48efa06bc8e650 100644 --- a/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp +++ b/src/coreclr/nativeaot/Runtime/unix/UnixNativeCodeManager.cpp @@ -378,7 +378,7 @@ bool UnixNativeCodeManager::IsUnwindable(PTR_VOID pvAddress) ASSERT(((uintptr_t)pvAddress & 1) == 0); #endif -#if defined(TARGET_ARM64) || defined(TARGET_ARM) +#if defined(TARGET_ARM64) MethodInfo methodInfo; FindMethodInfo(pvAddress, &methodInfo); pMethodInfo = &methodInfo; @@ -393,7 +393,74 @@ bool UnixNativeCodeManager::IsUnwindable(PTR_VOID pvAddress) #endif } -#if defined(TARGET_ARM) +// checks for known prolog instructions generated by ILC and returns +// 1 - in prolog +// 0 - not in prolog, +// -1 - unknown. +int UnixNativeCodeManager::IsInProlog(MethodInfo * pMethodInfo, PTR_VOID pvAddress) +{ +#if defined(TARGET_ARM64) + +// post/pre + + +// stp with signed offset +// x010 1001 00xx xxxx xxxx xxxx xxxx xxxx +#define STP_BITS1 0x29000000 +#define STP_MASK1 0x7FC00000 + +// stp with pre/post/no offset +// x010 100x x0xx xxxx xxxx xxxx xxxx xxxx +#define STP_BITS2 0x28000000 +#define STP_MASK2 0x7E400000 + +// add fp, sp, x +// mov fp, sp +// 1001 0001 0xxx xxxx xxxx xx11 1111 1101 +#define ADD_FP_SP_BITS 0x910003FD +#define ADD_FP_SP_MASK 0xFF8003FF + +#define STP_RT2_RT_MASK 0x7C1F +#define STP_RT2_RT_FP_LR 0x781D +#define STP_RN_MASK 0x3E0 +#define STP_RN_SP 0x3E0 +#define STP_RN_FP 0x3A0 + + UnixNativeMethodInfo * pNativeMethodInfo = (UnixNativeMethodInfo *)pMethodInfo; + ASSERT(pNativeMethodInfo != NULL); + + uint32_t* start = (uint32_t*)pNativeMethodInfo->pMethodStartAddress; + bool savedFpLr = false; + bool establishedFp = false; + + for (uint32_t* pInstr = (uint32_t*)start; pInstr < pvAddress && !(savedFpLr && establishedFp); pInstr++) + { + uint32_t instr = *pInstr; + + if (((instr & STP_MASK1) == STP_BITS1 || (instr & STP_MASK2) == STP_BITS2) && + ((instr & STP_RN_MASK) == STP_RN_SP || (instr & STP_RN_MASK) == STP_RN_FP)) + { + // SP/FP-relative store of pair of registers + savedFpLr |= (instr & STP_RT2_RT_MASK) == STP_RT2_RT_FP_LR; + } + else if ((instr & ADD_FP_SP_MASK) == ADD_FP_SP_BITS) + { + establishedFp = true; + } + else + { + // JIT generates other patterns into the prolog that we currently don't + // recognize (saving unpaired register, stack pointer adjustments). We + // don't need to recognize these patterns unless a compact unwinding code + // is generated for them in ILC. + // https://github.com/dotnet/runtime/issues/76371 + return -1; + } + } + + return savedFpLr && establishedFp ? 0 : 1; + +#elif defined(TARGET_ARM) // SUB SP, SP, # // 1011 0000 1xxx xxxx @@ -420,11 +487,6 @@ bool UnixNativeCodeManager::IsUnwindable(PTR_VOID pvAddress) #define SUB_W_SP_REG_BITS 0xEBAD0D00 #define SUB_W_SP_REG_MASK 0xFFEF8F00 -// ADD{S}.W FP, SP, # -// 1111 0x01 000x 1101 0xxx 1011 xxxx xxxx -#define ADD_W_FP_SP_BITS 0xF10D0B00 -#define ADD_W_FP_SP_MASK 0xFBEF8F00 - // PUSH // 1011 010x xxxx xxxx #define PUSH_BITS 0xB400 @@ -473,8 +535,7 @@ bool UnixNativeCodeManager::IsUnwindable(PTR_VOID pvAddress) // MOV R9, SP #define MOV_R9_SP 0x46E9 -static bool IsArmPrologInstruction(uint16_t* pInstr) -{ + uint16_t* pInstr = (uint16_t*)pvAddress; uint32_t instr = *pInstr; if ((instr & SUB_SP_IMM_MASK) == SUB_SP_IMM_BITS || @@ -490,7 +551,6 @@ static bool IsArmPrologInstruction(uint16_t* pInstr) if ((instr & SUB_W_SP_IMM_MASK) == SUB_W_SP_IMM_BITS || (instr & SUBW_SP_IMM_MASK) == SUBW_SP_IMM_BITS || (instr & SUB_W_SP_REG_MASK) == SUB_W_SP_REG_BITS || - (instr & ADD_W_FP_SP_MASK) == ADD_W_FP_SP_BITS || (instr & PUSH_W_MASK_T2) == PUSH_W_BITS_T2 || (instr & PUSH_W_MASK_T3) == PUSH_W_BITS_T3 || (instr & VPUSH_MASK_T1) == VPUSH_BITS_T1 || @@ -499,114 +559,6 @@ static bool IsArmPrologInstruction(uint16_t* pInstr) return 1; } - return 0; -} - -#endif - -// checks for known prolog instructions generated by ILC and returns -// 1 - in prolog -// 0 - not in prolog, -// -1 - unknown. -int UnixNativeCodeManager::IsInProlog(MethodInfo * pMethodInfo, PTR_VOID pvAddress) -{ -#if defined(TARGET_ARM64) - -// post/pre - - -// stp with signed offset -// x010 1001 00xx xxxx xxxx xxxx xxxx xxxx -#define STP_BITS1 0x29000000 -#define STP_MASK1 0x7FC00000 - -// stp with pre/post/no offset -// x010 100x x0xx xxxx xxxx xxxx xxxx xxxx -#define STP_BITS2 0x28000000 -#define STP_MASK2 0x7E400000 - -// add fp, sp, x -// mov fp, sp -// 1001 0001 0xxx xxxx xxxx xx11 1111 1101 -#define ADD_FP_SP_BITS 0x910003FD -#define ADD_FP_SP_MASK 0xFF8003FF - -#define STP_RT2_RT_MASK 0x7C1F -#define STP_RT2_RT_FP_LR 0x781D -#define STP_RN_MASK 0x3E0 -#define STP_RN_SP 0x3E0 -#define STP_RN_FP 0x3A0 - - UnixNativeMethodInfo * pNativeMethodInfo = (UnixNativeMethodInfo *)pMethodInfo; - ASSERT(pNativeMethodInfo != NULL); - - uint32_t* start = (uint32_t*)pNativeMethodInfo->pMethodStartAddress; - bool savedFpLr = false; - bool establishedFp = false; - - for (uint32_t* pInstr = (uint32_t*)start; pInstr < pvAddress && !(savedFpLr && establishedFp); pInstr++) - { - uint32_t instr = *pInstr; - - if (((instr & STP_MASK1) == STP_BITS1 || (instr & STP_MASK2) == STP_BITS2) && - ((instr & STP_RN_MASK) == STP_RN_SP || (instr & STP_RN_MASK) == STP_RN_FP)) - { - // SP/FP-relative store of pair of registers - savedFpLr |= (instr & STP_RT2_RT_MASK) == STP_RT2_RT_FP_LR; - } - else if ((instr & ADD_FP_SP_MASK) == ADD_FP_SP_BITS) - { - establishedFp = true; - } - else - { - // JIT generates other patterns into the prolog that we currently don't - // recognize (saving unpaired register, stack pointer adjustments). We - // don't need to recognize these patterns unless a compact unwinding code - // is generated for them in ILC. - // https://github.com/dotnet/runtime/issues/76371 - return -1; - } - } - - return savedFpLr && establishedFp ? 0 : 1; - -#elif defined(TARGET_ARM) - - UnixNativeMethodInfo * pNativeMethodInfo = (UnixNativeMethodInfo *)pMethodInfo; - ASSERT(pNativeMethodInfo != NULL); - - uint16_t* pInstr = (uint16_t*)pvAddress; - - // First check if the current instruction is any of the recognized prolog - // instructions. That may be a false positive but it's not going to be a - // false negative (with the exception of localloc pattern below). - if (IsArmPrologInstruction(pInstr)) - { - // Verify that everything in front of the instruction was also a prolog. - pInstr = (uint16_t*)pNativeMethodInfo->pMethodStartAddress; - while (pInstr < pvAddress) - { - if (!IsArmPrologInstruction(pInstr)) - { - return 0; - } - - uint16_t instr = *pInstr; - if (instr == MOV_R9_SP) - { - // The frame has been established, so anything that follows is - // not considered a prolog (ie. unwinding works). - return 0; - } - - // Skip over to next instruction - pInstr += (instr & 0xE000) == 0xE000 && (instr & 0xF800) != 0xE000 ? 2 : 1; - } - - return 1; - } - // The localloc pattern generated by JIT looks like: // // movw r4, #frameSize @@ -625,7 +577,7 @@ int UnixNativeCodeManager::IsInProlog(MethodInfo * pMethodInfo, PTR_VOID pvAddre // We can look ahead by couple of instructions and look for "mov sp, rXX". for (int c = 5; c >= 0; --c) { - uint16_t instr = *pInstr; + instr = *pInstr; if (instr == MOV_SP_R4) { return 1; @@ -641,7 +593,15 @@ int UnixNativeCodeManager::IsInProlog(MethodInfo * pMethodInfo, PTR_VOID pvAddre } // Skip over to next instruction - pInstr += (instr & 0xE000) == 0xE000 && (instr & 0xF800) != 0xE000 ? 2 : 1; + if ((instr & 0xE000) == 0xE000 && (instr & 0xF800) != 0xE000) + { + // 32-but Thumb instruction + pInstr += 2; + } + else + { + pInstr++; + } } return 0; @@ -977,49 +937,28 @@ int UnixNativeCodeManager::TrailingEpilogueInstructionsCount(MethodInfo * pMetho #define VPOP_BITS_T2 0xECBD0A00 #define VPOP_MASK_T2 0xFFBF0F00 - uint16_t *pInstr = (uint16_t *)pvAddress; - uint32_t instr; + uint32_t instr = *(uint16_t*)pvAddress; - while (1) + if ((instr & ADD_SP_IMM_MASK) == ADD_SP_IMM_BITS || + (instr & ADD_SP_REG_MASK) == ADD_SP_REG_BITS || + (instr & POP_MASK) == POP_BITS || + (instr & BX_LR_MASK) == BX_LR_BITS) { - instr = *pInstr; - - if ((instr & 0xE000) == 0xE000 && (instr & 0xF800) != 0xE000) - { - // 32-bit instruction - instr <<= 16; - instr |= *(pInstr + 1); - pInstr += 2; - } - else - { - pInstr++; - } - - // POP, VPOP and BX LR are definitely epilog - if ((instr & POP_MASK) == POP_BITS || - (instr & BX_LR_MASK) == BX_LR_BITS || - (instr & POP_W_MASK_T2) == POP_W_BITS_T2 || - (instr & POP_W_MASK_T3) == POP_W_BITS_T3 || - (instr & VPOP_MASK_T1) == VPOP_BITS_T1 || - (instr & VPOP_MASK_T2) == VPOP_BITS_T2) - { - return -1; - } - - // ADD SP, xxx may be part of epilog but in methods with frame - // pointer it can also appear in the body. Skip to the next - // instruction and check if it's still epilog. - if ((instr & ADD_SP_IMM_MASK) == ADD_SP_IMM_BITS || - (instr & ADD_SP_REG_MASK) == ADD_SP_REG_BITS || - (instr & ADD_W_SP_IMM_MASK) == ADD_W_SP_IMM_BITS || - (instr & ADDW_SP_IMM_MASK) == ADDW_SP_IMM_BITS || - (instr & ADD_W_SP_REG_MASK) == ADD_W_SP_REG_BITS) - { - continue; - } + return -1; + } - return 0; + instr <<= 16; + instr |= *((uint16_t*)pvAddress + 1); + + if ((instr & ADD_W_SP_IMM_MASK) == ADD_W_SP_IMM_BITS || + (instr & ADDW_SP_IMM_MASK) == ADDW_SP_IMM_BITS || + (instr & ADD_W_SP_REG_MASK) == ADD_W_SP_REG_BITS || + (instr & POP_W_MASK_T2) == POP_W_BITS_T2 || + (instr & POP_W_MASK_T3) == POP_W_BITS_T3 || + (instr & VPOP_MASK_T1) == VPOP_BITS_T1 || + (instr & VPOP_MASK_T2) == VPOP_BITS_T2) + { + return -1; } #endif diff --git a/src/coreclr/nativeaot/Runtime/unix/cgroupcpu.cpp b/src/coreclr/nativeaot/Runtime/unix/cgroupcpu.cpp index 9b30d817828302..322bc488cc1fac 100644 --- a/src/coreclr/nativeaot/Runtime/unix/cgroupcpu.cpp +++ b/src/coreclr/nativeaot/Runtime/unix/cgroupcpu.cpp @@ -36,6 +36,7 @@ Module Name: #include "cgroupcpu.h" #define CGROUP2_SUPER_MAGIC 0x63677270 +#define TMPFS_MAGIC 0x01021994 #define BASE_TEN 10 @@ -99,16 +100,12 @@ class CGroup if (result != 0) return 0; - if (stats.f_type == CGROUP2_SUPER_MAGIC) + switch (stats.f_type) { - return 2; - } - else - { - // Assume that if /sys/fs/cgroup exists and the file system type is not cgroup2fs, - // it is cgroup v1. Typically the file system type is tmpfs, but other values have - // been seen in the wild. - return 1; + case TMPFS_MAGIC: return 1; + case CGROUP2_SUPER_MAGIC: return 2; + default: + return 0; } #endif } diff --git a/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp b/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp index b6dd6ee812d2bb..1215431a83e0dd 100644 --- a/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp +++ b/src/coreclr/nativeaot/Runtime/windows/CoffNativeCodeManager.cpp @@ -19,23 +19,6 @@ #define GCINFODECODER_NO_EE #include "gcinfodecoder.cpp" -#ifdef TARGET_X86 -#define FEATURE_EH_FUNCLETS - -// Disable contracts -#define LIMITED_METHOD_CONTRACT -#define LIMITED_METHOD_DAC_CONTRACT -#define CONTRACTL -#define CONTRACTL_END -#define NOTHROW -#define GC_NOTRIGGER -#define HOST_NOCALLS - -#include "../../inc/gcdecoder.cpp" -#include "../../inc/gc_unwind_x86.h" -#include "../../vm/gc_unwind_x86.inl" -#endif - #define UBF_FUNC_KIND_MASK 0x03 #define UBF_FUNC_KIND_ROOT 0x00 #define UBF_FUNC_KIND_HANDLER 0x01 @@ -368,6 +351,7 @@ uint32_t CoffNativeCodeManager::GetCodeOffset(MethodInfo* pMethodInfo, PTR_VOID bool CoffNativeCodeManager::IsSafePoint(PTR_VOID pvAddress) { +#ifdef USE_GC_INFO_DECODER MethodInfo pMethodInfo; if (!FindMethodInfo(pvAddress, &pMethodInfo)) { @@ -377,7 +361,6 @@ bool CoffNativeCodeManager::IsSafePoint(PTR_VOID pvAddress) PTR_uint8_t gcInfo; uint32_t codeOffset = GetCodeOffset(&pMethodInfo, pvAddress, &gcInfo); -#ifdef USE_GC_INFO_DECODER GcInfoDecoder decoder( GCInfoToken(gcInfo), GcInfoDecoderFlags(DECODE_INTERRUPTIBILITY), @@ -386,11 +369,9 @@ bool CoffNativeCodeManager::IsSafePoint(PTR_VOID pvAddress) return decoder.IsInterruptible(); #else - // Extract the necessary information from the info block header - hdrInfo info; - DecodeGCHdrInfo(GCInfoToken(gcInfo), codeOffset, &info); - - return info.interruptible && info.prologOffs == hdrInfo::NOT_IN_PROLOG && info.epilogOffs == hdrInfo::NOT_IN_EPILOG; + // x86 has custom GC info, see DecodeGCHdrInfo in eetwain.cpp + PORTABILITY_ASSERT("IsSafePoint"); + RhFailFast(); #endif } @@ -400,20 +381,10 @@ void CoffNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, GCEnumContext * hCallback, bool isActiveStackFrame) { +#ifdef USE_GC_INFO_DECODER PTR_uint8_t gcInfo; uint32_t codeOffset = GetCodeOffset(pMethodInfo, safePointAddress, &gcInfo); - ICodeManagerFlags flags = (ICodeManagerFlags)0; - if (((CoffNativeMethodInfo *)pMethodInfo)->executionAborted) - flags = ICodeManagerFlags::ExecutionAborted; - - if (IsFilter(pMethodInfo)) - flags = (ICodeManagerFlags)(flags | ICodeManagerFlags::NoReportUntracked); - - if (isActiveStackFrame) - flags = (ICodeManagerFlags)(flags | ICodeManagerFlags::ActiveStackFrame); - -#ifdef USE_GC_INFO_DECODER if (!isActiveStackFrame) { // If we are not in the active method, we are currently pointing @@ -431,6 +402,16 @@ void CoffNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, codeOffset ); + ICodeManagerFlags flags = (ICodeManagerFlags)0; + if (((CoffNativeMethodInfo *)pMethodInfo)->executionAborted) + flags = ICodeManagerFlags::ExecutionAborted; + + if (IsFilter(pMethodInfo)) + flags = (ICodeManagerFlags)(flags | ICodeManagerFlags::NoReportUntracked); + + if (isActiveStackFrame) + flags = (ICodeManagerFlags)(flags | ICodeManagerFlags::ActiveStackFrame); + if (!decoder.EnumerateLiveSlots( pRegisterSet, isActiveStackFrame /* reportScratchSlots */, @@ -442,22 +423,9 @@ void CoffNativeCodeManager::EnumGcRefs(MethodInfo * pMethodInfo, assert(false); } #else - size_t unwindDataBlobSize; - CoffNativeMethodInfo* pNativeMethodInfo = (CoffNativeMethodInfo *) pMethodInfo; - PTR_VOID pUnwindDataBlob = GetUnwindDataBlob(m_moduleBase, pNativeMethodInfo->runtimeFunction, &unwindDataBlobSize); - PTR_uint8_t p = dac_cast(pUnwindDataBlob) + unwindDataBlobSize; - uint8_t unwindBlockFlags = *p++; - - ::EnumGcRefsX86(pRegisterSet, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->mainRuntimeFunction->BeginAddress), - codeOffset, - GCInfoToken(gcInfo), - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->runtimeFunction->BeginAddress), - (unwindBlockFlags & UBF_FUNC_KIND_MASK) != UBF_FUNC_KIND_ROOT, - (unwindBlockFlags & UBF_FUNC_KIND_MASK) == UBF_FUNC_KIND_FILTER, - flags, - hCallback->pCallback, - hCallback); + // x86 has custom GC info, see EnumGcRefs in eetwain.cpp + PORTABILITY_ASSERT("EnumGcRefs"); + RhFailFast(); #endif } @@ -506,13 +474,8 @@ uintptr_t CoffNativeCodeManager::GetConservativeUpperBoundForOutgoingArgs(Method // all outgoing arguments. upperBound = dac_cast(basePointer + slot); #else - hdrInfo info; - DecodeGCHdrInfo(GCInfoToken(p), 0, &info); - assert(info.revPInvokeOffset != INVALID_REV_PINVOKE_OFFSET); - upperBound = - info.ebpFrame ? - dac_cast(pRegisterSet->GetFP()) - info.revPInvokeOffset : - dac_cast(pRegisterSet->GetSP()) + info.revPInvokeOffset; + PORTABILITY_ASSERT("GetConservativeUpperBoundForOutgoingArgs"); + RhFailFast(); #endif } else @@ -572,25 +535,11 @@ uintptr_t CoffNativeCodeManager::GetConservativeUpperBoundForOutgoingArgs(Method NULL); upperBound = dac_cast(context.Sp); + #else - PTR_uint8_t gcInfo; - uint32_t codeOffset = GetCodeOffset(pMethodInfo, (PTR_VOID)pRegisterSet->IP, &gcInfo); - - hdrInfo infoBuf; - size_t infoSize = DecodeGCHdrInfo(GCInfoToken(gcInfo), codeOffset, &infoBuf); - PTR_CBYTE table = gcInfo + infoSize; - - REGDISPLAY registerSet = *pRegisterSet; - - ::UnwindStackFrameX86(®isterSet, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->mainRuntimeFunction->BeginAddress), - codeOffset, - &infoBuf, - table, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->runtimeFunction->BeginAddress), - (unwindBlockFlags & UBF_FUNC_KIND_MASK) != UBF_FUNC_KIND_ROOT, - true); - upperBound = dac_cast(registerSet.PCTAddr); + PORTABILITY_ASSERT("GetConservativeUpperBoundForOutgoingArgs"); + upperBound = NULL; + RhFailFast(); #endif } return upperBound; @@ -638,46 +587,21 @@ bool CoffNativeCodeManager::UnwindStackFrame(MethodInfo * pMethodInfo, } *ppPreviousTransitionFrame = *(PInvokeTransitionFrame**)(basePointer + slot); -#else - hdrInfo info; - DecodeGCHdrInfo(GCInfoToken(p), 0, &info); - assert(info.revPInvokeOffset != INVALID_REV_PINVOKE_OFFSET); - *ppPreviousTransitionFrame = - info.ebpFrame ? - *(PInvokeTransitionFrame**)(dac_cast(pRegisterSet->GetFP()) - info.revPInvokeOffset) : - *(PInvokeTransitionFrame**)(dac_cast(pRegisterSet->GetSP()) + info.revPInvokeOffset); -#endif if ((flags & USFF_StopUnwindOnTransitionFrame) != 0) { return true; } +#else + PORTABILITY_ASSERT("GetConservativeUpperBoundForOutgoingArgs"); + RhFailFast(); +#endif } else { *ppPreviousTransitionFrame = NULL; } -#if defined(TARGET_X86) - PTR_uint8_t gcInfo; - uint32_t codeOffset = GetCodeOffset(pMethodInfo, (PTR_VOID)pRegisterSet->IP, &gcInfo); - - hdrInfo infoBuf; - size_t infoSize = DecodeGCHdrInfo(GCInfoToken(gcInfo), codeOffset, &infoBuf); - PTR_CBYTE table = gcInfo + infoSize; - - if (!::UnwindStackFrameX86(pRegisterSet, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->mainRuntimeFunction->BeginAddress), - codeOffset, - &infoBuf, - table, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->runtimeFunction->BeginAddress), - (unwindBlockFlags & UBF_FUNC_KIND_MASK) != UBF_FUNC_KIND_ROOT, - true)) - { - return false; - } -#else CONTEXT context; KNONVOLATILE_CONTEXT_POINTERS contextPointers; @@ -711,7 +635,10 @@ bool CoffNativeCodeManager::UnwindStackFrame(MethodInfo * pMethodInfo, FOR_EACH_NONVOLATILE_REGISTER(REGDISPLAY_TO_CONTEXT); -#if defined(TARGET_AMD64) +#if defined(TARGET_X86) + PORTABILITY_ASSERT("CoffNativeCodeManager::UnwindStackFrame"); +#elif defined(TARGET_AMD64) + if (!(flags & USFF_GcUnwind)) { memcpy(&context.Xmm6, pRegisterSet->Xmm, sizeof(pRegisterSet->Xmm)); @@ -769,7 +696,7 @@ bool CoffNativeCodeManager::UnwindStackFrame(MethodInfo * pMethodInfo, for (int i = 8; i < 16; i++) pRegisterSet->D[i - 8] = context.V[i].Low; } -#endif +#endif // defined(TARGET_X86) FOR_EACH_NONVOLATILE_REGISTER(CONTEXT_TO_REGDISPLAY); @@ -778,8 +705,6 @@ bool CoffNativeCodeManager::UnwindStackFrame(MethodInfo * pMethodInfo, #undef REGDISPLAY_TO_CONTEXT #undef CONTEXT_TO_REGDISPLAY -#endif // defined(TARGET_X86) - return true; } @@ -807,6 +732,7 @@ bool CoffNativeCodeManager::GetReturnAddressHijackInfo(MethodInfo * pMethodIn PTR_PTR_VOID * ppvRetAddrLocation, // out GCRefKind * pRetValueKind) // out { +#ifdef USE_GC_INFO_DECODER CoffNativeMethodInfo * pNativeMethodInfo = (CoffNativeMethodInfo *)pMethodInfo; size_t unwindDataBlobSize; @@ -831,7 +757,6 @@ bool CoffNativeCodeManager::GetReturnAddressHijackInfo(MethodInfo * pMethodIn if ((unwindBlockFlags & UBF_FUNC_HAS_EHINFO) != 0) p += sizeof(int32_t); -#ifdef USE_GC_INFO_DECODER // Decode the GC info for the current method to determine its return type GcInfoDecoderFlags flags = DECODE_RETURN_KIND; #if defined(TARGET_ARM64) @@ -920,35 +845,8 @@ bool CoffNativeCodeManager::GetReturnAddressHijackInfo(MethodInfo * pMethodIn return false; #endif // defined(TARGET_AMD64) #else // defined(USE_GC_INFO_DECODER) - PTR_uint8_t gcInfo; - uint32_t codeOffset = GetCodeOffset(pMethodInfo, (PTR_VOID)pRegisterSet->IP, &gcInfo); - hdrInfo infoBuf; - size_t infoSize = DecodeGCHdrInfo(GCInfoToken(gcInfo), codeOffset, &infoBuf); - - // TODO: Hijack with saving the return value in FP stack - if (infoBuf.returnKind == RT_Float) - { - return false; - } - - REGDISPLAY registerSet = *pRegisterSet; - - if (!::UnwindStackFrameX86(®isterSet, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->mainRuntimeFunction->BeginAddress), - codeOffset, - &infoBuf, - gcInfo + infoSize, - (PTR_CBYTE)(m_moduleBase + pNativeMethodInfo->runtimeFunction->BeginAddress), - (unwindBlockFlags & UBF_FUNC_KIND_MASK) != UBF_FUNC_KIND_ROOT, - false)) - { - return false; - } - - *ppvRetAddrLocation = (PTR_PTR_VOID)registerSet.PCTAddr; - *pRetValueKind = GetGcRefKind(infoBuf.returnKind); - - return true; + PORTABILITY_ASSERT("GetReturnAddressHijackInfo"); + RhFailFast(); #endif } diff --git a/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp b/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp index 2842caf78ef685..06706ed1e8a4be 100644 --- a/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp +++ b/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp @@ -543,7 +543,7 @@ REDHAWK_PALEXPORT void REDHAWK_PALAPI PalHijack(HANDLE hThread, _In_opt_ void* p pThread->SetActivationPending(false); DWORD lastError = GetLastError(); - if (lastError != ERROR_INVALID_PARAMETER && lastError != ERROR_NOT_SUPPORTED) + if (lastError != ERROR_INVALID_PARAMETER) { // An unexpected failure has happened. It is a concern. ASSERT_UNCONDITIONALLY("Failed to queue an APC for unusual reason."); diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml b/src/coreclr/nativeaot/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml index 91859db1b0f80e..60ca0245fda089 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.xml @@ -1,4 +1,8 @@ + + + + diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeAugments.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeAugments.cs index 8421eea29887ec..441036959931d9 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeAugments.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/Augments/RuntimeAugments.cs @@ -138,11 +138,11 @@ public static Delegate CreateDelegate(RuntimeTypeHandle typeHandleForDelegate, I } // - // Helper to extract the artifact that identifies a reflectable delegate target in the runtime mapping tables. + // Helper to extract the artifact that uniquely identifies a method in the runtime mapping tables. // - public static IntPtr GetDelegateLdFtnResult(Delegate d, out RuntimeTypeHandle typeOfFirstParameterIfInstanceDelegate, out bool isOpenResolver) + public static IntPtr GetDelegateLdFtnResult(Delegate d, out RuntimeTypeHandle typeOfFirstParameterIfInstanceDelegate, out bool isOpenResolver, out bool isInterpreterEntrypoint) { - return d.GetDelegateLdFtnResult(out typeOfFirstParameterIfInstanceDelegate, out isOpenResolver); + return d.GetFunctionPointer(out typeOfFirstParameterIfInstanceDelegate, out isOpenResolver, out isInterpreterEntrypoint); } // Low level method that returns the loaded modules as array. ReadOnlySpan returning overload diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs index e5bfe42e497694..9f597e151d550f 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/InteropHelpers.cs @@ -359,23 +359,24 @@ internal static unsafe void FixupMethodCell(IntPtr hModule, MethodFixupCell* pCe if (charSetMangling == 0) { // Look for the user-provided entry point name only - pTarget = GetProcAddressWithMangling(hModule, methodName, pCell); + pTarget = Interop.Kernel32.GetProcAddress(hModule, methodName); } - else if (charSetMangling == CharSet.Ansi) + else + if (charSetMangling == CharSet.Ansi) { // For ANSI, look for the user-provided entry point name first. // If that does not exist, try the charset suffix. - pTarget = GetProcAddressWithMangling(hModule, methodName, pCell); + pTarget = Interop.Kernel32.GetProcAddress(hModule, methodName); if (pTarget == IntPtr.Zero) - pTarget = GetProcAddressWithSuffix(hModule, methodName, (byte)'A', pCell); + pTarget = GetProcAddressWithSuffix(hModule, methodName, (byte)'A'); } else { // For Unicode, look for the entry point name with the charset suffix first. // The 'W' API takes precedence over the undecorated one. - pTarget = GetProcAddressWithSuffix(hModule, methodName, (byte)'W', pCell); + pTarget = GetProcAddressWithSuffix(hModule, methodName, (byte)'W'); if (pTarget == IntPtr.Zero) - pTarget = GetProcAddressWithMangling(hModule, methodName, pCell); + pTarget = Interop.Kernel32.GetProcAddress(hModule, methodName); } #else pTarget = Interop.Sys.GetProcAddress(hModule, methodName); @@ -390,43 +391,23 @@ internal static unsafe void FixupMethodCell(IntPtr hModule, MethodFixupCell* pCe } #if TARGET_WINDOWS - private static unsafe IntPtr GetProcAddressWithMangling(IntPtr hModule, byte* methodName, MethodFixupCell* pCell) - { - IntPtr pMethod = Interop.Kernel32.GetProcAddress(hModule, methodName); -#if TARGET_X86 - if (pMethod == IntPtr.Zero && pCell->IsStdcall) - { - int nameLength = string.strlen(methodName); - // We need to add an extra bytes for the prefix, null terminator and stack size suffix: - // - 1 byte for '_' prefix - // - 1 byte for '@' suffix - // - up to 10 bytes for digits (maximum positive number representable by uint) - // - 1 byte for NULL termination character - byte* probedMethodName = stackalloc byte[nameLength + 13]; - probedMethodName[0] = (byte)'_'; - Unsafe.CopyBlock(probedMethodName + 1, methodName, (uint)nameLength); - probedMethodName[nameLength + 1] = (byte)'@'; - pCell->SignatureBytes.TryFormat(new Span(probedMethodName + 2 + nameLength, 10), out int bytesWritten); - probedMethodName[nameLength + 2 + bytesWritten] = 0; - pMethod = Interop.Kernel32.GetProcAddress(hModule, probedMethodName); - } -#else - _ = pCell; -#endif - return pMethod; - } - - private static unsafe IntPtr GetProcAddressWithSuffix(IntPtr hModule, byte* methodName, byte suffix, MethodFixupCell* pCell) + private static unsafe IntPtr GetProcAddressWithSuffix(IntPtr hModule, byte* methodName, byte suffix) { int nameLength = string.strlen(methodName); // We need to add an extra byte for the suffix, and an extra byte for the null terminator byte* probedMethodName = stackalloc byte[nameLength + 2]; - Unsafe.CopyBlock(probedMethodName, methodName, (uint)nameLength); - probedMethodName[nameLength] = suffix; + + for (int i = 0; i < nameLength; i++) + { + probedMethodName[i] = methodName[i]; + } + probedMethodName[nameLength + 1] = 0; - return GetProcAddressWithMangling(hModule, probedMethodName, pCell); + probedMethodName[nameLength] = suffix; + + return Interop.Kernel32.GetProcAddress(hModule, probedMethodName); } #endif @@ -644,16 +625,11 @@ internal unsafe struct MethodFixupCell public IntPtr Target; public IntPtr MethodName; public ModuleFixupCell* Module; - private uint Flags; + private int Flags; public CharSet CharSetMangling => (CharSet)(Flags & MethodFixupCellFlagsConstants.CharSetMask); -#if FEATURE_OBJCMARSHAL public bool IsObjectiveCMessageSend => (Flags & MethodFixupCellFlagsConstants.IsObjectiveCMessageSendMask) != 0; - public int ObjectiveCMessageSendFunction => (int)((Flags & MethodFixupCellFlagsConstants.ObjectiveCMessageSendFunctionMask) >> MethodFixupCellFlagsConstants.ObjectiveCMessageSendFunctionShift); -#elif TARGET_WINDOWS && TARGET_X86 - public bool IsStdcall => (Flags & MethodFixupCellFlagsConstants.IsStdcall) != 0; - public ushort SignatureBytes => (ushort)(Flags >> 16); -#endif + public int ObjectiveCMessageSendFunction => (Flags & MethodFixupCellFlagsConstants.ObjectiveCMessageSendFunctionMask) >> MethodFixupCellFlagsConstants.ObjectiveCMessageSendFunctionShift; } internal unsafe struct CustomMarshallerKey : IEquatable diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MathHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MathHelpers.cs index 85fef043acb0e1..7175ea9c00cbe2 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MathHelpers.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MathHelpers.cs @@ -4,7 +4,6 @@ using System; using System.Runtime; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using Internal.Runtime; @@ -14,7 +13,7 @@ namespace Internal.Runtime.CompilerHelpers /// Math helpers for generated code. The helpers marked with [RuntimeExport] and the type /// itself need to be public because they constitute a public contract with the .NET Native toolchain. /// - internal static partial class MathHelpers + internal static class MathHelpers { #if !TARGET_64BIT // @@ -145,9 +144,9 @@ public static ulong ULMulOvf(ulong i, ulong j) return ThrowULngOvf(); } - [LibraryImport(RuntimeLibrary)] - [SuppressGCTransition] - private static partial ulong RhpULMod(ulong i, ulong j); + [RuntimeImport(RuntimeLibrary, "RhpULMod")] + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern ulong RhpULMod(ulong i, ulong j); public static ulong ULMod(ulong i, ulong j) { @@ -157,9 +156,9 @@ public static ulong ULMod(ulong i, ulong j) return RhpULMod(i, j); } - [LibraryImport(RuntimeLibrary)] - [SuppressGCTransition] - private static partial long RhpLMod(long i, long j); + [RuntimeImport(RuntimeLibrary, "RhpLMod")] + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern long RhpLMod(long i, long j); public static long LMod(long i, long j) { @@ -171,9 +170,9 @@ public static long LMod(long i, long j) return RhpLMod(i, j); } - [LibraryImport(RuntimeLibrary)] - [SuppressGCTransition] - private static partial ulong RhpULDiv(ulong i, ulong j); + [RuntimeImport(RuntimeLibrary, "RhpULDiv")] + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern ulong RhpULDiv(ulong i, ulong j); public static ulong ULDiv(ulong i, ulong j) { @@ -183,9 +182,9 @@ public static ulong ULDiv(ulong i, ulong j) return RhpULDiv(i, j); } - [LibraryImport(RuntimeLibrary)] - [SuppressGCTransition] - private static partial long RhpLDiv(long i, long j); + [RuntimeImport(RuntimeLibrary, "RhpLDiv")] + [MethodImplAttribute(MethodImplOptions.InternalCall)] + private static extern long RhpLDiv(long i, long j); public static long LDiv(long i, long j) { diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MemoryHelpers.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MemoryHelpers.cs new file mode 100644 index 00000000000000..644fcf1a59940e --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerHelpers/MemoryHelpers.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime; + +namespace Internal.Runtime.CompilerHelpers +{ + /// + /// These methods are used to implement memcpy and memset intrinsics with null checks. + /// + internal static class MemoryHelpers + { + private static unsafe void MemSet(ref byte dest, byte value, nuint size) + { + if (size > 0) + { + _ = dest; + SpanHelpers.Fill(ref dest, size, value); + } + } + + private static unsafe void MemCopy(ref byte dest, ref byte src, nuint size) + { + if (size > 0) + { + _ = dest; + _ = src; + Buffer.Memmove(ref dest, ref src, size); + } + } + } +} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerServices/FunctionPointerOps.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerServices/FunctionPointerOps.cs index e77a106b252baf..a0f5312c92304e 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerServices/FunctionPointerOps.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/Internal/Runtime/CompilerServices/FunctionPointerOps.cs @@ -149,16 +149,5 @@ public static unsafe bool Compare(IntPtr functionPointerA, IntPtr functionPointe return pointerDefA->MethodFunctionPointer == pointerDefB->MethodFunctionPointer; } - - public static unsafe int GetHashCode(IntPtr functionPointer) - { - if (!IsGenericMethodPointer(functionPointer)) - { - return functionPointer.GetHashCode(); - } - - GenericMethodDescriptor* pointerDef = ConvertToGenericDescriptor(functionPointer); - return pointerDef->GetHashCode(); - } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 70928c997a5084..d6e8c3a7e90274 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -100,6 +100,7 @@ + @@ -176,12 +177,15 @@ + + + diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Array.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Array.NativeAot.cs index 61f70e212483c1..64ba6597446a81 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Array.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Array.NativeAot.cs @@ -494,7 +494,7 @@ private static unsafe void CopyImplValueTypeArrayNoInnerGcRefs(Array sourceArray // Copy scenario: ValueType-array to value-type array with no embedded gc-refs. nuint elementSize = sourceArray.ElementSize; - SpanHelpers.Memmove( + Buffer.Memmove( ref Unsafe.AddByteOffset(ref MemoryMarshal.GetArrayDataReference(destinationArray), (nuint)destinationIndex * elementSize), ref Unsafe.AddByteOffset(ref MemoryMarshal.GetArrayDataReference(sourceArray), (nuint)sourceIndex * elementSize), elementSize * (nuint)length); @@ -534,7 +534,7 @@ private static unsafe void CopyImplPrimitiveTypeWithWidening(Array sourceArray, if (sourceElementType == destElementType) { // Multidim arrays and enum->int copies can still reach this path. - SpanHelpers.Memmove(ref *data, ref *srcData, (nuint)length * srcElementSize); + Buffer.Memmove(ref *data, ref *srcData, (nuint)length * srcElementSize); return; } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Delegate.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Delegate.cs index bc8c517acb8ea9..1c565a045b8265 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Delegate.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Delegate.cs @@ -7,7 +7,6 @@ using System.Reflection; using System.Runtime; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; using System.Runtime.Serialization; using Internal.Reflection.Augments; @@ -41,19 +40,10 @@ protected Delegate([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.Al // New Delegate Implementation - private object _firstParameter; - private object _helperObject; - private nint _extraFunctionPointerOrData; - private IntPtr _functionPointer; - - // _helperObject may point to an array of delegates if this is a multicast delegate. We use this wrapper to distinguish between - // our own array of delegates and user provided Wrapper[]. As a added benefit, this wrapper also eliminates array co-variance - // overhead for our own array of delegates. - private struct Wrapper - { - public Wrapper(Delegate value) => Value = value; - public Delegate Value; - } + private object m_firstParameter; + private object m_helperObject; + private nint m_extraFunctionPointerOrData; + private IntPtr m_functionPointer; // WARNING: These constants are also declared in System.Private.TypeLoader\Internal\Runtime\TypeLoader\CallConverterThunk.cs // Do not change their values without updating the values in the calling convention converter component @@ -73,8 +63,14 @@ private protected virtual IntPtr GetThunk(int whichThunk) } /// + /// Used by various parts of the runtime as a replacement for Delegate.Method + /// + /// The Interop layer uses this to distinguish between different methods on a + /// single type, and to get the function pointer for delegates to static functions + /// /// The reflection apis use this api to figure out what MethodInfo is related /// to a delegate. + /// /// /// /// This value indicates which type an delegate's function pointer is associated with @@ -83,40 +79,57 @@ private protected virtual IntPtr GetThunk(int whichThunk) /// /// This value indicates if the returned pointer is an open resolver structure. /// - internal unsafe IntPtr GetDelegateLdFtnResult(out RuntimeTypeHandle typeOfFirstParameterIfInstanceDelegate, out bool isOpenResolver) + /// + /// Delegate points to an object array thunk (the delegate wraps a Func<object[], object> delegate). This + /// is typically a delegate pointing to the LINQ expression interpreter. + /// + /// + internal unsafe IntPtr GetFunctionPointer(out RuntimeTypeHandle typeOfFirstParameterIfInstanceDelegate, out bool isOpenResolver, out bool isInterpreterEntrypoint) { typeOfFirstParameterIfInstanceDelegate = default(RuntimeTypeHandle); isOpenResolver = false; + isInterpreterEntrypoint = false; - if (_extraFunctionPointerOrData != 0) + if (GetThunk(MulticastThunk) == m_functionPointer) + { + return IntPtr.Zero; + } + else if (GetThunk(ObjectArrayThunk) == m_functionPointer) { - if (GetThunk(OpenInstanceThunk) == _functionPointer) + isInterpreterEntrypoint = true; + return IntPtr.Zero; + } + else if (m_extraFunctionPointerOrData != 0) + { + if (GetThunk(OpenInstanceThunk) == m_functionPointer) { - typeOfFirstParameterIfInstanceDelegate = ((OpenMethodResolver*)_extraFunctionPointerOrData)->DeclaringType; + typeOfFirstParameterIfInstanceDelegate = ((OpenMethodResolver*)m_extraFunctionPointerOrData)->DeclaringType; isOpenResolver = true; } - return _extraFunctionPointerOrData; + return m_extraFunctionPointerOrData; } else { - if (_firstParameter != null) - typeOfFirstParameterIfInstanceDelegate = new RuntimeTypeHandle(_firstParameter.GetMethodTable()); + if (m_firstParameter != null) + typeOfFirstParameterIfInstanceDelegate = new RuntimeTypeHandle(m_firstParameter.GetMethodTable()); + + // TODO! Implementation issue for generic invokes here ... we need another IntPtr for uniqueness. - return _functionPointer; + return m_functionPointer; } } - // This function is known to the compiler. + // This function is known to the IL Transformer. private void InitializeClosedInstance(object firstParameter, IntPtr functionPointer) { if (firstParameter is null) throw new ArgumentException(SR.Arg_DlgtNullInst); - _functionPointer = functionPointer; - _firstParameter = firstParameter; + m_functionPointer = functionPointer; + m_firstParameter = firstParameter; } - // This function is known to the compiler. + // This function is known to the IL Transformer. private void InitializeClosedInstanceSlow(object firstParameter, IntPtr functionPointer) { // This method is like InitializeClosedInstance, but it handles ALL cases. In particular, it handles generic method with fun function pointers. @@ -126,15 +139,15 @@ private void InitializeClosedInstanceSlow(object firstParameter, IntPtr function if (!FunctionPointerOps.IsGenericMethodPointer(functionPointer)) { - _functionPointer = functionPointer; - _firstParameter = firstParameter; + m_functionPointer = functionPointer; + m_firstParameter = firstParameter; } else { - _firstParameter = this; - _functionPointer = GetThunk(ClosedInstanceThunkOverGenericMethod); - _extraFunctionPointerOrData = functionPointer; - _helperObject = firstParameter; + m_firstParameter = this; + m_functionPointer = GetThunk(ClosedInstanceThunkOverGenericMethod); + m_extraFunctionPointerOrData = functionPointer; + m_helperObject = firstParameter; } } @@ -153,28 +166,27 @@ private void InitializeClosedInstanceWithGVMResolution(object firstParameter, Ru } if (!FunctionPointerOps.IsGenericMethodPointer(functionResolution)) { - _functionPointer = functionResolution; - _firstParameter = firstParameter; + m_functionPointer = functionResolution; + m_firstParameter = firstParameter; } else { - _firstParameter = this; - _functionPointer = GetThunk(ClosedInstanceThunkOverGenericMethod); - _extraFunctionPointerOrData = functionResolution; - _helperObject = firstParameter; + m_firstParameter = this; + m_functionPointer = GetThunk(ClosedInstanceThunkOverGenericMethod); + m_extraFunctionPointerOrData = functionResolution; + m_helperObject = firstParameter; } return; } - // This function is known to the compiler. private void InitializeClosedInstanceToInterface(object firstParameter, IntPtr dispatchCell) { if (firstParameter is null) throw new NullReferenceException(); - _functionPointer = RuntimeImports.RhpResolveInterfaceMethod(firstParameter, dispatchCell); - _firstParameter = firstParameter; + m_functionPointer = RuntimeImports.RhpResolveInterfaceMethod(firstParameter, dispatchCell); + m_firstParameter = firstParameter; } // This is used to implement MethodInfo.CreateDelegate() in a desktop-compatible way. Yes, the desktop really @@ -183,53 +195,61 @@ private void InitializeClosedInstanceWithoutNullCheck(object firstParameter, Int { if (!FunctionPointerOps.IsGenericMethodPointer(functionPointer)) { - _functionPointer = functionPointer; - _firstParameter = firstParameter; + m_functionPointer = functionPointer; + m_firstParameter = firstParameter; } else { - _firstParameter = this; - _functionPointer = GetThunk(ClosedInstanceThunkOverGenericMethod); - _extraFunctionPointerOrData = functionPointer; - _helperObject = firstParameter; + m_firstParameter = this; + m_functionPointer = GetThunk(ClosedInstanceThunkOverGenericMethod); + m_extraFunctionPointerOrData = functionPointer; + m_helperObject = firstParameter; } } - // This function is known to the compiler. + // This function is known to the compiler backend. private void InitializeClosedStaticThunk(object firstParameter, IntPtr functionPointer, IntPtr functionPointerThunk) { - _extraFunctionPointerOrData = functionPointer; - _helperObject = firstParameter; - _functionPointer = functionPointerThunk; - _firstParameter = this; + m_extraFunctionPointerOrData = functionPointer; + m_helperObject = firstParameter; + m_functionPointer = functionPointerThunk; + m_firstParameter = this; } - // This function is known to the compiler. + // This function is known to the compiler backend. private void InitializeOpenStaticThunk(object _ /*firstParameter*/, IntPtr functionPointer, IntPtr functionPointerThunk) { // This sort of delegate is invoked by calling the thunk function pointer with the arguments to the delegate + a reference to the delegate object itself. - _firstParameter = this; - _functionPointer = functionPointerThunk; - _extraFunctionPointerOrData = functionPointer; + m_firstParameter = this; + m_functionPointer = functionPointerThunk; + m_extraFunctionPointerOrData = functionPointer; } private void InitializeOpenInstanceThunkDynamic(IntPtr functionPointer, IntPtr functionPointerThunk) { // This sort of delegate is invoked by calling the thunk function pointer with the arguments to the delegate + a reference to the delegate object itself. - _firstParameter = this; - _functionPointer = functionPointerThunk; - _extraFunctionPointerOrData = functionPointer; + m_firstParameter = this; + m_functionPointer = functionPointerThunk; + m_extraFunctionPointerOrData = functionPointer; } // This function is only ever called by the open instance method thunk, and in that case, - // _extraFunctionPointerOrData always points to an OpenMethodResolver + // m_extraFunctionPointerOrData always points to an OpenMethodResolver [MethodImpl(MethodImplOptions.NoInlining)] private IntPtr GetActualTargetFunctionPointer(object thisObject) { - return OpenMethodResolver.ResolveMethod(_extraFunctionPointerOrData, thisObject); + return OpenMethodResolver.ResolveMethod(m_extraFunctionPointerOrData, thisObject); } - internal bool IsDynamicDelegate() => GetThunk(MulticastThunk) == IntPtr.Zero; + internal bool IsDynamicDelegate() + { + if (this.GetThunk(MulticastThunk) == IntPtr.Zero) + { + return true; + } + + return false; + } [DebuggerGuidedStepThroughAttribute] protected virtual object? DynamicInvokeImpl(object?[]? args) @@ -237,7 +257,7 @@ private IntPtr GetActualTargetFunctionPointer(object thisObject) if (IsDynamicDelegate()) { // DynamicDelegate case - object? result = ((Func)_helperObject)(args); + object? result = ((Func)m_helperObject)(args); DebugAnnotations.PreviousCallContainsDebuggerStepInCode(); return result; } @@ -245,7 +265,7 @@ private IntPtr GetActualTargetFunctionPointer(object thisObject) { DynamicInvokeInfo dynamicInvokeInfo = ReflectionAugments.ReflectionCoreCallbacks.GetDelegateDynamicInvokeInfo(GetType()); - object? result = dynamicInvokeInfo.Invoke(_firstParameter, _functionPointer, + object? result = dynamicInvokeInfo.Invoke(m_firstParameter, m_functionPointer, args, binderBundle: null, wrapInTargetInvocationException: true); DebugAnnotations.PreviousCallContainsDebuggerStepInCode(); return result; @@ -254,53 +274,35 @@ private IntPtr GetActualTargetFunctionPointer(object thisObject) protected virtual MethodInfo GetMethodImpl() { - // Multi-cast delegates return the Method of the last delegate in the list - if (_helperObject is Wrapper[] invocationList) - { - int invocationCount = (int)_extraFunctionPointerOrData; - return invocationList[invocationCount - 1].Value.GetMethodImpl(); - } - - // Return the delegate Invoke method for marshalled function pointers and LINQ expressions - if ((_firstParameter is NativeFunctionPointerWrapper) || (_functionPointer == GetThunk(ObjectArrayThunk))) - { - return GetType().GetMethod("Invoke"); - } - return ReflectionAugments.ReflectionCoreCallbacks.GetDelegateMethod(this); } - public object? Target + public object Target { get { // Multi-cast delegates return the Target of the last delegate in the list - if (_helperObject is Wrapper[] invocationList) + if (m_functionPointer == GetThunk(MulticastThunk)) { - int invocationCount = (int)_extraFunctionPointerOrData; - return invocationList[invocationCount - 1].Value.Target; + Delegate[] invocationList = (Delegate[])m_helperObject; + int invocationCount = (int)m_extraFunctionPointerOrData; + return invocationList[invocationCount - 1].Target; } - // Closed static delegates place a value in _helperObject that they pass to the target method. - if (_functionPointer == GetThunk(ClosedStaticThunk) || - _functionPointer == GetThunk(ClosedInstanceThunkOverGenericMethod) || - _functionPointer == GetThunk(ObjectArrayThunk)) - return _helperObject; - - // Other non-closed thunks can be identified as the _firstParameter field points at this. - if (object.ReferenceEquals(_firstParameter, this)) - { - return null; - } + // Closed static delegates place a value in m_helperObject that they pass to the target method. + if (m_functionPointer == GetThunk(ClosedStaticThunk) || + m_functionPointer == GetThunk(ClosedInstanceThunkOverGenericMethod) || + m_functionPointer == GetThunk(ObjectArrayThunk)) + return m_helperObject; - // NativeFunctionPointerWrapper used by marshalled function pointers is not returned as a public target - if (_firstParameter is NativeFunctionPointerWrapper) + // Other non-closed thunks can be identified as the m_firstParameter field points at this. + if (object.ReferenceEquals(m_firstParameter, this)) { return null; } - // Closed instance delegates place a value in _firstParameter, and we've ruled out all other types of delegates - return _firstParameter; + // Closed instance delegates place a value in m_firstParameter, and we've ruled out all other types of delegates + return m_firstParameter; } } @@ -317,9 +319,13 @@ public object? Target // V1 api: Creates open delegates to static methods only, relaxed signature checking disallowed. public static Delegate CreateDelegate(Type type, [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] Type target, string method, bool ignoreCase, bool throwOnBindFailure) => ReflectionAugments.ReflectionCoreCallbacks.CreateDelegate(type, target, method, ignoreCase, throwOnBindFailure); - internal IntPtr TryGetOpenStaticFunctionPointer() => (GetThunk(OpenStaticThunk) == _functionPointer) ? _extraFunctionPointerOrData : 0; - - internal NativeFunctionPointerWrapper? TryGetNativeFunctionPointerWrapper() => _firstParameter as NativeFunctionPointerWrapper; + internal bool IsOpenStatic + { + get + { + return GetThunk(OpenStaticThunk) == m_functionPointer; + } + } internal static unsafe bool InternalEqualTypes(object a, object b) { @@ -344,9 +350,9 @@ internal static unsafe Delegate CreateObjectArrayDelegate(Type t, Func(RuntimeImports.RhNewObject(this.GetMethodTable())); + // First, allocate a new multicast delegate just like this one, i.e. same type as the this object + MulticastDelegate result = (MulticastDelegate)RuntimeImports.RhNewObject(this.GetMethodTable()); // Performance optimization - if this already points to a true multicast delegate, - // copy _functionPointer field rather than calling GetThunk to get it - result._functionPointer = thisIsMultiCastAlready ? _functionPointer : GetThunk(MulticastThunk); - result._firstParameter = result; - result._helperObject = invocationList; - result._extraFunctionPointerOrData = (IntPtr)invocationCount; + // copy _methodPtr and _methodPtrAux fields rather than calling into the EE to get them + if (thisIsMultiCastAlready) + { + result.m_functionPointer = this.m_functionPointer; + } + else + { + result.m_functionPointer = GetThunk(MulticastThunk); + } + result.m_firstParameter = result; + result.m_helperObject = invocationList; + result.m_extraFunctionPointerOrData = (IntPtr)invocationCount; return result; } - private static bool TrySetSlot(Wrapper[] a, int index, Delegate o) + private static bool TrySetSlot(Delegate[] a, int index, Delegate o) { - if (a[index].Value == null && System.Threading.Interlocked.CompareExchange(ref a[index].Value, o, null) == null) + if (a[index] == null && System.Threading.Interlocked.CompareExchange(ref a[index], o, null) == null) return true; // The slot may be already set because we have added and removed the same method before. // Optimize this case, because it's cheaper than copying the array. - if (a[index].Value is Delegate dd) + if (a[index] != null) { - if (object.ReferenceEquals(dd._firstParameter, o._firstParameter) && - object.ReferenceEquals(dd._helperObject, o._helperObject) && - dd._extraFunctionPointerOrData == o._extraFunctionPointerOrData && - dd._functionPointer == o._functionPointer) + MulticastDelegate d = (MulticastDelegate)o; + MulticastDelegate dd = (MulticastDelegate)a[index]; + + if (object.ReferenceEquals(dd.m_firstParameter, d.m_firstParameter) && + object.ReferenceEquals(dd.m_helperObject, d.m_helperObject) && + dd.m_extraFunctionPointerOrData == d.m_extraFunctionPointerOrData && + dd.m_functionPointer == d.m_functionPointer) { return true; } @@ -430,31 +446,35 @@ private static bool TrySetSlot(Wrapper[] a, int index, Delegate o) // to form a new delegate. protected virtual Delegate CombineImpl(Delegate? d) { - if (d is null) + if (d is null) // cast to object for a more efficient test return this; // Verify that the types are the same... if (!InternalEqualTypes(this, d)) throw new ArgumentException(SR.Arg_DlgtTypeMis); - if (IsDynamicDelegate()) + if (IsDynamicDelegate() && d.IsDynamicDelegate()) + { throw new InvalidOperationException(); + } + MulticastDelegate dFollow = (MulticastDelegate)d; + Delegate[]? resultList; int followCount = 1; - Wrapper[]? followList = d._helperObject as Wrapper[]; + Delegate[]? followList = dFollow.m_helperObject as Delegate[]; if (followList != null) - followCount = (int)d._extraFunctionPointerOrData; + followCount = (int)dFollow.m_extraFunctionPointerOrData; int resultCount; - Wrapper[]? resultList; - if (_helperObject is not Wrapper[] invocationList) + Delegate[]? invocationList = m_helperObject as Delegate[]; + if (invocationList == null) { resultCount = 1 + followCount; - resultList = new Wrapper[resultCount]; - resultList[0] = new Wrapper(this); + resultList = new Delegate[resultCount]; + resultList[0] = this; if (followList == null) { - resultList[1] = new Wrapper(d); + resultList[1] = dFollow; } else { @@ -465,7 +485,7 @@ protected virtual Delegate CombineImpl(Delegate? d) } else { - int invocationCount = (int)_extraFunctionPointerOrData; + int invocationCount = (int)m_extraFunctionPointerOrData; resultCount = invocationCount + followCount; resultList = null; if (resultCount <= invocationList.Length) @@ -473,14 +493,14 @@ protected virtual Delegate CombineImpl(Delegate? d) resultList = invocationList; if (followList == null) { - if (!TrySetSlot(resultList, invocationCount, d)) + if (!TrySetSlot(resultList, invocationCount, dFollow)) resultList = null; } else { for (int i = 0; i < followCount; i++) { - if (!TrySetSlot(resultList, invocationCount + i, followList[i].Value)) + if (!TrySetSlot(resultList, invocationCount + i, followList[i])) { resultList = null; break; @@ -495,14 +515,14 @@ protected virtual Delegate CombineImpl(Delegate? d) while (allocCount < resultCount) allocCount *= 2; - resultList = new Wrapper[allocCount]; + resultList = new Delegate[allocCount]; for (int i = 0; i < invocationCount; i++) resultList[i] = invocationList[i]; if (followList == null) { - resultList[invocationCount] = new Wrapper(d); + resultList[invocationCount] = dFollow; } else { @@ -514,13 +534,14 @@ protected virtual Delegate CombineImpl(Delegate? d) } } - private static Wrapper[] DeleteFromInvocationList(Wrapper[] invocationList, int invocationCount, int deleteIndex, int deleteCount) + private Delegate[] DeleteFromInvocationList(Delegate[] invocationList, int invocationCount, int deleteIndex, int deleteCount) { - int allocCount = invocationList.Length; + Delegate[] thisInvocationList = (Delegate[])m_helperObject; + int allocCount = thisInvocationList.Length; while (allocCount / 2 >= invocationCount - deleteCount) allocCount /= 2; - Wrapper[] newInvocationList = new Wrapper[allocCount]; + Delegate[] newInvocationList = new Delegate[allocCount]; for (int i = 0; i < deleteIndex; i++) newInvocationList[i] = invocationList[i]; @@ -531,11 +552,11 @@ private static Wrapper[] DeleteFromInvocationList(Wrapper[] invocationList, int return newInvocationList; } - private static bool EqualInvocationLists(Wrapper[] a, Wrapper[] b, int start, int count) + private static bool EqualInvocationLists(Delegate[] a, Delegate[] b, int start, int count) { for (int i = 0; i < count; i++) { - if (!(a[start + i].Value.Equals(b[i].Value))) + if (!(a[start + i].Equals(b[i]))) return false; } return true; @@ -551,31 +572,34 @@ private static bool EqualInvocationLists(Wrapper[] a, Wrapper[] b, int start, in // There is a special case were we are removing using a delegate as // the value we need to check for this case // - if (d is null) + MulticastDelegate? v = d as MulticastDelegate; + + if (v is null) return this; - if (d._helperObject is not Wrapper[] dInvocationList) + if (v.m_helperObject as Delegate[] == null) { - if (_helperObject is not Wrapper[] invocationList) + Delegate[]? invocationList = m_helperObject as Delegate[]; + if (invocationList == null) { // they are both not real Multicast - if (this.Equals(d)) + if (this.Equals(v)) return null; } else { - int invocationCount = (int)_extraFunctionPointerOrData; + int invocationCount = (int)m_extraFunctionPointerOrData; for (int i = invocationCount; --i >= 0;) { - if (d.Equals(invocationList[i].Value)) + if (v.Equals(invocationList[i])) { if (invocationCount == 2) { // Special case - only one value left, either at the beginning or the end - return invocationList[1 - i].Value; + return invocationList[1 - i]; } else { - Wrapper[] list = DeleteFromInvocationList(invocationList, invocationCount, i, 1); + Delegate[] list = DeleteFromInvocationList(invocationList, invocationCount, i, 1); return NewMulticastDelegate(list, invocationCount - 1, true); } } @@ -584,28 +608,29 @@ private static bool EqualInvocationLists(Wrapper[] a, Wrapper[] b, int start, in } else { - if (_helperObject is Wrapper[] invocationList) + Delegate[]? invocationList = m_helperObject as Delegate[]; + if (invocationList != null) { - int invocationCount = (int)_extraFunctionPointerOrData; - int dInvocationCount = (int)d._extraFunctionPointerOrData; - for (int i = invocationCount - dInvocationCount; i >= 0; i--) + int invocationCount = (int)m_extraFunctionPointerOrData; + int vInvocationCount = (int)v.m_extraFunctionPointerOrData; + for (int i = invocationCount - vInvocationCount; i >= 0; i--) { - if (EqualInvocationLists(invocationList, dInvocationList, i, dInvocationCount)) + if (EqualInvocationLists(invocationList, v.m_helperObject as Delegate[], i, vInvocationCount)) { - if (invocationCount - dInvocationCount == 0) + if (invocationCount - vInvocationCount == 0) { // Special case - no values left return null; } - else if (invocationCount - dInvocationCount == 1) + else if (invocationCount - vInvocationCount == 1) { // Special case - only one value left, either at the beginning or the end - return invocationList[i != 0 ? 0 : invocationCount - 1].Value; + return invocationList[i != 0 ? 0 : invocationCount - 1]; } else { - Wrapper[] list = DeleteFromInvocationList(invocationList, invocationCount, i, dInvocationCount); - return NewMulticastDelegate(list, invocationCount - dInvocationCount, true); + Delegate[] list = DeleteFromInvocationList(invocationList, invocationCount, i, vInvocationCount); + return NewMulticastDelegate(list, invocationCount - vInvocationCount, true); } } } @@ -617,19 +642,41 @@ private static bool EqualInvocationLists(Wrapper[] a, Wrapper[] b, int start, in public virtual Delegate[] GetInvocationList() { - if (_helperObject is Wrapper[] invocationList) + Delegate[] del; + Delegate[]? invocationList = m_helperObject as Delegate[]; + if (invocationList == null) + { + del = new Delegate[1]; + del[0] = this; + } + else { // Create an array of delegate copies and each // element into the array - int invocationCount = (int)_extraFunctionPointerOrData; + int invocationCount = (int)m_extraFunctionPointerOrData; + del = new Delegate[invocationCount]; - var del = new Delegate[invocationCount]; for (int i = 0; i < del.Length; i++) - del[i] = invocationList[i].Value; - return del; + del[i] = invocationList[i]; } + return del; + } + + private bool InvocationListEquals(MulticastDelegate d) + { + Delegate[] invocationList = (Delegate[])m_helperObject; + if (d.m_extraFunctionPointerOrData != m_extraFunctionPointerOrData) + return false; - return new Delegate[] { this }; + int invocationCount = (int)m_extraFunctionPointerOrData; + for (int i = 0; i < invocationCount; i++) + { + Delegate dd = invocationList[i]; + Delegate[] dInvocationList = (Delegate[])d.m_helperObject; + if (!dd.Equals(dInvocationList[i])) + return false; + } + return true; } public override bool Equals([NotNullWhen(true)] object? obj) @@ -641,92 +688,73 @@ public override bool Equals([NotNullWhen(true)] object? obj) if (!InternalEqualTypes(this, obj)) return false; - // Since this is a Delegate and we know the types are the same, obj should also be a Delegate - Debug.Assert(obj is Delegate, "Shouldn't have failed here since we already checked the types are the same!"); - var d = Unsafe.As(obj); + // Since this is a MulticastDelegate and we know + // the types are the same, obj should also be a + // MulticastDelegate + Debug.Assert(obj is MulticastDelegate, "Shouldn't have failed here since we already checked the types are the same!"); + var d = Unsafe.As(obj); - if (_helperObject is Wrapper[] invocationList) - { - if (d._extraFunctionPointerOrData != _extraFunctionPointerOrData) - return false; - - if (d._helperObject is not Wrapper[] dInvocationList) - return false; + // there are 2 kind of delegate kinds for comparison + // 1- Multicast (m_helperObject is Delegate[]) + // 2- Single-cast delegate, which can be compared with a structural comparison - int invocationCount = (int)_extraFunctionPointerOrData; - for (int i = 0; i < invocationCount; i++) - { - if (!invocationList[i].Value.Equals(dInvocationList[i].Value)) - return false; - } - return true; + IntPtr multicastThunk = GetThunk(MulticastThunk); + if (m_functionPointer == multicastThunk) + { + return d.m_functionPointer == multicastThunk && InvocationListEquals(d); } - - if (_firstParameter is NativeFunctionPointerWrapper nativeFunctionPointerWrapper) + else { - if (d._firstParameter is not NativeFunctionPointerWrapper dnativeFunctionPointerWrapper) + if (!object.ReferenceEquals(m_helperObject, d.m_helperObject) || + (!FunctionPointerOps.Compare(m_extraFunctionPointerOrData, d.m_extraFunctionPointerOrData)) || + (!FunctionPointerOps.Compare(m_functionPointer, d.m_functionPointer))) + { return false; + } - return nativeFunctionPointerWrapper.NativeFunctionPointer == dnativeFunctionPointerWrapper.NativeFunctionPointer; - } - - if (!object.ReferenceEquals(_helperObject, d._helperObject) || - (!FunctionPointerOps.Compare(_extraFunctionPointerOrData, d._extraFunctionPointerOrData)) || - (!FunctionPointerOps.Compare(_functionPointer, d._functionPointer))) - { - return false; - } + // Those delegate kinds with thunks put themselves into the m_firstParameter, so we can't + // blindly compare the m_firstParameter fields for equality. + if (object.ReferenceEquals(m_firstParameter, this)) + { + return object.ReferenceEquals(d.m_firstParameter, d); + } - // Those delegate kinds with thunks put themselves into the _firstParameter, so we can't - // blindly compare the _firstParameter fields for equality. - if (object.ReferenceEquals(_firstParameter, this)) - { - return object.ReferenceEquals(d._firstParameter, d); + return object.ReferenceEquals(m_firstParameter, d.m_firstParameter); } - - return object.ReferenceEquals(_firstParameter, d._firstParameter); } public override int GetHashCode() { - if (_helperObject is Wrapper[] invocationList) + Delegate[]? invocationList = m_helperObject as Delegate[]; + if (invocationList == null) { - int multiCastHash = 0; - for (int i = 0; i < (int)_extraFunctionPointerOrData; i++) - { - multiCastHash = multiCastHash * 33 + invocationList[i].Value.GetHashCode(); - } - return multiCastHash; + return base.GetHashCode(); } - - if (_firstParameter is NativeFunctionPointerWrapper nativeFunctionPointerWrapper) + else { - return nativeFunctionPointerWrapper.NativeFunctionPointer.GetHashCode(); - } - - int hash = RuntimeHelpers.GetHashCode(_helperObject) + - 7 * FunctionPointerOps.GetHashCode(_extraFunctionPointerOrData) + - 13 * FunctionPointerOps.GetHashCode(_functionPointer); + int hash = 0; + for (int i = 0; i < (int)m_extraFunctionPointerOrData; i++) + { + hash = hash * 33 + invocationList[i].GetHashCode(); + } - if (!object.ReferenceEquals(_firstParameter, this)) - { - hash += 17 * RuntimeHelpers.GetHashCode(_firstParameter); + return hash; } - - return hash; } - public bool HasSingleTarget => _helperObject is not Wrapper[]; + public bool HasSingleTarget => !(m_helperObject is Delegate[]); // Used by delegate invocation list enumerator internal Delegate? TryGetAt(int index) { - if (_helperObject is Wrapper[] invocationList) + if (!(m_helperObject is Delegate[] invocationList)) { - return ((uint)index < (uint)_extraFunctionPointerOrData) ? invocationList[index].Value : null; + return (index == 0) ? this : null; + } + else + { + return ((uint)index < (uint)m_extraFunctionPointerOrData) ? invocationList[index] : null; } - - return (index == 0) ? this : null; } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Diagnostics/Eventing/EventPipe.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Diagnostics/Eventing/EventPipe.NativeAot.cs new file mode 100644 index 00000000000000..8098c630dc9063 --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Diagnostics/Eventing/EventPipe.NativeAot.cs @@ -0,0 +1,163 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime; +using System.Runtime.InteropServices; + +#if FEATURE_PERFTRACING + +namespace System.Diagnostics.Tracing +{ + // + // NOTE: + // + // The implementation below takes some manual marshaling actions to ensure arguments are in + // primitive form before they are passed through to the underlying RuntimeImports.Rh* + // function. + // + // These extra steps are necessary only if the RuntimeImports mechanism represents "raw" + // calls into the native runtime (as has been the case at least in the distant past). + // + // If the RuntimeImports mechanism automatically applies rich p/invoke marshaling to all of + // these calls, then all of the manual steps below are unnecessary and can be removed (by + // making the RuntimeImports.Rh* function signatures generally match the corresponding + // EventPipeInternal function signatures; in other words, by making the RuntimeImports.Rh* + // functions look like the QCalls in EventPipe.CoreCLR.cs). + // + internal static partial class EventPipeInternal + { + // + // These PInvokes are used by the configuration APIs to interact with EventPipe. + // + private static unsafe ulong Enable( + char* outputFile, + EventPipeSerializationFormat format, + uint circularBufferSizeInMB, + EventPipeProviderConfigurationNative* providers, + uint numProviders) + { + return RuntimeImports.RhEventPipeInternal_Enable( + outputFile, + (int)format, + circularBufferSizeInMB, + providers, + numProviders); + } + + internal static void Disable(ulong sessionID) + { + RuntimeImports.RhEventPipeInternal_Disable(sessionID); + } + + // + // These PInvokes are used by EventSource to interact with the EventPipe. + // + +// private static extern unsafe IntPtr CreateProvider(string providerName, IntPtr callbackFunc, IntPtr callbackContext); + + internal static unsafe IntPtr CreateProvider(string providerName, + delegate* unmanaged callbackFunc, + void* callbackContext) + => CreateProvider(providerName, (IntPtr)callbackFunc, (IntPtr)callbackContext); + //internal static unsafe IntPtr CreateProvider(string providerName, IntPtr callbackFunc, IntPtr callbackContext); + + internal static unsafe IntPtr CreateProvider(string providerName, IntPtr callbackFunc, IntPtr callbackContext) + { + fixed (char* pProviderName = providerName) + { + return RuntimeImports.RhEventPipeInternal_CreateProvider( + pProviderName, + callbackFunc, + callbackContext); + } + } + + internal static unsafe IntPtr DefineEvent( + IntPtr provHandle, + uint eventID, + long keywords, + uint eventVersion, + uint level, + void *pMetadata, + uint metadataLength) + { + return RuntimeImports.RhEventPipeInternal_DefineEvent( + provHandle, + eventID, + keywords, + eventVersion, + level, + pMetadata, + metadataLength); + } + + internal static unsafe IntPtr GetProvider(string providerName) + { + fixed (char* pProviderName = providerName) + { + return RuntimeImports.RhEventPipeInternal_GetProvider(pProviderName); + } + } + + internal static void DeleteProvider(IntPtr provHandle) + { + RuntimeImports.RhEventPipeInternal_DeleteProvider(provHandle); + } + + internal static unsafe int EventActivityIdControl(uint controlCode, ref Guid activityId) + { + // + // Ensure that the address passed to native code is never on the managed heap, while still + // managing the supplied byref in an in/out manner. + // + Guid localActivityId = activityId; + try { return RuntimeImports.RhEventPipeInternal_EventActivityIdControl(controlCode, &localActivityId); } + finally { activityId = localActivityId; } + } + + internal static unsafe void WriteEventData( + IntPtr eventHandle, + EventProvider.EventData* pEventData, + uint dataCount, + Guid* activityId, + Guid* relatedActivityId) + { + RuntimeImports.RhEventPipeInternal_WriteEventData( + eventHandle, + pEventData, + dataCount, + activityId, + relatedActivityId); + } + + // + // These PInvokes are used as part of the EventPipeEventDispatcher. + // + internal static unsafe bool GetSessionInfo(ulong sessionID, EventPipeSessionInfo* pSessionInfo) + { + uint rawBool = RuntimeImports.RhEventPipeInternal_GetSessionInfo(sessionID, pSessionInfo); + return (rawBool != 0); + } + + internal static unsafe bool GetNextEvent(ulong sessionID, EventPipeEventInstanceData* pInstance) + { + uint rawBool = RuntimeImports.RhEventPipeInternal_GetNextEvent(sessionID, pInstance); + return (rawBool != 0); + } + + internal static bool SignalSession(ulong sessionID) + { + uint rawBool = RuntimeImports.RhEventPipeInternal_SignalSession(sessionID); + return (rawBool != 0); + } + + internal static bool WaitForSessionSignal(ulong sessionID, int timeoutMs) + { + uint rawBool = RuntimeImports.RhEventPipeInternal_WaitForSessionSignal(sessionID, timeoutMs); + return (rawBool != 0); + } + } +} + +#endif // FEATURE_PERFTRACING + diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Diagnostics/Eventing/NativeRuntimeEventSource.Threading.NativeSinks.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Diagnostics/Eventing/NativeRuntimeEventSource.Threading.NativeSinks.NativeAot.cs new file mode 100644 index 00000000000000..7f214ab0b2bb65 --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Diagnostics/Eventing/NativeRuntimeEventSource.Threading.NativeSinks.NativeAot.cs @@ -0,0 +1,152 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.Tracing; +using System.Runtime; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; + +using Internal.Runtime; +using Internal.Runtime.CompilerServices; + +namespace System.Diagnostics.Tracing +{ + // This is part of the NativeRuntimeEventsource, which is the managed version of the Microsoft-Windows-DotNETRuntime provider. + // It contains the runtime specific interop to native event sinks. + internal sealed partial class NativeRuntimeEventSource : EventSource + { + // We don't have these keywords defined from the genRuntimeEventSources.py, so we need to manually define them here. + public static partial class Keywords + { + public const EventKeywords ContentionKeyword = (EventKeywords)0x4000; + public const EventKeywords ThreadingKeyword = (EventKeywords)0x10000; + public const EventKeywords ThreadTransferKeyword = (EventKeywords)0x80000000; + public const EventKeywords WaitHandleKeyword = (EventKeywords)0x40000000000; + } + + [NonEvent] + internal static void LogContentionLockCreated(nint LockID, nint AssociatedObjectID, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogContentionLockCreated(LockID, AssociatedObjectID, ClrInstanceID); + } + + [NonEvent] + internal static void LogContentionStart(ContentionFlagsMap ContentionFlags, ushort ClrInstanceID, nint LockID, nint AssociatedObjectID, ulong LockOwnerThreadID) + { + RuntimeImports.NativeRuntimeEventSource_LogContentionStart((byte)ContentionFlags, ClrInstanceID, LockID, AssociatedObjectID, LockOwnerThreadID); + } + + [NonEvent] + internal static void LogContentionStop(ContentionFlagsMap ContentionFlags, ushort ClrInstanceID, double DurationNs) + { + RuntimeImports.NativeRuntimeEventSource_LogContentionStop((byte)ContentionFlags, ClrInstanceID, DurationNs); + } + + [NonEvent] + internal static void LogThreadPoolWorkerThreadStart(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(ActiveWorkerThreadCount, RetiredWorkerThreadCount, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolWorkerThreadStop(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(ActiveWorkerThreadCount, RetiredWorkerThreadCount, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolWorkerThreadWait(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(ActiveWorkerThreadCount, RetiredWorkerThreadCount, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolMinMaxThreads(ushort MinWorkerThreads, ushort MaxWorkerThreads, ushort MinIOCompletionThreads, ushort MaxIOCompletionThreads, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(MinWorkerThreads, MaxWorkerThreads, MinIOCompletionThreads, MaxIOCompletionThreads, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(Throughput, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint NewWorkerThreadCount, ThreadAdjustmentReasonMap Reason, ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(AverageThroughput, NewWorkerThreadCount, (uint)Reason, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolWorkerThreadAdjustmentStats( + double Duration, + double Throughput, + double ThreadPoolWorkerThreadWait, + double ThroughputWave, + double ThroughputErrorEstimate, + double AverageThroughputErrorEstimate, + double ThroughputRatio, + double Confidence, + double NewControlSetting, + ushort NewThreadWaveMagnitude, + ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats(Duration, Throughput, ThreadPoolWorkerThreadWait, ThroughputWave, + ThroughputErrorEstimate, AverageThroughputErrorEstimate, ThroughputRatio, Confidence, NewControlSetting, NewThreadWaveMagnitude, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolIOEnqueue( + IntPtr NativeOverlapped, + IntPtr Overlapped, + [MarshalAs(UnmanagedType.Bool)] bool MultiDequeues, + ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolIOEnqueue(NativeOverlapped, Overlapped, MultiDequeues, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolIODequeue( + IntPtr NativeOverlapped, + IntPtr Overlapped, + ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolIODequeue(NativeOverlapped, Overlapped, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolWorkingThreadCount( + uint Count, + ushort ClrInstanceID + ) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(Count, ClrInstanceID); + } + + [NonEvent] + internal static void LogThreadPoolIOPack( + IntPtr NativeOverlapped, + IntPtr Overlapped, + ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogThreadPoolIOPack(NativeOverlapped, Overlapped, ClrInstanceID); + } + + [NonEvent] + internal static void LogWaitHandleWaitStart( + WaitHandleWaitSourceMap WaitSource, + IntPtr AssociatedObjectID, + ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogWaitHandleWaitStart((byte)WaitSource, AssociatedObjectID, ClrInstanceID); + } + + [NonEvent] + internal static void LogWaitHandleWaitStop(ushort ClrInstanceID) + { + RuntimeImports.NativeRuntimeEventSource_LogWaitHandleWaitStop(ClrInstanceID); + } + } +} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs index 0ee16609157abc..62ed9a722671aa 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs @@ -695,9 +695,7 @@ public static unsafe IReadOnlyDictionary GetConfigurationVariabl Configurations = new Dictionary() }; -#pragma warning disable CS8500 // takes address of managed type - RuntimeImports.RhEnumerateConfigurationValues(&context, &ConfigCallback); -#pragma warning restore CS8500 + RuntimeImports.RhEnumerateConfigurationValues(Unsafe.AsPointer(ref context), &ConfigCallback); return context.Configurations!; } @@ -832,9 +830,7 @@ static T[] AllocateNewUninitializedArray(int length, bool pinned) throw new OverflowException(); T[]? array = null; -#pragma warning disable CS8500 // takes address of managed type - RuntimeImports.RhAllocateNewArray(MethodTable.Of(), (uint)length, (uint)flags, &array); -#pragma warning restore CS8500 + RuntimeImports.RhAllocateNewArray(MethodTable.Of(), (uint)length, (uint)flags, Unsafe.AsPointer(ref array)); if (array == null) throw new OutOfMemoryException(); @@ -861,9 +857,7 @@ public static unsafe T[] AllocateArray(int length, bool pinned = false) throw new OverflowException(); T[]? array = null; -#pragma warning disable CS8500 // takes address of managed type - RuntimeImports.RhAllocateNewArray(MethodTable.Of(), (uint)length, (uint)flags, &array); -#pragma warning restore CS8500 + RuntimeImports.RhAllocateNewArray(MethodTable.Of(), (uint)length, (uint)flags, Unsafe.AsPointer(ref array)); if (array == null) throw new OutOfMemoryException(); diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Object.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Object.NativeAot.cs index 9f8dbe11a212eb..727fbc9fbfdd2b 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Object.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Object.NativeAot.cs @@ -41,7 +41,7 @@ protected internal unsafe object MemberwiseClone() if (this.GetMethodTable()->ContainsGCPointers) Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount); else - SpanHelpers.Memmove(ref dst, ref src, byteCount); + Buffer.Memmove(ref dst, ref src, byteCount); return clone; } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs index f2c15681c288fe..fbf56e6e4603d2 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs @@ -36,7 +36,7 @@ public CustomMethodInvoker(Type thisType, Type[] parameterTypes, InvokerOptions if (!(thisObject == null && 0 != (_options & InvokerOptions.AllowNullThis))) ValidateThis(thisObject, _thisType.TypeHandle); - int argCount = arguments.Length; + int argCount = (arguments != null) ? arguments.Length : 0; if (argCount != _parameterTypes.Length) throw new TargetParameterCountException(); diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs index 9587d6f2d67fea..24a10f91faa7b4 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs @@ -1,34 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Globalization; - -using Internal.Reflection.Augments; - namespace System.Reflection { // Base class for runtime implemented Assembly public abstract class RuntimeAssembly : Assembly { - internal static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, CultureInfo culture, Version? version, bool throwOnFileNotFound) - { - AssemblyName mainAssemblyAn = mainAssembly.GetName(); - AssemblyName an = new AssemblyName(); - - an.CultureInfo = culture; - an.Name = mainAssemblyAn.Name + ".resources"; - an.SetPublicKeyToken(mainAssemblyAn.GetPublicKeyToken()); - an.Flags = mainAssemblyAn.Flags; - an.Version = version ?? mainAssemblyAn.Version; - - Assembly? retAssembly = ReflectionAugments.ReflectionCoreCallbacks.Load(an, throwOnFileNotFound); - - if (retAssembly == mainAssembly) - { - retAssembly = null; - } - - return retAssembly; - } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.NativeAot.cs new file mode 100644 index 00000000000000..f16c1ba3892381 --- /dev/null +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.NativeAot.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Reflection; +using System.Text; + +using Internal.Reflection.Augments; + +namespace System.Resources +{ + internal partial class ManifestBasedResourceGroveler + { + // Internal version of GetSatelliteAssembly that avoids throwing FileNotFoundException + private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, + CultureInfo culture, + Version? version) + { + AssemblyName mainAssemblyAn = mainAssembly.GetName(); + AssemblyName an = new AssemblyName(); + + an.CultureInfo = culture; + an.Name = mainAssemblyAn.Name + ".resources"; + an.SetPublicKeyToken(mainAssemblyAn.GetPublicKeyToken()); + an.Flags = mainAssemblyAn.Flags; + an.Version = version ?? mainAssemblyAn.Version; + + Assembly? retAssembly = ReflectionAugments.ReflectionCoreCallbacks.Load(an, false); + + if (retAssembly == mainAssembly) + { + retAssembly = null; + } + + return retAssembly; + } + } +} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NativeAot.cs index d91f79ce16a207..2babd66c359732 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NativeAot.cs @@ -1,16 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics.CodeAnalysis; - namespace System.Runtime.CompilerServices { public static partial class RuntimeFeature { - [FeatureSwitchDefinition("System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeSupported")] public static bool IsDynamicCodeSupported => false; - - [FeatureGuard(typeof(RequiresDynamicCodeAttribute))] public static bool IsDynamicCodeCompiled => false; } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.NativeAot.cs index 490997c1da90f5..a3ccfc5a8c431f 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.NativeAot.cs @@ -93,7 +93,7 @@ internal static unsafe void PtrToStructureImpl(IntPtr ptr, object structure) { nuint size = (nuint)RuntimeInteropData.GetStructUnsafeStructSize(structureTypeHandle); - SpanHelpers.Memmove(ref structure.GetRawData(), ref *(byte*)ptr, size); + Buffer.Memmove(ref structure.GetRawData(), ref *(byte*)ptr, size); } } @@ -180,7 +180,7 @@ public static unsafe void StructureToPtr(object structure, IntPtr ptr, bool fDel { nuint size = (nuint)RuntimeInteropData.GetStructUnsafeStructSize(structureTypeHandle); - SpanHelpers.Memmove(ref *(byte*)ptr, ref structure.GetRawData(), size); + Buffer.Memmove(ref *(byte*)ptr, ref structure.GetRawData(), size); } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeFunctionPointerWrapper.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeFunctionPointerWrapper.cs index 4571165530abfe..bb3ea2b5f10e27 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeFunctionPointerWrapper.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeFunctionPointerWrapper.cs @@ -12,9 +12,14 @@ internal abstract class NativeFunctionPointerWrapper { public NativeFunctionPointerWrapper(IntPtr nativeFunctionPointer) { - NativeFunctionPointer = nativeFunctionPointer; + m_nativeFunctionPointer = nativeFunctionPointer; } - public IntPtr NativeFunctionPointer { get; } + private IntPtr m_nativeFunctionPointer; + + public IntPtr NativeFunctionPointer + { + get { return m_nativeFunctionPointer; } + } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/PInvokeMarshal.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/PInvokeMarshal.cs index 0e1ad8d0486534..bf7fa122af6c5d 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/PInvokeMarshal.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/InteropServices/PInvokeMarshal.cs @@ -56,7 +56,7 @@ public static unsafe IntPtr GetFunctionPointerForDelegate(Delegate del) throw new ArgumentException(SR.Argument_NeedNonGenericType, "delegate"); #pragma warning restore CA2208 - NativeFunctionPointerWrapper? fpWrapper = del.TryGetNativeFunctionPointerWrapper(); + NativeFunctionPointerWrapper? fpWrapper = del.Target as NativeFunctionPointerWrapper; if (fpWrapper != null) { // @@ -104,71 +104,64 @@ internal unsafe struct ThunkContextData public IntPtr FunctionPtr; // Function pointer for open static delegates } - internal sealed unsafe class PInvokeDelegateThunk + internal sealed class PInvokeDelegateThunk { - public readonly IntPtr Thunk; // Thunk pointer - public readonly IntPtr ContextData; // ThunkContextData pointer which will be stored in the context slot of the thunk + public IntPtr Thunk; // Thunk pointer + public IntPtr ContextData; // ThunkContextData pointer which will be stored in the context slot of the thunk public PInvokeDelegateThunk(Delegate del) { + Thunk = RuntimeAugments.AllocateThunk(s_thunkPoolHeap); + Debug.Assert(Thunk != IntPtr.Zero); + if (Thunk == IntPtr.Zero) { - throw new OutOfMemoryException(); + // We've either run out of memory, or failed to allocate a new thunk due to some other bug. Now we should fail fast + Environment.FailFast("Insufficient number of thunks."); } - - // - // For open static delegates set target to ReverseOpenStaticDelegateStub which calls the static function pointer directly - // - IntPtr openStaticFunctionPointer = del.TryGetOpenStaticFunctionPointer(); - - // - // Allocate unmanaged memory for GCHandle of delegate and function pointer of open static delegate - // We will store this pointer on the context slot of thunk data - // - unsafe + else { - ContextData = (IntPtr)NativeMemory.AllocZeroed((nuint)(2 * IntPtr.Size)); + // + // Allocate unmanaged memory for GCHandle of delegate and function pointer of open static delegate + // We will store this pointer on the context slot of thunk data + // + unsafe + { + ContextData = (IntPtr)NativeMemory.Alloc((nuint)(2 * IntPtr.Size)); - ThunkContextData* thunkData = (ThunkContextData*)ContextData; + ThunkContextData* thunkData = (ThunkContextData*)ContextData; - // allocate a weak GChandle for the delegate - thunkData->Handle = GCHandle.Alloc(del, GCHandleType.WeakTrackResurrection); - thunkData->FunctionPtr = openStaticFunctionPointer; - } + // allocate a weak GChandle for the delegate + thunkData->Handle = GCHandle.Alloc(del, GCHandleType.Weak); - IntPtr pTarget = RuntimeInteropData.GetDelegateMarshallingStub(new RuntimeTypeHandle(del.GetMethodTable()), openStaticFunctionPointer != IntPtr.Zero); - Debug.Assert(pTarget != IntPtr.Zero); - - RuntimeAugments.SetThunkData(s_thunkPoolHeap, Thunk, ContextData, pTarget); + // if it is an open static delegate get the function pointer + if (del.IsOpenStatic) + thunkData->FunctionPtr = del.GetFunctionPointer(out RuntimeTypeHandle _, out bool _, out bool _); + else + thunkData->FunctionPtr = default; + } + } } ~PInvokeDelegateThunk() { - if (ContextData != IntPtr.Zero) + // Free the thunk + RuntimeAugments.FreeThunk(s_thunkPoolHeap, Thunk); + unsafe { - // free the GCHandle - GCHandle handle = ((ThunkContextData*)ContextData)->Handle; - if (handle.IsAllocated) + if (ContextData != IntPtr.Zero) { - // If the delegate is still alive, defer finalization. - if (handle.Target != null) + // free the GCHandle + GCHandle handle = ((ThunkContextData*)ContextData)->Handle; + if (handle.IsAllocated) { - GC.ReRegisterForFinalize(this); - return; + handle.Free(); } - handle.Free(); + // Free the allocated context data memory + NativeMemory.Free((void*)ContextData); } - - // Free the allocated context data memory - NativeMemory.Free((void*)ContextData); - } - - // Free the thunk - if (Thunk != IntPtr.Zero) - { - RuntimeAugments.FreeThunk(s_thunkPoolHeap, Thunk); } } } @@ -186,7 +179,19 @@ private static unsafe PInvokeDelegateThunk AllocateThunk(Delegate del) Debug.Assert(s_thunkPoolHeap != null); } - return new PInvokeDelegateThunk(del); + var delegateThunk = new PInvokeDelegateThunk(del); + + // + // For open static delegates set target to ReverseOpenStaticDelegateStub which calls the static function pointer directly + // + bool openStaticDelegate = del.IsOpenStatic; + + IntPtr pTarget = RuntimeInteropData.GetDelegateMarshallingStub(new RuntimeTypeHandle(del.GetMethodTable()), openStaticDelegate); + Debug.Assert(pTarget != IntPtr.Zero); + + RuntimeAugments.SetThunkData(s_thunkPoolHeap, delegateThunk.Thunk, delegateThunk.ContextData, pTarget); + + return delegateThunk; } /// diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs index 98604bc58561be..3f30983a74c3b2 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -42,9 +42,11 @@ internal static partial class RuntimeImports [LibraryImport(RuntimeLibrary)] [SuppressGCTransition] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial ulong RhpGetTickCount64(); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial IntPtr RhpGetCurrentThread(); [MethodImpl(MethodImplOptions.InternalCall)] @@ -164,12 +166,15 @@ internal static void RhWaitForPendingFinalizers(bool allowReentrantWait) internal static extern long RhGetLastGCDuration(int generation); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [MethodImpl(MethodImplOptions.InternalCall)] @@ -216,6 +221,7 @@ internal enum GCConfigurationType } [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial void RhEnumerateConfigurationValues(void* configurationContext, delegate* unmanaged callback); internal struct GCHeapHardLimitInfo @@ -231,15 +237,19 @@ internal struct GCHeapHardLimitInfo } [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial int RhRefreshMemoryLimit(GCHeapHardLimitInfo heapHardLimitInfo); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial int RhEnableNoGCRegionCallback(void* callback, long totalSize); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial long RhGetGenerationBudget(int generation); [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial long RhGetTotalAllocatedBytesPrecise(); [MethodImpl(MethodImplOptions.InternalCall)] @@ -365,6 +375,7 @@ internal static IntPtr RhHandleAllocDependent(object primary, object secondary) internal static extern int RhpGetThunkBlockSize(); [LibraryImport(RuntimeLibrary, EntryPoint = "RhAllocateThunksMapping")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial IntPtr RhAllocateThunksMapping(); // @@ -416,18 +427,22 @@ internal static IntPtr RhHandleAllocDependent(object primary, object secondary) // Busy spin for the given number of iterations. [LibraryImport(RuntimeLibrary, EntryPoint = "RhSpinWait")] [SuppressGCTransition] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial void RhSpinWait(int iterations); // Call RhSpinWait with a GC transition [LibraryImport(RuntimeLibrary, EntryPoint = "RhSpinWait")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial void RhLongSpinWait(int iterations); // Yield the cpu to another thread ready to process, if one is available. [LibraryImport(RuntimeLibrary, EntryPoint = "RhYield")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] private static partial int _RhYield(); internal static bool RhYield() { return (_RhYield() != 0); } [LibraryImport(RuntimeLibrary, EntryPoint = "RhFlushProcessWriteBuffers")] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static partial void RhFlushProcessWriteBuffers(); #if !TARGET_UNIX @@ -637,9 +652,170 @@ internal static IntPtr RhGetModuleSection(TypeManagerHandle module, ReadyToRunSe [RuntimeImport(RuntimeLibrary, "RhUnregisterForGCReporting")] internal static extern unsafe void RhUnregisterForGCReporting(GCFrameRegistration* pRegistration); + #if FEATURE_PERFTRACING + // + // EventPipeInternal helpers. + // + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial ulong RhEventPipeInternal_Enable( + char* outputFile, + int format, + uint circularBufferSizeInMB, + void* providers, + uint numProviders); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void RhEventPipeInternal_Disable(ulong sessionID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial IntPtr RhEventPipeInternal_CreateProvider(char* providerName, IntPtr callbackFunc, IntPtr callbackContext); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial IntPtr RhEventPipeInternal_DefineEvent( + IntPtr provHandle, + uint eventID, + long keywords, + uint eventVersion, + uint level, + void *pMetadata, + uint metadataLength); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial IntPtr RhEventPipeInternal_GetProvider(char* providerName); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void RhEventPipeInternal_DeleteProvider(IntPtr provHandle); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial int RhEventPipeInternal_EventActivityIdControl(uint controlCode, Guid* activityId); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial void RhEventPipeInternal_WriteEventData( + IntPtr eventHandle, + void* pEventData, + uint dataCount, + Guid* activityId, + Guid* relatedActivityId); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial uint RhEventPipeInternal_GetSessionInfo(ulong sessionID, void* pSessionInfo); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial uint RhEventPipeInternal_GetNextEvent(ulong sessionID, void* pInstance); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial uint RhEventPipeInternal_SignalSession(ulong sessionID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial uint RhEventPipeInternal_WaitForSessionSignal(ulong sessionID, int timeoutMs); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogContentionLockCreated(nint LockID, nint AssociatedObjectID, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogContentionStart(byte ContentionFlags, ushort ClrInstanceID, nint LockID, nint AssociatedObjectID, ulong LockOwnerThreadID); + [LibraryImport(RuntimeLibrary)] - internal static unsafe partial void NativeRuntimeEventSource_LogExceptionThrown(char* exceptionTypeName, char* exceptionMessage, IntPtr faultingIP, int hresult); + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogContentionStop(byte ContentionFlags, ushort ClrInstanceID, double DurationNs); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(uint ActiveWorkerThreadCount, uint RetiredWorkerThreadCount, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(ushort MinWorkerThreads, ushort MaxWorkerThreads, ushort MinIOCompletionThreads, ushort MaxIOCompletionThreads, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(double Throughput, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(double AverageThroughput, uint NewWorkerThreadCount, uint Reason, ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats( + double Duration, + double Throughput, + double ThreadPoolWorkerThreadWait, + double ThroughputWave, + double ThroughputErrorEstimate, + double AverageThroughputErrorEstimate, + double ThroughputRatio, + double Confidence, + double NewControlSetting, + ushort NewThreadWaveMagnitude, + ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolIOEnqueue( + IntPtr NativeOverlapped, + IntPtr Overlapped, + [MarshalAs(UnmanagedType.Bool)] bool MultiDequeues, + ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolIODequeue( + IntPtr NativeOverlapped, + IntPtr Overlapped, + ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount( + uint Count, + ushort ClrInstanceID + ); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogThreadPoolIOPack( + IntPtr NativeOverlapped, + IntPtr Overlapped, + ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static unsafe partial void NativeRuntimeEventSource_LogExceptionThrown(char* exceptionTypeName, char* exceptionMessage, IntPtr faultingIP, long hresult); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogWaitHandleWaitStart( + byte WaitSource, + IntPtr AssociatedObjectID, + ushort ClrInstanceID); + + [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] + internal static partial void NativeRuntimeEventSource_LogWaitHandleWaitStop(ushort ClrInstanceID); #endif // FEATURE_PERFTRACING // @@ -669,216 +845,265 @@ internal static IntPtr RhGetModuleSection(TypeManagerHandle module, ReadyToRunSe [RuntimeImport(RuntimeLibrary, "RhpCheckedXchg")] internal static extern object InterlockedExchange([NotNullIfNotNull(nameof(value))] ref object? location1, object? value); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "acos")] internal static extern double acos(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "acosf")] internal static extern float acosf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "acosh")] internal static extern double acosh(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "acoshf")] internal static extern float acoshf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "asin")] internal static extern double asin(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "asinf")] internal static extern float asinf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "asinh")] internal static extern double asinh(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "asinhf")] internal static extern float asinhf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "atan")] internal static extern double atan(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "atanf")] internal static extern float atanf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "atan2")] internal static extern double atan2(double y, double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "atan2f")] internal static extern float atan2f(float y, float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "atanh")] internal static extern double atanh(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "atanhf")] internal static extern float atanhf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "cbrt")] internal static extern double cbrt(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "cbrtf")] internal static extern float cbrtf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "ceil")] internal static extern double ceil(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "ceilf")] internal static extern float ceilf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "cos")] internal static extern double cos(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "cosf")] internal static extern float cosf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "cosh")] internal static extern double cosh(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "coshf")] internal static extern float coshf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "exp")] internal static extern double exp(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "expf")] internal static extern float expf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "floor")] internal static extern double floor(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "floorf")] internal static extern float floorf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "log")] internal static extern double log(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "logf")] internal static extern float logf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "log2")] internal static extern double log2(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "log2f")] internal static extern float log2f(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "log10")] internal static extern double log10(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "log10f")] internal static extern float log10f(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "pow")] internal static extern double pow(double x, double y); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "powf")] internal static extern float powf(float x, float y); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "sin")] internal static extern double sin(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "sinf")] internal static extern float sinf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "sinh")] internal static extern double sinh(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "sinhf")] internal static extern float sinhf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "sqrt")] internal static extern double sqrt(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "sqrtf")] internal static extern float sqrtf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "tan")] internal static extern double tan(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "tanf")] internal static extern float tanf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "tanh")] internal static extern double tanh(double x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "tanhf")] internal static extern float tanhf(float x); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "fmod")] internal static extern double fmod(double x, double y); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "fmodf")] internal static extern float fmodf(float x, float y); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "fma")] internal static extern double fma(double x, double y, double z); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "fmaf")] internal static extern float fmaf(float x, float y, float z); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "modf")] internal static extern unsafe double modf(double x, double* intptr); + [Intrinsic] [MethodImplAttribute(MethodImplOptions.InternalCall)] [RuntimeImport(RuntimeLibrary, "modff")] internal static extern unsafe float modff(float x, float* intptr); [LibraryImport(RuntimeImports.RuntimeLibrary)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial void* memmove(byte* dmem, byte* smem, nuint size); [LibraryImport(RuntimeImports.RuntimeLibrary)] - [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial void* memset(byte* mem, int value, nuint size); #if TARGET_X86 || TARGET_AMD64 [LibraryImport(RuntimeLibrary)] + [UnmanagedCallConv(CallConvs = [typeof(CallConvCdecl)])] internal static unsafe partial void RhCpuIdEx(int* cpuInfo, int functionId, int subFunctionId); #endif } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Lock.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Lock.NativeAot.cs index 7ac43d2257e786..78fc7745401990 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Lock.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/Lock.NativeAot.cs @@ -207,18 +207,14 @@ private static bool TryInitializeStatics() // Returns false until the static variable is lazy-initialized internal static bool IsSingleProcessor => s_isSingleProcessor; - // Used to transfer the state when inflating thin locks. The lock is considered unlocked if managedThreadId is zero, and - // locked otherwise. - internal void ResetForMonitor(int managedThreadId, uint recursionCount) + // Used to transfer the state when inflating thin locks + internal void InitializeLocked(int managedThreadId, uint recursionCount) { Debug.Assert(recursionCount == 0 || managedThreadId != 0); - Debug.Assert(!new State(this).UseTrivialWaits); _state = managedThreadId == 0 ? State.InitialStateValue : State.LockedStateValue; _owningThreadId = (uint)managedThreadId; _recursionCount = recursionCount; - - Debug.Assert(!new State(this).UseTrivialWaits); } internal struct ThreadId diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/SyncTable.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/SyncTable.cs index c75a5fe9de40d1..c3a273d32739a5 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/SyncTable.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Threading/SyncTable.cs @@ -274,7 +274,7 @@ public static void MoveThinLockToNewEntry(int syncIndex, int threadId, uint recu Debug.Assert(s_lock.IsHeldByCurrentThread); Debug.Assert((0 < syncIndex) && (syncIndex < s_unusedEntryIndex)); - s_entries[syncIndex].Lock.ResetForMonitor(threadId, recursionLevel); + s_entries[syncIndex].Lock.InitializeLocked(threadId, recursionLevel); } /// diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs index 968e97c425cf81..e8340e41191513 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs @@ -95,28 +95,43 @@ public override unsafe bool Equals([NotNullWhen(true)] object? obj) public override unsafe int GetHashCode() { - HashCode hashCode = default; - hashCode.Add((IntPtr)this.GetMethodTable()); + int hashCode = (int)this.GetMethodTable()->HashCode; + hashCode ^= GetHashCodeImpl(); + + return hashCode; + } + + private unsafe int GetHashCodeImpl() + { int numFields = __GetFieldHelper(GetNumFields, out _); if (numFields == UseFastHelper) - hashCode.AddBytes(GetSpanForField(this.GetMethodTable(), ref this.GetRawData())); - else - RegularGetValueTypeHashCode(ref hashCode, ref this.GetRawData(), numFields); + return FastGetValueTypeHashCodeHelper(this.GetMethodTable(), ref this.GetRawData()); - return hashCode.ToHashCode(); + return RegularGetValueTypeHashCode(ref this.GetRawData(), numFields); } - private static unsafe ReadOnlySpan GetSpanForField(MethodTable* type, ref byte data) + private static unsafe int FastGetValueTypeHashCodeHelper(MethodTable* type, ref byte data) { // Sanity check - if there are GC references, we should not be hashing bytes Debug.Assert(!type->ContainsGCPointers); - return new ReadOnlySpan(ref data, (int)type->ValueTypeSize); + + int size = (int)type->ValueTypeSize; + int hashCode = 0; + + for (int i = 0; i < size / 4; i++) + { + hashCode ^= Unsafe.As(ref Unsafe.Add(ref data, i * 4)); + } + + return hashCode; } - private unsafe void RegularGetValueTypeHashCode(ref HashCode hashCode, ref byte data, int numFields) + private unsafe int RegularGetValueTypeHashCode(ref byte data, int numFields) { + int hashCode = 0; + // We only take the hashcode for the first non-null field. That's what the CLR does. for (int i = 0; i < numFields; i++) { @@ -127,15 +142,15 @@ private unsafe void RegularGetValueTypeHashCode(ref HashCode hashCode, ref byte if (fieldType->ElementType == EETypeElementType.Single) { - hashCode.Add(Unsafe.As(ref fieldData)); + hashCode = Unsafe.As(ref fieldData).GetHashCode(); } else if (fieldType->ElementType == EETypeElementType.Double) { - hashCode.Add(Unsafe.As(ref fieldData)); + hashCode = Unsafe.As(ref fieldData).GetHashCode(); } else if (fieldType->IsPrimitive) { - hashCode.AddBytes(GetSpanForField(fieldType, ref fieldData)); + hashCode = FastGetValueTypeHashCodeHelper(fieldType, ref fieldData); } else if (fieldType->IsValueType) { @@ -149,7 +164,7 @@ private unsafe void RegularGetValueTypeHashCode(ref HashCode hashCode, ref byte var fieldValue = (ValueType)RuntimeImports.RhBox(fieldType, ref fieldData); if (fieldValue != null) { - hashCode.Add(fieldValue); + hashCode = fieldValue.GetHashCodeImpl(); } else { @@ -162,7 +177,7 @@ private unsafe void RegularGetValueTypeHashCode(ref HashCode hashCode, ref byte object fieldValue = Unsafe.As(ref fieldData); if (fieldValue != null) { - hashCode.Add(fieldValue); + hashCode = fieldValue.GetHashCode(); } else { @@ -172,6 +187,8 @@ private unsafe void RegularGetValueTypeHashCode(ref HashCode hashCode, ref byte } break; } + + return hashCode; } } } diff --git a/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/TypeLoader/ConstraintValidator.cs b/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/TypeLoader/ConstraintValidator.cs index d347b6d0f4af74..ccb1a9279f0a61 100644 --- a/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/TypeLoader/ConstraintValidator.cs +++ b/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Execution/TypeLoader/ConstraintValidator.cs @@ -13,9 +13,9 @@ internal static partial class ConstraintValidator { private static bool SatisfiesConstraints(this Type genericVariable, SigTypeContext typeContextOfConstraintDeclarer, Type typeArg) { - GenericParameterAttributes attributes = genericVariable.GenericParameterAttributes; + GenericParameterAttributes specialConstraints = genericVariable.GenericParameterAttributes & GenericParameterAttributes.SpecialConstraintMask; - if ((attributes & GenericParameterAttributes.NotNullableValueTypeConstraint) != 0) + if ((specialConstraints & GenericParameterAttributes.NotNullableValueTypeConstraint) != 0) { if (!typeArg.IsValueType) { @@ -30,19 +30,19 @@ private static bool SatisfiesConstraints(this Type genericVariable, SigTypeConte } } - if ((attributes & GenericParameterAttributes.ReferenceTypeConstraint) != 0) + if ((specialConstraints & GenericParameterAttributes.ReferenceTypeConstraint) != 0) { if (typeArg.IsValueType) return false; } - if ((attributes & GenericParameterAttributes.DefaultConstructorConstraint) != 0) + if ((specialConstraints & GenericParameterAttributes.DefaultConstructorConstraint) != 0) { if (!typeArg.HasExplicitOrImplicitPublicDefaultConstructor()) return false; } - if (typeArg.IsByRefLike && (attributes & (GenericParameterAttributes)0x20 /* GenericParameterAttributes.AllowByRefLike */) == 0) + if (typeArg.IsByRefLike && (specialConstraints & (GenericParameterAttributes)0x20 /* GenericParameterAttributes.AcceptByRefLike */) == 0) return false; // Now check general subtype constraints diff --git a/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Extensions/NonPortable/DelegateMethodInfoRetriever.cs b/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Extensions/NonPortable/DelegateMethodInfoRetriever.cs index 5669203a6f39d1..5ae75d1d55eb90 100644 --- a/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Extensions/NonPortable/DelegateMethodInfoRetriever.cs +++ b/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Internal/Reflection/Extensions/NonPortable/DelegateMethodInfoRetriever.cs @@ -17,7 +17,20 @@ public static class DelegateMethodInfoRetriever { public static MethodInfo GetDelegateMethodInfo(Delegate del) { - IntPtr originalLdFtnResult = RuntimeAugments.GetDelegateLdFtnResult(del, out RuntimeTypeHandle typeOfFirstParameterIfInstanceDelegate, out bool isOpenResolver); + Delegate[] invokeList = del.GetInvocationList(); + del = invokeList[invokeList.Length - 1]; + IntPtr originalLdFtnResult = RuntimeAugments.GetDelegateLdFtnResult(del, out RuntimeTypeHandle typeOfFirstParameterIfInstanceDelegate, out bool isOpenResolver, out bool isInterpreterEntrypoint); + + if (isInterpreterEntrypoint) + { + // This is a special kind of delegate where the invoke method is "ObjectArrayThunk". Typically, + // this will be a delegate that points the LINQ Expression interpreter. We could manufacture + // a MethodInfo based on the delegate's Invoke signature, but let's just throw for now. + throw new NotSupportedException(SR.DelegateGetMethodInfo_ObjectArrayDelegate); + } + + if (originalLdFtnResult == (IntPtr)0) + return null; QMethodDefinition methodHandle = default(QMethodDefinition); RuntimeTypeHandle[] genericMethodTypeArgumentHandles = null; @@ -66,7 +79,11 @@ public static MethodInfo GetDelegateMethodInfo(Delegate del) throw new NotSupportedException(SR.Format(SR.DelegateGetMethodInfo_NoDynamic_WithDisplayString, methodDisplayString)); } } - return (MethodInfo)ExecutionDomain.GetMethod(typeOfFirstParameterIfInstanceDelegate, methodHandle, genericMethodTypeArgumentHandles); + MethodBase methodBase = ExecutionDomain.GetMethod(typeOfFirstParameterIfInstanceDelegate, methodHandle, genericMethodTypeArgumentHandles); + MethodInfo methodInfo = methodBase as MethodInfo; + if (methodInfo != null) + return methodInfo; + return null; // GetMethod() returned a ConstructorInfo. } } } diff --git a/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Resources/Strings.resx b/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Resources/Strings.resx index e5432845a5206b..80023f62d226a7 100644 --- a/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Resources/Strings.resx +++ b/src/coreclr/nativeaot/System.Private.Reflection.Execution/src/Resources/Strings.resx @@ -204,6 +204,9 @@ Cannot retrieve a MethodInfo for this delegate because the necessary generic instantiation was not metadata-enabled. + + Cannot retrieve a MethodInfo for this delegate because the delegate target is an interpreted LINQ expression. + Could not retrieve the mapping of the interface '{0}' on type '{1}' because the type implements the interface abstractly. diff --git a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/EETypeCreator.cs b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/EETypeCreator.cs index 411f41d323372c..26137a0c61d551 100644 --- a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/EETypeCreator.cs +++ b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/EETypeCreator.cs @@ -104,14 +104,25 @@ public static int AlignUp(int val, int alignment) return result; } - public static unsafe void* AllocateMemory(int cbBytes) + public static unsafe void Memset(IntPtr destination, int length, byte value) { - return NativeMemory.Alloc((nuint)cbBytes); + byte* pbDest = (byte*)destination.ToPointer(); + while (length > 0) + { + *pbDest = value; + pbDest++; + length--; + } } - public static unsafe void FreeMemory(void* memoryPtrToFree) + public static unsafe IntPtr AllocateMemory(int cbBytes) { - NativeMemory.Free(memoryPtrToFree); + return (IntPtr)NativeMemory.Alloc((nuint)cbBytes); + } + + public static unsafe void FreeMemory(IntPtr memoryPtrToFree) + { + NativeMemory.Free((void*)memoryPtrToFree); } } @@ -121,12 +132,12 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo int arity, TypeBuilderState state) { bool successful = false; - void* eeTypePlusGCDesc = null; - void* writableData = null; - void* nonGcStaticData = null; - void* genericComposition = null; - void* threadStaticIndex = null; - nint gcStaticData = 0; + IntPtr eeTypePtrPlusGCDesc = IntPtr.Zero; + IntPtr writableDataPtr = IntPtr.Zero; + IntPtr gcStaticData = IntPtr.Zero; + IntPtr nonGcStaticData = IntPtr.Zero; + IntPtr genericComposition = IntPtr.Zero; + IntPtr threadStaticIndex = IntPtr.Zero; try { @@ -255,10 +266,10 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo int cbGCDescAligned = MemoryHelpers.AlignUp(cbGCDesc, IntPtr.Size); // Allocate enough space for the MethodTable + gcDescSize - eeTypePlusGCDesc = MemoryHelpers.AllocateMemory(cbGCDescAligned + cbEEType + cbOptionalFieldsSize); + eeTypePtrPlusGCDesc = MemoryHelpers.AllocateMemory(cbGCDescAligned + cbEEType + cbOptionalFieldsSize); // Get the MethodTable pointer, and the template MethodTable pointer - pEEType = (MethodTable*)((byte*)eeTypePlusGCDesc + cbGCDescAligned); + pEEType = (MethodTable*)(eeTypePtrPlusGCDesc + cbGCDescAligned); state.HalfBakedRuntimeTypeHandle = pEEType->ToRuntimeTypeHandle(); // Set basic MethodTable fields @@ -308,9 +319,9 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo *((void**)((byte*)pEEType + cbSealedVirtualSlotsTypeOffset)) = pTemplateEEType->GetSealedVirtualTable(); } - writableData = MemoryHelpers.AllocateMemory(WritableData.GetSize(IntPtr.Size)); - NativeMemory.Clear(writableData, (nuint)WritableData.GetSize(IntPtr.Size)); - pEEType->WritableData = writableData; + writableDataPtr = MemoryHelpers.AllocateMemory(WritableData.GetSize(IntPtr.Size)); + MemoryHelpers.Memset(writableDataPtr, WritableData.GetSize(IntPtr.Size), 0); + pEEType->WritableData = (void*)writableDataPtr; pEEType->DynamicTemplateType = pTemplateEEType; @@ -329,13 +340,13 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo if (arity > 1) { genericComposition = MemoryHelpers.AllocateMemory(MethodTable.GetGenericCompositionSize(arity)); - pEEType->SetGenericComposition((IntPtr)genericComposition); + pEEType->SetGenericComposition(genericComposition); } if (allocatedNonGCDataSize > 0) { nonGcStaticData = MemoryHelpers.AllocateMemory(allocatedNonGCDataSize); - NativeMemory.Clear(nonGcStaticData, (nuint)allocatedNonGCDataSize); + MemoryHelpers.Memset(nonGcStaticData, allocatedNonGCDataSize, 0); Debug.Assert(nonGCStaticDataOffset <= allocatedNonGCDataSize); pEEType->DynamicNonGcStaticsData = (IntPtr)((byte*)nonGcStaticData + nonGCStaticDataOffset); } @@ -348,7 +359,7 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo threadStaticIndex = MemoryHelpers.AllocateMemory(IntPtr.Size * 2); *(IntPtr*)threadStaticIndex = pEEType->PointerToTypeManager; *(((IntPtr*)threadStaticIndex) + 1) = (IntPtr)state.ThreadStaticOffset; - pEEType->DynamicThreadStaticsIndex = (IntPtr)threadStaticIndex; + pEEType->DynamicThreadStaticsIndex = threadStaticIndex; } if (state.GcDataSize != 0) @@ -357,7 +368,7 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo object obj = RuntimeAugments.RawNewObject(((MethodTable*)state.GcStaticDesc)->ToRuntimeTypeHandle()); gcStaticData = RuntimeAugments.RhHandleAlloc(obj, GCHandleType.Normal); - pEEType->DynamicGcStaticsData = (IntPtr)gcStaticData; + pEEType->DynamicGcStaticsData = gcStaticData; } if (state.Dictionary != null) @@ -372,16 +383,20 @@ private static void CreateEETypeWorker(MethodTable* pTemplateEEType, uint hashCo { if (!successful) { - if (gcStaticData != 0) + if (eeTypePtrPlusGCDesc != IntPtr.Zero) + MemoryHelpers.FreeMemory(eeTypePtrPlusGCDesc); + if (state.HalfBakedDictionary != IntPtr.Zero) + MemoryHelpers.FreeMemory(state.HalfBakedDictionary); + if (gcStaticData != IntPtr.Zero) RuntimeAugments.RhHandleFree(gcStaticData); - - MemoryHelpers.FreeMemory((void*)state.HalfBakedDictionary); - - MemoryHelpers.FreeMemory(threadStaticIndex); - MemoryHelpers.FreeMemory(nonGcStaticData); - MemoryHelpers.FreeMemory(genericComposition); - MemoryHelpers.FreeMemory(writableData); - MemoryHelpers.FreeMemory(eeTypePlusGCDesc); + if (genericComposition != IntPtr.Zero) + MemoryHelpers.FreeMemory(genericComposition); + if (nonGcStaticData != IntPtr.Zero) + MemoryHelpers.FreeMemory(nonGcStaticData); + if (writableDataPtr != IntPtr.Zero) + MemoryHelpers.FreeMemory(writableDataPtr); + if (threadStaticIndex != IntPtr.Zero) + MemoryHelpers.FreeMemory(threadStaticIndex); } } } diff --git a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/GenericDictionary.cs b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/GenericDictionary.cs index 7d2ead3b32aea2..c660b63d0c31eb 100644 --- a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/GenericDictionary.cs +++ b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/GenericDictionary.cs @@ -10,10 +10,10 @@ namespace Internal.Runtime.TypeLoader { - internal abstract unsafe class GenericDictionary + internal abstract class GenericDictionary { protected GenericDictionaryCell[] _cells; - protected void* _addressOfFirstCellSlot; + protected IntPtr _addressOfFirstCellSlot; public GenericDictionary(GenericDictionaryCell[] cells) { @@ -23,9 +23,9 @@ public GenericDictionary(GenericDictionaryCell[] cells) public abstract IntPtr Allocate(); - public void Finish(TypeBuilder typeBuilder) + public unsafe void Finish(TypeBuilder typeBuilder) { - Debug.Assert(_cells.Length == 0 || _addressOfFirstCellSlot != null); + Debug.Assert(_cells.Length == 0 || _addressOfFirstCellSlot != IntPtr.Zero); IntPtr* realCells = (IntPtr*)_addressOfFirstCellSlot; for (int i = 0; i < _cells.Length; i++) @@ -41,9 +41,9 @@ public GenericTypeDictionary(GenericDictionaryCell[] cells) : base(cells) { } - public override unsafe IntPtr Allocate() + public override IntPtr Allocate() { - Debug.Assert(_addressOfFirstCellSlot == null); + Debug.Assert(_addressOfFirstCellSlot == IntPtr.Zero); if (_cells.Length > 0) { @@ -51,7 +51,7 @@ public override unsafe IntPtr Allocate() _addressOfFirstCellSlot = MemoryHelpers.AllocateMemory(checked((int)(_cells.Length * IntPtr.Size))); } - return (IntPtr)_addressOfFirstCellSlot; + return _addressOfFirstCellSlot; } } @@ -63,20 +63,20 @@ public GenericMethodDictionary(GenericDictionaryCell[] cells) public override unsafe IntPtr Allocate() { - Debug.Assert(_addressOfFirstCellSlot == null); + Debug.Assert(_addressOfFirstCellSlot == IntPtr.Zero); // Method dictionaries start with a header containing the hash code, which is not part of the native layout. // The real first slot is located after the header. // Use checked typecast to int to ensure there aren't any overflows/truncations - void* dictionaryWithHeader = MemoryHelpers.AllocateMemory(checked((int)((_cells.Length + 1) * IntPtr.Size))); + IntPtr dictionaryWithHeader = MemoryHelpers.AllocateMemory(checked((int)((_cells.Length + 1) * IntPtr.Size))); // Put a magic hash code to indicate dynamically allocated method dictionary for // debugging purposes. *(int*)dictionaryWithHeader = 0xD1CC0DE; // DICCODE - _addressOfFirstCellSlot = (byte*)dictionaryWithHeader + IntPtr.Size; + _addressOfFirstCellSlot = IntPtr.Add(dictionaryWithHeader, IntPtr.Size); - return (IntPtr)_addressOfFirstCellSlot; + return _addressOfFirstCellSlot; } } } diff --git a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeBuilder.cs b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeBuilder.cs index 8ee004b11d42c7..0dbceb8be3c828 100644 --- a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeBuilder.cs +++ b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeBuilder.cs @@ -1080,7 +1080,7 @@ private unsafe IntPtr BuildGenericLookupTarget(TypeSystemContext typeSystemConte // The first is a pointer that points to the TypeManager indirection cell. // The second is the offset into the native layout info blob in that TypeManager, where the native signature is encoded. - IntPtr** lazySignature = (IntPtr**)signature; + IntPtr** lazySignature = (IntPtr**)signature.ToPointer(); typeManager = new TypeManagerHandle(lazySignature[0][0]); offset = checked((uint)new IntPtr(lazySignature[1]).ToInt32()); reader = TypeLoaderEnvironment.GetNativeLayoutInfoReader(typeManager); diff --git a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeLoaderEnvironment.LdTokenResultLookup.cs b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeLoaderEnvironment.LdTokenResultLookup.cs index 9ddec8a39d7aa3..d5e82d10d94cef 100644 --- a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeLoaderEnvironment.LdTokenResultLookup.cs +++ b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/Runtime/TypeLoader/TypeLoaderEnvironment.LdTokenResultLookup.cs @@ -43,7 +43,7 @@ internal struct DynamicMethodHandleInfo #region String conversions private static unsafe string GetStringFromMemoryInNativeFormat(IntPtr pointerToDataStream) { - byte* dataStream = (byte*)pointerToDataStream; + byte* dataStream = (byte*)pointerToDataStream.ToPointer(); uint stringLen = NativePrimitiveDecoder.DecodeUnsigned(ref dataStream); return Encoding.UTF8.GetString(dataStream, checked((int)stringLen)); } @@ -54,7 +54,7 @@ private static unsafe string GetStringFromMemoryInNativeFormat(IntPtr pointerToD /// /// /// - public unsafe IntPtr GetNativeFormatStringForString(string str) + public IntPtr GetNativeFormatStringForString(string str) { using (_typeLoaderLock.EnterScope()) { @@ -69,13 +69,13 @@ public unsafe IntPtr GetNativeFormatStringForString(string str) foreach (byte b in utf8Bytes) stringEncoder.WriteByte(b); - void* allocatedNativeFormatString = MemoryHelpers.AllocateMemory(stringEncoder.Size); + IntPtr allocatedNativeFormatString = MemoryHelpers.AllocateMemory(stringEncoder.Size); unsafe { - stringEncoder.Save((byte*)allocatedNativeFormatString, stringEncoder.Size); + stringEncoder.Save((byte*)allocatedNativeFormatString.ToPointer(), stringEncoder.Size); } - _nativeFormatStrings.Add(str, (IntPtr)allocatedNativeFormatString); - return (IntPtr)allocatedNativeFormatString; + _nativeFormatStrings.Add(str, allocatedNativeFormatString); + return allocatedNativeFormatString; } } @@ -197,12 +197,16 @@ public unsafe RuntimeFieldHandle GetRuntimeFieldHandleForComponents(RuntimeTypeH { if (!_runtimeFieldHandles.TryGetValue(key, out runtimeFieldHandle)) { - DynamicFieldHandleInfo* fieldData = (DynamicFieldHandleInfo*)MemoryHelpers.AllocateMemory(sizeof(DynamicFieldHandleInfo)); + IntPtr runtimeFieldHandleValue = MemoryHelpers.AllocateMemory(sizeof(DynamicFieldHandleInfo)); + if (runtimeFieldHandleValue == IntPtr.Zero) + throw new OutOfMemoryException(); + + DynamicFieldHandleInfo* fieldData = (DynamicFieldHandleInfo*)runtimeFieldHandleValue.ToPointer(); fieldData->DeclaringType = *(IntPtr*)&declaringTypeHandle; fieldData->FieldName = fieldName; // Special flag (lowest bit set) in the handle value to indicate it was dynamically allocated - IntPtr runtimeFieldHandleValue = (IntPtr)fieldData + 1; + runtimeFieldHandleValue++; runtimeFieldHandle = *(RuntimeFieldHandle*)&runtimeFieldHandleValue; _runtimeFieldHandles.Add(key, runtimeFieldHandle); @@ -224,9 +228,10 @@ private unsafe bool TryGetDynamicRuntimeFieldHandleComponents(RuntimeFieldHandle IntPtr runtimeFieldHandleValue = *(IntPtr*)&runtimeFieldHandle; // Special flag in the handle value to indicate it was dynamically allocated - Debug.Assert((runtimeFieldHandleValue & 0x1) == 0x1); + Debug.Assert((runtimeFieldHandleValue.ToInt64() & 0x1) == 0x1); + runtimeFieldHandleValue--; - DynamicFieldHandleInfo* fieldData = (DynamicFieldHandleInfo*)(runtimeFieldHandleValue - 1); + DynamicFieldHandleInfo* fieldData = (DynamicFieldHandleInfo*)runtimeFieldHandleValue.ToPointer(); declaringTypeHandle = *(RuntimeTypeHandle*)&(fieldData->DeclaringType); // FieldName points to the field name in NativeLayout format, so we parse it using a NativeParser @@ -292,8 +297,11 @@ public unsafe RuntimeMethodHandle GetRuntimeMethodHandleForComponents(RuntimeTyp int numGenericMethodArgs = genericMethodArgs == null ? 0 : genericMethodArgs.Length; // Use checked arithmetics to ensure there aren't any overflows/truncations sizeToAllocate = checked(sizeToAllocate + (numGenericMethodArgs > 0 ? sizeof(IntPtr) * (numGenericMethodArgs - 1) : 0)); + IntPtr runtimeMethodHandleValue = MemoryHelpers.AllocateMemory(sizeToAllocate); + if (runtimeMethodHandleValue == IntPtr.Zero) + throw new OutOfMemoryException(); - DynamicMethodHandleInfo* methodData = (DynamicMethodHandleInfo*)MemoryHelpers.AllocateMemory(sizeToAllocate); + DynamicMethodHandleInfo* methodData = (DynamicMethodHandleInfo*)runtimeMethodHandleValue.ToPointer(); methodData->DeclaringType = *(IntPtr*)&declaringTypeHandle; methodData->MethodName = methodName; methodData->MethodSignature = methodSignature; @@ -306,7 +314,7 @@ public unsafe RuntimeMethodHandle GetRuntimeMethodHandleForComponents(RuntimeTyp } // Special flag in the handle value to indicate it was dynamically allocated, and doesn't point into the InvokeMap blob - IntPtr runtimeMethodHandleValue = (IntPtr)methodData + 1; + runtimeMethodHandleValue++; runtimeMethodHandle = *(RuntimeMethodHandle*)&runtimeMethodHandleValue; _runtimeMethodHandles.Add(key, runtimeMethodHandle); @@ -338,12 +346,12 @@ public bool TryGetRuntimeMethodHandleComponents(RuntimeMethodHandle runtimeMetho private unsafe bool TryGetDynamicRuntimeMethodHandleComponents(RuntimeMethodHandle runtimeMethodHandle, out RuntimeTypeHandle declaringTypeHandle, out MethodNameAndSignature nameAndSignature, out RuntimeTypeHandle[] genericMethodArgs) { IntPtr runtimeMethodHandleValue = *(IntPtr*)&runtimeMethodHandle; + Debug.Assert((runtimeMethodHandleValue.ToInt64() & 0x1) == 0x1); // Special flag in the handle value to indicate it was dynamically allocated, and doesn't point into the InvokeMap blob - Debug.Assert((runtimeMethodHandleValue & 0x1) == 0x1); - - DynamicMethodHandleInfo* methodData = (DynamicMethodHandleInfo*)(runtimeMethodHandleValue - 1); + runtimeMethodHandleValue--; + DynamicMethodHandleInfo* methodData = (DynamicMethodHandleInfo*)runtimeMethodHandleValue.ToPointer(); declaringTypeHandle = *(RuntimeTypeHandle*)&(methodData->DeclaringType); genericMethodArgs = null; diff --git a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/TypeSystem/TypeSystemContext.Runtime.cs b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/TypeSystem/TypeSystemContext.Runtime.cs index 464a370a5f7be3..75252d276d49e7 100644 --- a/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/TypeSystem/TypeSystemContext.Runtime.cs +++ b/src/coreclr/nativeaot/System.Private.TypeLoader/src/Internal/TypeSystem/TypeSystemContext.Runtime.cs @@ -42,7 +42,7 @@ protected override RuntimeTypeHandle ConvertIntPtrToValue(IntPtr pointer) { unsafe { - return ((MethodTable*)pointer)->ToRuntimeTypeHandle(); + return ((MethodTable*)pointer.ToPointer())->ToRuntimeTypeHandle(); } } @@ -104,7 +104,7 @@ protected override RuntimeTypeHandle ConvertIntPtrToValue(IntPtr pointer) { unsafe { - return ((MethodTable*)pointer)->ToRuntimeTypeHandle(); + return ((MethodTable*)pointer.ToPointer())->ToRuntimeTypeHandle(); } } diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/InlineArrayAttribute.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/InlineArrayAttribute.cs deleted file mode 100644 index f7d47791c7e310..00000000000000 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/CompilerServices/InlineArrayAttribute.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Runtime.CompilerServices -{ - [AttributeUsage(AttributeTargets.Struct, AllowMultiple = false)] - public sealed class InlineArrayAttribute : Attribute - { - public InlineArrayAttribute(int length) - { - Length = length; - } - - public int Length { get; } - } -} diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs index d755bae45859aa..9bfc3314b2d3bb 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs +++ b/src/coreclr/nativeaot/Test.CoreLib/src/System/Runtime/RuntimeImports.cs @@ -40,13 +40,13 @@ internal static IntPtr RhHandleAlloc(object value, GCHandleType type) return h; } - [DllImport(RuntimeLibrary)] + [DllImport(RuntimeLibrary, CallingConvention = CallingConvention.Cdecl)] internal static extern unsafe IntPtr RhRegisterFrozenSegment(void* pSegmentStart, nuint allocSize, nuint commitSize, nuint reservedSize); - [DllImport(RuntimeLibrary)] + [DllImport(RuntimeLibrary, CallingConvention = CallingConvention.Cdecl)] internal static extern unsafe void RhUpdateFrozenSegment(IntPtr seg, void* allocated, void* committed); - [DllImport(RuntimeLibrary)] + [DllImport(RuntimeLibrary, CallingConvention = CallingConvention.Cdecl)] internal static extern void RhUnregisterFrozenSegment(IntPtr pSegmentHandle); [RuntimeImport(RuntimeLibrary, "RhpGetModuleSection")] diff --git a/src/coreclr/nativeaot/Test.CoreLib/src/Test.CoreLib.csproj b/src/coreclr/nativeaot/Test.CoreLib/src/Test.CoreLib.csproj index fc0d6ab6198c7f..ec2e93c06ad04e 100644 --- a/src/coreclr/nativeaot/Test.CoreLib/src/Test.CoreLib.csproj +++ b/src/coreclr/nativeaot/Test.CoreLib/src/Test.CoreLib.csproj @@ -218,7 +218,6 @@ - diff --git a/src/coreclr/pal/inc/unixasmmacrosamd64.inc b/src/coreclr/pal/inc/unixasmmacrosamd64.inc index bc6d770a51824a..bb1e70a27bef02 100644 --- a/src/coreclr/pal/inc/unixasmmacrosamd64.inc +++ b/src/coreclr/pal/inc/unixasmmacrosamd64.inc @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -#define C_VAR(Name) rip + C_FUNC(Name) - .macro NESTED_ENTRY Name, Section, Handler LEAF_ENTRY \Name, \Section .ifnc \Handler, NoHandler diff --git a/src/coreclr/pal/src/exception/machexception.cpp b/src/coreclr/pal/src/exception/machexception.cpp index 8f58cd4d627b4c..50db83248fe7ac 100644 --- a/src/coreclr/pal/src/exception/machexception.cpp +++ b/src/coreclr/pal/src/exception/machexception.cpp @@ -849,7 +849,7 @@ HijackFaultingThread( if (fIsStackOverflow) { // Allocate the minimal stack necessary for handling stack overflow - int stackOverflowStackSize = 15 * 4096; + int stackOverflowStackSize = 7 * 4096; // Align the size to virtual page size and add one virtual page as a stack guard stackOverflowStackSize = ALIGN_UP(stackOverflowStackSize, GetVirtualPageSize()) + GetVirtualPageSize(); void* stackOverflowHandlerStack = mmap(NULL, stackOverflowStackSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); diff --git a/src/coreclr/pal/src/include/pal/process.h b/src/coreclr/pal/src/include/pal/process.h index 1c48093af219da..5b0cd0739ed53c 100644 --- a/src/coreclr/pal/src/include/pal/process.h +++ b/src/coreclr/pal/src/include/pal/process.h @@ -155,10 +155,7 @@ BOOL PROCAbortInitialize(); Does not return --*/ -#if !defined(HOST_ARM) // PAL_NORETURN produces broken unwinding information for this method - // making crash dumps impossible to analyze PAL_NORETURN -#endif VOID PROCAbort(int signal = SIGABRT, siginfo_t* siginfo = nullptr); /*++ diff --git a/src/coreclr/pal/src/misc/cgroup.cpp b/src/coreclr/pal/src/misc/cgroup.cpp index f23ff4c970fe9b..ee3c0ae5843929 100644 --- a/src/coreclr/pal/src/misc/cgroup.cpp +++ b/src/coreclr/pal/src/misc/cgroup.cpp @@ -28,6 +28,7 @@ SET_DEFAULT_DEBUG_CHANNEL(MISC); #endif #define CGROUP2_SUPER_MAGIC 0x63677270 +#define TMPFS_MAGIC 0x01021994 #define BASE_TEN 10 @@ -93,16 +94,12 @@ class CGroup if (result != 0) return 0; - if (stats.f_type == CGROUP2_SUPER_MAGIC) + switch (stats.f_type) { - return 2; - } - else - { - // Assume that if /sys/fs/cgroup exists and the file system type is not cgroup2fs, - // it is cgroup v1. Typically the file system type is tmpfs, but other values have - // been seen in the wild. - return 1; + case TMPFS_MAGIC: return 1; + case CGROUP2_SUPER_MAGIC: return 2; + default: + return 0; } #endif } diff --git a/src/coreclr/pal/src/safecrt/vsprintf.cpp b/src/coreclr/pal/src/safecrt/vsprintf.cpp index 360222d5dc6798..b8ff745f563ceb 100644 --- a/src/coreclr/pal/src/safecrt/vsprintf.cpp +++ b/src/coreclr/pal/src/safecrt/vsprintf.cpp @@ -95,7 +95,7 @@ DLLEXPORT int __cdecl _vsnprintf_s ( retvalue = vsnprintf(string, sizeInBytes, format, ap); string[sizeInBytes - 1] = '\0'; /* we allow truncation if count == _TRUNCATE */ - if (retvalue >= (int)sizeInBytes && count == _TRUNCATE) + if (retvalue > (int)sizeInBytes && count == _TRUNCATE) { if (errno == ERANGE) { diff --git a/src/coreclr/pal/src/thread/process.cpp b/src/coreclr/pal/src/thread/process.cpp index 757ed25ade1d6f..b23eab001cca4e 100644 --- a/src/coreclr/pal/src/thread/process.cpp +++ b/src/coreclr/pal/src/thread/process.cpp @@ -1271,7 +1271,7 @@ RaiseFailFastException( ENTRY("RaiseFailFastException"); TerminateCurrentProcessNoExit(TRUE); - for (;;) PROCAbort(); + PROCAbort(); LOGEXIT("RaiseFailFastException"); PERF_EXIT(RaiseFailFastException); @@ -2541,9 +2541,7 @@ PROCCreateCrashDumpIfEnabled(int signal, siginfo_t* siginfo, bool serialize) Does not return --*/ -#if !defined(HOST_ARM) PAL_NORETURN -#endif VOID PROCAbort(int signal, siginfo_t* siginfo) { diff --git a/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp b/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp index 62b725208769c2..fb5ab3a2d7af41 100644 --- a/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp +++ b/src/coreclr/pal/tests/palsuite/c_runtime/_vsnprintf_s/test1/test1.cpp @@ -49,18 +49,6 @@ PALTEST(c_runtime__vsnprintf_s_test1_paltest_vsnprintf_test1, "c_runtime/_vsnpri Fail("ERROR: expected %s (up to %d chars), got %s\n", checkstr, 8, buf); } - char buf8[8] = {0}; - - ret = Testvsnprintf(buf8, 8, "abcdefgh"); - if (ret >= 0) - { - Fail("ERROR: expected negative return value, got %d", ret); - } - if (memcmp(buf8, "abcdefg\0", 8) != 0) - { - Fail("ERROR: Expected 7 chars + null terminator"); - } - PAL_Terminate(); return PASS; } diff --git a/src/coreclr/pal/tests/palsuite/paltestlist.txt b/src/coreclr/pal/tests/palsuite/paltestlist.txt index 0a0bd955f3adb6..878f00a2adf85c 100644 --- a/src/coreclr/pal/tests/palsuite/paltestlist.txt +++ b/src/coreclr/pal/tests/palsuite/paltestlist.txt @@ -2,6 +2,9 @@ c_runtime/atof/test1/paltest_atof_test1 c_runtime/atoi/test1/paltest_atoi_test1 c_runtime/bsearch/test1/paltest_bsearch_test1 c_runtime/bsearch/test2/paltest_bsearch_test2 +c_runtime/errno/test1/paltest_errno_test1 +c_runtime/errno/test2/paltest_errno_test2 +c_runtime/exit/test1/paltest_exit_test1 c_runtime/free/test1/paltest_free_test1 c_runtime/isalnum/test1/paltest_isalnum_test1 c_runtime/isalpha/test1/paltest_isalpha_test1 diff --git a/src/coreclr/runtime-prereqs.proj b/src/coreclr/runtime-prereqs.proj index 6bbe50f7d550a9..b1d1cf8b041df8 100644 --- a/src/coreclr/runtime-prereqs.proj +++ b/src/coreclr/runtime-prereqs.proj @@ -13,10 +13,6 @@ - - - - $(RuntimeBinDir)ilc-published/ + + false false false - false false true @@ -21,11 +22,16 @@ true true true + + <_hostOS>$(NETCoreSdkPortableRuntimeIdentifier.SubString(0, $(NETCoreSdkPortableRuntimeIdentifier.LastIndexOf('-')))) + <_hostArchitecture Condition="'$(OS)' != 'Windows_NT'">$(NETCoreSdkPortableRuntimeIdentifier.SubString($([MSBuild]::Add($(NETCoreSdkPortableRuntimeIdentifier.LastIndexOf('-')), 1)))) + <_hostArchitecture Condition="'$(OS)' == 'Windows_NT'">$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) + <_hostPackageName>runtime.$(_hostOS)-$(_hostArchitecture).Microsoft.DotNet.ILCompiler - + @@ -66,19 +72,6 @@ - - - - - - - <_XcodeVersion>$([System.Text.RegularExpressions.Regex]::Match($(_XcodeVersionString), '[1-9]\d*')) - - - - - - $(_CC_LDFLAGS.SubString(0, $(_CC_LDFLAGS.IndexOf(';')))) <_LDFLAGS>$(_CC_LDFLAGS.SubString($([MSBuild]::Add($(_CC_LDFLAGS.IndexOf(';')), 1)))) @@ -87,7 +80,7 @@ - + + + <_objWriterRidPlatformIndex>$(RuntimeIdentifier.LastIndexOf('-')) + $(RuntimeIdentifier.Substring(0, $(_objWriterRidPlatformIndex))) + $(RuntimeIdentifier.Substring($(_objWriterRidPlatformIndex)).TrimStart('-')) + + + linux + + $(ObjWriterRidWithoutPlatform)-$(ObjWriterRidPlatform) + + $(runtimelinuxarm64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimelinuxx64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimelinuxmuslarm64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimelinuxmuslx64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimewinarm64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimewinx64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimeosxarm64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimeosxx64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimefreebsdx64MicrosoftNETCoreRuntimeObjWriterVersion) + $(runtimefreebsdarm64MicrosoftNETCoreRuntimeObjWriterVersion) + true + + true + + $(ObjWriterVersion) + + + + + $(NetStandardLibraryVersion) + + + + PreserveNewest + false + false + + PreserveNewest diff --git a/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj b/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj index f7aa4cc8b93fd8..09e2a5bec4c8d4 100644 --- a/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj +++ b/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj @@ -6,9 +6,7 @@ AnyCPU false false - linux-x64;win-x64;osx-x64 - - $(RuntimeIdentifiers);freebsd-x64;freebsd-arm64 + linux-x64;win-x64;osx-x64;freebsd-x64;freebsd-arm64 Debug;Release;Checked true false diff --git a/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj b/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj index 657ac23590a039..522561c9c8a366 100644 --- a/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj +++ b/src/coreclr/tools/aot/crossgen2/crossgen2_publish.csproj @@ -7,9 +7,10 @@ + + false false false - false false true diff --git a/src/coreclr/tools/aot/ilc.sln b/src/coreclr/tools/aot/ilc.sln index a6b9c84c570776..3fe8026533f307 100644 --- a/src/coreclr/tools/aot/ilc.sln +++ b/src/coreclr/tools/aot/ilc.sln @@ -20,14 +20,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ILCompiler.Compiler.Tests", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ILCompiler.TypeSystem.Tests", "ILCompiler.TypeSystem.Tests\ILCompiler.TypeSystem.Tests.csproj", "{740CDFF4-B8EC-4A37-951B-C9FE9980EF2A}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mono.Linker.Tests", "Mono.Linker.Tests\Mono.Linker.Tests.csproj", "{4CF2ECD3-A1C3-4A28-AB08-A61C53114143}" +EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mono.Linker.Tests.Cases", "..\..\..\tools\illink\test\Mono.Linker.Tests.Cases\Mono.Linker.Tests.Cases.csproj", "{9DA153BF-51C4-4AD7-A355-9F9528843DC7}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Mono.Linker.Tests.Cases.Expectations", "..\..\..\tools\illink\test\Mono.Linker.Tests.Cases.Expectations\Mono.Linker.Tests.Cases.Expectations.csproj", "{219E0AC3-CDBF-4104-B324-85915DD16E25}" EndProject Project("{D954291E-2A0B-460D-934E-DC6B0785DB48}") = "ILLink.Shared", "..\..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.shproj", "{FF598E93-8E9E-4091-9F50-61A7572663AE}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ILCompiler.Trimming.Tests", "ILCompiler.Trimming.Tests\ILCompiler.Trimming.Tests.csproj", "{C331F49A-B2BA-46A4-975B-E922AA43FB6F}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Checked|Any CPU = Checked|Any CPU @@ -182,6 +182,24 @@ Global {740CDFF4-B8EC-4A37-951B-C9FE9980EF2A}.Release|x64.Build.0 = Release|x64 {740CDFF4-B8EC-4A37-951B-C9FE9980EF2A}.Release|x86.ActiveCfg = Release|Any CPU {740CDFF4-B8EC-4A37-951B-C9FE9980EF2A}.Release|x86.Build.0 = Release|Any CPU + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Checked|Any CPU.ActiveCfg = Checked|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Checked|Any CPU.Build.0 = Checked|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Checked|x64.ActiveCfg = Checked|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Checked|x64.Build.0 = Checked|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Checked|x86.ActiveCfg = Checked|x86 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Checked|x86.Build.0 = Checked|x86 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Debug|Any CPU.ActiveCfg = Debug|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Debug|Any CPU.Build.0 = Debug|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Debug|x64.ActiveCfg = Debug|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Debug|x64.Build.0 = Debug|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Debug|x86.ActiveCfg = Debug|x86 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Debug|x86.Build.0 = Debug|x86 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Release|Any CPU.ActiveCfg = Release|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Release|Any CPU.Build.0 = Release|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Release|x64.ActiveCfg = Release|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Release|x64.Build.0 = Release|x64 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Release|x86.ActiveCfg = Release|x86 + {4CF2ECD3-A1C3-4A28-AB08-A61C53114143}.Release|x86.Build.0 = Release|x86 {9DA153BF-51C4-4AD7-A355-9F9528843DC7}.Checked|Any CPU.ActiveCfg = Debug|Any CPU {9DA153BF-51C4-4AD7-A355-9F9528843DC7}.Checked|Any CPU.Build.0 = Debug|Any CPU {9DA153BF-51C4-4AD7-A355-9F9528843DC7}.Checked|x64.ActiveCfg = Debug|Any CPU @@ -218,24 +236,6 @@ Global {219E0AC3-CDBF-4104-B324-85915DD16E25}.Release|x64.Build.0 = Release|Any CPU {219E0AC3-CDBF-4104-B324-85915DD16E25}.Release|x86.ActiveCfg = Release|Any CPU {219E0AC3-CDBF-4104-B324-85915DD16E25}.Release|x86.Build.0 = Release|Any CPU - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Checked|Any CPU.ActiveCfg = Checked|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Checked|Any CPU.Build.0 = Checked|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Checked|x64.ActiveCfg = Checked|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Checked|x64.Build.0 = Checked|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Checked|x86.ActiveCfg = Checked|x86 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Checked|x86.Build.0 = Checked|x86 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Debug|Any CPU.ActiveCfg = Debug|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Debug|Any CPU.Build.0 = Debug|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Debug|x64.ActiveCfg = Debug|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Debug|x64.Build.0 = Debug|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Debug|x86.ActiveCfg = Debug|x86 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Debug|x86.Build.0 = Debug|x86 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Release|Any CPU.ActiveCfg = Release|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Release|Any CPU.Build.0 = Release|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Release|x64.ActiveCfg = Release|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Release|x64.Build.0 = Release|x64 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Release|x86.ActiveCfg = Release|x86 - {C331F49A-B2BA-46A4-975B-E922AA43FB6F}.Release|x86.Build.0 = Release|x86 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -244,7 +244,6 @@ Global SolutionGuid = {A484CF9D-B203-427F-9D15-A5BBC6013421} EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution - ..\..\..\tools\illink\test\Trimming.Tests.Shared\Trimming.Tests.Shared.projitems*{c331f49a-b2ba-46a4-975b-e922aa43fb6f}*SharedItemsImports = 5 ..\..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.projitems*{ff598e93-8e9e-4091-9f50-61a7572663ae}*SharedItemsImports = 13 ..\..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.projitems*{ffbd9619-de6f-4a98-8732-8a14ec3c1a18}*SharedItemsImports = 5 EndGlobalSection diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h index 754394f16ee9eb..5d659488b29257 100644 --- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h +++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h @@ -30,7 +30,7 @@ struct JitInterfaceCallbacks CORINFO_METHOD_HANDLE (* getUnboxedEntry)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, bool* requiresInstMethodTableArg); CORINFO_CLASS_HANDLE (* getDefaultComparerClass)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE elemType); CORINFO_CLASS_HANDLE (* getDefaultEqualityComparerClass)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE elemType); - void (* expandRawHandleIntrinsic)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult); + void (* expandRawHandleIntrinsic)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_GENERICHANDLE_RESULT* pResult); bool (* isIntrinsicType)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE classHnd); CorInfoCallConvExtension (* getUnmanagedCallConv)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* callSiteSig, bool* pSuppressGCTransition); bool (* pInvokeMarshalingRequired)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE method, CORINFO_SIG_INFO* callSiteSig); @@ -80,8 +80,8 @@ struct JitInterfaceCallbacks bool (* isObjectImmutable)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_OBJECT_HANDLE objPtr); bool (* getStringChar)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_OBJECT_HANDLE strObj, int index, uint16_t* value); CORINFO_CLASS_HANDLE (* getObjectType)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_OBJECT_HANDLE objPtr); - bool (* getReadyToRunHelper)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup); - void (* getReadyToRunDelegateCtorHelper)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pTargetMethod, unsigned int targetConstraint, CORINFO_CLASS_HANDLE delegateType, CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup); + bool (* getReadyToRunHelper)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, CORINFO_CONST_LOOKUP* pLookup); + void (* getReadyToRunDelegateCtorHelper)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pTargetMethod, unsigned int targetConstraint, CORINFO_CLASS_HANDLE delegateType, CORINFO_LOOKUP* pLookup); CorInfoInitClassResult (* initClass)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_FIELD_HANDLE field, CORINFO_METHOD_HANDLE method, CORINFO_CONTEXT_HANDLE context); void (* classMustBeLoadedBeforeCodeIsRun)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE cls); CORINFO_CLASS_HANDLE (* getBuiltinClass)(void * thisHandle, CorInfoExceptionClass** ppException, CorInfoClassId classId); @@ -115,7 +115,6 @@ struct JitInterfaceCallbacks void (* getVars)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, uint32_t* cVars, ICorDebugInfo::ILVarInfo** vars, bool* extendOthers); void (* setVars)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, uint32_t cVars, ICorDebugInfo::NativeVarInfo* vars); void (* reportRichMappings)(void * thisHandle, CorInfoExceptionClass** ppException, ICorDebugInfo::InlineTreeNode* inlineTreeNodes, uint32_t numInlineTreeNodes, ICorDebugInfo::RichOffsetMapping* mappings, uint32_t numMappings); - void (* reportMetadata)(void * thisHandle, CorInfoExceptionClass** ppException, const char* key, const void* value, size_t length); void* (* allocateArray)(void * thisHandle, CorInfoExceptionClass** ppException, size_t cBytes); void (* freeArray)(void * thisHandle, CorInfoExceptionClass** ppException, void* array); CORINFO_ARG_LIST_HANDLE (* getArgNext)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_ARG_LIST_HANDLE args); @@ -132,7 +131,6 @@ struct JitInterfaceCallbacks const char* (* getMethodNameFromMetadata)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn, const char** className, const char** namespaceName, const char** enclosingClassName); unsigned (* getMethodHash)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE ftn); bool (* getSystemVAmd64PassStructInRegisterDescriptor)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE structHnd, SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* structPassInRegDescPtr); - void (* getSwiftLowering)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering); uint32_t (* getLoongArch64PassStructInRegisterFlags)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE structHnd); uint32_t (* getRISCV64PassStructInRegisterFlags)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE structHnd); uint32_t (* getThreadTLSIndex)(void * thisHandle, CorInfoExceptionClass** ppException, void** ppIndirection); @@ -146,7 +144,7 @@ struct JitInterfaceCallbacks CORINFO_CLASS_HANDLE (* embedClassHandle)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE handle, void** ppIndirection); CORINFO_METHOD_HANDLE (* embedMethodHandle)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE handle, void** ppIndirection); CORINFO_FIELD_HANDLE (* embedFieldHandle)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_FIELD_HANDLE handle, void** ppIndirection); - void (* embedGenericHandle)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult); + void (* embedGenericHandle)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, CORINFO_GENERICHANDLE_RESULT* pResult); void (* getLocationOfThisType)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE context, CORINFO_LOOKUP_KIND* pLookupKind); void (* getAddressOfPInvokeTarget)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE method, CORINFO_CONST_LOOKUP* pLookup); void* (* GetCookieForPInvokeCalliSig)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_SIG_INFO* szMetaSig, void** ppIndirection); @@ -390,11 +388,10 @@ class JitInterfaceWrapper : public ICorJitInfo virtual void expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { CorInfoExceptionClass* pException = nullptr; - _callbacks->expandRawHandleIntrinsic(_thisHandle, &pException, pResolvedToken, callerHandle, pResult); + _callbacks->expandRawHandleIntrinsic(_thisHandle, &pException, pResolvedToken, pResult); if (pException != nullptr) throw pException; } @@ -876,11 +873,10 @@ class JitInterfaceWrapper : public ICorJitInfo CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { CorInfoExceptionClass* pException = nullptr; - bool temp = _callbacks->getReadyToRunHelper(_thisHandle, &pException, pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup); + bool temp = _callbacks->getReadyToRunHelper(_thisHandle, &pException, pResolvedToken, pGenericLookupKind, id, pLookup); if (pException != nullptr) throw pException; return temp; } @@ -889,11 +885,10 @@ class JitInterfaceWrapper : public ICorJitInfo CORINFO_RESOLVED_TOKEN* pTargetMethod, unsigned int targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { CorInfoExceptionClass* pException = nullptr; - _callbacks->getReadyToRunDelegateCtorHelper(_thisHandle, &pException, pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); + _callbacks->getReadyToRunDelegateCtorHelper(_thisHandle, &pException, pTargetMethod, targetConstraint, delegateType, pLookup); if (pException != nullptr) throw pException; } @@ -1219,16 +1214,6 @@ class JitInterfaceWrapper : public ICorJitInfo if (pException != nullptr) throw pException; } - virtual void reportMetadata( - const char* key, - const void* value, - size_t length) -{ - CorInfoExceptionClass* pException = nullptr; - _callbacks->reportMetadata(_thisHandle, &pException, key, value, length); - if (pException != nullptr) throw pException; -} - virtual void* allocateArray( size_t cBytes) { @@ -1372,15 +1357,6 @@ class JitInterfaceWrapper : public ICorJitInfo return temp; } - virtual void getSwiftLowering( - CORINFO_CLASS_HANDLE structHnd, - CORINFO_SWIFT_LOWERING* pLowering) -{ - CorInfoExceptionClass* pException = nullptr; - _callbacks->getSwiftLowering(_thisHandle, &pException, structHnd, pLowering); - if (pException != nullptr) throw pException; -} - virtual uint32_t getLoongArch64PassStructInRegisterFlags( CORINFO_CLASS_HANDLE structHnd) { @@ -1509,11 +1485,10 @@ class JitInterfaceWrapper : public ICorJitInfo virtual void embedGenericHandle( CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { CorInfoExceptionClass* pException = nullptr; - _callbacks->embedGenericHandle(_thisHandle, &pException, pResolvedToken, fEmbedParent, callerHandle, pResult); + _callbacks->embedGenericHandle(_thisHandle, &pException, pResolvedToken, fEmbedParent, pResult); if (pException != nullptr) throw pException; } diff --git a/src/coreclr/tools/r2rdump/CoreDisTools.cs b/src/coreclr/tools/r2rdump/CoreDisTools.cs index 32e88ae8d39d9f..6763c94859a6b7 100644 --- a/src/coreclr/tools/r2rdump/CoreDisTools.cs +++ b/src/coreclr/tools/r2rdump/CoreDisTools.cs @@ -29,6 +29,9 @@ public enum TargetArch [DllImport(_dll, CallingConvention = CallingConvention.Cdecl)] public static extern IntPtr InitBufferedDisasm(TargetArch Target); + [DllImport(_dll, CallingConvention = CallingConvention.Cdecl)] + public static extern void DumpCodeBlock(IntPtr Disasm, IntPtr Address, IntPtr Bytes, IntPtr Size); + [DllImport(_dll, CallingConvention = CallingConvention.Cdecl)] public static extern int DumpInstruction(IntPtr Disasm, IntPtr Address, IntPtr Bytes, IntPtr Size); @@ -233,11 +236,6 @@ public int GetInstruction(RuntimeFunction rtf, int imageOffset, int rtfOffset, o } int instrSize = CoreDisTools.GetInstruction(_disasm, rtf, imageOffset, rtfOffset, _reader.Image, out instruction); - if (instrSize == 0) - { - instruction = "Decode failure, aborting disassembly" + Environment.NewLine; - return rtf.Size - rtfOffset; - } // CoreDisTools dumps instructions in the following format: // diff --git a/src/coreclr/tools/r2rdump/R2RDump.csproj b/src/coreclr/tools/r2rdump/R2RDump.csproj index 9459b4d87693b1..3f5a93c94fbeb9 100644 --- a/src/coreclr/tools/r2rdump/R2RDump.csproj +++ b/src/coreclr/tools/r2rdump/R2RDump.csproj @@ -28,6 +28,10 @@ + + + $(NetStandardLibraryVersion) + PreserveNewest diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h index 104a7ae1fabd39..3ee4190e889065 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h @@ -423,16 +423,9 @@ struct Agnostic_CheckMethodModifier struct Agnostic_EmbedGenericHandle { Agnostic_CORINFO_RESOLVED_TOKEN ResolvedToken; - DWORDLONG hCallerHandle; DWORD fEmbedParent; }; -struct Agnostic_ExpandRawHandleIntrinsic -{ - Agnostic_CORINFO_RESOLVED_TOKENin ResolvedToken; - DWORDLONG hCallerHandle; -}; - struct Agnostic_CORINFO_GENERICHANDLE_RESULT { Agnostic_CORINFO_LOOKUP lookup; @@ -621,14 +614,6 @@ struct Agnostic_GetSystemVAmd64PassStructInRegisterDescriptor DWORD result; }; -struct Agnostic_GetSwiftLowering -{ - DWORD byReference; - DWORD loweredElements[MAX_SWIFT_LOWERED_ELEMENTS]; - DWORD offsets[MAX_SWIFT_LOWERED_ELEMENTS]; - DWORD numLoweredElements; -}; - struct Agnostic_ResolveVirtualMethodKey { DWORDLONG virtualMethod; @@ -694,7 +679,6 @@ struct GetReadyToRunHelper_TOKENin Agnostic_CORINFO_RESOLVED_TOKEN ResolvedToken; Agnostic_CORINFO_LOOKUP_KIND GenericLookupKind; DWORD id; - DWORDLONG callerHandle; }; struct GetReadyToRunHelper_TOKENout @@ -708,7 +692,6 @@ struct GetReadyToRunDelegateCtorHelper_TOKENIn Agnostic_CORINFO_RESOLVED_TOKEN TargetMethod; mdToken targetConstraint; DWORDLONG delegateType; - DWORDLONG callerHandle; }; struct Agnostic_RecordRelocation diff --git a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp index 74040dc5aa3bef..3c6653c41a1c3a 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp @@ -32,13 +32,7 @@ CompileResult::CompileResult() allocGCInfoDets.retval = nullptr; allocGCInfoDets.size = 0; - MethodFullName = nullptr; - TieringName = nullptr; memoryTracker = nullptr; - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) name = 0; -#include "jitmetadatalist.h" } CompileResult::~CompileResult() @@ -691,18 +685,6 @@ const char* relocationTypeToString(uint16_t fRelocType) // From corinfo.h case IMAGE_REL_BASED_REL32: return "rel32"; - case IMAGE_REL_SECREL: - return "secrel"; - case IMAGE_REL_TLSGD: - return "tlsgd"; - case IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21: - return "tlsdesc_high21"; - case IMAGE_REL_AARCH64_TLSDESC_LD64_LO12: - return "tlsdesc_lo12"; - case IMAGE_REL_AARCH64_TLSDESC_ADD_LO12: - return "tlsdesc_add_lo12"; - case IMAGE_REL_AARCH64_TLSDESC_CALL: - return "tlsdesc_call"; case IMAGE_REL_BASED_THUMB_BRANCH24: return "thumb_branch24"; default: @@ -863,7 +845,6 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b break; case IMAGE_REL_ARM64_PAGEBASE_REL21: // ADRP 21 bit PC-relative page address - case IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21: // ADRP 21 bit for TLSDesc { if ((section_begin <= address) && (address < section_end)) // A reloc for our section? { @@ -888,16 +869,6 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b } break; - case IMAGE_REL_AARCH64_TLSDESC_LD64_LO12: - case IMAGE_REL_AARCH64_TLSDESC_ADD_LO12: // TLSDESC ADD for corresponding ADRP - case IMAGE_REL_AARCH64_TLSDESC_CALL: - { - // These are patched later by linker during actual execution - // and do not need relocation. - wasRelocHandled = true; - } - break; - default: break; } @@ -925,19 +896,13 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b wasRelocHandled = true; } - else if (relocType == IMAGE_REL_TLSGD) - { - // These are patched later by linker during actual execution - // and do not need relocation. - wasRelocHandled = true; - } } if (wasRelocHandled) continue; // Now do all-platform relocations. - if ((tmp.fRelocType == IMAGE_REL_BASED_REL32) || (tmp.fRelocType == IMAGE_REL_SECREL)) + if (tmp.fRelocType == IMAGE_REL_BASED_REL32) { DWORDLONG fixupLocation = tmp.location; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.h b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.h index 72415f1d38f595..b7be4dcd89279e 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.h @@ -121,8 +121,6 @@ class CompileResult void dmpSetVars(DWORD key, const Agnostic_SetVars& value); bool repSetVars(CORINFO_METHOD_HANDLE* ftn, ULONG32* cVars, ICorDebugInfo::NativeVarInfo** vars); - void recMetadata(const char* key, const void* value); - void recSetPatchpointInfo(PatchpointInfo* patchpointInfo); void dmpSetPatchpointInfo(DWORD key, const Agnostic_SetPatchpointInfo& value); bool repSetPatchpointInfo(PatchpointInfo** patchpointInfo); @@ -217,15 +215,6 @@ class CompileResult #define DENSELWM(map, value) DenseLightWeightMap* map; #include "crlwmlist.h" -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) type name; -#include "jitmetadatalist.h" - - // Reported method full name from JIT (not available with release JIT) - const char* MethodFullName; - // Reported compilation tier from JIT - const char* TieringName; - // not persisted to disk. public: LightWeightMap* CallTargetTypes; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/jitmetadatalist.h b/src/coreclr/tools/superpmi/superpmi-shared/jitmetadatalist.h deleted file mode 100644 index f43f4300d73a1a..00000000000000 --- a/src/coreclr/tools/superpmi/superpmi-shared/jitmetadatalist.h +++ /dev/null @@ -1 +0,0 @@ -#include "../../../jit/jitmetadatalist.h" diff --git a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h index c3445291eeb931..a44faac3cee24e 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h @@ -125,7 +125,6 @@ LWM(GetExpectedTargetArchitecture, DWORD, DWORD) LWM(GetSharedCCtorHelper, DWORDLONG, DWORD) LWM(GetStringConfigValue, DWORD, DWORD) LWM(GetSystemVAmd64PassStructInRegisterDescriptor, DWORDLONG, Agnostic_GetSystemVAmd64PassStructInRegisterDescriptor) -LWM(GetSwiftLowering, DWORDLONG, Agnostic_GetSwiftLowering) LWM(GetLoongArch64PassStructInRegisterFlags, DWORDLONG, DWORD) LWM(GetRISCV64PassStructInRegisterFlags, DWORDLONG, DWORD) LWM(GetTailCallHelpers, Agnostic_GetTailCallHelpers, Agnostic_CORINFO_TAILCALL_HELPERS) @@ -147,7 +146,7 @@ LWM(InitClass, Agnostic_InitClass, DWORD) LWM(IsDelegateCreationAllowed, DLDL, DWORD) LWM(IsFieldStatic, DWORDLONG, DWORD) LWM(GetArrayOrStringLength, DWORDLONG, DWORD) -LWM(ExpandRawHandleIntrinsic, Agnostic_ExpandRawHandleIntrinsic, Agnostic_CORINFO_GENERICHANDLE_RESULT) +LWM(ExpandRawHandleIntrinsic, Agnostic_CORINFO_RESOLVED_TOKENin, Agnostic_CORINFO_GENERICHANDLE_RESULT) LWM(IsIntrinsicType, DWORDLONG, DWORD) LWM(IsSDArray, DWORDLONG, DWORD) LWM(GetStringLiteral, DLDDD, DD) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 70c5c627bd404c..c72b8b1eec1f6c 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -1670,15 +1670,14 @@ void MethodContext::repGetCallInfoFromMethodHandle(CORINFO_METHOD_HANDLE methodH LogException(EXCEPTIONCODE_MC, "Didn't find key %016" PRIX64 ".", methodHandle); } -void MethodContext::recExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) +void MethodContext::recExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_GENERICHANDLE_RESULT* pResult) { if (ExpandRawHandleIntrinsic == nullptr) - ExpandRawHandleIntrinsic = new LightWeightMap; + ExpandRawHandleIntrinsic = new LightWeightMap; - Agnostic_ExpandRawHandleIntrinsic key; + Agnostic_CORINFO_RESOLVED_TOKENin key; ZeroMemory(&key, sizeof(key)); // Zero key including any struct padding - key.ResolvedToken = SpmiRecordsHelper::CreateAgnostic_CORINFO_RESOLVED_TOKENin(pResolvedToken); - key.hCallerHandle = CastHandle(callerHandle); + key = SpmiRecordsHelper::CreateAgnostic_CORINFO_RESOLVED_TOKENin(pResolvedToken); Agnostic_CORINFO_GENERICHANDLE_RESULT value; value.lookup = SpmiRecordsHelper::StoreAgnostic_CORINFO_LOOKUP(&pResult->lookup); @@ -1688,21 +1687,19 @@ void MethodContext::recExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolve ExpandRawHandleIntrinsic->Add(key, value); DEBUG_REC(dmpExpandRawHandleIntrinsic(key, value)); } -void MethodContext::dmpExpandRawHandleIntrinsic(const Agnostic_ExpandRawHandleIntrinsic& key, const Agnostic_CORINFO_GENERICHANDLE_RESULT& result) +void MethodContext::dmpExpandRawHandleIntrinsic(const Agnostic_CORINFO_RESOLVED_TOKENin& key, const Agnostic_CORINFO_GENERICHANDLE_RESULT& result) { printf("ExpandRawHandleIntrinsic key: %s, value %s cth-%016" PRIx64 " ht-%u", - SpmiDumpHelper::DumpAgnostic_CORINFO_RESOLVED_TOKENin(key.ResolvedToken).c_str(), + SpmiDumpHelper::DumpAgnostic_CORINFO_RESOLVED_TOKENin(key).c_str(), SpmiDumpHelper::DumpAgnostic_CORINFO_LOOKUP(result.lookup).c_str(), result.compileTimeHandle, result.handleType); } -void MethodContext::repExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) +void MethodContext::repExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_GENERICHANDLE_RESULT* pResult) { - Agnostic_ExpandRawHandleIntrinsic key; + Agnostic_CORINFO_RESOLVED_TOKENin key; ZeroMemory(&key, sizeof(key)); // Zero key including any struct padding - - key.ResolvedToken = SpmiRecordsHelper::CreateAgnostic_CORINFO_RESOLVED_TOKENin(pResolvedToken); - key.hCallerHandle = CastHandle(callerHandle); + key = SpmiRecordsHelper::CreateAgnostic_CORINFO_RESOLVED_TOKENin(pResolvedToken); Agnostic_CORINFO_GENERICHANDLE_RESULT value = LookupByKeyOrMiss(ExpandRawHandleIntrinsic, key, ": key %x", pResolvedToken->token); @@ -2276,7 +2273,6 @@ CORINFO_CLASS_HANDLE MethodContext::repGetObjectType(CORINFO_OBJECT_HANDLE objPt void MethodContext::recGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup, bool result) { @@ -2288,7 +2284,6 @@ void MethodContext::recGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToke key.ResolvedToken = SpmiRecordsHelper::StoreAgnostic_CORINFO_RESOLVED_TOKEN(pResolvedToken, GetReadyToRunHelper); key.GenericLookupKind = SpmiRecordsHelper::CreateAgnostic_CORINFO_LOOKUP_KIND(pGenericLookupKind); key.id = (DWORD)id; - key.callerHandle = CastHandle(callerHandle); GetReadyToRunHelper_TOKENout value; value.Lookup = SpmiRecordsHelper::StoreAgnostic_CORINFO_CONST_LOOKUP(pLookup); value.result = result; @@ -2309,7 +2304,6 @@ void MethodContext::dmpGetReadyToRunHelper(GetReadyToRunHelper_TOKENin key, GetR bool MethodContext::repGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { AssertMapExistsNoMessage(GetReadyToRunHelper); @@ -2319,7 +2313,6 @@ bool MethodContext::repGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToke key.ResolvedToken = SpmiRecordsHelper::RestoreAgnostic_CORINFO_RESOLVED_TOKEN(pResolvedToken, GetReadyToRunHelper); key.GenericLookupKind = SpmiRecordsHelper::CreateAgnostic_CORINFO_LOOKUP_KIND(pGenericLookupKind); key.id = (DWORD)id; - key.callerHandle = CastHandle(callerHandle); GetReadyToRunHelper_TOKENout value = LookupByKeyOrMissNoMessage(GetReadyToRunHelper, key); @@ -2332,7 +2325,6 @@ bool MethodContext::repGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToke void MethodContext::recGetReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { if (GetReadyToRunDelegateCtorHelper == nullptr) @@ -2345,7 +2337,6 @@ void MethodContext::recGetReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* p SpmiRecordsHelper::StoreAgnostic_CORINFO_RESOLVED_TOKEN(pTargetMethod, GetReadyToRunDelegateCtorHelper); key.targetConstraint = targetConstraint; key.delegateType = CastHandle(delegateType); - key.callerHandle = CastHandle(callerHandle); Agnostic_CORINFO_LOOKUP value = SpmiRecordsHelper::StoreAgnostic_CORINFO_LOOKUP(pLookup); GetReadyToRunDelegateCtorHelper->Add(key, value); DEBUG_REC(dmpGetReadyToRunDelegateCtorHelper(key, value)); @@ -2362,7 +2353,6 @@ void MethodContext::dmpGetReadyToRunDelegateCtorHelper(GetReadyToRunDelegateCtor void MethodContext::repGetReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { AssertMapExistsNoMessage(GetReadyToRunDelegateCtorHelper); @@ -2373,7 +2363,6 @@ void MethodContext::repGetReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* p SpmiRecordsHelper::RestoreAgnostic_CORINFO_RESOLVED_TOKEN(pTargetMethod, GetReadyToRunDelegateCtorHelper); key.targetConstraint = targetConstraint; key.delegateType = CastHandle(delegateType); - key.callerHandle = CastHandle(callerHandle); Agnostic_CORINFO_LOOKUP value = LookupByKeyOrMissNoMessage(GetReadyToRunDelegateCtorHelper, key); @@ -3112,7 +3101,6 @@ CorInfoHelpFunc MethodContext::repGetNewHelper(CORINFO_CLASS_HANDLE classHandle void MethodContext::recEmbedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { if (EmbedGenericHandle == nullptr) @@ -3122,7 +3110,6 @@ void MethodContext::recEmbedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolve ZeroMemory(&key, sizeof(key)); // Zero key including any struct padding key.ResolvedToken = SpmiRecordsHelper::StoreAgnostic_CORINFO_RESOLVED_TOKEN(pResolvedToken, EmbedGenericHandle); key.fEmbedParent = (DWORD)fEmbedParent; - key.hCallerHandle = CastHandle(callerHandle); Agnostic_CORINFO_GENERICHANDLE_RESULT value; value.lookup = SpmiRecordsHelper::StoreAgnostic_CORINFO_LOOKUP(&pResult->lookup); @@ -3144,7 +3131,6 @@ void MethodContext::dmpEmbedGenericHandle(const Agnostic_EmbedGenericHandle& } void MethodContext::repEmbedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { AssertMapExistsNoMessage(EmbedGenericHandle); @@ -3153,7 +3139,6 @@ void MethodContext::repEmbedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolve ZeroMemory(&key, sizeof(key)); // Zero key including any struct padding key.ResolvedToken = SpmiRecordsHelper::RestoreAgnostic_CORINFO_RESOLVED_TOKEN(pResolvedToken, EmbedGenericHandle); key.fEmbedParent = (DWORD)fEmbedParent; - key.hCallerHandle = CastHandle(callerHandle); Agnostic_CORINFO_GENERICHANDLE_RESULT value = LookupByKeyOrMissNoMessage(EmbedGenericHandle, key); @@ -6235,56 +6220,6 @@ bool MethodContext::repGetSystemVAmd64PassStructInRegisterDescriptor( return value.result ? true : false; } -void MethodContext::recGetSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering) -{ - if (GetSwiftLowering == nullptr) - GetSwiftLowering = new LightWeightMap(); - - DWORDLONG key = CastHandle(structHnd); - - Agnostic_GetSwiftLowering value; - ZeroMemory(&value, sizeof(value)); - value.byReference = pLowering->byReference ? 1 : 0; - if (!pLowering->byReference) - { - value.numLoweredElements = static_cast(pLowering->numLoweredElements); - for (size_t i = 0; i < pLowering->numLoweredElements; i++) - { - value.loweredElements[i] = static_cast(pLowering->loweredElements[i]); - value.offsets[i] = pLowering->offsets[i]; - } - } - - GetSwiftLowering->Add(key, value); - DEBUG_REC(dmpGetSwiftLowering(key, value)); -} -void MethodContext::dmpGetSwiftLowering( - DWORDLONG key, const Agnostic_GetSwiftLowering& value) -{ - printf("GetSwiftLowering key structHnd-%016" PRIX64 ", value byReference-%u numLoweredElements-%u", key, - value.byReference, value.numLoweredElements); - for (size_t i = 0; i < value.numLoweredElements; i++) - { - printf(" [%zu] %u", i, value.loweredElements[i]); - } -} -void MethodContext::repGetSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering) -{ - DWORDLONG key = CastHandle(structHnd); - Agnostic_GetSwiftLowering value = LookupByKeyOrMiss(GetSwiftLowering, key, ": key %016" PRIX64 "", key); - - DEBUG_REP(dmpGetSwiftLowering(key, value)); - - pLowering->byReference = value.byReference != 0; - pLowering->numLoweredElements = value.numLoweredElements; - - for (size_t i = 0; i < pLowering->numLoweredElements; i++) - { - pLowering->loweredElements[i] = static_cast(value.loweredElements[i]); - pLowering->offsets[i] = value.offsets[i]; - } -} - void MethodContext::recGetLoongArch64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE structHnd, DWORD value) { if (GetLoongArch64PassStructInRegisterFlags == nullptr) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index cf9c235ae987ea..74128026015006 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -272,9 +272,9 @@ class MethodContext void dmpIsSDArray(DWORDLONG key, DWORD value); bool repIsSDArray(CORINFO_CLASS_HANDLE cls); - void recExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult); - void dmpExpandRawHandleIntrinsic(const Agnostic_ExpandRawHandleIntrinsic& key, const Agnostic_CORINFO_GENERICHANDLE_RESULT& result); - void repExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult); + void recExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_GENERICHANDLE_RESULT* pResult); + void dmpExpandRawHandleIntrinsic(const Agnostic_CORINFO_RESOLVED_TOKENin& key, const Agnostic_CORINFO_GENERICHANDLE_RESULT& result); + void repExpandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_GENERICHANDLE_RESULT* pResult); void recIsIntrinsicType(CORINFO_CLASS_HANDLE cls, bool result); void dmpIsIntrinsicType(DWORDLONG key, DWORD value); @@ -315,27 +315,23 @@ class MethodContext void recGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup, bool result); void dmpGetReadyToRunHelper(GetReadyToRunHelper_TOKENin key, GetReadyToRunHelper_TOKENout value); bool repGetReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup); void recGetReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup); void dmpGetReadyToRunDelegateCtorHelper(GetReadyToRunDelegateCtorHelper_TOKENIn key, Agnostic_CORINFO_LOOKUP pLookup); void repGetReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup); void recGetHelperFtn(CorInfoHelpFunc ftnNum, void** ppIndirection, void* result); @@ -427,13 +423,11 @@ class MethodContext void recEmbedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult); void dmpEmbedGenericHandle(const Agnostic_EmbedGenericHandle& key, const Agnostic_CORINFO_GENERICHANDLE_RESULT& value); void repEmbedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult); void recGetEHinfo(CORINFO_METHOD_HANDLE ftn, unsigned EHnumber, CORINFO_EH_CLAUSE* clause); @@ -763,10 +757,6 @@ class MethodContext bool repGetSystemVAmd64PassStructInRegisterDescriptor( CORINFO_CLASS_HANDLE structHnd, SYSTEMV_AMD64_CORINFO_STRUCT_REG_PASSING_DESCRIPTOR* structPassInRegDescPtr); - void recGetSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering); - void dmpGetSwiftLowering(DWORDLONG key, const Agnostic_GetSwiftLowering& value); - void repGetSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering); - void recGetLoongArch64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE structHnd, DWORD value); void dmpGetLoongArch64PassStructInRegisterFlags(DWORDLONG key, DWORD value); DWORD repGetLoongArch64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE structHnd); @@ -1164,7 +1154,6 @@ enum mcPackets Packet_HaveSameMethodDefinition = 213, Packet_NotifyMethodInfoUsage = 214, Packet_IsExactType = 215, - Packet_GetSwiftLowering = 216, }; void SetDebugDumpVariables(); diff --git a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp index d53002a42914b1..47c50535e450f1 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp @@ -259,12 +259,11 @@ CORINFO_CLASS_HANDLE interceptor_ICJI::getDefaultEqualityComparerClass(CORINFO_C } void interceptor_ICJI::expandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { mc->cr->AddCall("expandRawHandleIntrinsic"); - original_ICorJitInfo->expandRawHandleIntrinsic(pResolvedToken, callerHandle, pResult); - mc->recExpandRawHandleIntrinsic(pResolvedToken, callerHandle, pResult); + original_ICorJitInfo->expandRawHandleIntrinsic(pResolvedToken, pResult); + mc->recExpandRawHandleIntrinsic(pResolvedToken, pResult); } // Is the given type in System.Private.Corelib and marked with IntrinsicAttribute? @@ -781,24 +780,22 @@ CORINFO_CLASS_HANDLE interceptor_ICJI::getObjectType(CORINFO_OBJECT_HANDLE typeO bool interceptor_ICJI::getReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { mc->cr->AddCall("getReadyToRunHelper"); - bool result = original_ICorJitInfo->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup); - mc->recGetReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup, result); + bool result = original_ICorJitInfo->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, pLookup); + mc->recGetReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, pLookup, result); return result; } void interceptor_ICJI::getReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { mc->cr->AddCall("getReadyToRunDelegateCtorHelper"); - original_ICorJitInfo->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); - mc->recGetReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); + original_ICorJitInfo->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, pLookup); + mc->recGetReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, pLookup); } // This function tries to initialize the class (run the class constructor). @@ -1196,12 +1193,6 @@ void interceptor_ICJI::reportRichMappings(ICorDebugInfo::InlineTreeNode* inli original_ICorJitInfo->reportRichMappings(inlineTreeNodes, numInlineTreeNodes, mappings, numMappings); } -void interceptor_ICJI::reportMetadata(const char* key, const void* value, size_t length) -{ - mc->cr->AddCall("reportMetadata"); - original_ICorJitInfo->reportMetadata(key, value, length); -} - /*-------------------------- Misc ---------------------------------------*/ // Used to allocate memory that needs to handed to the EE. // For eg, use this to allocated memory for reporting debug info, @@ -1394,13 +1385,6 @@ bool interceptor_ICJI::getSystemVAmd64PassStructInRegisterDescriptor( return result; } -void interceptor_ICJI::getSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering) -{ - mc->cr->AddCall("getSwiftLowering"); - original_ICorJitInfo->getSwiftLowering(structHnd, pLowering); - mc->recGetSwiftLowering(structHnd, pLowering); -} - uint32_t interceptor_ICJI::getLoongArch64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE structHnd) { mc->cr->AddCall("getLoongArch64PassStructInRegisterFlags"); @@ -1533,12 +1517,11 @@ CORINFO_FIELD_HANDLE interceptor_ICJI::embedFieldHandle(CORINFO_FIELD_HANDLE han void interceptor_ICJI::embedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, // TRUE - embeds parent type handle of the field/method // handle - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { mc->cr->AddCall("embedGenericHandle"); - original_ICorJitInfo->embedGenericHandle(pResolvedToken, fEmbedParent, callerHandle, pResult); - mc->recEmbedGenericHandle(pResolvedToken, fEmbedParent, callerHandle, pResult); + original_ICorJitInfo->embedGenericHandle(pResolvedToken, fEmbedParent, pResult); + mc->recEmbedGenericHandle(pResolvedToken, fEmbedParent, pResult); } // Return information used to locate the exact enclosing type of the current method. diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp index ef1b277e805333..0f69dfae4a08c6 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp @@ -171,11 +171,10 @@ CORINFO_CLASS_HANDLE interceptor_ICJI::getDefaultEqualityComparerClass( void interceptor_ICJI::expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { mcs->AddCall("expandRawHandleIntrinsic"); - original_ICorJitInfo->expandRawHandleIntrinsic(pResolvedToken, callerHandle, pResult); + original_ICorJitInfo->expandRawHandleIntrinsic(pResolvedToken, pResult); } bool interceptor_ICJI::isIntrinsicType( @@ -565,22 +564,20 @@ bool interceptor_ICJI::getReadyToRunHelper( CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { mcs->AddCall("getReadyToRunHelper"); - return original_ICorJitInfo->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup); + return original_ICorJitInfo->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, pLookup); } void interceptor_ICJI::getReadyToRunDelegateCtorHelper( CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { mcs->AddCall("getReadyToRunDelegateCtorHelper"); - original_ICorJitInfo->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); + original_ICorJitInfo->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, pLookup); } CorInfoInitClassResult interceptor_ICJI::initClass( @@ -848,15 +845,6 @@ void interceptor_ICJI::reportRichMappings( original_ICorJitInfo->reportRichMappings(inlineTreeNodes, numInlineTreeNodes, mappings, numMappings); } -void interceptor_ICJI::reportMetadata( - const char* key, - const void* value, - size_t length) -{ - mcs->AddCall("reportMetadata"); - original_ICorJitInfo->reportMetadata(key, value, length); -} - void* interceptor_ICJI::allocateArray( size_t cBytes) { @@ -982,14 +970,6 @@ bool interceptor_ICJI::getSystemVAmd64PassStructInRegisterDescriptor( return original_ICorJitInfo->getSystemVAmd64PassStructInRegisterDescriptor(structHnd, structPassInRegDescPtr); } -void interceptor_ICJI::getSwiftLowering( - CORINFO_CLASS_HANDLE structHnd, - CORINFO_SWIFT_LOWERING* pLowering) -{ - mcs->AddCall("getSwiftLowering"); - original_ICorJitInfo->getSwiftLowering(structHnd, pLowering); -} - uint32_t interceptor_ICJI::getLoongArch64PassStructInRegisterFlags( CORINFO_CLASS_HANDLE structHnd) { @@ -1094,11 +1074,10 @@ CORINFO_FIELD_HANDLE interceptor_ICJI::embedFieldHandle( void interceptor_ICJI::embedGenericHandle( CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { mcs->AddCall("embedGenericHandle"); - original_ICorJitInfo->embedGenericHandle(pResolvedToken, fEmbedParent, callerHandle, pResult); + original_ICorJitInfo->embedGenericHandle(pResolvedToken, fEmbedParent, pResult); } void interceptor_ICJI::getLocationOfThisType( diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp index 55aef651273476..02bef7b549aca0 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp @@ -152,10 +152,9 @@ CORINFO_CLASS_HANDLE interceptor_ICJI::getDefaultEqualityComparerClass( void interceptor_ICJI::expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN* pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { - original_ICorJitInfo->expandRawHandleIntrinsic(pResolvedToken, callerHandle, pResult); + original_ICorJitInfo->expandRawHandleIntrinsic(pResolvedToken, pResult); } bool interceptor_ICJI::isIntrinsicType( @@ -496,20 +495,18 @@ bool interceptor_ICJI::getReadyToRunHelper( CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { - return original_ICorJitInfo->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup); + return original_ICorJitInfo->getReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, pLookup); } void interceptor_ICJI::getReadyToRunDelegateCtorHelper( CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { - original_ICorJitInfo->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); + original_ICorJitInfo->getReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, pLookup); } CorInfoInitClassResult interceptor_ICJI::initClass( @@ -744,14 +741,6 @@ void interceptor_ICJI::reportRichMappings( original_ICorJitInfo->reportRichMappings(inlineTreeNodes, numInlineTreeNodes, mappings, numMappings); } -void interceptor_ICJI::reportMetadata( - const char* key, - const void* value, - size_t length) -{ - original_ICorJitInfo->reportMetadata(key, value, length); -} - void* interceptor_ICJI::allocateArray( size_t cBytes) { @@ -861,13 +850,6 @@ bool interceptor_ICJI::getSystemVAmd64PassStructInRegisterDescriptor( return original_ICorJitInfo->getSystemVAmd64PassStructInRegisterDescriptor(structHnd, structPassInRegDescPtr); } -void interceptor_ICJI::getSwiftLowering( - CORINFO_CLASS_HANDLE structHnd, - CORINFO_SWIFT_LOWERING* pLowering) -{ - original_ICorJitInfo->getSwiftLowering(structHnd, pLowering); -} - uint32_t interceptor_ICJI::getLoongArch64PassStructInRegisterFlags( CORINFO_CLASS_HANDLE structHnd) { @@ -959,10 +941,9 @@ CORINFO_FIELD_HANDLE interceptor_ICJI::embedFieldHandle( void interceptor_ICJI::embedGenericHandle( CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { - original_ICorJitInfo->embedGenericHandle(pResolvedToken, fEmbedParent, callerHandle, pResult); + original_ICorJitInfo->embedGenericHandle(pResolvedToken, fEmbedParent, pResult); } void interceptor_ICJI::getLocationOfThisType( diff --git a/src/coreclr/tools/superpmi/superpmi/fileio.cpp b/src/coreclr/tools/superpmi/superpmi/fileio.cpp index e26723de6e3e1e..ed16485a038e8a 100644 --- a/src/coreclr/tools/superpmi/superpmi/fileio.cpp +++ b/src/coreclr/tools/superpmi/superpmi/fileio.cpp @@ -27,7 +27,10 @@ bool FileWriter::Printf(const char* fmt, ...) } else { - bool result = Print(pBuffer, static_cast(printed)); + DWORD numWritten; + bool result = + WriteFile(m_file.Get(), pBuffer, static_cast(printed), &numWritten, nullptr) && + (numWritten == static_cast(printed)); if (pBuffer != stackBuffer) delete[] pBuffer; @@ -38,75 +41,6 @@ bool FileWriter::Printf(const char* fmt, ...) } } -bool FileWriter::Print(const char* value, size_t numChars) -{ - DWORD numWritten; - bool result = - WriteFile(m_file.Get(), value, static_cast(numChars), &numWritten, nullptr) && - (numWritten == static_cast(numChars)); - return result; -} - -bool FileWriter::Print(const char* value) -{ - return Print(value, strlen(value)); -} - -bool FileWriter::Print(int value) -{ - return Printf("%d", value); -} - -bool FileWriter::Print(int64_t value) -{ - return Printf("%lld", value); -} - -bool FileWriter::Print(double value) -{ - return Printf("%f", value); -} - -bool FileWriter::PrintQuotedCsvField(const char* value) -{ - size_t numQuotes = 0; - for (const char* p = value; *p != '\0'; p++) - { - if (*p == '"') - { - numQuotes++; - } - } - - if (numQuotes == 0) - { - return Printf("\"%s\"", value); - } - else - { - size_t len = 2 + strlen(value) + numQuotes; - char* buffer = new char[len]; - - size_t index = 0; - buffer[index++] = '"'; - for (const char* p = value; *p != '\0'; p++) - { - if (*p == '"') - { - buffer[index++] = '"'; - } - buffer[index++] = *p; - } - - buffer[index++] = '"'; - assert(index == len); - - bool result = Print(buffer, len); - delete[] buffer; - return result; - } -} - bool FileWriter::CreateNew(const char* path, FileWriter* fw) { FileHandle handle(CreateFile(path, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr)); diff --git a/src/coreclr/tools/superpmi/superpmi/fileio.h b/src/coreclr/tools/superpmi/superpmi/fileio.h index 4a1434f972598a..a88e74d6ee00c5 100644 --- a/src/coreclr/tools/superpmi/superpmi/fileio.h +++ b/src/coreclr/tools/superpmi/superpmi/fileio.h @@ -93,12 +93,6 @@ class FileWriter { } - bool Print(const char* value, size_t numChars); - bool Print(const char* value); - bool Print(int value); - bool Print(int64_t value); - bool Print(double value); - bool PrintQuotedCsvField(const char* value); bool Printf(const char* fmt, ...); static bool CreateNew(const char* path, FileWriter* fw); diff --git a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp index f23c8e12f86618..15c17173abdda4 100644 --- a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp @@ -221,10 +221,10 @@ CORINFO_CLASS_HANDLE MyICJI::getDefaultEqualityComparerClass(CORINFO_CLASS_HANDL return result; } -void MyICJI::expandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) +void MyICJI::expandRawHandleIntrinsic(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_GENERICHANDLE_RESULT* pResult) { jitInstance->mc->cr->AddCall("expandRawHandleIntrinsic"); - jitInstance->mc->repExpandRawHandleIntrinsic(pResolvedToken, callerHandle, pResult); + jitInstance->mc->repExpandRawHandleIntrinsic(pResolvedToken, pResult); } // Is the given type in System.Private.Corelib and marked with IntrinsicAttribute? @@ -653,21 +653,19 @@ CORINFO_CLASS_HANDLE MyICJI::getObjectType(CORINFO_OBJECT_HANDLE objPtr) bool MyICJI::getReadyToRunHelper(CORINFO_RESOLVED_TOKEN* pResolvedToken, CORINFO_LOOKUP_KIND* pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP* pLookup) { jitInstance->mc->cr->AddCall("getReadyToRunHelper"); - return jitInstance->mc->repGetReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, callerHandle, pLookup); + return jitInstance->mc->repGetReadyToRunHelper(pResolvedToken, pGenericLookupKind, id, pLookup); } void MyICJI::getReadyToRunDelegateCtorHelper(CORINFO_RESOLVED_TOKEN* pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_LOOKUP* pLookup) { jitInstance->mc->cr->AddCall("getReadyToRunDelegateCtorHelper"); - jitInstance->mc->repGetReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, callerHandle, pLookup); + jitInstance->mc->repGetReadyToRunDelegateCtorHelper(pTargetMethod, targetConstraint, delegateType, pLookup); } // This function tries to initialize the class (run the class constructor). @@ -1029,37 +1027,6 @@ void MyICJI::reportRichMappings( freeArray(mappings); } -void MyICJI::reportMetadata(const char* key, const void* value, size_t length) -{ - jitInstance->mc->cr->AddCall("reportMetadata"); - - if (strcmp(key, "MethodFullName") == 0) - { - char* buf = static_cast(jitInstance->mc->cr->allocateMemory(length + 1)); - memcpy(buf, value, length + 1); - jitInstance->mc->cr->MethodFullName = buf; - return; - } - - if (strcmp(key, "TieringName") == 0) - { - char* buf = static_cast(jitInstance->mc->cr->allocateMemory(length + 1)); - memcpy(buf, value, length + 1); - jitInstance->mc->cr->TieringName = buf; - return; - } - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) \ - if ((strcmp(key, #name) == 0) && (length == sizeof(type))) \ - { \ - memcpy(&jitInstance->mc->cr->name, value, sizeof(type)); \ - return; \ - } - -#include "jitmetadatalist.h" -} - /*-------------------------- Misc ---------------------------------------*/ // Used to allocate memory that needs to handed to the EE. @@ -1227,12 +1194,6 @@ bool MyICJI::getSystemVAmd64PassStructInRegisterDescriptor( return jitInstance->mc->repGetSystemVAmd64PassStructInRegisterDescriptor(structHnd, structPassInRegDescPtr); } -void MyICJI::getSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering) -{ - jitInstance->mc->cr->AddCall("getSwiftLowering"); - jitInstance->mc->repGetSwiftLowering(structHnd, pLowering); -} - uint32_t MyICJI::getLoongArch64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE structHnd) { jitInstance->mc->cr->AddCall("getLoongArch64PassStructInRegisterFlags"); @@ -1340,11 +1301,10 @@ CORINFO_FIELD_HANDLE MyICJI::embedFieldHandle(CORINFO_FIELD_HANDLE handle, void* // void MyICJI::embedGenericHandle(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fEmbedParent, // TRUE - embeds parent type handle of the field/method handle - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT* pResult) { jitInstance->mc->cr->AddCall("embedGenericHandle"); - jitInstance->mc->repEmbedGenericHandle(pResolvedToken, fEmbedParent, callerHandle, pResult); + jitInstance->mc->repEmbedGenericHandle(pResolvedToken, fEmbedParent, pResult); } // Return information used to locate the exact enclosing type of the current method. diff --git a/src/coreclr/tools/superpmi/superpmi/jitinstance.cpp b/src/coreclr/tools/superpmi/superpmi/jitinstance.cpp index 8eb0e69e8b2e5b..7c55db51af20cf 100644 --- a/src/coreclr/tools/superpmi/superpmi/jitinstance.cpp +++ b/src/coreclr/tools/superpmi/superpmi/jitinstance.cpp @@ -459,7 +459,6 @@ ReplayResults JitInstance::CompileMethod(MethodContext* MethodToCompile, int mcI } mc->cr->secondsToCompile = stj.GetSeconds(); - param.results.CompileResults = mc->cr; UINT64 insCountAfter = 0; Instrumentor_GetInsCount(&insCountAfter); diff --git a/src/coreclr/tools/superpmi/superpmi/jitinstance.h b/src/coreclr/tools/superpmi/superpmi/jitinstance.h index b13fe46d641f40..03e283c1d5b62d 100644 --- a/src/coreclr/tools/superpmi/superpmi/jitinstance.h +++ b/src/coreclr/tools/superpmi/superpmi/jitinstance.h @@ -22,7 +22,6 @@ struct ReplayResults bool IsMinOpts = false; uint32_t NumCodeBytes = 0; uint64_t NumExecutedInstructions = 0; - CompileResult* CompileResults = nullptr; }; class JitInstance diff --git a/src/coreclr/tools/superpmi/superpmi/superpmi.cpp b/src/coreclr/tools/superpmi/superpmi/superpmi.cpp index f0b4b76fcc1c19..4ac03d18ce9a6d 100644 --- a/src/coreclr/tools/superpmi/superpmi/superpmi.cpp +++ b/src/coreclr/tools/superpmi/superpmi/superpmi.cpp @@ -137,81 +137,42 @@ static const char* ResultToString(ReplayResult result) } } -static void PrintDiffsCsvHeader(FileWriter& fw) +static bool PrintDiffsCsvHeader(FileWriter& fw) { - fw.Print("Context,Context size,Method full name,Tier name,Base result,Diff result,MinOpts,Has diff,Base size,Diff size,Base instructions,Diff instructions"); - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) fw.Print(",Base " #name ",Diff " #name); - -#include "jitmetadatalist.h" - - fw.Print("\n"); + return fw.Printf("Context,Context size,Base result,Diff result,MinOpts,Has diff,Base size,Diff size,Base instructions,Diff instructions\n"); } -static void PrintDiffsCsvRow( +static bool PrintDiffsCsvRow( FileWriter& fw, int context, uint32_t contextSize, const ReplayResults& baseRes, const ReplayResults& diffRes, bool hasDiff) { - fw.Printf("%d,%u,", context, contextSize); - fw.PrintQuotedCsvField(baseRes.CompileResults->MethodFullName == nullptr ? "" : baseRes.CompileResults->MethodFullName); - fw.Printf( - ",%s,%s,%s,%s,%s,%u,%u,%lld,%lld", - baseRes.CompileResults->TieringName == nullptr ? "" : baseRes.CompileResults->TieringName, + return fw.Printf("%d,%u,%s,%s,%s,%s,%u,%u,%lld,%lld\n", + context, contextSize, ResultToString(baseRes.Result), ResultToString(diffRes.Result), baseRes.IsMinOpts ? "True" : "False", hasDiff ? "True" : "False", baseRes.NumCodeBytes, diffRes.NumCodeBytes, baseRes.NumExecutedInstructions, diffRes.NumExecutedInstructions); - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) \ - fw.Print(","); \ - fw.Print(baseRes.CompileResults->name); \ - fw.Print(","); \ - fw.Print(diffRes.CompileResults->name); - -#include "jitmetadatalist.h" - - fw.Print("\n"); } -static void PrintReplayCsvHeader(FileWriter& fw) +static bool PrintReplayCsvHeader(FileWriter& fw) { - fw.Printf("Context,Context size,Method full name,Tier name,Result,MinOpts,Size,Instructions"); - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) fw.Print("," #name); - -#include "jitmetadatalist.h" - - fw.Print("\n"); + return fw.Printf("Context,Context size,Result,MinOpts,Size,Instructions\n"); } -static void PrintReplayCsvRow( +static bool PrintReplayCsvRow( FileWriter& fw, int context, uint32_t contextSize, const ReplayResults& res) { - fw.Printf("%d,%u,", context, contextSize); - fw.PrintQuotedCsvField(res.CompileResults->MethodFullName == nullptr ? "" : res.CompileResults->MethodFullName); - fw.Printf(",%s,%s,%s,%u,%lld", - res.CompileResults->TieringName == nullptr ? "" : res.CompileResults->TieringName, + return fw.Printf("%d,%u,%s,%s,%u,%lld\n", + context, contextSize, ResultToString(res.Result), res.IsMinOpts ? "True" : "False", res.NumCodeBytes, res.NumExecutedInstructions); - -#define JITMETADATAINFO(name, type, flags) -#define JITMETADATAMETRIC(name, type, flags) \ - fw.Print(","); \ - fw.Print(res.CompileResults->name); - -#include "jitmetadatalist.h" - - fw.Print("\n"); } // Run superpmi. The return value is as follows: diff --git a/src/coreclr/unwinder/i386/unwinder.cpp b/src/coreclr/unwinder/i386/unwinder.cpp index d8e7e7355681fa..5ee1193763a2d4 100644 --- a/src/coreclr/unwinder/i386/unwinder.cpp +++ b/src/coreclr/unwinder/i386/unwinder.cpp @@ -21,23 +21,15 @@ BOOL OOPStackUnwinderX86::Unwind(T_CONTEXT* pContextRecord, T_KNONVOLATILE_CONTE rd.pCurrentContextPointers = pContextPointers; } + CodeManState codeManState; + codeManState.dwIsSet = 0; + DWORD ControlPc = pContextRecord->Eip; EECodeInfo codeInfo; codeInfo.Init((PCODE) ControlPc); - GCInfoToken gcInfoToken = codeInfo.GetGCInfoToken(); - hdrInfo hdrInfoBody; - DWORD hdrInfoSize = (DWORD)DecodeGCHdrInfo(gcInfoToken, codeInfo.GetRelOffset(), &hdrInfoBody); - - if (!UnwindStackFrameX86(&rd, - PTR_CBYTE(codeInfo.GetSavedMethodCode()), - codeInfo.GetRelOffset(), - &hdrInfoBody, - dac_cast(gcInfoToken.Info) + hdrInfoSize, - PTR_CBYTE(codeInfo.GetJitManager()->GetFuncletStartAddress(&codeInfo)), - codeInfo.IsFunclet(), - true)) + if (!UnwindStackFrame(&rd, &codeInfo, UpdateAllRegs, &codeManState, NULL)) { return FALSE; } @@ -193,7 +185,10 @@ BOOL DacUnwindStackFrame(T_CONTEXT* pContextRecord, T_KNONVOLATILE_CONTEXT_POINT // language specific exception handler is returned. Otherwise, NULL is // returned. // +EXTERN_C +NTSYSAPI PEXCEPTION_ROUTINE +NTAPI RtlVirtualUnwind ( _In_ DWORD HandlerType, _In_ DWORD ImageBase, diff --git a/src/coreclr/vm/.vscode/c_cpp_properties.json b/src/coreclr/vm/.vscode/c_cpp_properties.json index d8abc20bf0bcbf..a1c7e61d179b48 100644 --- a/src/coreclr/vm/.vscode/c_cpp_properties.json +++ b/src/coreclr/vm/.vscode/c_cpp_properties.json @@ -71,6 +71,7 @@ "FEATURE_USE_ASM_GC_WRITE_BARRIERS", "FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP", "FEATURE_UTF8STRING=1", + "FEATURE_WIN32_REGISTRY", "TARGET_WINDOWS=1", "PROFILING_SUPPORTED_DATA", "UNICODE", diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index 30a04f06e62b8b..eb00b7c6a6757d 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -300,6 +300,7 @@ set(VM_SOURCES_WKS comdependenthandle.cpp comdynamic.cpp commodule.cpp + compatibilityswitch.cpp comsynchronizable.cpp comthreadpool.cpp comutilnative.cpp @@ -401,6 +402,7 @@ set(VM_HEADERS_WKS comdependenthandle.h comdynamic.h commodule.h + compatibilityswitch.h comsynchronizable.h comthreadpool.h comutilnative.h @@ -634,6 +636,7 @@ if(CLR_CMAKE_TARGET_ARCH_AMD64) ${ARCH_SOURCES_DIR}/AsmHelpers.asm ${ARCH_SOURCES_DIR}/CallDescrWorkerAMD64.asm ${ARCH_SOURCES_DIR}/ComCallPreStub.asm + ${ARCH_SOURCES_DIR}/CrtHelpers.asm ${ARCH_SOURCES_DIR}/GenericComCallStubs.asm ${ARCH_SOURCES_DIR}/GenericComPlusCallStubs.asm ${ARCH_SOURCES_DIR}/getstate.asm @@ -673,6 +676,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_ARM64) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/AsmHelpers.asm ${ARCH_SOURCES_DIR}/CallDescrWorkerARM64.asm + ${ARCH_SOURCES_DIR}/CrtHelpers.asm ${ARCH_SOURCES_DIR}/patchedcode.asm ${ARCH_SOURCES_DIR}/PInvokeStubs.asm ${ARCH_SOURCES_DIR}/thunktemplates.asm @@ -689,6 +693,7 @@ else(CLR_CMAKE_TARGET_WIN32) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S ${ARCH_SOURCES_DIR}/calldescrworkeramd64.S + ${ARCH_SOURCES_DIR}/crthelpers.S ${ARCH_SOURCES_DIR}/externalmethodfixupthunk.S ${ARCH_SOURCES_DIR}/getstate.S ${ARCH_SOURCES_DIR}/jithelpers_fast.S @@ -718,6 +723,7 @@ else(CLR_CMAKE_TARGET_WIN32) elseif(CLR_CMAKE_TARGET_ARCH_ARM) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S + ${ARCH_SOURCES_DIR}/crthelpers.S ${ARCH_SOURCES_DIR}/ehhelpers.S ${ARCH_SOURCES_DIR}/patchedcode.S ${ARCH_SOURCES_DIR}/pinvokestubs.S @@ -727,6 +733,7 @@ else(CLR_CMAKE_TARGET_WIN32) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S ${ARCH_SOURCES_DIR}/calldescrworkerarm64.S + ${ARCH_SOURCES_DIR}/crthelpers.S ${ARCH_SOURCES_DIR}/patchedcode.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S @@ -735,6 +742,7 @@ else(CLR_CMAKE_TARGET_WIN32) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S ${ARCH_SOURCES_DIR}/calldescrworkerloongarch64.S + ${ARCH_SOURCES_DIR}/crthelpers.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S ) @@ -742,6 +750,7 @@ else(CLR_CMAKE_TARGET_WIN32) set(VM_SOURCES_WKS_ARCH_ASM ${ARCH_SOURCES_DIR}/asmhelpers.S ${ARCH_SOURCES_DIR}/calldescrworkerriscv64.S + ${ARCH_SOURCES_DIR}/crthelpers.S ${ARCH_SOURCES_DIR}/pinvokestubs.S ${ARCH_SOURCES_DIR}/thunktemplates.S ) diff --git a/src/coreclr/vm/amd64/AsmHelpers.asm b/src/coreclr/vm/amd64/AsmHelpers.asm index 3fb2ca02e15a8f..aa1c443cf56f1c 100644 --- a/src/coreclr/vm/amd64/AsmHelpers.asm +++ b/src/coreclr/vm/amd64/AsmHelpers.asm @@ -634,6 +634,18 @@ NESTED_ENTRY ProfileTailcallNaked, _TEXT NESTED_END ProfileTailcallNaked, _TEXT +; EXTERN_C void moveOWord(LPVOID* src, LPVOID* target); +; +; MOVDQA is not an atomic operation. You need to call this function in a crst. +; +LEAF_ENTRY moveOWord, _TEXT + movdqa xmm0, [rcx] + movdqa [rdx], xmm0 + + ret +LEAF_END moveOWord, _TEXT + + extern JIT_InternalThrowFromHelper:proc LEAF_ENTRY SinglecastDelegateInvokeStub, _TEXT diff --git a/src/coreclr/vm/amd64/CrtHelpers.asm b/src/coreclr/vm/amd64/CrtHelpers.asm new file mode 100644 index 00000000000000..09f48fa5879bd1 --- /dev/null +++ b/src/coreclr/vm/amd64/CrtHelpers.asm @@ -0,0 +1,79 @@ +; Licensed to the .NET Foundation under one or more agreements. +; The .NET Foundation licenses this file to you under the MIT license. + +include AsmMacros.inc + +extern memset:proc +extern memmove:proc + +; JIT_MemSet/JIT_MemCpy +; +; It is IMPORTANT that the exception handling code is able to find these guys +; on the stack, but on windows platforms we can just defer to the platform +; implementation. +; + +; void JIT_MemSet(void* dest, int c, size_t count) +; +; Purpose: +; Sets the first "count" bytes of the block of memory pointed byte +; "dest" to the specified value (interpreted as an unsigned char). +; +; Entry: +; RCX: void* dest - Pointer to the block of memory to fill. +; RDX: int c - Value to be set. +; R8: size_t count - Number of bytes to be set to the value. +; +; Exit: +; +; Uses: +; +; Exceptions: +; +LEAF_ENTRY JIT_MemSet, _TEXT + test r8, r8 ; check if count is zero + jz Exit_MemSet ; if zero, no bytes to set + + cmp byte ptr [rcx], 0 ; check dest for null + + jmp memset ; forward to the CRT implementation + +Exit_MemSet: + ret + +LEAF_END_MARKED JIT_MemSet, _TEXT + +; void JIT_MemCpy(void* dest, const void* src, size_t count) +; +; Purpose: +; Copies the values of "count" bytes from the location pointed to +; by "src" to the memory block pointed by "dest". +; +; Entry: +; RCX: void* dest - Pointer to the destination array where content is to be copied. +; RDX: const void* src - Pointer to the source of the data to be copied. +; R8: size_t count - Number of bytes to copy. +; +; Exit: +; +; Uses: +; +; Exceptions: +; +LEAF_ENTRY JIT_MemCpy, _TEXT + test r8, r8 ; check if count is zero + jz Exit_MemCpy ; if zero, no bytes to copy + + cmp byte ptr [rcx], 0 ; check dest for null + cmp byte ptr [rdx], 0 ; check src for null + + ; Use memmove to handle overlapping buffers for better + ; compatibility with .NET Framework. Needing to handle + ; overlapping buffers in cpblk is undefined by the spec. + jmp memmove ; forward to the CRT implementation + +Exit_MemCpy: + ret + +LEAF_END_MARKED JIT_MemCpy, _TEXT + end diff --git a/src/coreclr/vm/amd64/JitHelpers_Fast.asm b/src/coreclr/vm/amd64/JitHelpers_Fast.asm index 70291c8307dc93..0f1b71b5ee93b3 100644 --- a/src/coreclr/vm/amd64/JitHelpers_Fast.asm +++ b/src/coreclr/vm/amd64/JitHelpers_Fast.asm @@ -56,14 +56,9 @@ extern JIT_InternalThrow:proc ; RDI - address of ref-field (assigned to) ; RSI - address of the data (source) ; RCX is trashed -; RAX is trashed -; -; NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -; if you add more trashed registers. -; +; RAX is trashed when FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP is defined ; Exit: ; RDI, RSI are incremented by SIZEOF(LPVOID) -; LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT mov rcx, [rsi] @@ -156,6 +151,8 @@ endif cmp rcx, [g_ephemeral_high] jnb Exit + ; do the following checks only if we are allowed to trash rax + ; otherwise we don't have enough registers ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP mov rax, rcx diff --git a/src/coreclr/vm/amd64/cgenamd64.cpp b/src/coreclr/vm/amd64/cgenamd64.cpp index f774d71a3b427f..261ecec5c46d8a 100644 --- a/src/coreclr/vm/amd64/cgenamd64.cpp +++ b/src/coreclr/vm/amd64/cgenamd64.cpp @@ -206,7 +206,7 @@ void HelperMethodFrame::UpdateRegDisplay(const PREGDISPLAY pRD, bool updateFloat #endif // TARGET_UNIX -#define CALLEE_SAVED_REGISTER(regname) pRD->pCurrentContextPointers->regname = (DWORD64 *)(TADDR *)m_MachState.m_Ptrs.p##regname; +#define CALLEE_SAVED_REGISTER(regname) pRD->pCurrentContextPointers->regname = m_MachState.m_Ptrs.p##regname; ENUM_CALLEE_SAVED_REGISTERS(); #undef CALLEE_SAVED_REGISTER diff --git a/src/coreclr/vm/amd64/crthelpers.S b/src/coreclr/vm/amd64/crthelpers.S new file mode 100644 index 00000000000000..82219e574092da --- /dev/null +++ b/src/coreclr/vm/amd64/crthelpers.S @@ -0,0 +1,74 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +.intel_syntax noprefix +#include "unixasmmacros.inc" +#include "asmconstants.h" + +// JIT_MemSet/JIT_MemCpy +// +// It is IMPORTANT that the exception handling code is able to find these guys +// on the stack, but on non-windows platforms we can just defer to the platform +// implementation. +// + +// void JIT_MemSet(void* dest, int c, size_t count) +// +// Purpose: +// Sets the first "count" bytes of the block of memory pointed byte +// "dest" to the specified value (interpreted as an unsigned char). +// +// Entry: +// RDI: void* dest - Pointer to the block of memory to fill. +// RSI: int c - Value to be set. +// RDX: size_t count - Number of bytes to be set to the value. +// +// Exit: +// +// Uses: +// +// Exceptions: +// +LEAF_ENTRY JIT_MemSet, _TEXT + test rdx, rdx // check if count is zero + jz Exit_MemSet // if zero, no bytes to set + + cmp byte ptr [rdi], 0 // check dest for null + + jmp C_PLTFUNC(memset) // forward to the CRT implementation + +Exit_MemSet: + ret + +LEAF_END_MARKED JIT_MemSet, _TEXT + +// void JIT_MemCpy(void* dest, const void* src, size_t count) +// +// Purpose: +// Copies the values of "count" bytes from the location pointed to +// by "src" to the memory block pointed by "dest". +// +// Entry: +// RDI: void* dest - Pointer to the destination array where content is to be copied. +// RSI: const void* src - Pointer to the source of the data to be copied. +// RDX: size_t count - Number of bytes to copy. +// +// Exit: +// +// Uses: +// +// Exceptions: +// +LEAF_ENTRY JIT_MemCpy, _TEXT + test rdx, rdx // check if count is zero + jz Exit_MemCpy // if zero, no bytes to set + + cmp byte ptr [rdi], 0 // check dest for null + cmp byte ptr [rsi], 0 // check src for null + + jmp C_PLTFUNC(memcpy) // forward to the CRT implementation + +Exit_MemCpy: + ret + +LEAF_END_MARKED JIT_MemCpy, _TEXT diff --git a/src/coreclr/vm/amd64/gmsamd64.cpp b/src/coreclr/vm/amd64/gmsamd64.cpp index 41c7b0c9afa211..bc1079c14ace10 100644 --- a/src/coreclr/vm/amd64/gmsamd64.cpp +++ b/src/coreclr/vm/amd64/gmsamd64.cpp @@ -136,7 +136,7 @@ void LazyMachState::unwindLazyState(LazyMachState* baseState, #else // !DACCESS_COMPILE -#define CALLEE_SAVED_REGISTER(regname) unwoundState->m_Ptrs.p##regname = PTR_TADDR(nonVolRegPtrs.regname); +#define CALLEE_SAVED_REGISTER(regname) unwoundState->m_Ptrs.p##regname = PTR_ULONG64(nonVolRegPtrs.regname); ENUM_CALLEE_SAVED_REGISTERS(); #undef CALLEE_SAVED_REGISTER diff --git a/src/coreclr/vm/amd64/jithelpers_fast.S b/src/coreclr/vm/amd64/jithelpers_fast.S index 96195353db2904..3a2d803a1460fb 100644 --- a/src/coreclr/vm/amd64/jithelpers_fast.S +++ b/src/coreclr/vm/amd64/jithelpers_fast.S @@ -10,15 +10,19 @@ // Entry: // RDI - address of ref-field (assigned to) // RSI - address of the data (source) -// RCX is trashed -// RAX is trashed // -// NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -// if you add more trashed registers. +// Note: RyuJIT assumes that all volatile registers can be trashed by +// the CORINFO_HELP_ASSIGN_BYREF helper (i.e. JIT_ByRefWriteBarrier) +// except RDI and RSI. This helper uses and defines RDI and RSI, so +// they remain as live GC refs or byrefs, and are not killed. // +// +// RCX is trashed +// RAX is trashed +// R10 is trashed +// R11 is trashed on Debug build // Exit: // RDI, RSI are incremented by SIZEOF(LPVOID) -// LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT mov rcx, [rsi] @@ -32,31 +36,31 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT // but if it isn't then it will just return. // // See if this is in GCHeap - cmp rdi, [C_VAR(g_lowest_address)] + PREPARE_EXTERNAL_VAR g_lowest_address, rax + cmp rdi, [rax] jb LOCAL_LABEL(NotInHeap_ByRefWriteBarrier) - cmp rdi, [C_VAR(g_highest_address)] + PREPARE_EXTERNAL_VAR g_highest_address, rax + cmp rdi, [rax] jnb LOCAL_LABEL(NotInHeap_ByRefWriteBarrier) #ifdef WRITE_BARRIER_CHECK - // we can only trash rcx in this function so in _DEBUG we need to save - // some scratch registers. - push r10 - push r11 - push rax - // **ALSO update the shadow GC heap if that is enabled** // Do not perform the work if g_GCShadow is 0 - cmp qword ptr [C_VAR(g_GCShadow)], 0 + PREPARE_EXTERNAL_VAR g_GCShadow, rax + cmp qword ptr [rax], 0 je LOCAL_LABEL(NoShadow_ByRefWriteBarrier) // If we end up outside of the heap don't corrupt random memory mov r10, rdi - sub r10, [C_VAR(g_lowest_address)] + PREPARE_EXTERNAL_VAR g_lowest_address, rax + sub r10, [rax] jb LOCAL_LABEL(NoShadow_ByRefWriteBarrier) // Check that our adjusted destination is somewhere in the shadow gc - add r10, [C_VAR(g_GCShadow)] - cmp r10, [C_VAR(g_GCShadowEnd)] + PREPARE_EXTERNAL_VAR g_GCShadow, rax + add r10, [rax] + PREPARE_EXTERNAL_VAR g_GCShadowEnd, rax + cmp r10, [rax] jnb LOCAL_LABEL(NoShadow_ByRefWriteBarrier) // Write ref into real GC @@ -87,57 +91,63 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT // Additionally we know for sure that we are inside the heap and therefore don't // need to replicate the above checks. LOCAL_LABEL(DoneShadow_ByRefWriteBarrier): - pop rax - pop r11 - pop r10 #endif #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // Update the write watch table if necessary - cmp byte ptr [C_VAR(g_sw_ww_enabled_for_gc_heap)], 0x0 + PREPARE_EXTERNAL_VAR g_sw_ww_enabled_for_gc_heap, rax + cmp byte ptr [rax], 0x0 je LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier) mov rax, rdi shr rax, 0xC // SoftwareWriteWatch::AddressToTableByteIndexShift - add rax, qword ptr [C_VAR(g_sw_ww_table)] + PREPARE_EXTERNAL_VAR g_sw_ww_table, r10 + add rax, qword ptr [r10] cmp byte ptr [rax], 0x0 jne LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier) mov byte ptr [rax], 0xFF #endif - // See if we can just quick out LOCAL_LABEL(CheckCardTable_ByRefWriteBarrier): - cmp rcx, [C_VAR(g_ephemeral_low)] + // See if we can just quick out + PREPARE_EXTERNAL_VAR g_ephemeral_low, rax + cmp rcx, [rax] jb LOCAL_LABEL(Exit_ByRefWriteBarrier) - cmp rcx, [C_VAR(g_ephemeral_high)] + PREPARE_EXTERNAL_VAR g_ephemeral_high, rax + cmp rcx, [rax] jnb LOCAL_LABEL(Exit_ByRefWriteBarrier) mov rax, rcx - mov cl, [C_VAR(g_region_shr)] + PREPARE_EXTERNAL_VAR g_region_shr, rcx + mov cl, [rcx] test cl, cl je LOCAL_LABEL(SkipCheck_ByRefWriteBarrier) // check if the source is in gen 2 - then it's not an ephemeral pointer shr rax, cl - add rax, [C_VAR(g_region_to_generation_table)] - cmp byte ptr [rax], 0x82 + PREPARE_EXTERNAL_VAR g_region_to_generation_table, r10 + mov r10, [r10] + cmp byte ptr [rax + r10], 0x82 je LOCAL_LABEL(Exit_ByRefWriteBarrier) // check if the destination happens to be in gen 0 mov rax, rdi shr rax, cl - add rax, [C_VAR(g_region_to_generation_table)] - cmp byte ptr [rax], 0 + cmp byte ptr [rax + r10], 0 je LOCAL_LABEL(Exit_ByRefWriteBarrier) LOCAL_LABEL(SkipCheck_ByRefWriteBarrier): - cmp byte ptr [C_VAR(g_region_use_bitwise_write_barrier)], 0 + PREPARE_EXTERNAL_VAR g_card_table, r10 + mov r10, [r10] + + PREPARE_EXTERNAL_VAR g_region_use_bitwise_write_barrier, rax + cmp byte ptr [rax], 0 je LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrier) // compute card table bit - mov rcx, rdi + mov ecx, edi mov al, 1 - shr rcx, 8 + shr ecx, 8 and cl, 7 shl al, cl @@ -149,51 +159,48 @@ LEAF_ENTRY JIT_ByRefWriteBarrier, _TEXT // Check if we need to update the card table // Calc pCardByte shr rcx, 0xB - add rcx, [C_VAR(g_card_table)] - // Check if this card table bit is already set - test byte ptr [rcx], al + test byte ptr [rcx + r10], al je LOCAL_LABEL(SetCardTableBit_ByRefWriteBarrier) REPRET LOCAL_LABEL(SetCardTableBit_ByRefWriteBarrier): - lock or byte ptr [rcx], al + lock or byte ptr [rcx + r10], al + jmp LOCAL_LABEL(CheckCardBundle_ByRefWriteBarrier) -LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrier): + LOCAL_LABEL(CheckCardTableByte_ByRefWriteBarrier): // move current rdi value into rcx and then increment the pointers mov rcx, rdi add rsi, 0x8 add rdi, 0x8 - // Check if we need to update the card table - // Calc pCardByte shr rcx, 0xB - add rcx, [C_VAR(g_card_table)] - - // Check if this card is dirty - cmp byte ptr [rcx], 0xFF - jne LOCAL_LABEL(UpdateCardTable_ByRefWriteBarrier) + cmp byte ptr [rcx + r10], 0xFF + jne LOCAL_LABEL(SetCardTableByte_ByRefWriteBarrier) REPRET - - LOCAL_LABEL(UpdateCardTable_ByRefWriteBarrier): - mov byte ptr [rcx], 0xFF + LOCAL_LABEL(SetCardTableByte_ByRefWriteBarrier): + mov byte ptr [rcx + r10], 0xFF LOCAL_LABEL(CheckCardBundle_ByRefWriteBarrier): #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES - // check if we need to update the card bundle table - // restore destination address from rdi - rdi has been incremented by 8 already - lea rcx, [rdi-8] - shr rcx, 0x15 - add rcx, [C_VAR(g_card_bundle_table)] + // Shift rcx by 0x0A more to get the card bundle byte (we shifted by 0x0B already) + shr rcx, 0x0A + + PREPARE_EXTERNAL_VAR g_card_bundle_table, rax + add rcx, [rax] + + // Check if this bundle byte is dirty cmp byte ptr [rcx], 0xFF - jne LOCAL_LABEL(UpdateCardBundleTable_ByRefWriteBarrier) + + jne LOCAL_LABEL(UpdateCardBundle_ByRefWriteBarrier) REPRET - LOCAL_LABEL(UpdateCardBundleTable_ByRefWriteBarrier): + LOCAL_LABEL(UpdateCardBundle_ByRefWriteBarrier): mov byte ptr [rcx], 0xFF #endif + ret .balign 16 diff --git a/src/coreclr/vm/amd64/unixasmhelpers.S b/src/coreclr/vm/amd64/unixasmhelpers.S index 77fe1384dcd9ee..4711ee9857f2c4 100644 --- a/src/coreclr/vm/amd64/unixasmhelpers.S +++ b/src/coreclr/vm/amd64/unixasmhelpers.S @@ -71,16 +71,14 @@ NESTED_ENTRY NDirectImportThunk, _TEXT, NoHandler // // Save integer parameter registers. // Make sure to preserve r11 as well as it is used to pass the stack argument size from JIT - // Make sure to preserve rax as well as it is used for the return buffer for Swift calls // PUSH_ARGUMENT_REGISTERS push_register r11 - push_register rax // - // Allocate space for XMM parameter registers and alignment + // Allocate space for XMM parameter registers // - alloc_stack 0x88 + alloc_stack 0x80 SAVE_FLOAT_ARGUMENT_REGISTERS 0 @@ -91,26 +89,34 @@ NESTED_ENTRY NDirectImportThunk, _TEXT, NoHandler // mov rdi, METHODDESC_REGISTER call C_FUNC(NDirectImportWorker) - mov r10, rax RESTORE_FLOAT_ARGUMENT_REGISTERS 0 // - // epilogue, r10 contains the native target address + // epilogue, rax contains the native target address // - free_stack 0x88 + free_stack 0x80 // - // Restore integer parameter registers, r11 and rax + // Restore integer parameter registers and r11 // - pop_register rax pop_register r11 POP_ARGUMENT_REGISTERS - jmp r10 - + TAILJMP_RAX NESTED_END NDirectImportThunk, _TEXT +// EXTERN_C void moveOWord(LPVOID* src, LPVOID* target); +// +// MOVDQA is not an atomic operation. You need to call this function in a crst. +// +LEAF_ENTRY moveOWord, _TEXT + movdqu xmm0, xmmword ptr [rdi] + movdqu xmmword ptr [rsi], xmm0 + + ret +LEAF_END moveOWord, _TEXT + //------------------------------------------------ // JIT_RareDisableHelper // diff --git a/src/coreclr/vm/appdomain.cpp b/src/coreclr/vm/appdomain.cpp index bb5d3d17e00534..feafd1f8abad6d 100644 --- a/src/coreclr/vm/appdomain.cpp +++ b/src/coreclr/vm/appdomain.cpp @@ -1352,7 +1352,7 @@ void SystemDomain::LoadBaseSystemClasses() // further loading of nonprimitive types may need casting support. // initialize cast cache here. CastCache::Initialize(); - ECall::PopulateManagedHelpers(); + ECall::PopulateManagedCastHelpers(); // used by IsImplicitInterfaceOfSZArray CoreLibBinder::GetClass(CLASS__IENUMERABLEGENERIC); diff --git a/src/coreclr/vm/arm/crthelpers.S b/src/coreclr/vm/arm/crthelpers.S new file mode 100644 index 00000000000000..db0ed192c4d60f --- /dev/null +++ b/src/coreclr/vm/arm/crthelpers.S @@ -0,0 +1,51 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// *********************************************************************** +// File: crthelpers.S +// +// *********************************************************************** + +#include "unixasmmacros.inc" +#include "asmconstants.h" + +.syntax unified +.thumb + +// JIT_MemSet/JIT_MemCpy +// +// It is IMPORANT that the exception handling code is able to find these guys +// on the stack, but to keep them from being tailcalled by VC++ we need to turn +// off optimization and it ends up being a wasteful implementation. +// +// Hence these assembly helpers. +// +//EXTERN_C void __stdcall JIT_MemSet(void* _dest, int c, size_t count) +LEAF_ENTRY JIT_MemSet, _TEXT + + cmp r2, #0 + it eq + bxeq lr + + ldrb r3, [r0] + + b C_PLTFUNC(memset) + +LEAF_END_MARKED JIT_MemSet, _TEXT + + +//EXTERN_C void __stdcall JIT_MemCpy(void* _dest, const void *_src, size_t count) +LEAF_ENTRY JIT_MemCpy, _TEXT +// + + cmp r2, #0 + it eq + bxeq lr + + ldrb r3, [r0] + ldrb r3, [r1] + + b C_PLTFUNC(memcpy) + +LEAF_END_MARKED JIT_MemCpy, _TEXT + diff --git a/src/coreclr/vm/arm/stubs.cpp b/src/coreclr/vm/arm/stubs.cpp index 1424dcecbd918d..d72c32201700e2 100644 --- a/src/coreclr/vm/arm/stubs.cpp +++ b/src/coreclr/vm/arm/stubs.cpp @@ -1619,7 +1619,7 @@ void InlinedCallFrame::UpdateRegDisplay(const PREGDISPLAY pRD, bool updateFloats // Update the frame pointer in the current context. pRD->pCurrentContext->R11 = m_pCalleeSavedFP; - pRD->pCurrentContextPointers->R11 = (DWORD *)&m_pCalleeSavedFP; + pRD->pCurrentContextPointers->R11 = &m_pCalleeSavedFP; // This is necessary to unwind methods with alloca. This needs to stay // in sync with definition of REG_SAVED_LOCALLOC_SP in the JIT. diff --git a/src/coreclr/vm/arm64/PInvokeStubs.asm b/src/coreclr/vm/arm64/PInvokeStubs.asm index 37699e790b241e..d7b432240d2db2 100644 --- a/src/coreclr/vm/arm64/PInvokeStubs.asm +++ b/src/coreclr/vm/arm64/PInvokeStubs.asm @@ -188,6 +188,8 @@ RarePath LEAF_END + INLINE_GETTHREAD_CONSTANT_POOL + ; ------------------------------------------------------------------ ; VarargPInvokeStub & VarargPInvokeGenILStub ; diff --git a/src/coreclr/vm/arm64/asmconstants.h b/src/coreclr/vm/arm64/asmconstants.h index cc19728d9e29ee..b52c164e1ab71f 100644 --- a/src/coreclr/vm/arm64/asmconstants.h +++ b/src/coreclr/vm/arm64/asmconstants.h @@ -96,11 +96,7 @@ ASMCONSTANTS_C_ASSERT(MachState__isValid == offsetof(MachState, _isValid)) #define LazyMachState_captureX19_X29 MachState__captureX19_X29 ASMCONSTANTS_C_ASSERT(LazyMachState_captureX19_X29 == offsetof(LazyMachState, captureX19_X29)) -#ifdef __APPLE__ -#define LazyMachState_captureSp (MachState__isValid+8+88) // padding for alignment -#else // __APPLE__ #define LazyMachState_captureSp (MachState__isValid+8) // padding for alignment -#endif // __APPLE ASMCONSTANTS_C_ASSERT(LazyMachState_captureSp == offsetof(LazyMachState, captureSp)) #define LazyMachState_captureIp (LazyMachState_captureSp+8) diff --git a/src/coreclr/vm/arm64/asmmacros.h b/src/coreclr/vm/arm64/asmmacros.h index 8dd56dc59045cd..4a0cb5dde49aba 100644 --- a/src/coreclr/vm/arm64/asmmacros.h +++ b/src/coreclr/vm/arm64/asmmacros.h @@ -313,17 +313,18 @@ __RedirectionFuncName SETS "|?RedirectedHandledJITCaseFor":CC:"$reason":CC:"@Thr MEND -;; ----------------------------------------------------------------------------- -;; -;; Macro to get a pointer to a threadlocal symbol for the currently executing thread -;; - +;----------------------------------------------------------------------------- +; Macro to get a pointer to the Thread* object for the currently executing thread +; __tls_array equ 0x58 ;; offsetof(TEB, ThreadLocalStoragePointer) - MACRO - INLINE_GET_TLS_VAR $destReg, $trashReg, $variable + EXTERN _tls_index + + GBLS __SECTIONREL_gCurrentThreadInfo +__SECTIONREL_gCurrentThreadInfo SETS "SECTIONREL_gCurrentThreadInfo" - EXTERN _tls_index + MACRO + INLINE_GETTHREAD $destReg, $trashReg ;; The following macro variables are just some assembler magic to get the name of the 32-bit version ;; of $trashReg. It does it by string manipulation. Replaces something like x3 with w3. @@ -331,27 +332,33 @@ __tls_array equ 0x58 ;; offsetof(TEB, ThreadLocalStoragePointer) TrashRegister32Bit SETS "$trashReg" TrashRegister32Bit SETS "w":CC:("$TrashRegister32Bit":RIGHT:((:LEN:TrashRegister32Bit) - 1)) - adrp $destReg, _tls_index - ldr $TrashRegister32Bit, [$destReg, _tls_index] + ldr $trashReg, =_tls_index + ldr $TrashRegister32Bit, [$trashReg] ldr $destReg, [xpr, #__tls_array] - ldr $destReg, [$destReg, $TrashRegister32Bit uxtw #3] - add $destReg, $destReg, #0, lsl #0xC - RELOC 0xA, $variable ;; IMAGE_REL_ARM64_SECREL_HIGH12A - add $destReg, $destReg, #0, lsl #0 - RELOC 0x9, $variable ;; IMAGE_REL_ARM64_SECREL_LOW12A + ldr $destReg, [$destReg, $trashReg lsl #3] + ldr $trashReg, =$__SECTIONREL_gCurrentThreadInfo + ldr $trashReg, [$trashReg] + ldr $destReg, [$destReg, $trashReg] ; return gCurrentThreadInfo.m_pThread MEND -;; ----------------------------------------------------------------------------- -;; -;; Macro to get a pointer to the Thread* object for the currently executing thread -;; +;----------------------------------------------------------------------------- +; INLINE_GETTHREAD_CONSTANT_POOL macro has to be used after the last function in the .asm file that used +; INLINE_GETTHREAD. Optionally, it can be also used after any function that used INLINE_GETTHREAD +; to improve density, or to reduce distance between the constant pool and its use. +; SETALIAS gCurrentThreadInfo, ?gCurrentThreadInfo@@3UThreadLocalInfo@@A MACRO - INLINE_GETTHREAD $destReg, $trashReg + INLINE_GETTHREAD_CONSTANT_POOL EXTERN $gCurrentThreadInfo - INLINE_GET_TLS_VAR $destReg, $trashReg, $gCurrentThreadInfo - ldr $destReg, [$destReg] ;; return gCurrentThreadInfo.m_pThread + ;; Section relocs are 32 bits. Using an extra DCD initialized to zero for 8-byte alignment. +$__SECTIONREL_gCurrentThreadInfo + DCD $gCurrentThreadInfo + RELOC 8, $gCurrentThreadInfo ;; SECREL + DCD 0 + +__SECTIONREL_gCurrentThreadInfo SETS "$__SECTIONREL_gCurrentThreadInfo":CC:"_" + MEND diff --git a/src/coreclr/vm/arm64/crthelpers.S b/src/coreclr/vm/arm64/crthelpers.S new file mode 100644 index 00000000000000..e123fc82808d16 --- /dev/null +++ b/src/coreclr/vm/arm64/crthelpers.S @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "unixasmmacros.inc" + +// JIT_MemSet/JIT_MemCpy +// +// It is IMPORTANT that the exception handling code is able to find these guys +// on the stack, but on non-windows platforms we can just defer to the platform +// implementation. +// +LEAF_ENTRY JIT_MemSet, _TEXT + cbz x2, LOCAL_LABEL(JIT_MemSet_ret) + + ldrb wzr, [x0] + + b C_PLTFUNC(memset) + +LOCAL_LABEL(JIT_MemSet_ret): + ret lr +LEAF_END_MARKED JIT_MemSet, _TEXT + +LEAF_ENTRY JIT_MemCpy, _TEXT + cbz x2, LOCAL_LABEL(JIT_MemCpy_ret) + + ldrb wzr, [x0] + ldrb wzr, [x1] + + b C_PLTFUNC(memcpy) + +LOCAL_LABEL(JIT_MemCpy_ret): + ret lr +LEAF_END_MARKED JIT_MemCpy, _TEXT diff --git a/src/coreclr/vm/arm64/crthelpers.asm b/src/coreclr/vm/arm64/crthelpers.asm new file mode 100644 index 00000000000000..d4d13351365c95 --- /dev/null +++ b/src/coreclr/vm/arm64/crthelpers.asm @@ -0,0 +1,81 @@ +; Licensed to the .NET Foundation under one or more agreements. +; The .NET Foundation licenses this file to you under the MIT license. + +#include "ksarm64.h" +#include "asmconstants.h" +#include "asmmacros.h" + + IMPORT memset + IMPORT memmove + +; JIT_MemSet/JIT_MemCpy +; +; It is IMPORTANT that the exception handling code is able to find these guys +; on the stack, but on windows platforms we can just defer to the platform +; implementation. +; + +; void JIT_MemSet(void* dest, int c, size_t count) +; +; Purpose: +; Sets the first "count" bytes of the block of memory pointed byte +; "dest" to the specified value (interpreted as an unsigned char). +; +; Entry: +; RCX: void* dest - Pointer to the block of memory to fill. +; RDX: int c - Value to be set. +; R8: size_t count - Number of bytes to be set to the value. +; +; Exit: +; +; Uses: +; +; Exceptions: +; + + TEXTAREA + + LEAF_ENTRY JIT_MemSet + cbz x2, JIT_MemSet_ret ; check if count is zero, no bytes to set + + ldrb wzr, [x0] ; check dest for null + + b memset ; forward to the CRT implementation + +JIT_MemSet_ret + ret lr + + LEAF_END_MARKED JIT_MemSet + +; void JIT_MemCpy(void* dest, const void* src, size_t count) +; +; Purpose: +; Copies the values of "count" bytes from the location pointed to +; by "src" to the memory block pointed by "dest". +; +; Entry: +; RCX: void* dest - Pointer to the destination array where content is to be copied. +; RDX: const void* src - Pointer to the source of the data to be copied. +; R8: size_t count - Number of bytes to copy. +; +; Exit: +; +; Uses: +; +; Exceptions: +; + LEAF_ENTRY JIT_MemCpy + cbz x2, JIT_MemCpy_ret ; check if count is zero, no bytes to set + + ldrb wzr, [x0] ; check dest for null + ldrb wzr, [x1] ; check src for null + + b memmove ; forward to the CRT implementation + +JIT_MemCpy_ret + ret lr + + LEAF_END_MARKED JIT_MemCpy + +; Must be at very end of file + END diff --git a/src/coreclr/vm/arm64/gmscpu.h b/src/coreclr/vm/arm64/gmscpu.h index 000eed14b4743b..887a41b4f07c10 100644 --- a/src/coreclr/vm/arm64/gmscpu.h +++ b/src/coreclr/vm/arm64/gmscpu.h @@ -25,11 +25,6 @@ struct MachState { TADDR _pc; // program counter after the function returns TADDR _sp; // stack pointer after the function returns BOOL _isValid; -#ifdef __APPLE__ - // libunwind on macOS doesn't support context pointers and we cannot modify the captureX19_X29, - // so we store the unwound values in a separate array. - ULONG64 unwoundX19_X29[NUM_NONVOLATILE_CONTEXT_POINTERS]; // preserved registers -#endif // __APPLE__ BOOL isValid() { LIMITED_METHOD_DAC_CONTRACT; return _isValid; } TADDR GetRetAddr() { LIMITED_METHOD_DAC_CONTRACT; return _pc; } @@ -60,10 +55,6 @@ inline void LazyMachState::setLazyStateFromUnwind(MachState* copy) _sp = copy->_sp; _pc = copy->_pc; -#ifdef __APPLE__ - memcpy(unwoundX19_X29, copy->unwoundX19_X29, sizeof(unwoundX19_X29)); -#endif // __APPLE__ - // Capture* has already been set, so there is no need to touch it // loop over the nonvolatile context pointers and make @@ -89,6 +80,7 @@ inline void LazyMachState::setLazyStateFromUnwind(MachState* copy) } *pDst++ = valueSrc; + captureX19_X29[i] = copy->captureX19_X29[i]; } diff --git a/src/coreclr/vm/arm64/patchedcode.S b/src/coreclr/vm/arm64/patchedcode.S index ae8d07ab1f9442..2c1199be69a78e 100644 --- a/src/coreclr/vm/arm64/patchedcode.S +++ b/src/coreclr/vm/arm64/patchedcode.S @@ -42,9 +42,6 @@ LEAF_END JIT_PatchedCodeStart, _TEXT // x15 : trashed // x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // -// NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -// if you add more trashed registers. -// WRITE_BARRIER_ENTRY JIT_ByRefWriteBarrier ldr x15, [x13], 8 diff --git a/src/coreclr/vm/arm64/patchedcode.asm b/src/coreclr/vm/arm64/patchedcode.asm index 4bb8aa196818d7..bd4f57cc6810c8 100644 --- a/src/coreclr/vm/arm64/patchedcode.asm +++ b/src/coreclr/vm/arm64/patchedcode.asm @@ -75,9 +75,6 @@ wbs_GCShadowEnd ; x14 : incremented by 8 ; x15 : trashed ; x17 : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP -; -; NOTE: Keep in sync with RBM_CALLEE_TRASH_WRITEBARRIER_BYREF and RBM_CALLEE_GCTRASH_WRITEBARRIER_BYREF -; if you add more trashed registers. ; WRITE_BARRIER_ENTRY JIT_ByRefWriteBarrier diff --git a/src/coreclr/vm/arm64/stubs.cpp b/src/coreclr/vm/arm64/stubs.cpp index 03783f016a52d3..871a115097b602 100644 --- a/src/coreclr/vm/arm64/stubs.cpp +++ b/src/coreclr/vm/arm64/stubs.cpp @@ -378,19 +378,19 @@ void LazyMachState::unwindLazyState(LazyMachState* baseState, } } while (true); -#ifdef __APPLE__ - unwoundstate->unwoundX19_X29[0] = context.X19; - unwoundstate->unwoundX19_X29[1] = context.X20; - unwoundstate->unwoundX19_X29[2] = context.X21; - unwoundstate->unwoundX19_X29[3] = context.X22; - unwoundstate->unwoundX19_X29[4] = context.X23; - unwoundstate->unwoundX19_X29[5] = context.X24; - unwoundstate->unwoundX19_X29[6] = context.X25; - unwoundstate->unwoundX19_X29[7] = context.X26; - unwoundstate->unwoundX19_X29[8] = context.X27; - unwoundstate->unwoundX19_X29[9] = context.X28; - unwoundstate->unwoundX19_X29[10] = context.Fp; -#endif // __APPLE__ +#ifdef TARGET_UNIX + unwoundstate->captureX19_X29[0] = context.X19; + unwoundstate->captureX19_X29[1] = context.X20; + unwoundstate->captureX19_X29[2] = context.X21; + unwoundstate->captureX19_X29[3] = context.X22; + unwoundstate->captureX19_X29[4] = context.X23; + unwoundstate->captureX19_X29[5] = context.X24; + unwoundstate->captureX19_X29[6] = context.X25; + unwoundstate->captureX19_X29[7] = context.X26; + unwoundstate->captureX19_X29[8] = context.X27; + unwoundstate->captureX19_X29[9] = context.X28; + unwoundstate->captureX19_X29[10] = context.Fp; +#endif #ifdef DACCESS_COMPILE // For DAC builds, we update the registers directly since we dont have context pointers @@ -505,20 +505,20 @@ void HelperMethodFrame::UpdateRegDisplay(const PREGDISPLAY pRD, bool updateFloat pRD->pCurrentContext->Pc = pRD->ControlPC; pRD->pCurrentContext->Sp = pRD->SP; -#ifdef __APPLE__ - pRD->pCurrentContext->X19 = (DWORD64)(m_MachState.unwoundX19_X29[0]); - pRD->pCurrentContext->X20 = (DWORD64)(m_MachState.unwoundX19_X29[1]); - pRD->pCurrentContext->X21 = (DWORD64)(m_MachState.unwoundX19_X29[2]); - pRD->pCurrentContext->X22 = (DWORD64)(m_MachState.unwoundX19_X29[3]); - pRD->pCurrentContext->X23 = (DWORD64)(m_MachState.unwoundX19_X29[4]); - pRD->pCurrentContext->X24 = (DWORD64)(m_MachState.unwoundX19_X29[5]); - pRD->pCurrentContext->X25 = (DWORD64)(m_MachState.unwoundX19_X29[6]); - pRD->pCurrentContext->X26 = (DWORD64)(m_MachState.unwoundX19_X29[7]); - pRD->pCurrentContext->X27 = (DWORD64)(m_MachState.unwoundX19_X29[8]); - pRD->pCurrentContext->X28 = (DWORD64)(m_MachState.unwoundX19_X29[9]); - pRD->pCurrentContext->Fp = (DWORD64)(m_MachState.unwoundX19_X29[10]); +#ifdef TARGET_UNIX + pRD->pCurrentContext->X19 = m_MachState.ptrX19_X29[0] ? *m_MachState.ptrX19_X29[0] : m_MachState.captureX19_X29[0]; + pRD->pCurrentContext->X20 = m_MachState.ptrX19_X29[1] ? *m_MachState.ptrX19_X29[1] : m_MachState.captureX19_X29[1]; + pRD->pCurrentContext->X21 = m_MachState.ptrX19_X29[2] ? *m_MachState.ptrX19_X29[2] : m_MachState.captureX19_X29[2]; + pRD->pCurrentContext->X22 = m_MachState.ptrX19_X29[3] ? *m_MachState.ptrX19_X29[3] : m_MachState.captureX19_X29[3]; + pRD->pCurrentContext->X23 = m_MachState.ptrX19_X29[4] ? *m_MachState.ptrX19_X29[4] : m_MachState.captureX19_X29[4]; + pRD->pCurrentContext->X24 = m_MachState.ptrX19_X29[5] ? *m_MachState.ptrX19_X29[5] : m_MachState.captureX19_X29[5]; + pRD->pCurrentContext->X25 = m_MachState.ptrX19_X29[6] ? *m_MachState.ptrX19_X29[6] : m_MachState.captureX19_X29[6]; + pRD->pCurrentContext->X26 = m_MachState.ptrX19_X29[7] ? *m_MachState.ptrX19_X29[7] : m_MachState.captureX19_X29[7]; + pRD->pCurrentContext->X27 = m_MachState.ptrX19_X29[8] ? *m_MachState.ptrX19_X29[8] : m_MachState.captureX19_X29[8]; + pRD->pCurrentContext->X28 = m_MachState.ptrX19_X29[9] ? *m_MachState.ptrX19_X29[9] : m_MachState.captureX19_X29[9]; + pRD->pCurrentContext->Fp = m_MachState.ptrX19_X29[10] ? *m_MachState.ptrX19_X29[10] : m_MachState.captureX19_X29[10]; pRD->pCurrentContext->Lr = NULL; // Unwind again to get Caller's PC -#else // __APPLE__ +#else // TARGET_UNIX pRD->pCurrentContext->X19 = *m_MachState.ptrX19_X29[0]; pRD->pCurrentContext->X20 = *m_MachState.ptrX19_X29[1]; pRD->pCurrentContext->X21 = *m_MachState.ptrX19_X29[2]; @@ -531,7 +531,7 @@ void HelperMethodFrame::UpdateRegDisplay(const PREGDISPLAY pRD, bool updateFloat pRD->pCurrentContext->X28 = *m_MachState.ptrX19_X29[9]; pRD->pCurrentContext->Fp = *m_MachState.ptrX19_X29[10]; pRD->pCurrentContext->Lr = NULL; // Unwind again to get Caller's PC -#endif // __APPLE__ +#endif #if !defined(DACCESS_COMPILE) pRD->pCurrentContextPointers->X19 = m_MachState.ptrX19_X29[0]; @@ -731,7 +731,7 @@ void InlinedCallFrame::UpdateRegDisplay(const PREGDISPLAY pRD, bool updateFloats // Update the frame pointer in the current context. - pRD->pCurrentContextPointers->Fp = (DWORD64 *)&m_pCalleeSavedFP; + pRD->pCurrentContextPointers->Fp = &m_pCalleeSavedFP; LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK InlinedCallFrame::UpdateRegDisplay(pc:%p, sp:%p)\n", pRD->ControlPC, pRD->SP)); diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index 547db8d05971fe..0796e59a15c290 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -3402,6 +3402,24 @@ MethodDesc *Module::FindMethod(mdToken pMethod) RETURN pMDRet; } +// +// GetPropertyInfoForMethodDef wraps the metadata function of the same name. +// + +HRESULT Module::GetPropertyInfoForMethodDef(mdMethodDef md, mdProperty *ppd, LPCSTR *pName, ULONG *pSemantic) +{ + CONTRACTL + { + INSTANCE_CHECK; + NOTHROW; + GC_NOTRIGGER; + MODE_ANY; + } + CONTRACTL_END; + + return GetMDImport()->GetPropertyInfoForMethodDef(md, ppd, pName, pSemantic); +} + // Return true if this module has any live (jitted) JMC functions. // If a module has no jitted JMC functions, then it's as if it's a // non-user module. @@ -4659,178 +4677,22 @@ PTR_VOID ReflectionModule::GetRvaField(RVA field) // virtual // VASigCookies // =========================================================================== -static bool TypeSignatureContainsGenericVariables(SigParser& sp); -static bool MethodSignatureContainsGenericVariables(SigParser& sp); - -static bool TypeSignatureContainsGenericVariables(SigParser& sp) -{ - STANDARD_VM_CONTRACT; - - CorElementType et = ELEMENT_TYPE_END; - IfFailThrow(sp.GetElemType(&et)); - - if (CorIsPrimitiveType(et)) - return false; - - switch (et) - { - case ELEMENT_TYPE_OBJECT: - case ELEMENT_TYPE_STRING: - case ELEMENT_TYPE_TYPEDBYREF: - return false; - - case ELEMENT_TYPE_BYREF: - case ELEMENT_TYPE_PTR: - case ELEMENT_TYPE_SZARRAY: - return TypeSignatureContainsGenericVariables(sp); - - case ELEMENT_TYPE_VALUETYPE: - case ELEMENT_TYPE_CLASS: - IfFailThrow(sp.GetToken(NULL)); // Skip RID - return false; - - case ELEMENT_TYPE_FNPTR: - return MethodSignatureContainsGenericVariables(sp); - - case ELEMENT_TYPE_ARRAY: - { - if (TypeSignatureContainsGenericVariables(sp)) - return true; - - uint32_t rank; - IfFailThrow(sp.GetData(&rank)); // Get rank - if (rank) - { - uint32_t nsizes; - IfFailThrow(sp.GetData(&nsizes)); // Get # of sizes - while (nsizes--) - { - IfFailThrow(sp.GetData(NULL)); // Skip size - } - - uint32_t nlbounds; - IfFailThrow(sp.GetData(&nlbounds)); // Get # of lower bounds - while (nlbounds--) - { - IfFailThrow(sp.GetData(NULL)); // Skip lower bounds - } - } - } - return false; - - case ELEMENT_TYPE_GENERICINST: - { - if (TypeSignatureContainsGenericVariables(sp)) - return true; - - uint32_t argCnt; - IfFailThrow(sp.GetData(&argCnt)); // Get number of parameters - while (argCnt--) - { - if (TypeSignatureContainsGenericVariables(sp)) - return true; - } - } - return false; - - case ELEMENT_TYPE_INTERNAL: - IfFailThrow(sp.GetPointer(NULL)); - return false; - - case ELEMENT_TYPE_VAR: - case ELEMENT_TYPE_MVAR: - return true; - - default: - // Return conservative answer for unhandled elements - _ASSERTE(!"Unexpected element type."); - return true; - } -} - -static bool MethodSignatureContainsGenericVariables(SigParser& sp) -{ - STANDARD_VM_CONTRACT; - - uint32_t callConv = 0; - IfFailThrow(sp.GetCallingConvInfo(&callConv)); - - if (callConv & IMAGE_CEE_CS_CALLCONV_GENERIC) - { - // Generic signatures should never show up here, return conservative answer. - _ASSERTE(!"Unexpected generic signature."); - return true; - } - - uint32_t numArgs = 0; - IfFailThrow(sp.GetData(&numArgs)); - - // iterate over the return type and parameters - for (uint32_t i = 0; i <= numArgs; i++) - { - if (TypeSignatureContainsGenericVariables(sp)) - return true; - } - - return false; -} - //========================================================================== // Enregisters a VASig. //========================================================================== -VASigCookie *Module::GetVASigCookie(Signature vaSignature, const SigTypeContext* typeContext) +VASigCookie *Module::GetVASigCookie(Signature vaSignature) { CONTRACT(VASigCookie*) { INSTANCE_CHECK; - STANDARD_VM_CHECK; + THROWS; + GC_TRIGGERS; + MODE_ANY; POSTCONDITION(CheckPointer(RETVAL)); INJECT_FAULT(COMPlusThrowOM()); } CONTRACT_END; - SigTypeContext emptyContext; - - Module* pLoaderModule = this; - if (!typeContext->IsEmpty()) - { - // Strip the generic context if it is not actually used by the signature. It is nececessary for both: - // - Performance: allow more sharing of vasig cookies - // - Functionality: built-in runtime marshalling is disallowed for generic signatures - SigParser sigParser = vaSignature.CreateSigParser(); - if (MethodSignatureContainsGenericVariables(sigParser)) - { - pLoaderModule = ClassLoader::ComputeLoaderModuleWorker(this, mdTokenNil, typeContext->m_classInst, typeContext->m_methodInst); - } - else - { - typeContext = &emptyContext; - } - } - else - { -#ifdef _DEBUG - // The method signature should not contain any generic variables if the generic context is not provided. - SigParser sigParser = vaSignature.CreateSigParser(); - _ASSERTE(!MethodSignatureContainsGenericVariables(sigParser)); -#endif - } - - VASigCookie *pCookie = GetVASigCookieWorker(this, pLoaderModule, vaSignature, typeContext); - - RETURN pCookie; -} - -VASigCookie *Module::GetVASigCookieWorker(Module* pDefiningModule, Module* pLoaderModule, Signature vaSignature, const SigTypeContext* typeContext) -{ - CONTRACT(VASigCookie*) - { - STANDARD_VM_CHECK; - POSTCONDITION(CheckPointer(RETVAL)); - INJECT_FAULT(COMPlusThrowOM()); - } - CONTRACT_END; - VASigCookieBlock *pBlock; VASigCookie *pCookie; @@ -4838,70 +4700,39 @@ VASigCookie *Module::GetVASigCookieWorker(Module* pDefiningModule, Module* pLoad // First, see if we already enregistered this sig. // Note that we're outside the lock here, so be a bit careful with our logic - for (pBlock = pLoaderModule->m_pVASigCookieBlock; pBlock != NULL; pBlock = pBlock->m_Next) + for (pBlock = m_pVASigCookieBlock; pBlock != NULL; pBlock = pBlock->m_Next) { for (UINT i = 0; i < pBlock->m_numcookies; i++) { if (pBlock->m_cookies[i].signature.GetRawSig() == vaSignature.GetRawSig()) { - _ASSERTE(pBlock->m_cookies[i].classInst.GetNumArgs() == typeContext->m_classInst.GetNumArgs()); - _ASSERTE(pBlock->m_cookies[i].methodInst.GetNumArgs() == typeContext->m_methodInst.GetNumArgs()); - - bool instMatch = true; - - for (DWORD j = 0; j < pBlock->m_cookies[i].classInst.GetNumArgs(); j++) - { - if (pBlock->m_cookies[i].classInst[j] != typeContext->m_classInst[j]) - { - instMatch = false; - break; - } - } - - if (instMatch) - { - for (DWORD j = 0; j < pBlock->m_cookies[i].methodInst.GetNumArgs(); j++) - { - if (pBlock->m_cookies[i].methodInst[j] != typeContext->m_methodInst[j]) - { - instMatch = false; - break; - } - } - } - - if (instMatch) - { - pCookie = &(pBlock->m_cookies[i]); - break; - } + pCookie = &(pBlock->m_cookies[i]); + break; } } } - + if (!pCookie) { // If not, time to make a new one. // Compute the size of args first, outside of the lock. - MetaSig metasig(vaSignature, pDefiningModule, typeContext); + // @TODO GENERICS: We may be calling a varargs method from a + // generic type/method. Using an empty context will make such a + // case cause an unexpected exception. To make this work, + // we need to create a specialized signature for every instantiation + SigTypeContext typeContext; + + MetaSig metasig(vaSignature, this, &typeContext); ArgIterator argit(&metasig); // Upper estimate of the vararg size DWORD sizeOfArgs = argit.SizeOfArgStack(); - // Prepare instantiation - LoaderAllocator *pLoaderAllocator = pLoaderModule->GetLoaderAllocator(); - - DWORD classInstCount = typeContext->m_classInst.GetNumArgs(); - DWORD methodInstCount = typeContext->m_methodInst.GetNumArgs(); - pLoaderAllocator->EnsureInstantiation(pDefiningModule, typeContext->m_classInst); - pLoaderAllocator->EnsureInstantiation(pDefiningModule, typeContext->m_methodInst); - // enable gc before taking lock { - CrstHolder ch(&pLoaderModule->m_Crst); + CrstHolder ch(&m_Crst); // Note that we were possibly racing to create the cookie, and another thread // may have already created it. We could put another check @@ -4909,57 +4740,32 @@ VASigCookie *Module::GetVASigCookieWorker(Module* pDefiningModule, Module* pLoad // occasional duplicate cookie instead. // Is the first block in the list full? - if (pLoaderModule->m_pVASigCookieBlock && pLoaderModule->m_pVASigCookieBlock->m_numcookies + if (m_pVASigCookieBlock && m_pVASigCookieBlock->m_numcookies < VASigCookieBlock::kVASigCookieBlockSize) { // Nope, reserve a new slot in the existing block. - pCookie = &(pLoaderModule->m_pVASigCookieBlock->m_cookies[pLoaderModule->m_pVASigCookieBlock->m_numcookies]); + pCookie = &(m_pVASigCookieBlock->m_cookies[m_pVASigCookieBlock->m_numcookies]); } else { // Yes, create a new block. VASigCookieBlock *pNewBlock = new VASigCookieBlock(); - pNewBlock->m_Next = pLoaderModule->m_pVASigCookieBlock; + pNewBlock->m_Next = m_pVASigCookieBlock; pNewBlock->m_numcookies = 0; - pLoaderModule->m_pVASigCookieBlock = pNewBlock; + m_pVASigCookieBlock = pNewBlock; pCookie = &(pNewBlock->m_cookies[0]); } // Now, fill in the new cookie (assuming we had enough memory to create one.) - pCookie->pModule = pDefiningModule; + pCookie->pModule = this; pCookie->pNDirectILStub = NULL; pCookie->sizeOfArgs = sizeOfArgs; pCookie->signature = vaSignature; - pCookie->pLoaderModule = pLoaderModule; - - AllocMemTracker amt; - - if (classInstCount != 0) - { - TypeHandle* pClassInst = (TypeHandle*)(void*)amt.Track(pLoaderAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(classInstCount) * S_SIZE_T(sizeof(TypeHandle)))); - for (DWORD i = 0; i < classInstCount; i++) - { - pClassInst[i] = typeContext->m_classInst[i]; - } - pCookie->classInst = Instantiation(pClassInst, classInstCount); - } - - if (methodInstCount != 0) - { - TypeHandle* pMethodInst = (TypeHandle*)(void*)amt.Track(pLoaderAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(methodInstCount) * S_SIZE_T(sizeof(TypeHandle)))); - for (DWORD i = 0; i < methodInstCount; i++) - { - pMethodInst[i] = typeContext->m_methodInst[i]; - } - pCookie->methodInst = Instantiation(pMethodInst, methodInstCount); - } - - amt.SuppressRelease(); // Finally, now that it's safe for asynchronous readers to see it, // update the count. - pLoaderModule->m_pVASigCookieBlock->m_numcookies++; + m_pVASigCookieBlock->m_numcookies++; } } diff --git a/src/coreclr/vm/ceeload.h b/src/coreclr/vm/ceeload.h index 1b487dd9fa871f..18335c5a5f01a8 100644 --- a/src/coreclr/vm/ceeload.h +++ b/src/coreclr/vm/ceeload.h @@ -338,10 +338,7 @@ struct VASigCookie unsigned sizeOfArgs; // size of argument list Volatile pNDirectILStub; // will be use if target is NDirect (tag == 0) PTR_Module pModule; - PTR_Module pLoaderModule; Signature signature; - Instantiation classInst; - Instantiation methodInst; }; // @@ -1324,6 +1321,8 @@ class Module : public ModuleBase MethodDesc *FindMethodThrowing(mdToken pMethod); MethodDesc *FindMethod(mdToken pMethod); + HRESULT GetPropertyInfoForMethodDef(mdMethodDef md, mdProperty *ppd, LPCSTR *pName, ULONG *pSemantic); + public: // Debugger stuff @@ -1361,9 +1360,7 @@ class Module : public ModuleBase void NotifyEtwLoadFinished(HRESULT hr); // Enregisters a VASig. - VASigCookie *GetVASigCookie(Signature vaSignature, const SigTypeContext* typeContext); -private: - static VASigCookie *GetVASigCookieWorker(Module* pDefiningModule, Module* pLoaderModule, Signature vaSignature, const SigTypeContext* typeContext); + VASigCookie *GetVASigCookie(Signature vaSignature); public: #ifndef DACCESS_COMPILE diff --git a/src/coreclr/vm/ceeload.inl b/src/coreclr/vm/ceeload.inl index 18d7557d6e85bd..ff446d4ec79912 100644 --- a/src/coreclr/vm/ceeload.inl +++ b/src/coreclr/vm/ceeload.inl @@ -64,7 +64,7 @@ inline void LookupMap::SetValueAt(PTR_TADDR pValue, SIZE_T value, TADDR flags) { WRAPPER_NO_CONTRACT; - VolatileStore(pValue, dac_cast(value | flags)); + VolatileStore(pValue, value | flags); } #endif // DACCESS_COMPILE diff --git a/src/coreclr/vm/ceemain.cpp b/src/coreclr/vm/ceemain.cpp index 0e546ffa120630..813616e529c802 100644 --- a/src/coreclr/vm/ceemain.cpp +++ b/src/coreclr/vm/ceemain.cpp @@ -869,6 +869,8 @@ void EEStartupHelper() // Set up the sync block SyncBlockCache::Start(); + StackwalkCache::Init(); + // This isn't done as part of InitializeGarbageCollector() above because it // requires write barriers to have been set up on x86, which happens as part // of InitJITHelpers1. diff --git a/src/coreclr/vm/classlayoutinfo.cpp b/src/coreclr/vm/classlayoutinfo.cpp index 468dc63d59de36..9dd6fb4881b23b 100644 --- a/src/coreclr/vm/classlayoutinfo.cpp +++ b/src/coreclr/vm/classlayoutinfo.cpp @@ -1023,6 +1023,7 @@ EEClassNativeLayoutInfo* EEClassNativeLayoutInfo::CollectNativeLayoutFieldMetada { // The intrinsic Vector type has a special size. Copy the native size and alignment // from the managed size and alignment. + // Crossgen scenarios block Vector from even being loaded, so only do this check when not in crossgen. if (pMT->HasSameTypeDefAs(CoreLibBinder::GetClass(CLASS__VECTORT))) { pNativeLayoutInfo->m_size = pEEClassLayoutInfo->GetManagedSize(); diff --git a/src/coreclr/vm/clrtocomcall.cpp b/src/coreclr/vm/clrtocomcall.cpp index d47445f8f64fd5..c604a6c8a90116 100644 --- a/src/coreclr/vm/clrtocomcall.cpp +++ b/src/coreclr/vm/clrtocomcall.cpp @@ -505,7 +505,7 @@ UINT32 CLRToCOMLateBoundWorker( ULONG uSemantic; // See if there is property information for this member. - hr = pItfMT->GetMDImport()->GetPropertyInfoForMethodDef(pItfMD->GetMemberDef(), &propToken, &strMemberName, &uSemantic); + hr = pItfMT->GetModule()->GetPropertyInfoForMethodDef(pItfMD->GetMemberDef(), &propToken, &strMemberName, &uSemantic); if (hr != S_OK) { // Non-property method diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index e9394c267c3804..52917161fb0026 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -1297,6 +1297,9 @@ void EEJitManager::SetCpuInfo() CPUCompileFlags.Set(InstructionSet_VectorT512); } + // TODO-XArch: Add support for 512-bit Vector + _ASSERTE(!CPUCompileFlags.IsSet(InstructionSet_VectorT512)); + if (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableHWIntrinsic)) { CPUCompileFlags.Set(InstructionSet_X86Base); @@ -4935,6 +4938,14 @@ void ExecutionManager::Unload(LoaderAllocator *pLoaderAllocator) // a size of 0 is a signal to Nirvana to flush the entire cache FlushInstructionCache(GetCurrentProcess(),0,0); + /* StackwalkCacheEntry::EIP is an address into code. Since we are + unloading the code, we need to invalidate the cache. Otherwise, + its possible that another appdomain might generate code at the very + same address, and we might incorrectly think that the old + StackwalkCacheEntry corresponds to it. So flush the cache. + */ + StackwalkCache::Invalidate(pLoaderAllocator); + JumpStubCache * pJumpStubCache = (JumpStubCache *) pLoaderAllocator->m_pJumpStubCache; if (pJumpStubCache != NULL) { diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 9adec20c01e8cd..6fe87885da111f 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -2619,20 +2619,21 @@ class EECodeInfo PTR_RUNTIME_FUNCTION GetFunctionEntry(); BOOL IsFunclet() { WRAPPER_NO_CONTRACT; return GetJitManager()->IsFunclet(this); } EECodeInfo GetMainFunctionInfo(); -#endif // FEATURE_EH_FUNCLETS + ULONG GetFixedStackSize(); + +#if defined(TARGET_AMD64) + BOOL HasFrameRegister(); +#endif // TARGET_AMD64 -#if defined(TARGET_X86) +#else // FEATURE_EH_FUNCLETS ULONG GetFixedStackSize() { WRAPPER_NO_CONTRACT; return GetCodeManager()->GetFrameSize(GetGCInfoToken()); } -#endif // TARGET_X86 +#endif // FEATURE_EH_FUNCLETS #if defined(TARGET_AMD64) - BOOL HasFrameRegister(); - ULONG GetFixedStackSize(); - void GetOffsetsFromUnwindInfo(ULONG* pRSPOffset, ULONG* pRBPOffset); ULONG GetFrameOffsetFromUnwindInfo(); #if defined(_DEBUG) && defined(HAVE_GCCOVER) diff --git a/src/coreclr/vm/comdelegate.cpp b/src/coreclr/vm/comdelegate.cpp index ef4021039a66b8..7e7a4de4b02af0 100644 --- a/src/coreclr/vm/comdelegate.cpp +++ b/src/coreclr/vm/comdelegate.cpp @@ -2019,7 +2019,7 @@ void COMDelegate::ThrowIfInvalidUnmanagedCallersOnlyUsage(MethodDesc* pMD) // Arguments - Scenarios involving UnmanagedCallersOnly are handled during the jit. bool unmanagedCallersOnlyRequiresMarshalling = false; - if (NDirect::MarshalingRequired(pMD, NULL, NULL, NULL, unmanagedCallersOnlyRequiresMarshalling)) + if (NDirect::MarshalingRequired(pMD, NULL, NULL, unmanagedCallersOnlyRequiresMarshalling)) EX_THROW(EEResourceException, (kInvalidProgramException, W("InvalidProgram_NonBlittableTypes"))); } diff --git a/src/coreclr/vm/commtmemberinfomap.cpp b/src/coreclr/vm/commtmemberinfomap.cpp index 7b2626a24c7631..8bc185e9d81a4c 100644 --- a/src/coreclr/vm/commtmemberinfomap.cpp +++ b/src/coreclr/vm/commtmemberinfomap.cpp @@ -689,7 +689,7 @@ void ComMTMemberInfoMap::GetMethodPropsForMeth( rProps[ix].bFunction2Getter = FALSE; // See if there is property information for this member. - hr = pMeth->GetMDImport()->GetPropertyInfoForMethodDef(pMeth->GetMemberDef(), &pd, &pPropName, &uSemantic); + hr = pMeth->GetModule()->GetPropertyInfoForMethodDef(pMeth->GetMemberDef(), &pd, &pPropName, &uSemantic); IfFailThrow(hr); if (hr == S_OK) diff --git a/src/coreclr/vm/compatibilityswitch.cpp b/src/coreclr/vm/compatibilityswitch.cpp new file mode 100644 index 00000000000000..aedddbc7cbbf4b --- /dev/null +++ b/src/coreclr/vm/compatibilityswitch.cpp @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + + +#include "common.h" +#include "clrconfig.h" +#include "compatibilityswitch.h" + +FCIMPL1(StringObject*, CompatibilitySwitch::GetValue, StringObject* switchNameUNSAFE) { + CONTRACTL { + FCALL_CHECK; + } + CONTRACTL_END; + + if (!switchNameUNSAFE) + FCThrowRes(kArgumentNullException, W("Arg_InvalidSwitchName")); + + STRINGREF name = (STRINGREF) switchNameUNSAFE; + VALIDATEOBJECTREF(name); + + STRINGREF refName = NULL; + + HELPER_METHOD_FRAME_BEGIN_RET_1(name); + CLRConfig::ConfigStringInfo info; + info.name = name->GetBuffer(); + info.options = CLRConfig::LookupOptions::Default; + LPWSTR strVal = CLRConfig::GetConfigValue(info); + refName = StringObject::NewString(strVal); + HELPER_METHOD_FRAME_END(); + + return (StringObject*)OBJECTREFToObject(refName); +} +FCIMPLEND diff --git a/src/coreclr/vm/compatibilityswitch.h b/src/coreclr/vm/compatibilityswitch.h new file mode 100644 index 00000000000000..bd291ee3e4428b --- /dev/null +++ b/src/coreclr/vm/compatibilityswitch.h @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + + + +#ifndef _COMPATIBILITYSWITCH_H_ +#define _COMPATIBILITYSWITCH_H_ + +#include "object.h" +#include "typehandle.h" +#include "fcall.h" +#include "field.h" +#include "typectxt.h" + +class CompatibilitySwitch +{ +public: + static FCDECL1(StringObject*, GetValue, StringObject *switchNameUNSAFE); +}; + + +#endif + diff --git a/src/coreclr/vm/comutilnative.cpp b/src/coreclr/vm/comutilnative.cpp index 6c7e2468d2744b..027c4ae8903aed 100644 --- a/src/coreclr/vm/comutilnative.cpp +++ b/src/coreclr/vm/comutilnative.cpp @@ -1681,136 +1681,226 @@ BOOL CanCompareBitsOrUseFastGetHashCode(MethodTable* mt) return canCompareBitsOrUseFastGetHashCode; } -extern "C" BOOL QCALLTYPE MethodTable_CanCompareBitsOrUseFastGetHashCode(MethodTable * mt) +NOINLINE static FC_BOOL_RET CanCompareBitsHelper(MethodTable* mt, OBJECTREF objRef) { - QCALL_CONTRACT; + FC_INNER_PROLOG(ValueTypeHelper::CanCompareBits); + + _ASSERTE(mt != NULL); + _ASSERTE(objRef != NULL); BOOL ret = FALSE; - BEGIN_QCALL; + HELPER_METHOD_FRAME_BEGIN_RET_ATTRIB_1(Frame::FRAME_ATTR_EXACT_DEPTH|Frame::FRAME_ATTR_CAPTURE_DEPTH_2, objRef); ret = CanCompareBitsOrUseFastGetHashCode(mt); - END_QCALL; + HELPER_METHOD_FRAME_END(); + FC_INNER_EPILOG(); - return ret; + FC_RETURN_BOOL(ret); } -enum ValueTypeHashCodeStrategy +// Return true if the valuetype does not contain pointer, is tightly packed, +// does not have floating point number field and does not override Equals method. +FCIMPL1(FC_BOOL_RET, ValueTypeHelper::CanCompareBits, Object* obj) { - None, - ReferenceField, - DoubleField, - SingleField, - FastGetHashCode, - ValueTypeOverride, -}; + FCALL_CONTRACT; + + _ASSERTE(obj != NULL); + MethodTable* mt = obj->GetMethodTable(); -static ValueTypeHashCodeStrategy GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize, MethodTable** fieldMTOut) + if (mt->HasCheckedCanCompareBitsOrUseFastGetHashCode()) + { + FC_RETURN_BOOL(mt->CanCompareBitsOrUseFastGetHashCode()); + } + + OBJECTREF objRef(obj); + + FC_INNER_RETURN(FC_BOOL_RET, CanCompareBitsHelper(mt, objRef)); +} +FCIMPLEND + +static INT32 FastGetValueTypeHashCodeHelper(MethodTable *mt, void *pObjRef) +{ + CONTRACTL + { + NOTHROW; + GC_NOTRIGGER; + MODE_COOPERATIVE; + } CONTRACTL_END; + + INT32 hashCode = 0; + INT32 *pObj = (INT32*)pObjRef; + + // this is a struct with no refs and no "strange" offsets, just go through the obj and xor the bits + INT32 size = mt->GetNumInstanceFieldBytes(); + for (INT32 i = 0; i < (INT32)(size / sizeof(INT32)); i++) + hashCode ^= *pObj++; + + return hashCode; +} + +static INT32 RegularGetValueTypeHashCode(MethodTable *mt, void *pObjRef) { CONTRACTL { THROWS; GC_TRIGGERS; - MODE_PREEMPTIVE; + MODE_COOPERATIVE; } CONTRACTL_END; - // Should be handled by caller - _ASSERTE(!mt->CanCompareBitsOrUseFastGetHashCode()); + INT32 hashCode = 0; - ValueTypeHashCodeStrategy ret = ValueTypeHashCodeStrategy::None; + GCPROTECT_BEGININTERIOR(pObjRef); - // Grab the first non-null field and return its hash code or 'it' as hash code - ApproxFieldDescIterator fdIterator(mt, ApproxFieldDescIterator::INSTANCE_FIELDS); + BOOL canUseFastGetHashCodeHelper = FALSE; + if (mt->HasCheckedCanCompareBitsOrUseFastGetHashCode()) + { + canUseFastGetHashCodeHelper = mt->CanCompareBitsOrUseFastGetHashCode(); + } + else + { + canUseFastGetHashCodeHelper = CanCompareBitsOrUseFastGetHashCode(mt); + } - FieldDesc *field; - while ((field = fdIterator.Next()) != NULL) + // While we should not get here directly from ValueTypeHelper::GetHashCode, if we recurse we need to + // be able to handle getting the hashcode for an embedded structure whose hashcode is computed by the fast path. + if (canUseFastGetHashCodeHelper) { - _ASSERTE(!field->IsRVA()); - if (field->IsObjRef()) - { - GCX_COOP(); - // if we get an object reference we get the hash code out of that - if (*(Object**)((BYTE *)objHandle.Get()->UnBox() + *fieldOffset + field->GetOffsetUnsafe()) != NULL) - { - *fieldOffset += field->GetOffsetUnsafe(); - ret = ValueTypeHashCodeStrategy::ReferenceField; - } - else - { - // null object reference, try next - continue; - } - } - else + hashCode = FastGetValueTypeHashCodeHelper(mt, pObjRef); + } + else + { + // it's looking ugly so we'll use the old behavior in managed code. Grab the first non-null + // field and return its hash code or 'it' as hash code + // Note that the old behavior has already been broken for value types + // that is qualified for CanUseFastGetHashCodeHelper. So maybe we should + // change the implementation here to use all fields instead of just the 1st one. + // + // + // check this approximation - we may be losing exact type information + ApproxFieldDescIterator fdIterator(mt, ApproxFieldDescIterator::INSTANCE_FIELDS); + + FieldDesc *field; + while ((field = fdIterator.Next()) != NULL) { - CorElementType fieldType = field->GetFieldType(); - if (fieldType == ELEMENT_TYPE_R8) + _ASSERTE(!field->IsRVA()); + if (field->IsObjRef()) { - *fieldOffset += field->GetOffsetUnsafe(); - ret = ValueTypeHashCodeStrategy::DoubleField; - } - else if (fieldType == ELEMENT_TYPE_R4) - { - *fieldOffset += field->GetOffsetUnsafe(); - ret = ValueTypeHashCodeStrategy::SingleField; - } - else if (fieldType != ELEMENT_TYPE_VALUETYPE) - { - *fieldOffset += field->GetOffsetUnsafe(); - *fieldSize = field->LoadSize(); - ret = ValueTypeHashCodeStrategy::FastGetHashCode; + // if we get an object reference we get the hash code out of that + if (*(Object**)((BYTE *)pObjRef + field->GetOffsetUnsafe()) != NULL) + { + PREPARE_SIMPLE_VIRTUAL_CALLSITE(METHOD__OBJECT__GET_HASH_CODE, (*(Object**)((BYTE *)pObjRef + field->GetOffsetUnsafe()))); + DECLARE_ARGHOLDER_ARRAY(args, 1); + args[ARGNUM_0] = PTR_TO_ARGHOLDER(*(Object**)((BYTE *)pObjRef + field->GetOffsetUnsafe())); + CALL_MANAGED_METHOD(hashCode, INT32, args); + } + else + { + // null object reference, try next + continue; + } } else { - // got another value type. Get the type - TypeHandle fieldTH = field->GetFieldTypeHandleThrowing(); - _ASSERTE(!fieldTH.IsNull()); - MethodTable* fieldMT = fieldTH.GetMethodTable(); - if (CanCompareBitsOrUseFastGetHashCode(fieldMT)) + CorElementType fieldType = field->GetFieldType(); + if (fieldType == ELEMENT_TYPE_R8) + { + PREPARE_NONVIRTUAL_CALLSITE(METHOD__DOUBLE__GET_HASH_CODE); + DECLARE_ARGHOLDER_ARRAY(args, 1); + args[ARGNUM_0] = PTR_TO_ARGHOLDER(((BYTE *)pObjRef + field->GetOffsetUnsafe())); + CALL_MANAGED_METHOD(hashCode, INT32, args); + } + else if (fieldType == ELEMENT_TYPE_R4) { - *fieldOffset += field->GetOffsetUnsafe(); - *fieldSize = field->LoadSize(); - ret = ValueTypeHashCodeStrategy::FastGetHashCode; + PREPARE_NONVIRTUAL_CALLSITE(METHOD__SINGLE__GET_HASH_CODE); + DECLARE_ARGHOLDER_ARRAY(args, 1); + args[ARGNUM_0] = PTR_TO_ARGHOLDER(((BYTE *)pObjRef + field->GetOffsetUnsafe())); + CALL_MANAGED_METHOD(hashCode, INT32, args); } - else if (HasOverriddenMethod(fieldMT, - CoreLibBinder::GetClass(CLASS__VALUE_TYPE), - CoreLibBinder::GetMethod(METHOD__VALUE_TYPE__GET_HASH_CODE)->GetSlot())) + else if (fieldType != ELEMENT_TYPE_VALUETYPE) { - *fieldOffset += field->GetOffsetUnsafe(); - *fieldMTOut = fieldMT; - ret = ValueTypeHashCodeStrategy::ValueTypeOverride; + UINT fieldSize = field->LoadSize(); + INT32 *pValue = (INT32*)((BYTE *)pObjRef + field->GetOffsetUnsafe()); + for (INT32 j = 0; j < (INT32)(fieldSize / sizeof(INT32)); j++) + hashCode ^= *pValue++; } else { - *fieldOffset += field->GetOffsetUnsafe(); - ret = GetHashCodeStrategy(fieldMT, objHandle, fieldOffset, fieldSize, fieldMTOut); + // got another value type. Get the type + TypeHandle fieldTH = field->GetFieldTypeHandleThrowing(); + _ASSERTE(!fieldTH.IsNull()); + hashCode = RegularGetValueTypeHashCode(fieldTH.GetMethodTable(), (BYTE *)pObjRef + field->GetOffsetUnsafe()); } } + break; } - break; } - return ret; + GCPROTECT_END(); + + return hashCode; } -extern "C" INT32 QCALLTYPE ValueType_GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize, MethodTable** fieldMT) +// The default implementation of GetHashCode() for all value types. +// Note that this implementation reveals the value of the fields. +// So if the value type contains any sensitive information it should +// implement its own GetHashCode(). +FCIMPL1(INT32, ValueTypeHelper::GetHashCode, Object* objUNSAFE) { - QCALL_CONTRACT; + FCALL_CONTRACT; + + if (objUNSAFE == NULL) + FCThrow(kNullReferenceException); - ValueTypeHashCodeStrategy ret = ValueTypeHashCodeStrategy::None; - *fieldOffset = 0; - *fieldSize = 0; - *fieldMT = NULL; + OBJECTREF obj = ObjectToOBJECTREF(objUNSAFE); + VALIDATEOBJECTREF(obj); - BEGIN_QCALL; + INT32 hashCode = 0; + MethodTable *pMT = objUNSAFE->GetMethodTable(); - ret = GetHashCodeStrategy(mt, objHandle, fieldOffset, fieldSize, fieldMT); + // We don't want to expose the method table pointer in the hash code + // Let's use the typeID instead. + UINT32 typeID = pMT->LookupTypeID(); + if (typeID == TypeIDProvider::INVALID_TYPE_ID) + { + // If the typeID has yet to be generated, fall back to GetTypeID + // This only needs to be done once per MethodTable + HELPER_METHOD_FRAME_BEGIN_RET_1(obj); + typeID = pMT->GetTypeID(); + HELPER_METHOD_FRAME_END(); + } - END_QCALL; + // To get less colliding and more evenly distributed hash codes, + // we munge the class index with two big prime numbers + hashCode = typeID * 711650207 + 2506965631U; - return ret; + BOOL canUseFastGetHashCodeHelper = FALSE; + if (pMT->HasCheckedCanCompareBitsOrUseFastGetHashCode()) + { + canUseFastGetHashCodeHelper = pMT->CanCompareBitsOrUseFastGetHashCode(); + } + else + { + HELPER_METHOD_FRAME_BEGIN_RET_1(obj); + canUseFastGetHashCodeHelper = CanCompareBitsOrUseFastGetHashCode(pMT); + HELPER_METHOD_FRAME_END(); + } + + if (canUseFastGetHashCodeHelper) + { + hashCode ^= FastGetValueTypeHashCodeHelper(pMT, obj->UnBox()); + } + else + { + HELPER_METHOD_FRAME_BEGIN_RET_1(obj); + hashCode ^= RegularGetValueTypeHashCode(pMT, obj->UnBox()); + HELPER_METHOD_FRAME_END(); + } + + return hashCode; } +FCIMPLEND FCIMPL1(UINT32, MethodTableNative::GetNumInstanceFieldBytes, MethodTable* mt) { diff --git a/src/coreclr/vm/comutilnative.h b/src/coreclr/vm/comutilnative.h index 0f305e0af90072..80d35da7b72141 100644 --- a/src/coreclr/vm/comutilnative.h +++ b/src/coreclr/vm/comutilnative.h @@ -245,14 +245,18 @@ class COMInterlocked extern "C" void QCALLTYPE Interlocked_MemoryBarrierProcessWide(); +class ValueTypeHelper { +public: + static FCDECL1(FC_BOOL_RET, CanCompareBits, Object* obj); + static FCDECL1(INT32, GetHashCode, Object* objRef); +}; + class MethodTableNative { public: static FCDECL1(UINT32, GetNumInstanceFieldBytes, MethodTable* mt); }; extern "C" BOOL QCALLTYPE MethodTable_AreTypesEquivalent(MethodTable* mta, MethodTable* mtb); -extern "C" BOOL QCALLTYPE MethodTable_CanCompareBitsOrUseFastGetHashCode(MethodTable* mt); -extern "C" INT32 QCALLTYPE ValueType_GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize, MethodTable** fieldMT); class StreamNative { public: diff --git a/src/coreclr/vm/corelib.cpp b/src/coreclr/vm/corelib.cpp index c0a217593adc5b..c4b0b43450e568 100644 --- a/src/coreclr/vm/corelib.cpp +++ b/src/coreclr/vm/corelib.cpp @@ -29,6 +29,7 @@ #include "floatdouble.h" #include "floatsingle.h" #include "comdatetime.h" +#include "compatibilityswitch.h" #include "debugdebugger.h" #include "assemblynative.hpp" #include "comthreadpool.h" diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index c52c58954165a2..bd4a2090166522 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -569,6 +569,12 @@ DEFINE_METHOD(OBJECT, GET_TYPE, GetType, DEFINE_METHOD(OBJECT, GET_HASH_CODE, GetHashCode, IM_RetInt) DEFINE_METHOD(OBJECT, EQUALS, Equals, IM_Obj_RetBool) +// DEFINE_CLASS(DOUBLE, System, Double) +DEFINE_METHOD(DOUBLE, GET_HASH_CODE, GetHashCode, IM_RetInt) + +// DEFINE_CLASS(SINGLE, System, Single) +DEFINE_METHOD(SINGLE, GET_HASH_CODE, GetHashCode, IM_RetInt) + DEFINE_CLASS(__CANON, System, __Canon) BEGIN_ILLINK_FEATURE_SWITCH(System.Runtime.InteropServices.BuiltInComInterop.IsSupported, true, true) @@ -633,11 +639,6 @@ DEFINE_METHOD(RUNTIME_HELPERS, ALLOC_TAILCALL_ARG_BUFFER, AllocTailCallArgB DEFINE_METHOD(RUNTIME_HELPERS, GET_TAILCALL_INFO, GetTailCallInfo, NoSig) DEFINE_METHOD(RUNTIME_HELPERS, DISPATCH_TAILCALLS, DispatchTailCalls, NoSig) -DEFINE_CLASS(SPAN_HELPERS, System, SpanHelpers) -DEFINE_METHOD(SPAN_HELPERS, MEMSET, Fill, SM_RefByte_Byte_UIntPtr_RetVoid) -DEFINE_METHOD(SPAN_HELPERS, MEMZERO, ClearWithoutReferences, SM_RefByte_UIntPtr_RetVoid) -DEFINE_METHOD(SPAN_HELPERS, MEMCOPY, Memmove, SM_RefByte_RefByte_UIntPtr_RetVoid) - DEFINE_CLASS(UNSAFE, CompilerServices, Unsafe) DEFINE_METHOD(UNSAFE, AS_POINTER, AsPointer, NoSig) DEFINE_METHOD(UNSAFE, BYREF_IS_NULL, IsNullRef, NoSig) diff --git a/src/coreclr/vm/debugdebugger.cpp b/src/coreclr/vm/debugdebugger.cpp index a6bdd075afc484..bb5bed368de9fe 100644 --- a/src/coreclr/vm/debugdebugger.cpp +++ b/src/coreclr/vm/debugdebugger.cpp @@ -33,13 +33,82 @@ #define IMAGE_DEBUG_TYPE_EMBEDDED_PORTABLE_PDB 17 #ifndef DACCESS_COMPILE +//---------------------------------------------------------------------------- +// +// FindMostRecentUserCodeOnStack - find out the most recent user managed code on stack +// +// +// Arguments: +// pContext - [optional] pointer to the context to be restored the user code's context if found +// +// Return Value: +// The most recent user managed code or NULL if not found. +// +// Note: +// It is a heuristic approach to get the address of the user managed code that calls into +// BCL like System.Diagnostics.Debugger.Break assuming that we can find the original user +// code caller with stack walking. +// +// DoWatsonForUserBreak has the address returned from the helper frame that points to an +// internal BCL helpful function doing permission check. From bucketing perspetive it is +// more preferable to report the user managed code that invokes Debugger.Break instead. +// +// User managed code is managed code in non-system assembly. Currently, only CoreLib +// is marked as system assembly. +// +//---------------------------------------------------------------------------- +UINT_PTR FindMostRecentUserCodeOnStack(void) +{ + CONTRACTL + { + NOTHROW; + GC_NOTRIGGER; + CAN_TAKE_LOCK; + } + CONTRACTL_END; + + Thread * pThread = GetThread(); + UINT_PTR address = NULL; + + CONTEXT ctx; + REGDISPLAY rd; + SetUpRegdisplayForStackWalk(pThread, &ctx, &rd); + StackFrameIterator frameIter; + frameIter.Init(pThread, pThread->GetFrame(), &rd, FUNCTIONSONLY | LIGHTUNWIND); + + while (frameIter.IsValid()) + { + MethodDesc * pMD = frameIter.m_crawl.GetFunction(); + + // Is it not a system assembly? User manged user will not be in system assembly. + if ((pMD != NULL) && (!pMD->GetAssembly()->IsSystem())) + { + CrawlFrame * pCF = &(frameIter.m_crawl); + address = (UINT_PTR)GetControlPC(pCF->GetRegisterSet()); + break; + } + + if (frameIter.Next() != SWA_CONTINUE) + { + break; + } + } + + return address; +} + + +// This does a user break, triggered by System.Diagnostics.Debugger.Break, or the IL opcode for break. // // Notes: // If a managed debugger is attached, this should send the managed UserBreak event. // Else if a native debugger is attached, this should send a native break event (kernel32!DebugBreak) // Else, this should invoke Watson. // +// Historical trivia: +// - In whidbey, this would still invoke Watson if a native-only debugger is attached. +// - In arrowhead, the managed debugging pipeline switched to be built on the native pipeline. FCIMPL0(void, DebugDebugger::Break) { FCALL_CONTRACT; @@ -848,7 +917,7 @@ void DebugStackTrace::GetStackFramesHelper(Frame *pStartFrame, pData->TargetThread->GetInternal() == GetThread()) { // Null target thread specifies current thread. - GetThread()->StackWalkFrames(GetStackFramesCallback, pData, FUNCTIONSONLY | QUICKUNWIND, pStartFrame); + GetThread()->StackWalkFrames(GetStackFramesCallback, pData, FUNCTIONSONLY, pStartFrame); } else { @@ -1083,18 +1152,9 @@ void DebugStackTrace::GetStackFramesFromException(OBJECTREF * e, // to spot. DWORD dwNativeOffset; - UINT_PTR ip = cur.ip; -#if defined(DACCESS_COMPILE) && defined(TARGET_AMD64) - // Compensate for a bug in the old EH that for a frame that faulted - // has the ip pointing to an address before the faulting instruction - if (g_isNewExceptionHandlingEnabled && (i == 0) && ((cur.flags & STEF_IP_ADJUSTED) == 0)) - { - ip -= 1; - } -#endif // DACCESS_COMPILE && TARGET_AMD64 - if (ip) + if (cur.ip) { - EECodeInfo codeInfo(ip); + EECodeInfo codeInfo(cur.ip); dwNativeOffset = codeInfo.GetRelOffset(); } else @@ -1105,7 +1165,7 @@ void DebugStackTrace::GetStackFramesFromException(OBJECTREF * e, pData->pElements[i].InitPass1( dwNativeOffset, pMD, - (PCODE)ip, + (PCODE)cur.ip, cur.flags); #ifndef DACCESS_COMPILE pData->pElements[i].InitPass2(); diff --git a/src/coreclr/vm/dispatchinfo.cpp b/src/coreclr/vm/dispatchinfo.cpp index 8b769c71bcc055..eb0c83f7a6ce56 100644 --- a/src/coreclr/vm/dispatchinfo.cpp +++ b/src/coreclr/vm/dispatchinfo.cpp @@ -2578,9 +2578,10 @@ bool DispatchInfo::IsPropertyAccessorVisible(bool fIsSetter, OBJECTREF* pMemberI // Check to see if the new method is a property accessor. mdToken tkMember = mdTokenNil; - if (pMDForProperty->GetMDImport()->GetPropertyInfoForMethodDef(pMDForProperty->GetMemberDef(), &tkMember, NULL, NULL) == S_OK) + MethodTable *pDeclaringMT = pMDForProperty->GetMethodTable(); + if (pMDForProperty->GetModule()->GetPropertyInfoForMethodDef(pMDForProperty->GetMemberDef(), &tkMember, NULL, NULL) == S_OK) { - if (IsMemberVisibleFromCom(pMDForProperty->GetMethodTable(), tkMember, pMDForProperty->GetMemberDef())) + if (IsMemberVisibleFromCom(pDeclaringMT, tkMember, pMDForProperty->GetMemberDef())) return true; } } diff --git a/src/coreclr/vm/dllimport.cpp b/src/coreclr/vm/dllimport.cpp index 12ca187ecbeacd..d395b8e32cf5c4 100644 --- a/src/coreclr/vm/dllimport.cpp +++ b/src/coreclr/vm/dllimport.cpp @@ -113,7 +113,7 @@ StubSigDesc::StubSigDesc(MethodDesc *pMD) INDEBUG(InitDebugNames()); } -StubSigDesc::StubSigDesc(MethodDesc* pMD, const Signature& sig, Module* pModule, Module* pLoaderModule) +StubSigDesc::StubSigDesc(MethodDesc* pMD, const Signature& sig, Module* pModule) { CONTRACTL { @@ -135,13 +135,13 @@ StubSigDesc::StubSigDesc(MethodDesc* pMD, const Signature& sig, Module* pModule, m_tkMethodDef = pMD->GetMemberDef(); SigTypeContext::InitTypeContext(pMD, &m_typeContext); m_pMetadataModule = pMD->GetModule(); - m_pLoaderModule = pLoaderModule == NULL ? pMD->GetLoaderModule() : pLoaderModule; // Used for ILStubCache selection and MethodTable creation. + m_pLoaderModule = pMD->GetLoaderModule(); // Used for ILStubCache selection and MethodTable creation. } else { m_tkMethodDef = mdMethodDefNil; m_pMetadataModule = m_pModule; - m_pLoaderModule = pLoaderModule == NULL ? m_pModule : pLoaderModule; + m_pLoaderModule = m_pModule; } INDEBUG(InitDebugNames()); @@ -3180,7 +3180,6 @@ BOOL NDirect::MarshalingRequired( _In_opt_ MethodDesc* pMD, _In_opt_ PCCOR_SIGNATURE pSig, _In_opt_ Module* pModule, - _In_opt_ SigTypeContext* pTypeContext, _In_ bool unmanagedCallersOnlyRequiresMarshalling) { CONTRACTL @@ -3261,6 +3260,8 @@ BOOL NDirect::MarshalingRequired( mdParamDef *pParamTokenArray = (mdParamDef *)_alloca(numArgs * sizeof(mdParamDef)); IMDInternalImport *pMDImport = pModule->GetMDImport(); + SigTypeContext emptyTypeContext; + mdMethodDef methodToken = mdMethodDefNil; if (pMD != NULL) { @@ -3320,7 +3321,7 @@ BOOL NDirect::MarshalingRequired( case ELEMENT_TYPE_VALUETYPE: case ELEMENT_TYPE_GENERICINST: { - TypeHandle hndArgType = arg.GetTypeHandleThrowing(pModule, pTypeContext); + TypeHandle hndArgType = arg.GetTypeHandleThrowing(pModule, &emptyTypeContext); bool isValidGeneric = IsValidForGenericMarshalling(hndArgType.GetMethodTable(), false, runtimeMarshallingEnabled); if(!hndArgType.IsValueType() || !isValidGeneric) return true; @@ -4191,10 +4192,8 @@ namespace pHashParams, pParams->m_dwStubFlags, pParams->m_pModule, - pParams->m_pLoaderModule, pParams->m_sig.GetRawSig(), pParams->m_sig.GetRawSigLen(), - pParams->m_pTypeContext, pamTracker, bILStubCreator, pLastMD); @@ -4264,7 +4263,8 @@ static void CreateNDirectStubAccessMetadata( { if (unmgdCallConv == CorInfoCallConvExtension::Managed || unmgdCallConv == CorInfoCallConvExtension::Fastcall || - unmgdCallConv == CorInfoCallConvExtension::FastcallMemberFunction) + unmgdCallConv == CorInfoCallConvExtension::FastcallMemberFunction || + unmgdCallConv == CorInfoCallConvExtension::Swift) { COMPlusThrow(kTypeLoadException, IDS_INVALID_PINVOKE_CALLCONV); } @@ -5041,21 +5041,6 @@ namespace } else { - if (!pSigDesc->m_typeContext.IsEmpty()) - { - // For generic calli, we only support blittable types - if (SF_IsCALLIStub(dwStubFlags) - && NDirect::MarshalingRequired(NULL, pStubMD->GetSig(), pSigDesc->m_pModule, &pSigDesc->m_typeContext)) - { - COMPlusThrow(kMarshalDirectiveException, IDS_EE_BADMARSHAL_GENERICS_RESTRICTION); - } - // We don't want to support generic varargs, so block it - else if (SF_IsVarArgStub(dwStubFlags)) - { - COMPlusThrow(kNotSupportedException, BFA_GENCODE_NOT_BE_VARARG); - } - } - CreateNDirectStubWorker(pss, pSigDesc, nlType, @@ -6042,7 +6027,7 @@ PCODE GetILStubForCalli(VASigCookie *pVASigCookie, MethodDesc *pMD) } } - LoaderHeap *pHeap = pVASigCookie->pLoaderModule->GetLoaderAllocator()->GetHighFrequencyHeap(); + LoaderHeap *pHeap = pVASigCookie->pModule->GetLoaderAllocator()->GetHighFrequencyHeap(); PCOR_SIGNATURE new_sig = (PCOR_SIGNATURE)(void *)pHeap->AllocMem(S_SIZE_T(signature.GetRawSigLen())); CopyMemory(new_sig, signature.GetRawSig(), signature.GetRawSigLen()); @@ -6080,8 +6065,7 @@ PCODE GetILStubForCalli(VASigCookie *pVASigCookie, MethodDesc *pMD) nlType = nltAnsi; } - StubSigDesc sigDesc(pMD, signature, pVASigCookie->pModule, pVASigCookie->pLoaderModule); - sigDesc.InitTypeContext(pVASigCookie->classInst, pVASigCookie->methodInst); + StubSigDesc sigDesc(pMD, signature, pVASigCookie->pModule); MethodDesc* pStubMD = NDirect::CreateCLRToNativeILStub(&sigDesc, nlType, diff --git a/src/coreclr/vm/dllimport.h b/src/coreclr/vm/dllimport.h index 111c7436c9473c..256b950799336e 100644 --- a/src/coreclr/vm/dllimport.h +++ b/src/coreclr/vm/dllimport.h @@ -16,9 +16,9 @@ struct StubSigDesc { public: StubSigDesc(MethodDesc* pMD); - StubSigDesc(MethodDesc* pMD, const Signature& sig, Module* pModule, Module* pLoaderModule = NULL); - StubSigDesc(MethodTable* pMT, const Signature& sig, Module* pModule); - StubSigDesc(const Signature& sig, Module* pModule); + StubSigDesc(MethodDesc* pMD, const Signature& sig, Module* m_pModule); + StubSigDesc(MethodTable* pMT, const Signature& sig, Module* m_pModule); + StubSigDesc(const Signature& sig, Module* m_pModule); MethodDesc *m_pMD; MethodTable *m_pMT; @@ -56,17 +56,6 @@ struct StubSigDesc } } #endif // _DEBUG - -#ifndef DACCESS_COMPILE - void InitTypeContext(Instantiation classInst, Instantiation methodInst) - { - LIMITED_METHOD_CONTRACT; - - _ASSERTE(m_typeContext.IsEmpty()); - - m_typeContext = SigTypeContext(classInst, methodInst); - } -#endif }; //======================================================================= @@ -103,7 +92,6 @@ class NDirect _In_opt_ MethodDesc* pMD, _In_opt_ PCCOR_SIGNATURE pSig = NULL, _In_opt_ Module* pModule = NULL, - _In_opt_ SigTypeContext* pTypeContext = NULL, _In_ bool unmanagedCallersOnlyRequiresMarshalling = true); static void PopulateNDirectMethodDesc(_Inout_ NDirectMethodDesc* pNMD); diff --git a/src/coreclr/vm/ecall.cpp b/src/coreclr/vm/ecall.cpp index 7a9538d8ea7dd9..37ac50d124f6f6 100644 --- a/src/coreclr/vm/ecall.cpp +++ b/src/coreclr/vm/ecall.cpp @@ -96,7 +96,7 @@ void ECall::PopulateManagedStringConstructors() INDEBUG(fInitialized = true); } -void ECall::PopulateManagedHelpers() +void ECall::PopulateManagedCastHelpers() { STANDARD_VM_CONTRACT; @@ -144,18 +144,6 @@ void ECall::PopulateManagedHelpers() pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__CASTHELPERS__LDELEMAREF)); pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_LDELEMA_REF, pDest); - - pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__SPAN_HELPERS__MEMSET)); - pDest = pMD->GetMultiCallableAddrOfCode(); - SetJitHelperFunction(CORINFO_HELP_MEMSET, pDest); - - pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__SPAN_HELPERS__MEMZERO)); - pDest = pMD->GetMultiCallableAddrOfCode(); - SetJitHelperFunction(CORINFO_HELP_MEMZERO, pDest); - - pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__SPAN_HELPERS__MEMCOPY)); - pDest = pMD->GetMultiCallableAddrOfCode(); - SetJitHelperFunction(CORINFO_HELP_MEMCPY, pDest); } static CrstStatic gFCallLock; diff --git a/src/coreclr/vm/ecall.h b/src/coreclr/vm/ecall.h index 792eea633e8f7a..bc9d63ae467137 100644 --- a/src/coreclr/vm/ecall.h +++ b/src/coreclr/vm/ecall.h @@ -94,7 +94,7 @@ class ECall static void PopulateManagedStringConstructors(); - static void PopulateManagedHelpers(); + static void PopulateManagedCastHelpers(); #ifdef DACCESS_COMPILE // Enumerates all gFCallMethods for minidumps. diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h index e72f08ead0e6f1..1af6a5055e6c33 100644 --- a/src/coreclr/vm/ecalllist.h +++ b/src/coreclr/vm/ecalllist.h @@ -76,6 +76,11 @@ FCFuncStart(gStringFuncs) FCDynamicSig(COR_CTOR_METHOD_NAME, &gsig_IM_PtrSByt_Int_Int_Encoding_RetVoid, ECall::CtorSBytePtrStartLengthEncodingManaged) FCFuncEnd() +FCFuncStart(gValueTypeFuncs) + FCFuncElement("CanCompareBits", ValueTypeHelper::CanCompareBits) + FCFuncElement("GetHashCode", ValueTypeHelper::GetHashCode) +FCFuncEnd() + FCFuncStart(gDiagnosticsDebugger) FCFuncElement("BreakInternal", DebugDebugger::Break) FCFuncElement("get_IsAttached", DebugDebugger::IsDebuggerAttached) @@ -93,6 +98,10 @@ FCFuncStart(gEnvironmentFuncs) FCFuncElement("get_TickCount64", SystemNative::GetTickCount64) FCFuncElement("set_ExitCode", SystemNative::SetExitCode) FCFuncElement("get_ExitCode", SystemNative::GetExitCode) + + FCFuncElementSig("FailFast", &gsig_SM_Str_RetVoid, SystemNative::FailFast) + FCFuncElementSig("FailFast", &gsig_SM_Str_Exception_RetVoid, SystemNative::FailFastWithException) + FCFuncElementSig("FailFast", &gsig_SM_Str_Exception_Str_RetVoid, SystemNative::FailFastWithExceptionAndSource) FCFuncEnd() FCFuncStart(gExceptionFuncs) @@ -251,6 +260,10 @@ FCFuncStart(gCOMCustomAttributeFuncs) FCFuncElement("_GetPropertyOrFieldData", COMCustomAttribute::GetPropertyOrFieldData) FCFuncEnd() +FCFuncStart(gCompatibilitySwitchFuncs) + FCFuncElement("GetValueInternal", CompatibilitySwitch::GetValue) +FCFuncEnd() + FCFuncStart(gRuntimeAssemblyFuncs) FCFuncElement("FCallIsDynamic", AssemblyNative::IsDynamic) FCFuncElement("GetManifestModule", AssemblyHandle::GetManifestModule) @@ -463,7 +476,8 @@ FCFuncStart(gRuntimeHelpers) FCFuncElement("PrepareDelegate", ReflectionInvocation::PrepareDelegate) FCFuncElement("GetHashCode", ObjectNative::GetHashCode) FCFuncElement("TryGetHashCode", ObjectNative::TryGetHashCode) - FCFuncElement("ContentEquals", ObjectNative::ContentEquals) + FCFuncElement("Equals", ObjectNative::Equals) + FCFuncElement("AllocateUninitializedClone", ObjectNative::AllocateUninitializedClone) FCFuncElement("EnsureSufficientExecutionStack", ReflectionInvocation::EnsureSufficientExecutionStack) FCFuncElement("TryEnsureSufficientExecutionStack", ReflectionInvocation::TryEnsureSufficientExecutionStack) FCFuncElement("AllocTailCallArgBuffer", TailCallHelp::AllocTailCallArgBuffer) @@ -561,6 +575,7 @@ FCClassElement("AssemblyLoadContext", "System.Runtime.Loader", gAssemblyLoadCont FCClassElement("Buffer", "System", gBufferFuncs) FCClassElement("CastHelpers", "System.Runtime.CompilerServices", gCastHelpers) FCClassElement("ComAwareWeakReference", "System", gComAwareWeakReferenceFuncs) +FCClassElement("CompatibilitySwitch", "System.Runtime.Versioning", gCompatibilitySwitchFuncs) FCClassElement("CustomAttribute", "System.Reflection", gCOMCustomAttributeFuncs) FCClassElement("Debugger", "System.Diagnostics", gDiagnosticsDebugger) FCClassElement("Delegate", "System", gDelegateFuncs) @@ -604,6 +619,7 @@ FCClassElement("Thread", "System.Threading", gThreadFuncs) FCClassElement("ThreadPool", "System.Threading", gThreadPoolFuncs) FCClassElement("Type", "System", gSystem_Type) FCClassElement("TypedReference", "System", gTypedReferenceFuncs) +FCClassElement("ValueType", "System", gValueTypeFuncs) #ifdef FEATURE_COMINTEROP FCClassElement("Variant", "System", gVariantFuncs) #endif diff --git a/src/coreclr/vm/eeconfig.cpp b/src/coreclr/vm/eeconfig.cpp index 29017c18d3c74f..bcbe20a723977b 100644 --- a/src/coreclr/vm/eeconfig.cpp +++ b/src/coreclr/vm/eeconfig.cpp @@ -113,8 +113,10 @@ HRESULT EEConfig::Init() fJitEnableOptionalRelocs = false; fPInvokeRestoreEsp = (DWORD)-1; + fNgenBindOptimizeNonGac = false; fStressLog = false; fForceEnc = false; + fProbeForStackOverflow = true; INDEBUG(fStressLog = true;) @@ -177,6 +179,12 @@ HRESULT EEConfig::Init() DoubleArrayToLargeObjectHeapThreshold = 1000; #endif +#if defined(TARGET_X86) || defined(TARGET_AMD64) + dwDisableStackwalkCache = 0; +#else // TARGET_X86 + dwDisableStackwalkCache = 1; +#endif // TARGET_X86 + #ifdef _DEBUG // interop logging m_TraceWrapper = 0; @@ -474,6 +482,9 @@ HRESULT EEConfig::sync() DoubleArrayToLargeObjectHeapThreshold = CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_DoubleArrayToLargeObjectHeap, DoubleArrayToLargeObjectHeapThreshold); #endif + dwDisableStackwalkCache = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_DisableStackwalkCache, dwDisableStackwalkCache); + + #ifdef _DEBUG IfFailRet (CLRConfig::GetConfigValue(CLRConfig::INTERNAL_BreakOnClassLoad, (LPWSTR*) &pszBreakOnClassLoad)); pszBreakOnClassLoad = NarrowWideChar((LPWSTR)pszBreakOnClassLoad); diff --git a/src/coreclr/vm/eeconfig.h b/src/coreclr/vm/eeconfig.h index 54af05d20b1645..ea28adf7efed7d 100644 --- a/src/coreclr/vm/eeconfig.h +++ b/src/coreclr/vm/eeconfig.h @@ -309,6 +309,12 @@ class EEConfig unsigned int GetDoubleArrayToLargeObjectHeapThreshold() const { LIMITED_METHOD_CONTRACT; return DoubleArrayToLargeObjectHeapThreshold; } #endif + inline bool ProbeForStackOverflow() const + { + LIMITED_METHOD_CONTRACT; + return fProbeForStackOverflow; + } + #ifdef TEST_DATA_CONSISTENCY // get the value of fTestDataConsistency, which controls whether we test that we can correctly detect // held locks in DAC builds. This is determined by an environment variable. @@ -416,6 +422,10 @@ class EEConfig // Loader bool ExcludeReadyToRun(LPCUTF8 assemblyName) const; + bool NgenBindOptimizeNonGac() const { LIMITED_METHOD_CONTRACT; return fNgenBindOptimizeNonGac; } + + DWORD DisableStackwalkCache() const {LIMITED_METHOD_CONTRACT; return dwDisableStackwalkCache; } + bool StressLog() const { LIMITED_METHOD_CONTRACT; return fStressLog; } bool ForceEnc() const { LIMITED_METHOD_CONTRACT; return fForceEnc; } bool DebugAssembliesModifiable() const { LIMITED_METHOD_CONTRACT; return fDebugAssembliesModifiable; } @@ -584,9 +594,15 @@ class EEConfig // Assemblies which cannot use Ready to Run images. AssemblyNamesList * pReadyToRunExcludeList; + bool fNgenBindOptimizeNonGac; + bool fStressLog; bool fForceEnc; bool fDebugAssembliesModifiable; + bool fProbeForStackOverflow; + + // Stackwalk optimization flag + DWORD dwDisableStackwalkCache; #ifdef _DEBUG // interop logging diff --git a/src/coreclr/vm/eetwain.cpp b/src/coreclr/vm/eetwain.cpp index 545bdf7f721025..b0886fcadebee4 100644 --- a/src/coreclr/vm/eetwain.cpp +++ b/src/coreclr/vm/eetwain.cpp @@ -9,6 +9,8 @@ #include "dbginterface.h" #include "gcenv.h" +#define RETURN_ADDR_OFFS 1 // in DWORDS + #ifdef USE_GC_INFO_DECODER #include "gcinfodecoder.h" #endif @@ -17,7 +19,50 @@ #include "gccover.h" #endif // HAVE_GCCOVER -#ifdef TARGET_X86 +#include "argdestination.h" + +#define X86_INSTR_TEST_ESP_SIB 0x24 +#define X86_INSTR_PUSH_0 0x6A // push 00, entire instruction is 0x6A00 +#define X86_INSTR_PUSH_IMM 0x68 // push NNNN, +#define X86_INSTR_W_PUSH_IND_IMM 0x35FF // push [NNNN] +#define X86_INSTR_CALL_REL32 0xE8 // call rel32 +#define X86_INSTR_W_CALL_IND_IMM 0x15FF // call [addr32] +#define X86_INSTR_NOP 0x90 // nop +#define X86_INSTR_NOP2 0x9090 // 2-byte nop +#define X86_INSTR_NOP3_1 0x9090 // 1st word of 3-byte nop +#define X86_INSTR_NOP3_3 0x90 // 3rd byte of 3-byte nop +#define X86_INSTR_NOP4 0x90909090 // 4-byte nop +#define X86_INSTR_NOP5_1 0x90909090 // 1st dword of 5-byte nop +#define X86_INSTR_NOP5_5 0x90 // 5th byte of 5-byte nop +#define X86_INSTR_INT3 0xCC // int3 +#define X86_INSTR_HLT 0xF4 // hlt +#define X86_INSTR_PUSH_EAX 0x50 // push eax +#define X86_INSTR_PUSH_EBP 0x55 // push ebp +#define X86_INSTR_W_MOV_EBP_ESP 0xEC8B // mov ebp, esp +#define X86_INSTR_POP_ECX 0x59 // pop ecx +#define X86_INSTR_RET 0xC2 // ret imm16 +#define X86_INSTR_RETN 0xC3 // ret +#define X86_INSTR_XOR 0x33 // xor +#define X86_INSTR_w_TEST_ESP_EAX 0x0485 // test [esp], eax +#define X86_INSTR_w_TEST_ESP_DWORD_OFFSET_EAX 0x8485 // test [esp-dwOffset], eax +#define X86_INSTR_w_LEA_ESP_EBP_BYTE_OFFSET 0x658d // lea esp, [ebp-bOffset] +#define X86_INSTR_w_LEA_ESP_EBP_DWORD_OFFSET 0xa58d // lea esp, [ebp-dwOffset] +#define X86_INSTR_w_LEA_EAX_ESP_BYTE_OFFSET 0x448d // lea eax, [esp-bOffset] +#define X86_INSTR_w_LEA_EAX_ESP_DWORD_OFFSET 0x848d // lea eax, [esp-dwOffset] +#define X86_INSTR_JMP_NEAR_REL32 0xE9 // near jmp rel32 +#define X86_INSTR_w_JMP_FAR_IND_IMM 0x25FF // far jmp [addr32] + +#ifndef USE_GC_INFO_DECODER + + +#ifdef _DEBUG +// For dumping of verbose info. +#ifndef DACCESS_COMPILE +static bool trFixContext = false; +#endif +static bool trEnumGCRefs = false; +static bool dspPtr = false; // prints the live ptrs as reported +#endif // NOTE: enabling compiler optimizations, even for debug builds. // Comment this out in order to be able to fully debug methods here. @@ -25,235 +70,843 @@ #pragma optimize("tg", on) #endif -void promoteVarArgs(PTR_BYTE argsStart, PTR_VASigCookie varArgSig, GCCONTEXT* ctx); +__forceinline unsigned decodeUnsigned(PTR_CBYTE& src) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + +#ifdef DACCESS_COMPILE + PTR_CBYTE begin = src; +#endif + + BYTE byte = *src++; + unsigned value = byte & 0x7f; + while (byte & 0x80) + { +#ifdef DACCESS_COMPILE + // In DAC builds, the target data may be corrupt. Rather than return incorrect data + // and risk wasting time in a potentially long loop, we want to fail early and gracefully. + // The data is encoded with 7 value-bits per byte, and so we may need to read a maximum + // of 5 bytes (7*5=35) to read a full 32-bit integer. + if ((src - begin) > 5) + { + DacError(CORDBG_E_TARGET_INCONSISTENT); + } +#endif -#include "gc_unwind_x86.inl" + byte = *src++; + value <<= 7; + value += byte & 0x7f; + } + return value; +} -#endif // TARGET_X86 +__forceinline int decodeSigned(PTR_CBYTE& src) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; -#include "argdestination.h" +#ifdef DACCESS_COMPILE + PTR_CBYTE begin = src; +#endif + + BYTE byte = *src++; + BYTE first = byte; + int value = byte & 0x3f; + while (byte & 0x80) + { +#ifdef DACCESS_COMPILE + // In DAC builds, the target data may be corrupt. Rather than return incorrect data + // and risk wasting time in a potentially long loop, we want to fail early and gracefully. + // The data is encoded with 7 value-bits per byte, and so we may need to read a maximum + // of 5 bytes (7*5=35) to read a full 32-bit integer. + if ((src - begin) > 5) + { + DacError(CORDBG_E_TARGET_INCONSISTENT); + } +#endif + + byte = *src++; + value <<= 7; + value += byte & 0x7f; + } + if (first & 0x40) + value = -value; + return value; +} +// Fast versions of the above, with one iteration of the loop unrolled +#define fastDecodeUnsigned(src) (((*(src) & 0x80) == 0) ? (unsigned) (*(src)++) : decodeUnsigned((src))) +#define fastDecodeSigned(src) (((*(src) & 0xC0) == 0) ? (unsigned) (*(src)++) : decodeSigned((src))) + +// Fast skipping past encoded integers #ifndef DACCESS_COMPILE -#ifndef FEATURE_EH_FUNCLETS +#define fastSkipUnsigned(src) { while ((*(src)++) & 0x80) { } } +#define fastSkipSigned(src) { while ((*(src)++) & 0x80) { } } +#else +// In DAC builds we want to trade-off a little perf in the common case for reliaiblity against corrupt data. +#define fastSkipUnsigned(src) (decodeUnsigned(src)) +#define fastSkipSigned(src) (decodeSigned(src)) +#endif + /***************************************************************************** * - * Setup context to enter an exception handler (a 'catch' block). - * This is the last chance for the runtime support to do fixups in - * the context before execution continues inside a filter, catch handler, - * or finally. + * Decodes the X86 GcInfo header and returns the decoded information + * in the hdrInfo struct. + * curOffset is the code offset within the active method used in the + * computation of PrologOffs/EpilogOffs. + * Returns the size of the header (number of bytes decoded). */ -void EECodeManager::FixContext( ContextType ctxType, - EHContext *ctx, - EECodeInfo *pCodeInfo, - DWORD dwRelOffset, - DWORD nestingLevel, - OBJECTREF thrownObject, - CodeManState *pState, - size_t ** ppShadowSP, - size_t ** ppEndRegion) +size_t DecodeGCHdrInfo(GCInfoToken gcInfoToken, + unsigned curOffset, + hdrInfo * infoPtr) { CONTRACTL { NOTHROW; GC_NOTRIGGER; + HOST_NOCALLS; + SUPPORTS_DAC; } CONTRACTL_END; - _ASSERTE((ctxType == FINALLY_CONTEXT) == (thrownObject == NULL)); + PTR_CBYTE table = (PTR_CBYTE) gcInfoToken.Info; +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xFEEF); +#endif - _ASSERTE(sizeof(CodeManStateBuf) <= sizeof(pState->stateBuf)); - CodeManStateBuf * stateBuf = (CodeManStateBuf*)pState->stateBuf; + infoPtr->methodSize = fastDecodeUnsigned(table); - /* Extract the necessary information from the info block header */ + _ASSERTE(curOffset >= 0); + _ASSERTE(curOffset <= infoPtr->methodSize); - stateBuf->hdrInfoSize = (DWORD)DecodeGCHdrInfo(pCodeInfo->GetGCInfoToken(), - dwRelOffset, - &stateBuf->hdrInfoBody); - pState->dwIsSet = 1; + /* Decode the InfoHdr */ -#ifdef _DEBUG - if (trFixContext) { - printf("FixContext [%s][%s] for %s.%s: ", - stateBuf->hdrInfoBody.ebpFrame?"ebp":" ", - stateBuf->hdrInfoBody.interruptible?"int":" ", - "UnknownClass","UnknownMethod"); - fflush(stdout); - } -#endif + InfoHdr header; + table = decodeHeader(table, gcInfoToken.Version, &header); - /* make sure that we have an ebp stack frame */ + BOOL hasArgTabOffset = FALSE; + if (header.untrackedCnt == HAS_UNTRACKED) + { + hasArgTabOffset = TRUE; + header.untrackedCnt = fastDecodeUnsigned(table); + } - _ASSERTE(stateBuf->hdrInfoBody.ebpFrame); - _ASSERTE(stateBuf->hdrInfoBody.handlers); // @TODO : This will always be set. Remove it + if (header.varPtrTableSize == HAS_VARPTR) + { + hasArgTabOffset = TRUE; + header.varPtrTableSize = fastDecodeUnsigned(table); + } - TADDR baseSP; - GetHandlerFrameInfo(&stateBuf->hdrInfoBody, ctx->Ebp, - ctxType == FILTER_CONTEXT ? ctx->Esp : IGNORE_VAL, - ctxType == FILTER_CONTEXT ? (DWORD) IGNORE_VAL : nestingLevel, - &baseSP, - &nestingLevel); + if (header.gsCookieOffset == HAS_GS_COOKIE_OFFSET) + { + header.gsCookieOffset = fastDecodeUnsigned(table); + } - _ASSERTE((size_t)ctx->Ebp >= baseSP); - _ASSERTE(baseSP >= (size_t)ctx->Esp); + if (header.syncStartOffset == HAS_SYNC_OFFSET) + { + header.syncStartOffset = decodeUnsigned(table); + header.syncEndOffset = decodeUnsigned(table); - ctx->Esp = (DWORD)baseSP; + _ASSERTE(header.syncStartOffset != INVALID_SYNC_OFFSET && header.syncEndOffset != INVALID_SYNC_OFFSET); + _ASSERTE(header.syncStartOffset < header.syncEndOffset); + } - // EE will write Esp to **pShadowSP before jumping to handler + if (header.revPInvokeOffset == HAS_REV_PINVOKE_FRAME_OFFSET) + { + header.revPInvokeOffset = fastDecodeUnsigned(table); + } - PTR_TADDR pBaseSPslots = - GetFirstBaseSPslotPtr(ctx->Ebp, &stateBuf->hdrInfoBody); - *ppShadowSP = (size_t *)&pBaseSPslots[-(int) nestingLevel ]; - pBaseSPslots[-(int)(nestingLevel+1)] = 0; // Zero out the next slot + /* Some sanity checks on header */ - // EE will write the end offset of the filter - if (ctxType == FILTER_CONTEXT) - *ppEndRegion = (size_t *)pBaseSPslots + 1; + _ASSERTE( header.prologSize + + (size_t)(header.epilogCount*header.epilogSize) <= infoPtr->methodSize); + _ASSERTE( header.epilogCount == 1 || !header.epilogAtEnd); - /* This is just a simple assignment of throwObject to ctx->Eax, - just pretend the cast goo isn't there. - */ + _ASSERTE( header.untrackedCnt <= header.argCount+header.frameSize); - *((OBJECTREF*)&(ctx->Eax)) = thrownObject; -} + _ASSERTE( header.ebpSaved || !(header.ebpFrame || header.doubleAlign)); + _ASSERTE(!header.ebpFrame || !header.doubleAlign ); + _ASSERTE( header.ebpFrame || !header.security ); + _ASSERTE( header.ebpFrame || !header.handlers ); + _ASSERTE( header.ebpFrame || !header.localloc ); + _ASSERTE( header.ebpFrame || !header.editNcontinue); // : Esp frames NYI for EnC -#endif // !FEATURE_EH_FUNCLETS + /* Initialize the infoPtr struct */ + infoPtr->argSize = header.argCount * 4; + infoPtr->ebpFrame = header.ebpFrame; + infoPtr->interruptible = header.interruptible; + infoPtr->returnKind = (ReturnKind) header.returnKind; + infoPtr->prologSize = header.prologSize; + infoPtr->epilogSize = header.epilogSize; + infoPtr->epilogCnt = header.epilogCount; + infoPtr->epilogEnd = header.epilogAtEnd; + infoPtr->untrackedCnt = header.untrackedCnt; + infoPtr->varPtrTableSize = header.varPtrTableSize; + infoPtr->gsCookieOffset = header.gsCookieOffset; + infoPtr->syncStartOffset = header.syncStartOffset; + infoPtr->syncEndOffset = header.syncEndOffset; + infoPtr->revPInvokeOffset = header.revPInvokeOffset; -/*****************************************************************************/ + infoPtr->doubleAlign = header.doubleAlign; + infoPtr->handlers = header.handlers; + infoPtr->localloc = header.localloc; + infoPtr->editNcontinue = header.editNcontinue; + infoPtr->varargs = header.varargs; + infoPtr->profCallbacks = header.profCallbacks; + infoPtr->genericsContext = header.genericsContext; + infoPtr->genericsContextIsMethodDesc = header.genericsContextIsMethodDesc; + infoPtr->isSpeculativeStackWalk = false; -bool VarIsInReg(ICorDebugInfo::VarLoc varLoc) -{ - LIMITED_METHOD_CONTRACT; + /* Are we within the prolog of the method? */ - switch(varLoc.vlType) + if (curOffset < infoPtr->prologSize) { - case ICorDebugInfo::VLT_REG: - case ICorDebugInfo::VLT_REG_REG: - case ICorDebugInfo::VLT_REG_STK: - return true; - - default: - return false; + infoPtr->prologOffs = curOffset; + } + else + { + infoPtr->prologOffs = hdrInfo::NOT_IN_PROLOG; } -} -#ifdef FEATURE_REMAP_FUNCTION -/***************************************************************************** - * Last chance for the runtime support to do fixups in the context - * before execution continues inside an EnC updated function. - * It also adjusts ESP and munges on the stack. So the caller has to make - * sure that this stack region is not needed (by doing a localloc). - * Also, if this returns EnC_FAIL, we should not have munged the - * context ie. transcated commit - * The plan of attack is: - * 1) Error checking up front. If we get through here, everything - * else should work - * 2) Get all the info about current variables, registers, etc - * 3) zero out the stack frame - this'll initialize _all_ variables - * 4) Put the variables from step 3 into their new locations. - * - * Note that while we use the ShuffleVariablesGet/Set methods, they don't - * have any info/logic that's internal to the runtime: another codemanger - * could easily duplicate what they do, which is why we're calling into them. - */ + /* Assume we're not in the epilog of the method */ -HRESULT EECodeManager::FixContextForEnC(PCONTEXT pCtx, - EECodeInfo * pOldCodeInfo, - const ICorDebugInfo::NativeVarInfo * oldMethodVars, - SIZE_T oldMethodVarsCount, - EECodeInfo * pNewCodeInfo, - const ICorDebugInfo::NativeVarInfo * newMethodVars, - SIZE_T newMethodVarsCount) -{ - CONTRACTL { - DISABLED(NOTHROW); - DISABLED(GC_NOTRIGGER); - } CONTRACTL_END; + infoPtr->epilogOffs = hdrInfo::NOT_IN_EPILOG; - HRESULT hr = S_OK; + /* Are we within an epilog of the method? */ - // Grab a copy of the context before the EnC update. - T_CONTEXT oldCtx = *pCtx; + if (infoPtr->epilogCnt) + { + unsigned epilogStart; -#if defined(TARGET_X86) + if (infoPtr->epilogCnt > 1 || !infoPtr->epilogEnd) + { +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xFACE); +#endif + epilogStart = 0; + for (unsigned i = 0; i < infoPtr->epilogCnt; i++) + { + epilogStart += fastDecodeUnsigned(table); + if (curOffset > epilogStart && + curOffset < epilogStart + infoPtr->epilogSize) + { + infoPtr->epilogOffs = curOffset - epilogStart; + } + } + } + else + { + epilogStart = infoPtr->methodSize - infoPtr->epilogSize; - /* Extract the necessary information from the info block header */ + if (curOffset > epilogStart && + curOffset < epilogStart + infoPtr->epilogSize) + { + infoPtr->epilogOffs = curOffset - epilogStart; + } + } - hdrInfo oldInfo, newInfo; + infoPtr->syncEpilogStart = epilogStart; + } - DecodeGCHdrInfo(pOldCodeInfo->GetGCInfoToken(), - pOldCodeInfo->GetRelOffset(), - &oldInfo); + unsigned argTabOffset = INVALID_ARGTAB_OFFSET; + if (hasArgTabOffset) + { + argTabOffset = fastDecodeUnsigned(table); + } + infoPtr->argTabOffset = argTabOffset; - DecodeGCHdrInfo(pNewCodeInfo->GetGCInfoToken(), - pNewCodeInfo->GetRelOffset(), - &newInfo); + size_t frameDwordCount = header.frameSize; - //1) Error checking up front. If we get through here, everything - // else should work + /* Set the rawStackSize to the number of bytes that it bumps ESP */ - if (!oldInfo.editNcontinue || !newInfo.editNcontinue) { - LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC EnC_INFOLESS_METHOD\n")); - return CORDBG_E_ENC_INFOLESS_METHOD; - } + infoPtr->rawStkSize = (UINT)(frameDwordCount * sizeof(size_t)); - if (!oldInfo.ebpFrame || !newInfo.ebpFrame) { - LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC Esp frames NYI\n")); - return E_FAIL; // Esp frames NYI - } + /* Calculate the callee saves regMask and adjust stackSize to */ + /* include the callee saves register spills */ - if (pCtx->Esp != pCtx->Ebp - oldInfo.stackSize + sizeof(DWORD)) { - LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC stack should be empty\n")); - return E_FAIL; // stack should be empty - @TODO : Barring localloc + unsigned savedRegs = RM_NONE; + unsigned savedRegsCount = 0; + + if (header.ediSaved) + { + savedRegsCount++; + savedRegs |= RM_EDI; + } + if (header.esiSaved) + { + savedRegsCount++; + savedRegs |= RM_ESI; + } + if (header.ebxSaved) + { + savedRegsCount++; + savedRegs |= RM_EBX; + } + if (header.ebpSaved) + { + savedRegsCount++; + savedRegs |= RM_EBP; } - if (oldInfo.handlers) + infoPtr->savedRegMask = (RegMask)savedRegs; + + infoPtr->savedRegsCountExclFP = savedRegsCount; + if (header.ebpFrame || header.doubleAlign) { - bool hasInnerFilter; - TADDR baseSP; - FrameType frameType = GetHandlerFrameInfo(&oldInfo, pCtx->Ebp, - pCtx->Esp, IGNORE_VAL, - &baseSP, NULL, &hasInnerFilter); - _ASSERTE(frameType != FR_INVALID); - _ASSERTE(!hasInnerFilter); // FixContextForEnC() is called for bottommost funclet + _ASSERTE(header.ebpSaved); + infoPtr->savedRegsCountExclFP = savedRegsCount - 1; + } - // If the method is in a fuclet, and if the framesize grows, we are in trouble. + frameDwordCount += savedRegsCount; - if (frameType != FR_NORMAL) - { - /* @TODO : What if the new method offset is in a fuclet, - and the old is not, or the nesting level changed, etc */ + infoPtr->stackSize = (UINT)(frameDwordCount * sizeof(size_t)); - if (oldInfo.stackSize != newInfo.stackSize) { - LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC stack size mismatch\n")); - return CORDBG_E_ENC_IN_FUNCLET; - } - } - } + _ASSERTE(infoPtr->gsCookieOffset == INVALID_GS_COOKIE_OFFSET || + (infoPtr->gsCookieOffset < infoPtr->stackSize) && + ((header.gsCookieOffset % sizeof(void*)) == 0)); - /* @TODO: Check if we have grown out of space for locals, in the face of localloc */ - _ASSERTE(!oldInfo.localloc && !newInfo.localloc); + return table - PTR_CBYTE(gcInfoToken.Info); +} - // @TODO: If nesting level grows above the MAX_EnC_HANDLER_NESTING_LEVEL, - // we should return EnC_NESTED_HANLDERS - _ASSERTE(oldInfo.handlers && newInfo.handlers); +/*****************************************************************************/ - LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: Checks out\n")); +// We do a "pop eax; jmp eax" to return from a fault or finally handler +const size_t END_FIN_POP_STACK = sizeof(TADDR); -#elif defined(TARGET_AMD64) || defined(TARGET_ARM64) +inline +size_t GetLocallocSPOffset(hdrInfo * info) +{ + LIMITED_METHOD_DAC_CONTRACT; - // Strategy for zeroing out the frame on x64: - // - // The stack frame looks like this (stack grows up) - // - // ======================================= - // <--- RSP == RBP (invariant: localalloc disallowed before remap) - // Arguments for next call (if there is one) - // PSPSym (optional) - // JIT temporaries (if any) - // Security object (if any) + _ASSERTE(info->localloc && info->ebpFrame); + + unsigned position = info->savedRegsCountExclFP + + 1; + return position * sizeof(TADDR); +} + +inline +size_t GetParamTypeArgOffset(hdrInfo * info) +{ + LIMITED_METHOD_DAC_CONTRACT; + + _ASSERTE((info->genericsContext || info->handlers) && info->ebpFrame); + + unsigned position = info->savedRegsCountExclFP + + info->localloc + + 1; // For CORINFO_GENERICS_CTXT_FROM_PARAMTYPEARG + return position * sizeof(TADDR); +} + +inline size_t GetStartShadowSPSlotsOffset(hdrInfo * info) +{ + LIMITED_METHOD_DAC_CONTRACT; + + _ASSERTE(info->handlers && info->ebpFrame); + + return GetParamTypeArgOffset(info) + + sizeof(TADDR); // Slot for end-of-last-executed-filter +} + +/***************************************************************************** + * Returns the start of the hidden slots for the shadowSP for functions + * with exception handlers. There is one slot per nesting level starting + * near Ebp and is zero-terminated after the active slots. + */ + +inline +PTR_TADDR GetFirstBaseSPslotPtr(TADDR ebp, hdrInfo * info) +{ + LIMITED_METHOD_DAC_CONTRACT; + + _ASSERTE(info->handlers && info->ebpFrame); + + size_t offsetFromEBP = GetStartShadowSPSlotsOffset(info) + + sizeof(TADDR); // to get to the *start* of the next slot + + return PTR_TADDR(ebp - offsetFromEBP); +} + +inline size_t GetEndShadowSPSlotsOffset(hdrInfo * info, unsigned maxHandlerNestingLevel) +{ + LIMITED_METHOD_DAC_CONTRACT; + + _ASSERTE(info->handlers && info->ebpFrame); + + unsigned numberOfShadowSPSlots = maxHandlerNestingLevel + + 1 + // For zero-termination + 1; // For a filter (which can be active at the same time as a catch/finally handler + + return GetStartShadowSPSlotsOffset(info) + + (numberOfShadowSPSlots * sizeof(TADDR)); +} + +/***************************************************************************** + * returns the base frame pointer corresponding to the target nesting level. + */ + +inline +TADDR GetOutermostBaseFP(TADDR ebp, hdrInfo * info) +{ + LIMITED_METHOD_DAC_CONTRACT; + + // we are not taking into account double alignment. We are + // safe because the jit currently bails on double alignment if there + // are handles or localalloc + _ASSERTE(!info->doubleAlign); + if (info->localloc) + { + // If the function uses localloc we will fetch the ESP from the localloc + // slot. + PTR_TADDR pLocalloc = PTR_TADDR(ebp - GetLocallocSPOffset(info)); + + return (*pLocalloc); + } + else + { + // Default, go back all the method's local stack size + return ebp - info->stackSize + sizeof(int); + } +} + +/***************************************************************************** + * + * For functions with handlers, checks if it is currently in a handler. + * Either of unwindESP or unwindLevel will specify the target nesting level. + * If unwindLevel is specified, info about the funclet at that nesting level + * will be returned. (Use if you are interested in a specific nesting level.) + * If unwindESP is specified, info for nesting level invoked before the stack + * reached unwindESP will be returned. (Use if you have a specific ESP value + * during stack walking.) + * + * *pBaseSP is set to the base SP (base of the stack on entry to + * the current funclet) corresponding to the target nesting level. + * *pNestLevel is set to the nesting level of the target nesting level (useful + * if unwindESP!=IGNORE_VAL + * *pHasInnerFilter will be set to true (only when unwindESP!=IGNORE_VAL) if a filter + * is currently active, but the target nesting level is an outer nesting level. + * *pHadInnerFilter - was the last use of the frame to execute a filter. + * This mainly affects GC lifetime reporting. + */ + +enum FrameType +{ + FR_NORMAL, // Normal method frame - no exceptions currently active + FR_FILTER, // Frame-let of a filter + FR_HANDLER, // Frame-let of a callable catch/fault/finally + + FR_INVALID, // Invalid frame (for speculative stackwalks) +}; + +enum { IGNORE_VAL = -1 }; + +FrameType GetHandlerFrameInfo(hdrInfo * info, + TADDR frameEBP, + TADDR unwindESP, + DWORD unwindLevel, + TADDR * pBaseSP = NULL, /* OUT */ + DWORD * pNestLevel = NULL, /* OUT */ + bool * pHasInnerFilter = NULL, /* OUT */ + bool * pHadInnerFilter = NULL) /* OUT */ +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + HOST_NOCALLS; + SUPPORTS_DAC; + } CONTRACTL_END; + + _ASSERTE(info->ebpFrame && info->handlers); + // One and only one of them should be IGNORE_VAL + _ASSERTE((unwindESP == (TADDR) IGNORE_VAL) != + (unwindLevel == (DWORD) IGNORE_VAL)); + _ASSERTE(pHasInnerFilter == NULL || unwindESP != (TADDR) IGNORE_VAL); + + // Many of the conditions that we'd like to assert cannot be asserted in the case that we're + // in the middle of a stackwalk seeded by a profiler, since such seeds can't be trusted + // (profilers are external, untrusted sources). So during profiler walks, we test the condition + // and throw an exception if it's not met. Otherwise, we just assert the condition. + #define FAIL_IF_SPECULATIVE_WALK(condition) \ + if (info->isSpeculativeStackWalk) \ + { \ + if (!(condition)) \ + { \ + return FR_INVALID; \ + } \ + } \ + else \ + { \ + _ASSERTE(condition); \ + } + + PTR_TADDR pFirstBaseSPslot = GetFirstBaseSPslotPtr(frameEBP, info); + TADDR baseSP = GetOutermostBaseFP(frameEBP, info); + bool nonLocalHandlers = false; // Are the funclets invoked by EE (instead of managed code itself) + bool hasInnerFilter = false; + bool hadInnerFilter = false; + + /* Get the last non-zero slot >= unwindESP, or lvl curSlotVal || + (baseSP == curSlotVal && pSlot == pFirstBaseSPslot)); + + if (curSlotVal == LCL_FINALLY_MARK) + { + // Locally called finally + baseSP -= sizeof(TADDR); + } + else + { + // Is this a funclet we unwound before (can only happen with filters) ? + // If unwindESP is specified, normally we expect it to be the last entry in the shadow slot array. + // Or, if there is a filter, we expect unwindESP to be the second last entry. However, this may + // not be the case in DAC builds. For example, the user can use .cxr in an EH clause to set a + // CONTEXT captured in the try clause. In this case, unwindESP will be the ESP of the parent + // function, but the shadow slot array will contain the SP of the EH clause, which is closer to + // the leaf than the parent method. + + if (unwindESP != (TADDR) IGNORE_VAL && + unwindESP > END_FIN_POP_STACK + + (curSlotVal & ~ICodeManager::SHADOW_SP_BITS)) + { + // In non-DAC builds, the only time unwindESP is closer to the root than entries in the shadow + // slot array is when the last entry in the array is for a filter. Also, filters can't have + // nested handlers. + if ((pSlot[0] & ICodeManager::SHADOW_SP_IN_FILTER) && + (pSlot[-1] == 0) && + !(baseSP & ICodeManager::SHADOW_SP_IN_FILTER)) + { + if (pSlot[0] & ICodeManager::SHADOW_SP_FILTER_DONE) + hadInnerFilter = true; + else + hasInnerFilter = true; + break; + } + else + { +#if defined(DACCESS_COMPILE) + // In DAC builds, this could happen. We just need to bail out of this loop early. + break; +#else // !DACCESS_COMPILE + // In non-DAC builds, this is an error. + FAIL_IF_SPECULATIVE_WALK(FALSE); +#endif // DACCESS_COMPILE + } + } + + nonLocalHandlers = true; + baseSP = curSlotVal; + } + } +#endif // FEATURE_EH_FUNCLETS + + if (unwindESP != (TADDR) IGNORE_VAL) + { + FAIL_IF_SPECULATIVE_WALK(baseSP >= unwindESP || + baseSP == unwindESP - sizeof(TADDR)); // About to locally call a finally + + if (baseSP < unwindESP) // About to locally call a finally + baseSP = unwindESP; + } + else + { + FAIL_IF_SPECULATIVE_WALK(lvl == unwindLevel); // unwindLevel must be currently active on stack + } + + if (pBaseSP) + *pBaseSP = baseSP & ~ICodeManager::SHADOW_SP_BITS; + + if (pNestLevel) + { + *pNestLevel = (DWORD)lvl; + } + + if (pHasInnerFilter) + *pHasInnerFilter = hasInnerFilter; + + if (pHadInnerFilter) + *pHadInnerFilter = hadInnerFilter; + + if (baseSP & ICodeManager::SHADOW_SP_IN_FILTER) + { + FAIL_IF_SPECULATIVE_WALK(!hasInnerFilter); // nested filters not allowed + return FR_FILTER; + } + else if (nonLocalHandlers) + { + return FR_HANDLER; + } + else + { + return FR_NORMAL; + } + + #undef FAIL_IF_SPECULATIVE_WALK +} + +// Returns the number of bytes at the beginning of the stack frame that shouldn't be +// modified by an EnC. This is everything except the space for locals and temporaries. +inline size_t GetSizeOfFrameHeaderForEnC(hdrInfo * info) +{ + WRAPPER_NO_CONTRACT; + + // See comment above Compiler::lvaAssignFrameOffsets() in src\jit\il\lclVars.cpp + // for frame layout + + // EnC supports increasing the maximum handler nesting level by always + // assuming that the max is MAX_EnC_HANDLER_NESTING_LEVEL. Methods with + // a higher max cannot be updated by EnC + + // Take the offset (from EBP) of the last slot of the header, plus one for the EBP slot itself + // to get the total size of the header. + return sizeof(TADDR) + + GetEndShadowSPSlotsOffset(info, MAX_EnC_HANDLER_NESTING_LEVEL); +} +#endif // !USE_GC_INFO_DECODER + +#ifndef DACCESS_COMPILE +#ifndef FEATURE_EH_FUNCLETS + +/***************************************************************************** + * + * Setup context to enter an exception handler (a 'catch' block). + * This is the last chance for the runtime support to do fixups in + * the context before execution continues inside a filter, catch handler, + * or finally. + */ +void EECodeManager::FixContext( ContextType ctxType, + EHContext *ctx, + EECodeInfo *pCodeInfo, + DWORD dwRelOffset, + DWORD nestingLevel, + OBJECTREF thrownObject, + CodeManState *pState, + size_t ** ppShadowSP, + size_t ** ppEndRegion) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + _ASSERTE((ctxType == FINALLY_CONTEXT) == (thrownObject == NULL)); + + _ASSERTE(sizeof(CodeManStateBuf) <= sizeof(pState->stateBuf)); + CodeManStateBuf * stateBuf = (CodeManStateBuf*)pState->stateBuf; + + /* Extract the necessary information from the info block header */ + + stateBuf->hdrInfoSize = (DWORD)DecodeGCHdrInfo(pCodeInfo->GetGCInfoToken(), + dwRelOffset, + &stateBuf->hdrInfoBody); + pState->dwIsSet = 1; + +#ifdef _DEBUG + if (trFixContext) { + printf("FixContext [%s][%s] for %s.%s: ", + stateBuf->hdrInfoBody.ebpFrame?"ebp":" ", + stateBuf->hdrInfoBody.interruptible?"int":" ", + "UnknownClass","UnknownMethod"); + fflush(stdout); + } +#endif + + /* make sure that we have an ebp stack frame */ + + _ASSERTE(stateBuf->hdrInfoBody.ebpFrame); + _ASSERTE(stateBuf->hdrInfoBody.handlers); // @TODO : This will always be set. Remove it + + TADDR baseSP; + GetHandlerFrameInfo(&stateBuf->hdrInfoBody, ctx->Ebp, + ctxType == FILTER_CONTEXT ? ctx->Esp : IGNORE_VAL, + ctxType == FILTER_CONTEXT ? (DWORD) IGNORE_VAL : nestingLevel, + &baseSP, + &nestingLevel); + + _ASSERTE((size_t)ctx->Ebp >= baseSP); + _ASSERTE(baseSP >= (size_t)ctx->Esp); + + ctx->Esp = (DWORD)baseSP; + + // EE will write Esp to **pShadowSP before jumping to handler + + PTR_TADDR pBaseSPslots = + GetFirstBaseSPslotPtr(ctx->Ebp, &stateBuf->hdrInfoBody); + *ppShadowSP = (size_t *)&pBaseSPslots[-(int) nestingLevel ]; + pBaseSPslots[-(int)(nestingLevel+1)] = 0; // Zero out the next slot + + // EE will write the end offset of the filter + if (ctxType == FILTER_CONTEXT) + *ppEndRegion = (size_t *)pBaseSPslots + 1; + + /* This is just a simple assignment of throwObject to ctx->Eax, + just pretend the cast goo isn't there. + */ + + *((OBJECTREF*)&(ctx->Eax)) = thrownObject; +} + +#endif // !FEATURE_EH_FUNCLETS + + + + + +/*****************************************************************************/ + +bool VarIsInReg(ICorDebugInfo::VarLoc varLoc) +{ + LIMITED_METHOD_CONTRACT; + + switch(varLoc.vlType) + { + case ICorDebugInfo::VLT_REG: + case ICorDebugInfo::VLT_REG_REG: + case ICorDebugInfo::VLT_REG_STK: + return true; + + default: + return false; + } +} + +#ifdef FEATURE_REMAP_FUNCTION +/***************************************************************************** + * Last chance for the runtime support to do fixups in the context + * before execution continues inside an EnC updated function. + * It also adjusts ESP and munges on the stack. So the caller has to make + * sure that this stack region is not needed (by doing a localloc). + * Also, if this returns EnC_FAIL, we should not have munged the + * context ie. transcated commit + * The plan of attack is: + * 1) Error checking up front. If we get through here, everything + * else should work + * 2) Get all the info about current variables, registers, etc + * 3) zero out the stack frame - this'll initialize _all_ variables + * 4) Put the variables from step 3 into their new locations. + * + * Note that while we use the ShuffleVariablesGet/Set methods, they don't + * have any info/logic that's internal to the runtime: another codemanger + * could easily duplicate what they do, which is why we're calling into them. + */ + +HRESULT EECodeManager::FixContextForEnC(PCONTEXT pCtx, + EECodeInfo * pOldCodeInfo, + const ICorDebugInfo::NativeVarInfo * oldMethodVars, + SIZE_T oldMethodVarsCount, + EECodeInfo * pNewCodeInfo, + const ICorDebugInfo::NativeVarInfo * newMethodVars, + SIZE_T newMethodVarsCount) +{ + CONTRACTL { + DISABLED(NOTHROW); + DISABLED(GC_NOTRIGGER); + } CONTRACTL_END; + + HRESULT hr = S_OK; + + // Grab a copy of the context before the EnC update. + T_CONTEXT oldCtx = *pCtx; + +#if defined(TARGET_X86) + + /* Extract the necessary information from the info block header */ + + hdrInfo oldInfo, newInfo; + + DecodeGCHdrInfo(pOldCodeInfo->GetGCInfoToken(), + pOldCodeInfo->GetRelOffset(), + &oldInfo); + + DecodeGCHdrInfo(pNewCodeInfo->GetGCInfoToken(), + pNewCodeInfo->GetRelOffset(), + &newInfo); + + //1) Error checking up front. If we get through here, everything + // else should work + + if (!oldInfo.editNcontinue || !newInfo.editNcontinue) { + LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC EnC_INFOLESS_METHOD\n")); + return CORDBG_E_ENC_INFOLESS_METHOD; + } + + if (!oldInfo.ebpFrame || !newInfo.ebpFrame) { + LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC Esp frames NYI\n")); + return E_FAIL; // Esp frames NYI + } + + if (pCtx->Esp != pCtx->Ebp - oldInfo.stackSize + sizeof(DWORD)) { + LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC stack should be empty\n")); + return E_FAIL; // stack should be empty - @TODO : Barring localloc + } + + if (oldInfo.handlers) + { + bool hasInnerFilter; + TADDR baseSP; + FrameType frameType = GetHandlerFrameInfo(&oldInfo, pCtx->Ebp, + pCtx->Esp, IGNORE_VAL, + &baseSP, NULL, &hasInnerFilter); + _ASSERTE(frameType != FR_INVALID); + _ASSERTE(!hasInnerFilter); // FixContextForEnC() is called for bottommost funclet + + // If the method is in a fuclet, and if the framesize grows, we are in trouble. + + if (frameType != FR_NORMAL) + { + /* @TODO : What if the new method offset is in a fuclet, + and the old is not, or the nesting level changed, etc */ + + if (oldInfo.stackSize != newInfo.stackSize) { + LOG((LF_ENC, LL_INFO100, "**Error** EECM::FixContextForEnC stack size mismatch\n")); + return CORDBG_E_ENC_IN_FUNCLET; + } + } + } + + /* @TODO: Check if we have grown out of space for locals, in the face of localloc */ + _ASSERTE(!oldInfo.localloc && !newInfo.localloc); + + // @TODO: If nesting level grows above the MAX_EnC_HANDLER_NESTING_LEVEL, + // we should return EnC_NESTED_HANLDERS + _ASSERTE(oldInfo.handlers && newInfo.handlers); + + LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: Checks out\n")); + +#elif defined(TARGET_AMD64) || defined(TARGET_ARM64) + + // Strategy for zeroing out the frame on x64: + // + // The stack frame looks like this (stack grows up) + // + // ======================================= + // <--- RSP == RBP (invariant: localalloc disallowed before remap) + // Arguments for next call (if there is one) + // PSPSym (optional) + // JIT temporaries (if any) + // Security object (if any) // Local variables (if any) // --------------------------------------- // Frame header (stuff we must preserve, such as bool for synchronized @@ -264,1110 +917,4257 @@ HRESULT EECodeManager::FixContextForEnC(PCONTEXT pCtx, // since fixed-frame size doesn't include this. // ======================================= // - // Goal: Zero out everything AFTER (above) frame header. + // Goal: Zero out everything AFTER (above) frame header. + // + // How do we find this stuff? + // + // EECodeInfo::GetFixedStackSize() gives us the full size from the top ("Arguments + // for next call") all the way down to and including Return Address. + // + // GetSizeOfEditAndContinuePreservedArea() gives us the size in bytes of the + // frame header at the bottom. + // + // So we start at RSP, and zero out: + // GetFixedStackSize() - GetSizeOfEditAndContinuePreservedArea() bytes. + // + // We'll need to restore PSPSym; location gotten from GCInfo. + // We'll need to copy security object; location gotten from GCInfo. + // + // On ARM64 the JIT generates a slightly different frame and we do not have + // the invariant FP == SP, since the FP needs to point at the saved fp/lr + // pair for ETW stack walks. The frame there looks something like: + // ======================================= + // Arguments for next call (if there is one) <- SP + // JIT temporaries + // Locals + // PSPSym + // --------------------------------------- ^ zeroed area + // MonitorAcquired (for synchronized methods) + // Saved FP <- FP + // Saved LR + // --------------------------------------- ^ preserved area + // Arguments + // + // The JIT reports the size of the "preserved" area, which includes + // MonitorAcquired when it is present. It could also include other local + // values that need to be preserved across EnC transitions, but no explicit + // treatment of these is necessary here beyond preserving the values in + // this region. + + // GCInfo for old method + GcInfoDecoder oldGcDecoder( + pOldCodeInfo->GetGCInfoToken(), + GcInfoDecoderFlags(DECODE_SECURITY_OBJECT | DECODE_PSP_SYM | DECODE_EDIT_AND_CONTINUE), + 0 // Instruction offset (not needed) + ); + + // GCInfo for new method + GcInfoDecoder newGcDecoder( + pNewCodeInfo->GetGCInfoToken(), + GcInfoDecoderFlags(DECODE_SECURITY_OBJECT | DECODE_PSP_SYM | DECODE_EDIT_AND_CONTINUE), + 0 // Instruction offset (not needed) + ); + + UINT32 oldSizeOfPreservedArea = oldGcDecoder.GetSizeOfEditAndContinuePreservedArea(); + UINT32 newSizeOfPreservedArea = newGcDecoder.GetSizeOfEditAndContinuePreservedArea(); + + LOG((LF_CORDB, LL_INFO100, "EECM::FixContextForEnC: Got old and new EnC preserved area sizes of %u and %u\n", oldSizeOfPreservedArea, newSizeOfPreservedArea)); + // This ensures the JIT generated EnC compliant code. + if ((oldSizeOfPreservedArea == NO_SIZE_OF_EDIT_AND_CONTINUE_PRESERVED_AREA) || + (newSizeOfPreservedArea == NO_SIZE_OF_EDIT_AND_CONTINUE_PRESERVED_AREA)) + { + _ASSERTE(!"FixContextForEnC called on a non-EnC-compliant method frame"); + return CORDBG_E_ENC_INFOLESS_METHOD; + } + + TADDR oldStackBase = GetSP(&oldCtx); + + LOG((LF_CORDB, LL_INFO100, "EECM::FixContextForEnC: Old SP=%p, FP=%p\n", (void*)oldStackBase, (void*)GetFP(&oldCtx))); + +#if defined(TARGET_AMD64) + // Note: we cannot assert anything about the relationship between oldFixedStackSize + // and newFixedStackSize. It's possible the edited frame grows (new locals) or + // shrinks (less temporaries). + DWORD oldFixedStackSize = pOldCodeInfo->GetFixedStackSize(); + DWORD newFixedStackSize = pNewCodeInfo->GetFixedStackSize(); + + // This verifies no localallocs were used in the old method. + // JIT is required to emit frame register for EnC-compliant code + _ASSERTE(pOldCodeInfo->HasFrameRegister()); + _ASSERTE(pNewCodeInfo->HasFrameRegister()); + +#elif defined(TARGET_ARM64) + DWORD oldFixedStackSize = oldGcDecoder.GetSizeOfEditAndContinueFixedStackFrame(); + DWORD newFixedStackSize = newGcDecoder.GetSizeOfEditAndContinueFixedStackFrame(); +#else + PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); +#endif + + LOG((LF_CORDB, LL_INFO100, "EECM::FixContextForEnC: Old and new fixed stack sizes are %u and %u\n", oldFixedStackSize, newFixedStackSize)); + +#if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) + // win-x64: SP == FP before localloc + if (oldStackBase != GetFP(&oldCtx)) + { + return E_FAIL; + } +#else + // All other 64-bit targets use frame chaining with the FP stored right below the + // return address (LR is always pushed on arm64). FP + 16 == SP + oldFixedStackSize + // gives the caller's SP before stack alloc. + if (GetFP(&oldCtx) + 16 != oldStackBase + oldFixedStackSize) + { + return E_FAIL; + } +#endif + + // EnC remap inside handlers is not supported + if (pOldCodeInfo->IsFunclet() || pNewCodeInfo->IsFunclet()) + return CORDBG_E_ENC_IN_FUNCLET; + + if (oldSizeOfPreservedArea != newSizeOfPreservedArea) + { + _ASSERTE(!"FixContextForEnC called with method whose frame header size changed from old to new version."); + return E_FAIL; + } + + TADDR callerSP = oldStackBase + oldFixedStackSize; + +#ifdef _DEBUG + // If the old method has a PSPSym, then its value should == initial-SP (i.e. + // oldStackBase) for x64 and callerSP for arm64 + INT32 nOldPspSymStackSlot = oldGcDecoder.GetPSPSymStackSlot(); + if (nOldPspSymStackSlot != NO_PSP_SYM) + { +#if defined(TARGET_AMD64) + TADDR oldPSP = *PTR_TADDR(oldStackBase + nOldPspSymStackSlot); + _ASSERTE(oldPSP == oldStackBase); +#else + TADDR oldPSP = *PTR_TADDR(callerSP + nOldPspSymStackSlot); + _ASSERTE(oldPSP == callerSP); +#endif + } +#endif // _DEBUG + +#else + PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); +#endif + + // 2) Get all the info about current variables, registers, etc + + const ICorDebugInfo::NativeVarInfo * pOldVar; + + // sorted by varNumber + ICorDebugInfo::NativeVarInfo * oldMethodVarsSorted = NULL; + ICorDebugInfo::NativeVarInfo * oldMethodVarsSortedBase = NULL; + ICorDebugInfo::NativeVarInfo *newMethodVarsSorted = NULL; + ICorDebugInfo::NativeVarInfo *newMethodVarsSortedBase = NULL; + + SIZE_T *rgVal1 = NULL; + SIZE_T *rgVal2 = NULL; + + { + SIZE_T local; + + // We'll need to sort the old native var info by variable number, since the + // order of them isn't necc. the same. We'll use the number as the key. + // We will assume we may have hidden arguments (which have negative values as the index) + + unsigned oldNumVars = unsigned(-ICorDebugInfo::UNKNOWN_ILNUM); + for (pOldVar = oldMethodVars, local = 0; + local < oldMethodVarsCount; + local++, pOldVar++) + { + DWORD varNumber = pOldVar->varNumber; + if (signed(varNumber) >= 0) + { + // This is an explicit (not special) var, so add its varNumber + 1 to our + // max count ("+1" because varNumber is zero-based). + oldNumVars = max(oldNumVars, unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) + varNumber + 1); + } + } + + oldMethodVarsSortedBase = new (nothrow) ICorDebugInfo::NativeVarInfo[oldNumVars]; + if (!oldMethodVarsSortedBase) + { + hr = E_FAIL; + goto ErrExit; + } + oldMethodVarsSorted = oldMethodVarsSortedBase + (-ICorDebugInfo::UNKNOWN_ILNUM); + + memset((void *)oldMethodVarsSortedBase, 0, oldNumVars * sizeof(ICorDebugInfo::NativeVarInfo)); + + for (local = 0; local < oldNumVars;local++) + oldMethodVarsSortedBase[local].loc.vlType = ICorDebugInfo::VLT_INVALID; + + BYTE **rgVCs = NULL; + DWORD oldMethodOffset = pOldCodeInfo->GetRelOffset(); + + for (pOldVar = oldMethodVars, local = 0; + local < oldMethodVarsCount; + local++, pOldVar++) + { + DWORD varNumber = pOldVar->varNumber; + + _ASSERTE(varNumber + unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) < oldNumVars); + + // Only care about old local variables alive at oldMethodOffset + if (pOldVar->startOffset <= oldMethodOffset && + pOldVar->endOffset > oldMethodOffset) + { + // Indexing should be performed with a signed value - could be negative. + oldMethodVarsSorted[(int32_t)varNumber] = *pOldVar; + } + } + + // 3) Next sort the new var info by varNumber. We want to do this here, since + // we're allocating memory (which may fail) - do this before going to step 2 + + // First, count the new vars the same way we did the old vars above. + + const ICorDebugInfo::NativeVarInfo * pNewVar; + + unsigned newNumVars = unsigned(-ICorDebugInfo::UNKNOWN_ILNUM); + for (pNewVar = newMethodVars, local = 0; + local < newMethodVarsCount; + local++, pNewVar++) + { + DWORD varNumber = pNewVar->varNumber; + if (signed(varNumber) >= 0) + { + // This is an explicit (not special) var, so add its varNumber + 1 to our + // max count ("+1" because varNumber is zero-based). + newNumVars = max(newNumVars, unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) + varNumber + 1); + } + } + + // sorted by varNumber + newMethodVarsSortedBase = new (nothrow) ICorDebugInfo::NativeVarInfo[newNumVars]; + if (!newMethodVarsSortedBase) + { + hr = E_FAIL; + goto ErrExit; + } + newMethodVarsSorted = newMethodVarsSortedBase + (-ICorDebugInfo::UNKNOWN_ILNUM); + + memset(newMethodVarsSortedBase, 0, newNumVars * sizeof(ICorDebugInfo::NativeVarInfo)); + for (local = 0; local < newNumVars;local++) + newMethodVarsSortedBase[local].loc.vlType = ICorDebugInfo::VLT_INVALID; + + DWORD newMethodOffset = pNewCodeInfo->GetRelOffset(); + + for (pNewVar = newMethodVars, local = 0; + local < newMethodVarsCount; + local++, pNewVar++) + { + DWORD varNumber = pNewVar->varNumber; + + _ASSERTE(varNumber + unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) < newNumVars); + + // Only care about new local variables alive at newMethodOffset + if (pNewVar->startOffset <= newMethodOffset && + pNewVar->endOffset > newMethodOffset) + { + // Indexing should be performed with a signed valued - could be negative. + newMethodVarsSorted[(int32_t)varNumber] = *pNewVar; + } + } + + _ASSERTE(newNumVars >= oldNumVars || + !"Not allowed to reduce the number of locals between versions!"); + + LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: gathered info!\n")); + + rgVal1 = new (nothrow) SIZE_T[newNumVars]; + if (rgVal1 == NULL) + { + hr = E_FAIL; + goto ErrExit; + } + + rgVal2 = new (nothrow) SIZE_T[newNumVars]; + if (rgVal2 == NULL) + { + hr = E_FAIL; + goto ErrExit; + } + + // 4) Next we'll zero them out, so any variables that aren't in scope + // in the old method, but are in scope in the new, will have the + // default, zero, value. + + memset(rgVal1, 0, sizeof(SIZE_T) * newNumVars); + memset(rgVal2, 0, sizeof(SIZE_T) * newNumVars); + + unsigned varsToGet = (oldNumVars > newNumVars) + ? newNumVars + : oldNumVars; + + // 2) Get all the info about current variables, registers, etc. + + hr = g_pDebugInterface->GetVariablesFromOffset(pOldCodeInfo->GetMethodDesc(), + varsToGet, + oldMethodVarsSortedBase, + oldMethodOffset, + &oldCtx, + rgVal1, + rgVal2, + newNumVars, + &rgVCs); + if (FAILED(hr)) + { + goto ErrExit; + } + + + LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: got vars!\n")); + + /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + * IMPORTANT : Once we start munging on the context, we cannot return + * EnC_FAIL, as this should be a transacted commit, + **=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ + +#if defined(TARGET_X86) + // Zero out all the registers as some may hold new variables. + pCtx->Eax = pCtx->Ecx = pCtx->Edx = pCtx->Ebx = pCtx->Esi = pCtx->Edi = 0; + + // 3) zero out the stack frame - this'll initialize _all_ variables + + /*------------------------------------------------------------------------- + * Adjust the stack height + */ + pCtx->Esp -= (newInfo.stackSize - oldInfo.stackSize); + + // Zero-init the local and tempory section of new stack frame being careful to avoid + // touching anything in the frame header. + // This is necessary to ensure that any JIT temporaries in the old version can't be mistaken + // for ObjRefs now. + size_t frameHeaderSize = GetSizeOfFrameHeaderForEnC( &newInfo ); + _ASSERTE( frameHeaderSize <= oldInfo.stackSize ); + _ASSERTE( GetSizeOfFrameHeaderForEnC( &oldInfo ) == frameHeaderSize ); + +#elif defined(TARGET_AMD64) && !defined(UNIX_AMD64_ABI) + + // Next few statements zero out all registers that may end up holding new variables. + + // volatile int registers (JIT may use these to enregister variables) + pCtx->Rax = pCtx->Rcx = pCtx->Rdx = pCtx->R8 = pCtx->R9 = pCtx->R10 = pCtx->R11 = 0; + + // volatile float registers + pCtx->Xmm1.High = pCtx->Xmm1.Low = 0; + pCtx->Xmm2.High = pCtx->Xmm2.Low = 0; + pCtx->Xmm3.High = pCtx->Xmm3.Low = 0; + pCtx->Xmm4.High = pCtx->Xmm4.Low = 0; + pCtx->Xmm5.High = pCtx->Xmm5.Low = 0; + + // 3) zero out the stack frame - this'll initialize _all_ variables + + /*------------------------------------------------------------------------- + * Adjust the stack height + */ + + TADDR newStackBase = callerSP - newFixedStackSize; + + SetSP(pCtx, newStackBase); + + // We want to zero-out everything pushed after the frame header. This way we'll zero + // out locals (both old & new) and temporaries. This is necessary to ensure that any + // JIT temporaries in the old version can't be mistaken for ObjRefs now. (I am told + // this last point is less of an issue on x64 as it is on x86, but zeroing out the + // temporaries is still the cleanest, most robust way to go.) + size_t frameHeaderSize = newSizeOfPreservedArea; + _ASSERTE(frameHeaderSize <= oldFixedStackSize); + _ASSERTE(frameHeaderSize <= newFixedStackSize); + + // For EnC-compliant x64 code, FP == SP. Since SP changed above, update FP now + pCtx->Rbp = newStackBase; + +#else +#if defined(TARGET_ARM64) + // Zero out volatile part of stack frame + // x0-x17 + memset(&pCtx->X[0], 0, sizeof(pCtx->X[0]) * 18); + // v0-v7 + memset(&pCtx->V[0], 0, sizeof(pCtx->V[0]) * 8); + // v16-v31 + memset(&pCtx->V[16], 0, sizeof(pCtx->V[0]) * 16); +#elif defined(TARGET_AMD64) + // SysV ABI + pCtx->Rax = pCtx->Rdi = pCtx->Rsi = pCtx->Rdx = pCtx->Rcx = pCtx->R8 = pCtx->R9 = 0; + + // volatile float registers + memset(&pCtx->Xmm0, 0, sizeof(pCtx->Xmm0) * 16); +#else + PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); +#endif + + TADDR newStackBase = callerSP - newFixedStackSize; + + SetSP(pCtx, newStackBase); + + size_t frameHeaderSize = newSizeOfPreservedArea; + _ASSERTE(frameHeaderSize <= oldFixedStackSize); + _ASSERTE(frameHeaderSize <= newFixedStackSize); + + // EnC prolog saves only FP (and LR on arm64), and FP points to saved FP for frame chaining. + // These should already be set up from previous version. + _ASSERTE(GetFP(pCtx) == callerSP - 16); +#endif + + // Perform some debug-only sanity checks on stack variables. Some checks are + // performed differently between X86/AMD64. + +#ifdef _DEBUG + for( unsigned i = 0; i < newNumVars; i++ ) + { + // Make sure that stack variables existing in both old and new methods did not + // move. This matters if the address of a local is used in the remapped method. + // For example: + // + // static unsafe void Main(string[] args) + // { + // int x; + // int* p = &x; + // <- Edit made here - cannot move address of x + // *p = 5; + // } + // + if ((i + unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) < oldNumVars) && // Does variable exist in old method? + (oldMethodVarsSorted[i].loc.vlType == ICorDebugInfo::VLT_STK) && // Is the variable on the stack? + (newMethodVarsSorted[i].loc.vlType == ICorDebugInfo::VLT_STK)) + { + SIZE_T * pOldVarStackLocation = NativeVarStackAddr(oldMethodVarsSorted[i].loc, &oldCtx); + SIZE_T * pNewVarStackLocation = NativeVarStackAddr(newMethodVarsSorted[i].loc, pCtx); + _ASSERTE(pOldVarStackLocation == pNewVarStackLocation); + } + + // Sanity-check that the range we're clearing contains all of the stack variables + +#if defined(TARGET_X86) + const ICorDebugInfo::VarLoc &varLoc = newMethodVarsSortedBase[i].loc; + if( varLoc.vlType == ICorDebugInfo::VLT_STK ) + { + // This is an EBP frame, all stack variables should be EBP relative + _ASSERTE( varLoc.vlStk.vlsBaseReg == ICorDebugInfo::REGNUM_EBP ); + // Generic special args may show up as locals with positive offset from EBP, so skip them + if( varLoc.vlStk.vlsOffset <= 0 ) + { + // Normal locals must occur after the header on the stack + _ASSERTE( unsigned(-varLoc.vlStk.vlsOffset) >= frameHeaderSize ); + // Value must occur before the top of the stack + _ASSERTE( unsigned(-varLoc.vlStk.vlsOffset) < newInfo.stackSize ); + } + + // Ideally we'd like to verify that the stack locals (if any) start at exactly the end + // of the header. However, we can't easily determine the size of value classes here, + // and so (since the stack grows towards 0) can't easily determine where the end of + // the local lies. + } +#elif defined(TARGET_AMD64) || defined(TARGET_ARM64) + switch(newMethodVarsSortedBase[i].loc.vlType) + { + default: + // No validation here for non-stack locals + break; + + case ICorDebugInfo::VLT_STK_BYREF: + { + // For byrefs, verify that the ptr will be zeroed out + + SIZE_T regOffs = GetRegOffsInCONTEXT(newMethodVarsSortedBase[i].loc.vlStk.vlsBaseReg); + TADDR baseReg = *(TADDR *)(regOffs + (BYTE*)pCtx); + TADDR addrOfPtr = baseReg + newMethodVarsSortedBase[i].loc.vlStk.vlsOffset; + + _ASSERTE( + // The ref must exist in the portion we'll zero-out + ( + (newStackBase <= addrOfPtr) && + (addrOfPtr < newStackBase + (newFixedStackSize - frameHeaderSize)) + ) || + // OR in the caller's frame (for parameters) + (addrOfPtr >= newStackBase + newFixedStackSize)); + + // Deliberately fall through, so that we also verify that the value that the ptr + // points to will be zeroed out + // ... + } + __fallthrough; + + case ICorDebugInfo::VLT_STK: + case ICorDebugInfo::VLT_STK2: + case ICorDebugInfo::VLT_REG_STK: + case ICorDebugInfo::VLT_STK_REG: + SIZE_T * pVarStackLocation = NativeVarStackAddr(newMethodVarsSortedBase[i].loc, pCtx); + _ASSERTE (pVarStackLocation != NULL); + _ASSERTE( + // The value must exist in the portion we'll zero-out + ( + (newStackBase <= (TADDR) pVarStackLocation) && + ((TADDR) pVarStackLocation < newStackBase + (newFixedStackSize - frameHeaderSize)) + ) || + // OR in the caller's frame (for parameters) + ((TADDR) pVarStackLocation >= newStackBase + newFixedStackSize)); + break; + } +#else // !X86, !X64, !ARM64 + PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); +#endif + } + +#endif // _DEBUG + + // Clear the local and temporary stack space + +#if defined(TARGET_X86) + memset((void*)(size_t)(pCtx->Esp), 0, newInfo.stackSize - frameHeaderSize ); +#elif defined(TARGET_AMD64) || defined(TARGET_ARM64) + memset((void*)newStackBase, 0, newFixedStackSize - frameHeaderSize); + + // Restore PSPSym for the new function. Its value should be set to our new FP. But + // first, we gotta find PSPSym's location on the stack + INT32 nNewPspSymStackSlot = newGcDecoder.GetPSPSymStackSlot(); + if (nNewPspSymStackSlot != NO_PSP_SYM) + { +#if defined(TARGET_AMD64) + *PTR_TADDR(newStackBase + nNewPspSymStackSlot) = newStackBase; +#elif defined(TARGET_ARM64) + *PTR_TADDR(callerSP + nNewPspSymStackSlot) = callerSP; +#else + PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); +#endif + } +#else // !X86, !X64, !ARM64 + PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); +#endif + + // 4) Put the variables from step 3 into their new locations. + + LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: set vars!\n")); + + // Move the old variables into their new places. + + hr = g_pDebugInterface->SetVariablesAtOffset(pNewCodeInfo->GetMethodDesc(), + newNumVars, + newMethodVarsSortedBase, + newMethodOffset, + pCtx, // place them into the new context + rgVal1, + rgVal2, + rgVCs); + + /*-----------------------------------------------------------------------*/ + } +ErrExit: + if (oldMethodVarsSortedBase) + delete[] oldMethodVarsSortedBase; + if (newMethodVarsSortedBase) + delete[] newMethodVarsSortedBase; + if (rgVal1 != NULL) + delete[] rgVal1; + if (rgVal2 != NULL) + delete[] rgVal2; + + LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: exiting!\n")); + + return hr; +} +#endif // !FEATURE_METADATA_UPDATER + +#endif // #ifndef DACCESS_COMPILE + +#ifdef USE_GC_INFO_DECODER +/***************************************************************************** + * + * Is the function currently at a "GC safe point" ? + */ +bool EECodeManager::IsGcSafe( EECodeInfo *pCodeInfo, + DWORD dwRelOffset) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); + + GcInfoDecoder gcInfoDecoder( + gcInfoToken, + DECODE_INTERRUPTIBILITY, + dwRelOffset + ); + + return gcInfoDecoder.IsInterruptible(); +} + +#if defined(TARGET_ARM) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) +bool EECodeManager::HasTailCalls( EECodeInfo *pCodeInfo) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); + + GcInfoDecoder gcInfoDecoder( + gcInfoToken, + DECODE_HAS_TAILCALLS, + 0 + ); + + return gcInfoDecoder.HasTailCalls(); +} +#endif // TARGET_ARM || TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64 + +#if defined(TARGET_AMD64) && defined(_DEBUG) + +struct FindEndOfLastInterruptibleRegionState +{ + unsigned curOffset; + unsigned endOffset; + unsigned lastRangeOffset; +}; + +bool FindEndOfLastInterruptibleRegionCB ( + UINT32 startOffset, + UINT32 stopOffset, + LPVOID hCallback) +{ + FindEndOfLastInterruptibleRegionState *pState = (FindEndOfLastInterruptibleRegionState*)hCallback; + // - // How do we find this stuff? + // If the current range doesn't overlap the given range, keep searching. // - // EECodeInfo::GetFixedStackSize() gives us the full size from the top ("Arguments - // for next call") all the way down to and including Return Address. + if ( startOffset >= pState->endOffset + || stopOffset < pState->curOffset) + { + return false; + } + // - // GetSizeOfEditAndContinuePreservedArea() gives us the size in bytes of the - // frame header at the bottom. + // If the range overlaps the end, then the last point is the end. // - // So we start at RSP, and zero out: - // GetFixedStackSize() - GetSizeOfEditAndContinuePreservedArea() bytes. + if ( stopOffset > pState->endOffset + /*&& startOffset < pState->endOffset*/) + { + // The ranges should be sorted in increasing order. + CONSISTENCY_CHECK(startOffset >= pState->lastRangeOffset); + + pState->lastRangeOffset = pState->endOffset; + return true; + } + // - // We'll need to restore PSPSym; location gotten from GCInfo. - // We'll need to copy security object; location gotten from GCInfo. + // See if the end of this range is the closet to the end that we've found + // so far. // - // On ARM64 the JIT generates a slightly different frame and we do not have - // the invariant FP == SP, since the FP needs to point at the saved fp/lr - // pair for ETW stack walks. The frame there looks something like: - // ======================================= - // Arguments for next call (if there is one) <- SP - // JIT temporaries - // Locals - // PSPSym - // --------------------------------------- ^ zeroed area - // MonitorAcquired (for synchronized methods) - // Saved FP <- FP - // Saved LR - // --------------------------------------- ^ preserved area - // Arguments + if (stopOffset > pState->lastRangeOffset) + pState->lastRangeOffset = stopOffset; + + return false; +} + +/* + Locates the end of the last interruptible region in the given code range. + Returns 0 if the entire range is uninterruptible. Returns the end point + if the entire range is interruptible. +*/ +unsigned EECodeManager::FindEndOfLastInterruptibleRegion(unsigned curOffset, + unsigned endOffset, + GCInfoToken gcInfoToken) +{ +#ifndef DACCESS_COMPILE + GcInfoDecoder gcInfoDecoder( + gcInfoToken, + DECODE_FOR_RANGES_CALLBACK + ); + + FindEndOfLastInterruptibleRegionState state; + state.curOffset = curOffset; + state.endOffset = endOffset; + state.lastRangeOffset = 0; + + gcInfoDecoder.EnumerateInterruptibleRanges(&FindEndOfLastInterruptibleRegionCB, &state); + + return state.lastRangeOffset; +#else + DacNotImpl(); + return NULL; +#endif // #ifndef DACCESS_COMPILE +} + +#endif // TARGET_AMD64 && _DEBUG + + +#else // !USE_GC_INFO_DECODER + +/***************************************************************************** + * + * Is the function currently at a "GC safe point" ? + */ +bool EECodeManager::IsGcSafe( EECodeInfo *pCodeInfo, + DWORD dwRelOffset) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } CONTRACTL_END; + + hdrInfo info; + BYTE * table; + + /* Extract the necessary information from the info block header */ + + table = (BYTE *)DecodeGCHdrInfo(pCodeInfo->GetGCInfoToken(), + dwRelOffset, + &info); + + /* workaround: prevent interruption within prolog/epilog */ + + if (info.prologOffs != hdrInfo::NOT_IN_PROLOG || info.epilogOffs != hdrInfo::NOT_IN_EPILOG) + return false; + +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xBEEF); +#endif + + return (info.interruptible); +} + + +/*****************************************************************************/ +static +PTR_CBYTE skipToArgReg(const hdrInfo& info, PTR_CBYTE table) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } CONTRACTL_END; + +#ifdef _DEBUG + PTR_CBYTE tableStart = table; +#else + if (info.argTabOffset != INVALID_ARGTAB_OFFSET) + { + return table + info.argTabOffset; + } +#endif + + unsigned count; + +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xBEEF); +#endif + + /* Skip over the untracked frame variable table */ + + count = info.untrackedCnt; + while (count-- > 0) { + fastSkipSigned(table); + } + +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xCAFE); +#endif + + /* Skip over the frame variable lifetime table */ + + count = info.varPtrTableSize; + while (count-- > 0) { + fastSkipUnsigned(table); fastSkipUnsigned(table); fastSkipUnsigned(table); + } + +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *) == 0xBABE); +#endif + +#ifdef _DEBUG + if (info.argTabOffset != INVALID_ARGTAB_OFFSET) + { + CONSISTENCY_CHECK_MSGF((info.argTabOffset == (unsigned) (table - tableStart)), + ("table = %p, tableStart = %p, info.argTabOffset = %d", table, tableStart, info.argTabOffset)); + } +#endif + + return table; +} + +/*****************************************************************************/ + +#define regNumToMask(regNum) RegMask(1<<(regNum)) + +/***************************************************************************** + Helper for scanArgRegTable() and scanArgRegTableI() for regMasks + */ + +void * getCalleeSavedReg(PREGDISPLAY pContext, regNum reg) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + switch (reg) + { + case REGI_EBP: return pContext->GetEbpLocation(); + case REGI_EBX: return pContext->GetEbxLocation(); + case REGI_ESI: return pContext->GetEsiLocation(); + case REGI_EDI: return pContext->GetEdiLocation(); + + default: _ASSERTE(!"bad info.thisPtrResult"); return NULL; + } +} + +/***************************************************************************** + These functions converts the bits in the GC encoding to RegMask + */ + +inline +RegMask convertCalleeSavedRegsMask(unsigned inMask) // EBP,EBX,ESI,EDI +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + _ASSERTE((inMask & 0x0F) == inMask); + + unsigned outMask = RM_NONE; + if (inMask & 0x1) outMask |= RM_EDI; + if (inMask & 0x2) outMask |= RM_ESI; + if (inMask & 0x4) outMask |= RM_EBX; + if (inMask & 0x8) outMask |= RM_EBP; + + return (RegMask) outMask; +} + +inline +RegMask convertAllRegsMask(unsigned inMask) // EAX,ECX,EDX,EBX, EBP,ESI,EDI +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + _ASSERTE((inMask & 0xEF) == inMask); + + unsigned outMask = RM_NONE; + if (inMask & 0x01) outMask |= RM_EAX; + if (inMask & 0x02) outMask |= RM_ECX; + if (inMask & 0x04) outMask |= RM_EDX; + if (inMask & 0x08) outMask |= RM_EBX; + if (inMask & 0x20) outMask |= RM_EBP; + if (inMask & 0x40) outMask |= RM_ESI; + if (inMask & 0x80) outMask |= RM_EDI; + + return (RegMask)outMask; +} + +/***************************************************************************** + * scan the register argument table for the not fully interruptible case. + this function is called to find all live objects (pushed arguments) + and to get the stack base for EBP-less methods. + + NOTE: If info->argTabResult is NULL, info->argHnumResult indicates + how many bits in argMask are valid + If info->argTabResult is non-NULL, then the argMask field does + not fit in 32-bits and the value in argMask meaningless. + Instead argHnum specifies the number of (variable-length) elements + in the array, and argTabBytes specifies the total byte size of the + array. [ Note this is an extremely rare case ] + */ + +#ifdef _PREFAST_ +#pragma warning(push) +#pragma warning(disable:21000) // Suppress PREFast warning about overly large function +#endif +static +unsigned scanArgRegTable(PTR_CBYTE table, + unsigned curOffs, + hdrInfo * info) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } CONTRACTL_END; + + regNum thisPtrReg = REGI_NA; +#ifdef _DEBUG + bool isCall = false; +#endif + unsigned regMask = 0; // EBP,EBX,ESI,EDI + unsigned argMask = 0; + unsigned argHnum = 0; + PTR_CBYTE argTab = 0; + unsigned argTabBytes = 0; + unsigned stackDepth = 0; + + unsigned iregMask = 0; // EBP,EBX,ESI,EDI + unsigned iargMask = 0; + unsigned iptrMask = 0; + +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xBABE); +#endif + + unsigned scanOffs = 0; + + _ASSERTE(scanOffs <= info->methodSize); + + if (info->ebpFrame) { + /* + Encoding table for methods with an EBP frame and + that are not fully interruptible + + The encoding used is as follows: + + this pointer encodings: + + 01000000 this pointer in EBX + 00100000 this pointer in ESI + 00010000 this pointer in EDI + + tiny encoding: + + 0bsdDDDD + requires code delta < 16 (4-bits) + requires pushed argmask == 0 + + where DDDD is code delta + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + d indicates that register EDI is a live pointer + + small encoding: + + 1DDDDDDD bsdAAAAA + + requires code delta < 120 (7-bits) + requires pushed argmask < 64 (5-bits) + + where DDDDDDD is code delta + AAAAA is the pushed args mask + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + d indicates that register EDI is a live pointer + + medium encoding + + 0xFD aaaaaaaa AAAAdddd bseDDDDD + + requires code delta < 0x1000000000 (9-bits) + requires pushed argmask < 0x1000000000000 (12-bits) + + where DDDDD is the upper 5-bits of the code delta + dddd is the low 4-bits of the code delta + AAAA is the upper 4-bits of the pushed arg mask + aaaaaaaa is the low 8-bits of the pushed arg mask + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + e indicates that register EDI is a live pointer + + medium encoding with interior pointers + + 0xF9 DDDDDDDD bsdAAAAAA iiiIIIII + + requires code delta < (8-bits) + requires pushed argmask < (5-bits) + + where DDDDDDD is the code delta + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + d indicates that register EDI is a live pointer + AAAAA is the pushed arg mask + iii indicates that EBX,EDI,ESI are interior pointers + IIIII indicates that bits is the arg mask are interior + pointers + + large encoding + + 0xFE [0BSD0bsd][32-bit code delta][32-bit argMask] + + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + d indicates that register EDI is a live pointer + B indicates that register EBX is an interior pointer + S indicates that register ESI is an interior pointer + D indicates that register EDI is an interior pointer + requires pushed argmask < 32-bits + + large encoding with interior pointers + + 0xFA [0BSD0bsd][32-bit code delta][32-bit argMask][32-bit interior pointer mask] + + + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + d indicates that register EDI is a live pointer + B indicates that register EBX is an interior pointer + S indicates that register ESI is an interior pointer + D indicates that register EDI is an interior pointer + requires pushed argmask < 32-bits + requires pushed iArgmask < 32-bits + + huge encoding This is the only encoding that supports + a pushed argmask which is greater than + 32-bits. + + 0xFB [0BSD0bsd][32-bit code delta] + [32-bit table count][32-bit table size] + [pushed ptr offsets table...] + + b indicates that register EBX is a live pointer + s indicates that register ESI is a live pointer + d indicates that register EDI is a live pointer + B indicates that register EBX is an interior pointer + S indicates that register ESI is an interior pointer + D indicates that register EDI is an interior pointer + the list count is the number of entries in the list + the list size gives the byte-length of the list + the offsets in the list are variable-length + */ + while (scanOffs < curOffs) + { + iregMask = 0; + iargMask = 0; + argTab = NULL; +#ifdef _DEBUG + isCall = true; +#endif + + /* Get the next byte and check for a 'special' entry */ + + unsigned encType = *table++; +#if defined(DACCESS_COMPILE) + // In this scenario, it is invalid to have a zero byte in the GC info encoding (refer to the + // comments above). At least one bit has to be set. For example, a byte can represent which + // register is the "this" pointer, and this byte has to be 0x10, 0x20, or 0x40. Having a zero + // byte indicates there is most likely some sort of DAC error, and it may lead to problems such as + // infinite loops. So we bail out early instead. + if (encType == 0) + { + DacError(CORDBG_E_TARGET_INCONSISTENT); + UNREACHABLE(); + } +#endif // DACCESS_COMPILE + + switch (encType) + { + unsigned val, nxt; + + default: + + /* A tiny or small call entry */ + val = encType; + if ((val & 0x80) == 0x00) { + if (val & 0x0F) { + /* A tiny call entry */ + scanOffs += (val & 0x0F); + regMask = (val & 0x70) >> 4; + argMask = 0; + argHnum = 0; + } + else { + /* This pointer liveness encoding */ + regMask = (val & 0x70) >> 4; + if (regMask == 0x1) + thisPtrReg = REGI_EDI; + else if (regMask == 0x2) + thisPtrReg = REGI_ESI; + else if (regMask == 0x4) + thisPtrReg = REGI_EBX; + else + _ASSERTE(!"illegal encoding for 'this' pointer liveness"); + } + } + else { + /* A small call entry */ + scanOffs += (val & 0x7F); + val = *table++; + regMask = val >> 5; + argMask = val & 0x1F; + argHnum = 5; + } + break; + + case 0xFD: // medium encoding + + argMask = *table++; + val = *table++; + argMask |= ((val & 0xF0) << 4); + argHnum = 12; + nxt = *table++; + scanOffs += (val & 0x0F) + ((nxt & 0x1F) << 4); + regMask = nxt >> 5; // EBX,ESI,EDI + + break; + + case 0xF9: // medium encoding with interior pointers + + scanOffs += *table++; + val = *table++; + argMask = val & 0x1F; + argHnum = 5; + regMask = val >> 5; + val = *table++; + iargMask = val & 0x1F; + iregMask = val >> 5; + + break; + + case 0xFE: // large encoding + case 0xFA: // large encoding with interior pointers + + val = *table++; + regMask = val & 0x7; + iregMask = val >> 4; + scanOffs += *dac_cast(table); table += sizeof(DWORD); + argMask = *dac_cast(table); table += sizeof(DWORD); + argHnum = 31; + if (encType == 0xFA) // read iargMask + { + iargMask = *dac_cast(table); table += sizeof(DWORD); + } + break; + + case 0xFB: // huge encoding This is the only partially interruptible + // encoding that supports a pushed ArgMask + // which is greater than 32-bits. + // The ArgMask is encoded using the argTab + val = *table++; + regMask = val & 0x7; + iregMask = val >> 4; + scanOffs += *dac_cast(table); table += sizeof(DWORD); + argHnum = *dac_cast(table); table += sizeof(DWORD); + argTabBytes = *dac_cast(table); table += sizeof(DWORD); + argTab = table; table += argTabBytes; + + argMask = 0; + break; + + case 0xFF: + scanOffs = curOffs + 1; + break; + + } // end case + + // iregMask & iargMask are subsets of regMask & argMask respectively + + _ASSERTE((iregMask & regMask) == iregMask); + _ASSERTE((iargMask & argMask) == iargMask); + + } // end while + + } + else { + +/* + * Encoding table for methods with an ESP frame and are not fully interruptible + * This encoding does not support a pushed ArgMask greater than 32 + * + * The encoding used is as follows: + * + * push 000DDDDD ESP push one item with 5-bit delta + * push 00100000 [pushCount] ESP push multiple items + * reserved 0011xxxx + * skip 01000000 [Delta] Skip Delta, arbitrary sized delta + * skip 0100DDDD Skip small Delta, for call (DDDD != 0) + * pop 01CCDDDD ESP pop CC items with 4-bit delta (CC != 00) + * call 1PPPPPPP Call Pattern, P=[0..79] + * call 1101pbsd DDCCCMMM Call RegMask=pbsd,ArgCnt=CCC, + * ArgMask=MMM Delta=commonDelta[DD] + * call 1110pbsd [ArgCnt] [ArgMask] Call ArgCnt,RegMask=pbsd,[32-bit ArgMask] + * call 11111000 [PBSDpbsd][32-bit delta][32-bit ArgCnt] + * [32-bit PndCnt][32-bit PndSize][PndOffs...] + * iptr 11110000 [IPtrMask] Arbitrary 32-bit Interior Pointer Mask + * thisptr 111101RR This pointer is in Register RR + * 00=EDI,01=ESI,10=EBX,11=EBP + * reserved 111100xx xx != 00 + * reserved 111110xx xx != 00 + * reserved 11111xxx xxx != 000 && xxx != 111(EOT) + * + * The value 11111111 [0xFF] indicates the end of the table. + * + * An offset (at which stack-walking is performed) without an explicit encoding + * is assumed to be a trivial call-site (no GC registers, stack empty before and + * after) to avoid having to encode all trivial calls. + * + * Note on the encoding used for interior pointers + * + * The iptr encoding must immediately precede a call encoding. It is used to + * transform a normal GC pointer addresses into an interior pointers for GC purposes. + * The mask supplied to the iptr encoding is read from the least signicant bit + * to the most signicant bit. (i.e the lowest bit is read first) + * + * p indicates that register EBP is a live pointer + * b indicates that register EBX is a live pointer + * s indicates that register ESI is a live pointer + * d indicates that register EDI is a live pointer + * P indicates that register EBP is an interior pointer + * B indicates that register EBX is an interior pointer + * S indicates that register ESI is an interior pointer + * D indicates that register EDI is an interior pointer + * + * As an example the following sequence indicates that EDI.ESI and the 2nd pushed pointer + * in ArgMask are really interior pointers. The pointer in ESI in a normal pointer: + * + * iptr 11110000 00010011 => read Interior Ptr, Interior Ptr, Normal Ptr, Normal Ptr, Interior Ptr + * call 11010011 DDCCC011 RRRR=1011 => read EDI is a GC-pointer, ESI is a GC-pointer. EBP is a GC-pointer + * MMM=0011 => read two GC-pointers arguments on the stack (nested call) + * + * Since the call instruction mentions 5 GC-pointers we list them in the required order: + * EDI, ESI, EBP, 1st-pushed pointer, 2nd-pushed pointer + * + * And we apply the Interior Pointer mask mmmm=10011 to the above five ordered GC-pointers + * we learn that EDI and ESI are interior GC-pointers and that the second push arg is an + * interior GC-pointer. + */ + +#if defined(DACCESS_COMPILE) + DWORD cbZeroBytes = 0; +#endif // DACCESS_COMPILE + + while (scanOffs <= curOffs) + { + unsigned callArgCnt; + unsigned skip; + unsigned newRegMask, inewRegMask; + unsigned newArgMask, inewArgMask; + unsigned oldScanOffs = scanOffs; + + if (iptrMask) + { + // We found this iptrMask in the previous iteration. + // This iteration must be for a call. Set these variables + // so that they are available at the end of the loop + + inewRegMask = iptrMask & 0x0F; // EBP,EBX,ESI,EDI + inewArgMask = iptrMask >> 4; + + iptrMask = 0; + } + else + { + // Zero out any stale values. + + inewRegMask = 0; + inewArgMask = 0; + } + + /* Get the next byte and decode it */ + + unsigned val = *table++; +#if defined(DACCESS_COMPILE) + // In this scenario, a 0 means that there is a push at the current offset. For a struct with + // two double fields, the JIT may use two movq instructions to push the struct onto the stack, and + // the JIT will encode 4 pushes at the same code offset. This means that we can have up to 4 + // consecutive bytes of 0 without changing the code offset. Having more than 4 consecutive bytes + // of zero indicates that there is most likely some sort of DAC error, and it may lead to problems + // such as infinite loops. So we bail out early instead. + if (val == 0) + { + cbZeroBytes += 1; + if (cbZeroBytes > 4) + { + DacError(CORDBG_E_TARGET_INCONSISTENT); + UNREACHABLE(); + } + } + else + { + cbZeroBytes = 0; + } +#endif // DACCESS_COMPILE + +#ifdef _DEBUG + if (scanOffs != curOffs) + isCall = false; +#endif + + /* Check pushes, pops, and skips */ + + if (!(val & 0x80)) { + + // iptrMask can immediately precede only calls + + _ASSERTE(inewRegMask == 0); + _ASSERTE(inewArgMask == 0); + + if (!(val & 0x40)) { + + unsigned pushCount; + + if (!(val & 0x20)) + { + // + // push 000DDDDD ESP push one item, 5-bit delta + // + pushCount = 1; + scanOffs += val & 0x1f; + } + else + { + // + // push 00100000 [pushCount] ESP push multiple items + // + _ASSERTE(val == 0x20); + pushCount = fastDecodeUnsigned(table); + } + + if (scanOffs > curOffs) + { + scanOffs = oldScanOffs; + goto FINISHED; + } + + stackDepth += pushCount; + } + else if ((val & 0x3f) != 0) { + // + // pop 01CCDDDD pop CC items, 4-bit delta + // + scanOffs += val & 0x0f; + if (scanOffs > curOffs) + { + scanOffs = oldScanOffs; + goto FINISHED; + } + stackDepth -= (val & 0x30) >> 4; + + } else if (scanOffs < curOffs) { + // + // skip 01000000 [Delta] Skip arbitrary sized delta + // + skip = fastDecodeUnsigned(table); + scanOffs += skip; + } + else // don't process a skip if we are already at curOffs + goto FINISHED; + + /* reset regs and args state since we advance past last call site */ + + regMask = 0; + iregMask = 0; + argMask = 0; + iargMask = 0; + argHnum = 0; + + } + else /* It must be a call, thisptr, or iptr */ + { + switch ((val & 0x70) >> 4) { + default: // case 0-4, 1000xxxx through 1100xxxx + // + // call 1PPPPPPP Call Pattern, P=[0..79] + // + decodeCallPattern((val & 0x7f), &callArgCnt, + &newRegMask, &newArgMask, &skip); + // If we've already reached curOffs and the skip amount + // is non-zero then we are done + if ((scanOffs == curOffs) && (skip > 0)) + goto FINISHED; + // otherwise process this call pattern + scanOffs += skip; + if (scanOffs > curOffs) + goto FINISHED; +#ifdef _DEBUG + isCall = true; +#endif + regMask = newRegMask; + argMask = newArgMask; argTab = NULL; + iregMask = inewRegMask; + iargMask = inewArgMask; + stackDepth -= callArgCnt; + argHnum = 2; // argMask is known to be <= 3 + break; + + case 5: + // + // call 1101RRRR DDCCCMMM Call RegMask=RRRR,ArgCnt=CCC, + // ArgMask=MMM Delta=commonDelta[DD] + // + newRegMask = val & 0xf; // EBP,EBX,ESI,EDI + val = *table++; // read next byte + skip = callCommonDelta[val>>6]; + // If we've already reached curOffs and the skip amount + // is non-zero then we are done + if ((scanOffs == curOffs) && (skip > 0)) + goto FINISHED; + // otherwise process this call encoding + scanOffs += skip; + if (scanOffs > curOffs) + goto FINISHED; +#ifdef _DEBUG + isCall = true; +#endif + regMask = newRegMask; + iregMask = inewRegMask; + callArgCnt = (val >> 3) & 0x7; + stackDepth -= callArgCnt; + argMask = (val & 0x7); argTab = NULL; + iargMask = inewArgMask; + argHnum = 3; + break; + + case 6: + // + // call 1110RRRR [ArgCnt] [ArgMask] + // Call ArgCnt,RegMask=RRR,ArgMask + // +#ifdef _DEBUG + isCall = true; +#endif + regMask = val & 0xf; // EBP,EBX,ESI,EDI + iregMask = inewRegMask; + callArgCnt = fastDecodeUnsigned(table); + stackDepth -= callArgCnt; + argMask = fastDecodeUnsigned(table); argTab = NULL; + iargMask = inewArgMask; + argHnum = sizeof(argMask) * 8; // The size of argMask in bits + break; + + case 7: + switch (val & 0x0C) + { + case 0x00: + // + // 0xF0 iptr 11110000 [IPtrMask] Arbitrary Interior Pointer Mask + // + iptrMask = fastDecodeUnsigned(table); + break; + + case 0x04: + // + // 0xF4 thisptr 111101RR This pointer is in Register RR + // 00=EDI,01=ESI,10=EBX,11=EBP + // + { + static const regNum calleeSavedRegs[] = + { REGI_EDI, REGI_ESI, REGI_EBX, REGI_EBP }; + thisPtrReg = calleeSavedRegs[val&0x3]; + } + break; + + case 0x08: + // + // 0xF8 call 11111000 [PBSDpbsd][32-bit delta][32-bit ArgCnt] + // [32-bit PndCnt][32-bit PndSize][PndOffs...] + // + val = *table++; + skip = *dac_cast(table); table += sizeof(DWORD); +// [VSUQFE 4670] + // If we've already reached curOffs and the skip amount + // is non-zero then we are done + if ((scanOffs == curOffs) && (skip > 0)) + goto FINISHED; +// [VSUQFE 4670] + scanOffs += skip; + if (scanOffs > curOffs) + goto FINISHED; +#ifdef _DEBUG + isCall = true; +#endif + regMask = val & 0xF; + iregMask = val >> 4; + callArgCnt = *dac_cast(table); table += sizeof(DWORD); + stackDepth -= callArgCnt; + argHnum = *dac_cast(table); table += sizeof(DWORD); + argTabBytes = *dac_cast(table); table += sizeof(DWORD); + argTab = table; + table += argTabBytes; + break; + + case 0x0C: + // + // 0xFF end 11111111 End of table marker + // + _ASSERTE(val==0xff); + goto FINISHED; + + default: + _ASSERTE(!"reserved GC encoding"); + break; + } + break; + + } // end switch + + } // end else (!(val & 0x80)) + + // iregMask & iargMask are subsets of regMask & argMask respectively + + _ASSERTE((iregMask & regMask) == iregMask); + _ASSERTE((iargMask & argMask) == iargMask); + + } // end while + + } // end else ebp-less frame + +FINISHED: + + // iregMask & iargMask are subsets of regMask & argMask respectively + + _ASSERTE((iregMask & regMask) == iregMask); + _ASSERTE((iargMask & argMask) == iargMask); + + if (scanOffs != curOffs) + { + /* must have been a boring call */ + info->regMaskResult = RM_NONE; + info->argMaskResult = ptrArgTP(0); + info->iregMaskResult = RM_NONE; + info->iargMaskResult = ptrArgTP(0); + info->argHnumResult = 0; + info->argTabResult = NULL; + info->argTabBytes = 0; + } + else + { + info->regMaskResult = convertCalleeSavedRegsMask(regMask); + info->argMaskResult = ptrArgTP(argMask); + info->argHnumResult = argHnum; + info->iregMaskResult = convertCalleeSavedRegsMask(iregMask); + info->iargMaskResult = ptrArgTP(iargMask); + info->argTabResult = argTab; + info->argTabBytes = argTabBytes; + } + +#ifdef _DEBUG + if (scanOffs != curOffs) { + isCall = false; + } + _ASSERTE(thisPtrReg == REGI_NA || (!isCall || (regNumToMask(thisPtrReg) & info->regMaskResult))); +#endif + info->thisPtrResult = thisPtrReg; + + _ASSERTE(int(stackDepth) < INT_MAX); // check that it did not underflow + return (stackDepth * sizeof(unsigned)); +} +#ifdef _PREFAST_ +#pragma warning(pop) +#endif + + +/***************************************************************************** + * scan the register argument table for the fully interruptible case. + this function is called to find all live objects (pushed arguments) + and to get the stack base for fully interruptible methods. + Returns size of things pushed on the stack for ESP frames + + Arguments: + table - The pointer table + curOffsRegs - The current code offset that should be used for reporting registers + curOffsArgs - The current code offset that should be used for reporting args + info - Incoming arg used to determine if there's a frame, and to save results + */ + +static +unsigned scanArgRegTableI(PTR_CBYTE table, + unsigned curOffsRegs, + unsigned curOffsArgs, + hdrInfo * info) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } CONTRACTL_END; + + regNum thisPtrReg = REGI_NA; + unsigned ptrRegs = 0; // The mask of registers that contain pointers + unsigned iptrRegs = 0; // The subset of ptrRegs that are interior pointers + unsigned ptrOffs = 0; // The code offset of the table entry we are currently looking at + unsigned argCnt = 0; // The number of args that have been pushed + + ptrArgTP ptrArgs(0); // The mask of stack values that contain pointers. + ptrArgTP iptrArgs(0); // The subset of ptrArgs that are interior pointers. + ptrArgTP argHigh(0); // The current mask position that corresponds to the top of the stack. + + bool isThis = false; + bool iptr = false; + + // The comment before the call to scanArgRegTableI in EnumGCRefs + // describes why curOffsRegs can be smaller than curOffsArgs. + _ASSERTE(curOffsRegs <= curOffsArgs); + +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xBABE); +#endif + + bool hasPartialArgInfo; + +#ifndef UNIX_X86_ABI + hasPartialArgInfo = info->ebpFrame; +#else + // For x86/Linux, interruptible code always has full arg info // - // The JIT reports the size of the "preserved" area, which includes - // MonitorAcquired when it is present. It could also include other local - // values that need to be preserved across EnC transitions, but no explicit - // treatment of these is necessary here beyond preserving the values in - // this region. + // This should be aligned with emitFullArgInfo setting at + // emitter::emitEndCodeGen (in JIT) + hasPartialArgInfo = false; +#endif + + /* + Encoding table for methods that are fully interruptible + + The encoding used is as follows: + + ptr reg dead 00RRRDDD [RRR != 100] + ptr reg live 01RRRDDD [RRR != 100] + + non-ptr arg push 10110DDD [SSS == 110] + ptr arg push 10SSSDDD [SSS != 110] && [SSS != 111] + ptr arg pop 11CCCDDD [CCC != 000] && [CCC != 110] && [CCC != 111] + little delta skip 11000DDD [CCC == 000] + bigger delta skip 11110BBB [CCC == 110] + + The values used in the encodings are as follows: + + DDD code offset delta from previous entry (0-7) + BBB bigger delta 000=8,001=16,010=24,...,111=64 + RRR register number (EAX=000,ECX=001,EDX=010,EBX=011, + EBP=101,ESI=110,EDI=111), ESP=100 is reserved + SSS argument offset from base of stack. This is + redundant for frameless methods as we can + infer it from the previous pushes+pops. However, + for EBP-methods, we only report GC pushes, and + so we need SSS + CCC argument count being popped (includes only ptrs for EBP methods) + + The following are the 'large' versions: + + large delta skip 10111000 [0xB8] , encodeUnsigned(delta) + + large ptr arg push 11111000 [0xF8] , encodeUnsigned(pushCount) + large non-ptr arg push 11111001 [0xF9] , encodeUnsigned(pushCount) + large ptr arg pop 11111100 [0xFC] , encodeUnsigned(popCount) + large arg dead 11111101 [0xFD] , encodeUnsigned(popCount) for caller-pop args. + Any GC args go dead after the call, + but are still sitting on the stack + + this pointer prefix 10111100 [0xBC] the next encoding is a ptr live + or a ptr arg push + and contains the this pointer + + interior or by-ref 10111111 [0xBF] the next encoding is a ptr live + pointer prefix or a ptr arg push + and contains an interior + or by-ref pointer + + + The value 11111111 [0xFF] indicates the end of the table. + */ + +#if defined(DACCESS_COMPILE) + bool fLastByteIsZero = false; +#endif // DACCESS_COMPILE + + /* Have we reached the instruction we're looking for? */ + + while (ptrOffs <= curOffsArgs) + { + unsigned val; + + int isPop; + unsigned argOfs; + + unsigned regMask; + + // iptrRegs & iptrArgs are subsets of ptrRegs & ptrArgs respectively + + _ASSERTE((iptrRegs & ptrRegs) == iptrRegs); + _ASSERTE((iptrArgs & ptrArgs) == iptrArgs); + + /* Now find the next 'life' transition */ + + val = *table++; +#if defined(DACCESS_COMPILE) + // In this scenario, a zero byte means that EAX is going dead at the current offset. Since EAX + // can't go dead more than once at any given offset, it's invalid to have two consecutive bytes + // of zero. If this were to happen, then it means that there is most likely some sort of DAC + // error, and it may lead to problems such as infinite loops. So we bail out early instead. + if ((val == 0) && fLastByteIsZero) + { + DacError(CORDBG_E_TARGET_INCONSISTENT); + UNREACHABLE(); + } + fLastByteIsZero = (val == 0); +#endif // DACCESS_COMPILE + + if (!(val & 0x80)) + { + /* A small 'regPtr' encoding */ + + regNum reg; + + ptrOffs += (val ) & 0x7; + if (ptrOffs > curOffsArgs) { + iptr = isThis = false; + goto REPORT_REFS; + } + else if (ptrOffs > curOffsRegs) { + iptr = isThis = false; + continue; + } + + reg = (regNum)((val >> 3) & 0x7); + regMask = 1 << reg; // EAX,ECX,EDX,EBX,---,EBP,ESI,EDI + +#if 0 + printf("regMask = %04X -> %04X\n", ptrRegs, + (val & 0x40) ? (ptrRegs | regMask) + : (ptrRegs & ~regMask)); +#endif + + /* The register is becoming live/dead here */ + + if (val & 0x40) + { + /* Becomes Live */ + _ASSERTE((ptrRegs & regMask) == 0); + + ptrRegs |= regMask; + + if (isThis) + { + thisPtrReg = reg; + } + if (iptr) + { + iptrRegs |= regMask; + } + } + else + { + /* Becomes Dead */ + _ASSERTE((ptrRegs & regMask) != 0); + + ptrRegs &= ~regMask; + + if (reg == thisPtrReg) + { + thisPtrReg = REGI_NA; + } + if (iptrRegs & regMask) + { + iptrRegs &= ~regMask; + } + } + iptr = isThis = false; + continue; + } + + /* This is probably an argument push/pop */ + + argOfs = (val & 0x38) >> 3; + + /* 6 [110] and 7 [111] are reserved for other encodings */ + if (argOfs < 6) + { + + /* A small argument encoding */ + + ptrOffs += (val & 0x07); + if (ptrOffs > curOffsArgs) { + iptr = isThis = false; + goto REPORT_REFS; + } + isPop = (val & 0x40); + + ARG: + + if (isPop) + { + if (argOfs == 0) + continue; // little skip encoding + + /* We remove (pop) the top 'argOfs' entries */ + + _ASSERTE(argOfs || argOfs <= argCnt); + + /* adjust # of arguments */ + + argCnt -= argOfs; + _ASSERTE(argCnt < MAX_PTRARG_OFS); + +// printf("[%04X] popping %u args: mask = %04X\n", ptrOffs, argOfs, (int)ptrArgs); + + do + { + _ASSERTE(!isZero(argHigh)); + + /* Do we have an argument bit that's on? */ + + if (intersect(ptrArgs, argHigh)) + { + /* Turn off the bit */ + + setDiff(ptrArgs, argHigh); + setDiff(iptrArgs, argHigh); + + /* We've removed one more argument bit */ + + argOfs--; + } + else if (hasPartialArgInfo) + argCnt--; + else /* full arg info && not a ref */ + argOfs--; + + /* Continue with the next lower bit */ + + argHigh >>= 1; + } + while (argOfs); + + _ASSERTE(!hasPartialArgInfo || + isZero(argHigh) || + (argHigh == CONSTRUCT_ptrArgTP(1, (argCnt-1)))); + + if (hasPartialArgInfo) + { + // We always leave argHigh pointing to the next ptr arg. + // So, while argHigh is non-zero, and not a ptrArg, we shift right (and subtract + // one arg from our argCnt) until it is a ptrArg. + while (!intersect(argHigh, ptrArgs) && (!isZero(argHigh))) + { + argHigh >>= 1; + argCnt--; + } + } + + } + else + { + /* Add a new ptr arg entry at stack offset 'argOfs' */ + + if (argOfs >= MAX_PTRARG_OFS) + { + _ASSERTE_ALL_BUILDS(!"scanArgRegTableI: args pushed 'too deep'"); + } + else + { + /* Full arg info reports all pushes, and thus + argOffs has to be consistent with argCnt */ + + _ASSERTE(hasPartialArgInfo || argCnt == argOfs); + + /* store arg count */ + + argCnt = argOfs + 1; + _ASSERTE((argCnt < MAX_PTRARG_OFS)); + + /* Compute the appropriate argument offset bit */ + + ptrArgTP argMask = CONSTRUCT_ptrArgTP(1, argOfs); + +// printf("push arg at offset %02u --> mask = %04X\n", argOfs, (int)argMask); + + /* We should never push twice at the same offset */ + + _ASSERTE(!intersect( ptrArgs, argMask)); + _ASSERTE(!intersect(iptrArgs, argMask)); + + /* We should never push within the current highest offset */ + + // _ASSERTE(argHigh < argMask); + + /* This is now the highest bit we've set */ + + argHigh = argMask; + + /* Set the appropriate bit in the argument mask */ + + ptrArgs |= argMask; + + if (iptr) + iptrArgs |= argMask; + } + + iptr = isThis = false; + } + continue; + } + else if (argOfs == 6) + { + if (val & 0x40) { + /* Bigger delta 000=8,001=16,010=24,...,111=64 */ + ptrOffs += (((val & 0x07) + 1) << 3); + } + else { + /* non-ptr arg push */ + _ASSERTE(!hasPartialArgInfo); + ptrOffs += (val & 0x07); + if (ptrOffs > curOffsArgs) { + iptr = isThis = false; + goto REPORT_REFS; + } + argHigh = CONSTRUCT_ptrArgTP(1, argCnt); + argCnt++; + _ASSERTE(argCnt < MAX_PTRARG_OFS); + } + continue; + } + + /* argOfs was 7 [111] which is reserved for the larger encodings */ + + _ASSERTE(argOfs==7); + + switch (val) + { + case 0xFF: + iptr = isThis = false; + goto REPORT_REFS; // the method might loop !!! + + case 0xB8: + val = fastDecodeUnsigned(table); + ptrOffs += val; + continue; + + case 0xBC: + isThis = true; + break; + + case 0xBF: + iptr = true; + break; + + case 0xF8: + case 0xFC: + isPop = val & 0x04; + argOfs = fastDecodeUnsigned(table); + goto ARG; + + case 0xFD: { + argOfs = fastDecodeUnsigned(table); + _ASSERTE(argOfs && argOfs <= argCnt); + + // Kill the top "argOfs" pointers. + + ptrArgTP argMask; + for(argMask = CONSTRUCT_ptrArgTP(1, argCnt); (argOfs != 0); argMask >>= 1) + { + _ASSERTE(!isZero(argMask) && !isZero(ptrArgs)); // there should be remaining pointers + + if (intersect(ptrArgs, argMask)) + { + setDiff(ptrArgs, argMask); + setDiff(iptrArgs, argMask); + argOfs--; + } + } + + // For partial arg info, need to find the next highest pointer for argHigh + + if (hasPartialArgInfo) + { + for(argHigh = ptrArgTP(0); !isZero(argMask); argMask >>= 1) + { + if (intersect(ptrArgs, argMask)) { + argHigh = argMask; + break; + } + } + } + } break; + + case 0xF9: + argOfs = fastDecodeUnsigned(table); + argCnt += argOfs; + break; + + default: + _ASSERTE(!"Unexpected special code %04X"); + } + } + + /* Report all live pointer registers */ +REPORT_REFS: + + _ASSERTE((iptrRegs & ptrRegs) == iptrRegs); // iptrRegs is a subset of ptrRegs + _ASSERTE((iptrArgs & ptrArgs) == iptrArgs); // iptrArgs is a subset of ptrArgs + + /* Save the current live register, argument set, and argCnt */ + + info->regMaskResult = convertAllRegsMask(ptrRegs); + info->argMaskResult = ptrArgs; + info->argHnumResult = 0; + info->iregMaskResult = convertAllRegsMask(iptrRegs); + info->iargMaskResult = iptrArgs; + + info->thisPtrResult = thisPtrReg; + _ASSERTE(thisPtrReg == REGI_NA || (regNumToMask(thisPtrReg) & info->regMaskResult)); + + if (hasPartialArgInfo) + { + return 0; + } + else + { + _ASSERTE(int(argCnt) < INT_MAX); // check that it did not underflow + return (argCnt * sizeof(unsigned)); + } +} + +/*****************************************************************************/ + +unsigned GetPushedArgSize(hdrInfo * info, PTR_CBYTE table, DWORD curOffs) +{ + SUPPORTS_DAC; + + unsigned sz; + + if (info->interruptible) + { + sz = scanArgRegTableI(skipToArgReg(*info, table), + curOffs, + curOffs, + info); + } + else + { + sz = scanArgRegTable(skipToArgReg(*info, table), + curOffs, + info); + } + + return sz; +} + +/*****************************************************************************/ + +inline +void TRASH_CALLEE_UNSAVED_REGS(PREGDISPLAY pContext) +{ + LIMITED_METHOD_DAC_CONTRACT; + +#ifdef _DEBUG + /* This is not completely correct as we lose the current value, but + it should not really be useful to anyone. */ + static DWORD s_badData = 0xDEADBEEF; + pContext->SetEaxLocation(&s_badData); + pContext->SetEcxLocation(&s_badData); + pContext->SetEdxLocation(&s_badData); +#endif //_DEBUG +} + +/***************************************************************************** + * Sizes of certain i386 instructions which are used in the prolog/epilog + */ + +// Can we use sign-extended byte to encode the imm value, or do we need a dword +#define CAN_COMPRESS(val) ((INT8)(val) == (INT32)(val)) + +#define SZ_ADD_REG(val) ( 2 + (CAN_COMPRESS(val) ? 1 : 4)) +#define SZ_AND_REG(val) SZ_ADD_REG(val) +#define SZ_POP_REG 1 +#define SZ_LEA(offset) SZ_ADD_REG(offset) +#define SZ_MOV_REG_REG 2 + +bool IsMarkerInstr(BYTE val) +{ + SUPPORTS_DAC; + +#ifdef _DEBUG + if (val == X86_INSTR_INT3) + { + return true; + } +#ifdef HAVE_GCCOVER + else // GcCover might have stomped on the instruction + { + if (GCStress::IsEnabled()) + { + if (IsGcCoverageInterruptInstructionVal(val)) + { + return true; + } + } + } +#endif // HAVE_GCCOVER +#endif // _DEBUG + + return false; +} + +/* Check if the given instruction opcode is the one we expect. + This is a "necessary" but not "sufficient" check as it ignores the check + if the instruction is one of our special markers (for debugging and GcStress) */ + +bool CheckInstrByte(BYTE val, BYTE expectedValue) +{ + SUPPORTS_DAC; + return ((val == expectedValue) || IsMarkerInstr(val)); +} + +/* Similar to CheckInstrByte(). Use this to check a masked opcode (ignoring + optional bits in the opcode encoding). + valPattern is the masked out value. + expectedPattern is the mask value we expect. + val is the actual instruction opcode + */ +bool CheckInstrBytePattern(BYTE valPattern, BYTE expectedPattern, BYTE val) +{ + SUPPORTS_DAC; + + _ASSERTE((valPattern & val) == valPattern); + + return ((valPattern == expectedPattern) || IsMarkerInstr(val)); +} + +/* Similar to CheckInstrByte() */ + +bool CheckInstrWord(WORD val, WORD expectedValue) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + return ((val == expectedValue) || IsMarkerInstr(val & 0xFF)); +} + +// Use this to check if the instruction at offset "walkOffset" has already +// been executed +// "actualHaltOffset" is the offset when the code was suspended +// It is assumed that there is linear control flow from offset 0 to "actualHaltOffset". +// +// This has been factored out just so that the intent of the comparison +// is clear (compared to the opposite intent) + +bool InstructionAlreadyExecuted(unsigned walkOffset, unsigned actualHaltOffset) +{ + SUPPORTS_DAC; + return (walkOffset < actualHaltOffset); +} + +// skips past a "arith REG, IMM" +inline unsigned SKIP_ARITH_REG(int val, PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + + unsigned delta = 0; + if (val != 0) + { +#ifdef _DEBUG + // Confirm that arith instruction is at the correct place + _ASSERTE(CheckInstrBytePattern(base[offset ] & 0xFD, 0x81, base[offset]) && + CheckInstrBytePattern(base[offset+1] & 0xC0, 0xC0, base[offset+1])); + // only use DWORD form if needed + _ASSERTE(((base[offset] & 2) != 0) == CAN_COMPRESS(val) || + IsMarkerInstr(base[offset])); +#endif + delta = 2 + (CAN_COMPRESS(val) ? 1 : 4); + } + return(offset + delta); +} + +inline unsigned SKIP_PUSH_REG(PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + + // Confirm it is a push instruction + _ASSERTE(CheckInstrBytePattern(base[offset] & 0xF8, 0x50, base[offset])); + return(offset + 1); +} + +inline unsigned SKIP_POP_REG(PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + + // Confirm it is a pop instruction + _ASSERTE(CheckInstrBytePattern(base[offset] & 0xF8, 0x58, base[offset])); + return(offset + 1); +} + +inline unsigned SKIP_MOV_REG_REG(PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + + // Confirm it is a move instruction + // Note that only the first byte may have been stomped on by IsMarkerInstr() + // So we can check the second byte directly + _ASSERTE(CheckInstrBytePattern(base[offset] & 0xFD, 0x89, base[offset]) && + (base[offset+1] & 0xC0) == 0xC0); + return(offset + 2); +} + +inline unsigned SKIP_LEA_ESP_EBP(int val, PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + +#ifdef _DEBUG + // Confirm it is the right instruction + // Note that only the first byte may have been stomped on by IsMarkerInstr() + // So we can check the second byte directly + WORD wOpcode = *(PTR_WORD)base; + _ASSERTE((CheckInstrWord(wOpcode, X86_INSTR_w_LEA_ESP_EBP_BYTE_OFFSET) && + (val == *(PTR_SBYTE)(base+2)) && + CAN_COMPRESS(val)) || + (CheckInstrWord(wOpcode, X86_INSTR_w_LEA_ESP_EBP_DWORD_OFFSET) && + (val == *(PTR_INT32)(base+2)) && + !CAN_COMPRESS(val))); +#endif + + unsigned delta = 2 + (CAN_COMPRESS(val) ? 1 : 4); + return(offset + delta); +} + +inline unsigned SKIP_LEA_EAX_ESP(int val, PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + +#ifdef _DEBUG + WORD wOpcode = *(PTR_WORD)(base + offset); + if (CheckInstrWord(wOpcode, X86_INSTR_w_LEA_EAX_ESP_BYTE_OFFSET)) + { + _ASSERTE(val == *(PTR_SBYTE)(base + offset + 3)); + _ASSERTE(CAN_COMPRESS(val)); + } + else + { + _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_w_LEA_EAX_ESP_DWORD_OFFSET)); + _ASSERTE(val == *(PTR_INT32)(base + offset + 3)); + _ASSERTE(!CAN_COMPRESS(val)); + } +#endif + + unsigned delta = 3 + (CAN_COMPRESS(-val) ? 1 : 4); + return(offset + delta); +} + +inline unsigned SKIP_HELPER_CALL(PTR_CBYTE base, unsigned offset) +{ + LIMITED_METHOD_DAC_CONTRACT; + + unsigned delta; + + if (CheckInstrByte(base[offset], X86_INSTR_CALL_REL32)) + { + delta = 5; + } + else + { +#ifdef _DEBUG + WORD wOpcode = *(PTR_WORD)(base+offset); + _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_W_CALL_IND_IMM)); +#endif + delta = 6; + } + + return(offset+delta); +} + +unsigned SKIP_ALLOC_FRAME(int size, PTR_CBYTE base, unsigned offset) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } CONTRACTL_END; + + _ASSERTE(size != 0); + + if (size == sizeof(void*)) + { + // JIT emits "push eax" instead of "sub esp,4" + return SKIP_PUSH_REG(base, offset); + } + + const int STACK_PROBE_PAGE_SIZE_BYTES = 4096; + const int STACK_PROBE_BOUNDARY_THRESHOLD_BYTES = 1024; + + int lastProbedLocToFinalSp = size; + + if (size < STACK_PROBE_PAGE_SIZE_BYTES) + { + // sub esp, size + offset = SKIP_ARITH_REG(size, base, offset); + } + else + { + WORD wOpcode = *(PTR_WORD)(base + offset); + + if (CheckInstrWord(wOpcode, X86_INSTR_w_TEST_ESP_DWORD_OFFSET_EAX)) + { + // In .NET 5.0 and earlier for frames that have size smaller than 0x3000 bytes + // JIT emits one or two 'test eax, [esp-dwOffset]' instructions before adjusting the stack pointer. + _ASSERTE(size < 0x3000); + + // test eax, [esp-0x1000] + offset += 7; + lastProbedLocToFinalSp -= 0x1000; + + if (size >= 0x2000) + { +#ifdef _DEBUG + wOpcode = *(PTR_WORD)(base + offset); + _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_w_TEST_ESP_DWORD_OFFSET_EAX)); +#endif + //test eax, [esp-0x2000] + offset += 7; + lastProbedLocToFinalSp -= 0x1000; + } + + // sub esp, size + offset = SKIP_ARITH_REG(size, base, offset); + } + else + { + bool pushedStubParam = false; + + if (CheckInstrByte(base[offset], X86_INSTR_PUSH_EAX)) + { + // push eax + offset = SKIP_PUSH_REG(base, offset); + pushedStubParam = true; + } + + if (CheckInstrByte(base[offset], X86_INSTR_XOR)) + { + // In .NET Core 3.1 and earlier for frames that have size greater than or equal to 0x3000 bytes + // JIT emits the following loop. + _ASSERTE(size >= 0x3000); + + offset += 2; + // xor eax, eax 2 + // [nop] 0-3 + // loop: + // test [esp + eax], eax 3 + // sub eax, 0x1000 5 + // cmp eax, -size 5 + // jge loop 2 + + // R2R images that support ReJIT may have extra nops we need to skip over. + while (offset < 5) + { + if (CheckInstrByte(base[offset], X86_INSTR_NOP)) + { + offset++; + } + else + { + break; + } + } + + offset += 15; + + if (pushedStubParam) + { + // pop eax + offset = SKIP_POP_REG(base, offset); + } + + // sub esp, size + return SKIP_ARITH_REG(size, base, offset); + } + else + { + // In .NET 5.0 and later JIT emits a call to JIT_StackProbe helper. + + if (pushedStubParam) + { + // lea eax, [esp-size+4] + offset = SKIP_LEA_EAX_ESP(-size + 4, base, offset); + // call JIT_StackProbe + offset = SKIP_HELPER_CALL(base, offset); + // pop eax + offset = SKIP_POP_REG(base, offset); + // sub esp, size + return SKIP_ARITH_REG(size, base, offset); + } + else + { + // lea eax, [esp-size] + offset = SKIP_LEA_EAX_ESP(-size, base, offset); + // call JIT_StackProbe + offset = SKIP_HELPER_CALL(base, offset); + // mov esp, eax + return SKIP_MOV_REG_REG(base, offset); + } + } + } + } + + if (lastProbedLocToFinalSp + STACK_PROBE_BOUNDARY_THRESHOLD_BYTES > STACK_PROBE_PAGE_SIZE_BYTES) + { +#ifdef _DEBUG + WORD wOpcode = *(PTR_WORD)(base + offset); + _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_w_TEST_ESP_EAX)); +#endif + // test [esp], eax + offset += 3; + } + + return offset; +} + +#endif // !USE_GC_INFO_DECODER + + +#if defined(FEATURE_EH_FUNCLETS) + +void EECodeManager::EnsureCallerContextIsValid( PREGDISPLAY pRD, StackwalkCacheEntry* pCacheEntry, EECodeInfo * pCodeInfo /*= NULL*/ ) +{ + CONTRACTL + { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } + CONTRACTL_END; + + if( !pRD->IsCallerContextValid ) + { +#if !defined(DACCESS_COMPILE) && defined(HAS_QUICKUNWIND) + if (pCacheEntry != NULL) + { + // lightened schema: take stack unwind info from stackwalk cache + QuickUnwindStackFrame(pRD, pCacheEntry, EnsureCallerStackFrameIsValid); + } + else +#endif // !DACCESS_COMPILE + { + // We need to make a copy here (instead of switching the pointers), in order to preserve the current context + *(pRD->pCallerContext) = *(pRD->pCurrentContext); + *(pRD->pCallerContextPointers) = *(pRD->pCurrentContextPointers); + + Thread::VirtualUnwindCallFrame(pRD->pCallerContext, pRD->pCallerContextPointers, pCodeInfo); + } + + pRD->IsCallerContextValid = TRUE; + } + + _ASSERTE( pRD->IsCallerContextValid ); +} + +size_t EECodeManager::GetCallerSp( PREGDISPLAY pRD ) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + SUPPORTS_DAC; + } CONTRACTL_END; + + // Don't add usage of this field. This is only temporary. + // See ExceptionTracker::InitializeCrawlFrame() for more information. + if (!pRD->IsCallerSPValid) + { + EnsureCallerContextIsValid(pRD, NULL); + } + + return GetSP(pRD->pCallerContext); +} + +#endif // FEATURE_EH_FUNCLETS + +#ifdef HAS_QUICKUNWIND +/* + * Light unwind the current stack frame, using provided cache entry. + * pPC, Esp and pEbp of pContext are updated. + */ + +// static +void EECodeManager::QuickUnwindStackFrame(PREGDISPLAY pRD, StackwalkCacheEntry *pCacheEntry, QuickUnwindFlag flag) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + _ASSERTE(pCacheEntry); + _ASSERTE(GetControlPC(pRD) == (PCODE)(pCacheEntry->IP)); + +#if defined(TARGET_X86) + _ASSERTE(flag == UnwindCurrentStackFrame); + + _ASSERTE(!pCacheEntry->fUseEbp || pCacheEntry->fUseEbpAsFrameReg); + + if (pCacheEntry->fUseEbpAsFrameReg) + { + _ASSERTE(pCacheEntry->fUseEbp); + TADDR curEBP = GetRegdisplayFP(pRD); + + // EBP frame, update ESP through EBP, since ESPOffset may vary + pRD->SetEbpLocation(PTR_DWORD(curEBP)); + pRD->SP = curEBP + sizeof(void*); + } + else + { + _ASSERTE(!pCacheEntry->fUseEbp); + // ESP frame, update up to retAddr using ESPOffset + pRD->SP += pCacheEntry->ESPOffset; + } + pRD->PCTAddr = (TADDR)pRD->SP; + pRD->ControlPC = *PTR_PCODE(pRD->PCTAddr); + pRD->SP += sizeof(void*) + pCacheEntry->argSize; + +#elif defined(TARGET_AMD64) + if (pRD->IsCallerContextValid) + { + pRD->pCurrentContext->Rbp = pRD->pCallerContext->Rbp; + pRD->pCurrentContext->Rsp = pRD->pCallerContext->Rsp; + pRD->pCurrentContext->Rip = pRD->pCallerContext->Rip; + } + else + { + PCONTEXT pSourceCtx = NULL; + PCONTEXT pTargetCtx = NULL; + if (flag == UnwindCurrentStackFrame) + { + pTargetCtx = pRD->pCurrentContext; + pSourceCtx = pRD->pCurrentContext; + } + else + { + pTargetCtx = pRD->pCallerContext; + pSourceCtx = pRD->pCurrentContext; + } + + // Unwind RBP. The offset is relative to the current sp. + if (pCacheEntry->RBPOffset == 0) + { + pTargetCtx->Rbp = pSourceCtx->Rbp; + } + else + { + pTargetCtx->Rbp = *(UINT_PTR*)(pSourceCtx->Rsp + pCacheEntry->RBPOffset); + } + + // Adjust the sp. From this pointer onwards pCurrentContext->Rsp is the caller sp. + pTargetCtx->Rsp = pSourceCtx->Rsp + pCacheEntry->RSPOffset; + + // Retrieve the return address. + pTargetCtx->Rip = *(UINT_PTR*)((pTargetCtx->Rsp) - sizeof(UINT_PTR)); + } + + if (flag == UnwindCurrentStackFrame) + { + SyncRegDisplayToCurrentContext(pRD); + pRD->IsCallerContextValid = FALSE; + pRD->IsCallerSPValid = FALSE; // Don't add usage of this field. This is only temporary. + } + +#else // !TARGET_X86 && !TARGET_AMD64 + PORTABILITY_ASSERT("EECodeManager::QuickUnwindStackFrame is not implemented on this platform."); +#endif // !TARGET_X86 && !TARGET_AMD64 +} +#endif // HAS_QUICKUNWIND + +/*****************************************************************************/ +#ifdef TARGET_X86 // UnwindStackFrame +/*****************************************************************************/ + +const RegMask CALLEE_SAVED_REGISTERS_MASK[] = +{ + RM_EDI, // first register to be pushed + RM_ESI, + RM_EBX, + RM_EBP // last register to be pushed +}; + +static void SetLocation(PREGDISPLAY pRD, int ind, PDWORD loc) +{ +#ifdef FEATURE_EH_FUNCLETS + static const SIZE_T OFFSET_OF_CALLEE_SAVED_REGISTERS[] = + { + offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Edi), // first register to be pushed + offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Esi), + offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Ebx), + offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Ebp), // last register to be pushed + }; + + SIZE_T offsetOfRegPtr = OFFSET_OF_CALLEE_SAVED_REGISTERS[ind]; + *(LPVOID*)(PBYTE(pRD->pCurrentContextPointers) + offsetOfRegPtr) = loc; +#else + static const SIZE_T OFFSET_OF_CALLEE_SAVED_REGISTERS[] = + { + offsetof(REGDISPLAY, pEdi), // first register to be pushed + offsetof(REGDISPLAY, pEsi), + offsetof(REGDISPLAY, pEbx), + offsetof(REGDISPLAY, pEbp), // last register to be pushed + }; + + SIZE_T offsetOfRegPtr = OFFSET_OF_CALLEE_SAVED_REGISTERS[ind]; + *(LPVOID*)(PBYTE(pRD) + offsetOfRegPtr) = loc; +#endif +} + +/*****************************************************************************/ + +void UnwindEspFrameEpilog( + PREGDISPLAY pContext, + hdrInfo * info, + PTR_CBYTE epilogBase, + unsigned flags) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + _ASSERTE(info->epilogOffs != hdrInfo::NOT_IN_EPILOG); + _ASSERTE(!info->ebpFrame && !info->doubleAlign); + _ASSERTE(info->epilogOffs > 0); + + int offset = 0; + unsigned ESP = pContext->SP; + + if (info->rawStkSize) + { + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + { + /* We have NOT executed the "ADD ESP, FrameSize", + so manually adjust stack pointer */ + ESP += info->rawStkSize; + } + + // We have already popped off the frame (excluding the callee-saved registers) + + if (epilogBase[0] == X86_INSTR_POP_ECX) + { + // We may use "POP ecx" for doing "ADD ESP, 4", + // or we may not (in the case of JMP epilogs) + _ASSERTE(info->rawStkSize == sizeof(void*)); + offset = SKIP_POP_REG(epilogBase, offset); + } + else + { + // "add esp, rawStkSize" + offset = SKIP_ARITH_REG(info->rawStkSize, epilogBase, offset); + } + } + + /* Remaining callee-saved regs are at ESP. Need to update + regsMask as well to exclude registers which have already been popped. */ + + const RegMask regsMask = info->savedRegMask; + + /* Increment "offset" in steps to see which callee-saved + registers have already been popped */ + + for (unsigned i = ARRAY_SIZE(CALLEE_SAVED_REGISTERS_MASK); i > 0; i--) + { + RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i - 1]; + + if (!(regMask & regsMask)) + continue; + + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + { + /* We have NOT yet popped off the register. + Get the value from the stack if needed */ + if ((flags & UpdateAllRegs) || (regMask == RM_EBP)) + { + SetLocation(pContext, i - 1, PTR_DWORD((TADDR)ESP)); + } + + /* Adjust ESP */ + ESP += sizeof(void*); + } + + offset = SKIP_POP_REG(epilogBase, offset); + } + + //CEE_JMP generates an epilog similar to a normal CEE_RET epilog except for the last instruction + _ASSERTE(CheckInstrBytePattern(epilogBase[offset] & X86_INSTR_RET, X86_INSTR_RET, epilogBase[offset]) //ret + || CheckInstrBytePattern(epilogBase[offset], X86_INSTR_JMP_NEAR_REL32, epilogBase[offset]) //jmp ret32 + || CheckInstrWord(*PTR_WORD(epilogBase + offset), X86_INSTR_w_JMP_FAR_IND_IMM)); //jmp [addr32] + + /* Finally we can set pPC */ + pContext->PCTAddr = (TADDR)ESP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); + + pContext->SP = ESP; +} + +/*****************************************************************************/ + +void UnwindEbpDoubleAlignFrameEpilog( + PREGDISPLAY pContext, + hdrInfo * info, + PTR_CBYTE epilogBase, + unsigned flags) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + _ASSERTE(info->epilogOffs != hdrInfo::NOT_IN_EPILOG); + _ASSERTE(info->ebpFrame || info->doubleAlign); + + _ASSERTE(info->argSize < 0x10000); // "ret" only has a 2 byte operand + + /* See how many instructions we have executed in the + epilog to determine which callee-saved registers + have already been popped */ + int offset = 0; + + unsigned ESP = pContext->SP; + + bool needMovEspEbp = false; + + if (info->doubleAlign) + { + // add esp, rawStkSize + + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + ESP += info->rawStkSize; + _ASSERTE(info->rawStkSize != 0); + offset = SKIP_ARITH_REG(info->rawStkSize, epilogBase, offset); + + // We also need "mov esp, ebp" after popping the callee-saved registers + needMovEspEbp = true; + } + else + { + bool needLea = false; + + if (info->localloc) + { + // ESP may be variable if a localloc was actually executed. We will reset it. + // lea esp, [ebp-calleeSavedRegs] + + needLea = true; + } + else if (info->savedRegsCountExclFP == 0) + { + // We will just generate "mov esp, ebp" and be done with it. + + if (info->rawStkSize != 0) + { + needMovEspEbp = true; + } + } + else if (info->rawStkSize == 0) + { + // do nothing before popping the callee-saved registers + } + else if (info->rawStkSize == sizeof(void*)) + { + // "pop ecx" will make ESP point to the callee-saved registers + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + ESP += sizeof(void*); + offset = SKIP_POP_REG(epilogBase, offset); + } + else + { + // We need to make ESP point to the callee-saved registers + // lea esp, [ebp-calleeSavedRegs] + + needLea = true; + } + + if (needLea) + { + // lea esp, [ebp-calleeSavedRegs] + + unsigned calleeSavedRegsSize = info->savedRegsCountExclFP * sizeof(void*); + + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + ESP = GetRegdisplayFP(pContext) - calleeSavedRegsSize; - // GCInfo for old method - GcInfoDecoder oldGcDecoder( - pOldCodeInfo->GetGCInfoToken(), - GcInfoDecoderFlags(DECODE_SECURITY_OBJECT | DECODE_PSP_SYM | DECODE_EDIT_AND_CONTINUE), - 0 // Instruction offset (not needed) - ); + offset = SKIP_LEA_ESP_EBP(-int(calleeSavedRegsSize), epilogBase, offset); + } + } - // GCInfo for new method - GcInfoDecoder newGcDecoder( - pNewCodeInfo->GetGCInfoToken(), - GcInfoDecoderFlags(DECODE_SECURITY_OBJECT | DECODE_PSP_SYM | DECODE_EDIT_AND_CONTINUE), - 0 // Instruction offset (not needed) - ); + for (unsigned i = STRING_LENGTH(CALLEE_SAVED_REGISTERS_MASK); i > 0; i--) + { + RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i - 1]; + _ASSERTE(regMask != RM_EBP); - UINT32 oldSizeOfPreservedArea = oldGcDecoder.GetSizeOfEditAndContinuePreservedArea(); - UINT32 newSizeOfPreservedArea = newGcDecoder.GetSizeOfEditAndContinuePreservedArea(); + if ((info->savedRegMask & regMask) == 0) + continue; - LOG((LF_CORDB, LL_INFO100, "EECM::FixContextForEnC: Got old and new EnC preserved area sizes of %u and %u\n", oldSizeOfPreservedArea, newSizeOfPreservedArea)); - // This ensures the JIT generated EnC compliant code. - if ((oldSizeOfPreservedArea == NO_SIZE_OF_EDIT_AND_CONTINUE_PRESERVED_AREA) || - (newSizeOfPreservedArea == NO_SIZE_OF_EDIT_AND_CONTINUE_PRESERVED_AREA)) + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + { + if (flags & UpdateAllRegs) + { + SetLocation(pContext, i - 1, PTR_DWORD((TADDR)ESP)); + } + ESP += sizeof(void*); + } + + offset = SKIP_POP_REG(epilogBase, offset); + } + + if (needMovEspEbp) { - _ASSERTE(!"FixContextForEnC called on a non-EnC-compliant method frame"); - return CORDBG_E_ENC_INFOLESS_METHOD; + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + ESP = GetRegdisplayFP(pContext); + + offset = SKIP_MOV_REG_REG(epilogBase, offset); } - TADDR oldStackBase = GetSP(&oldCtx); + // Have we executed the pop EBP? + if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) + { + pContext->SetEbpLocation(PTR_DWORD(TADDR(ESP))); + ESP += sizeof(void*); + } + offset = SKIP_POP_REG(epilogBase, offset); - LOG((LF_CORDB, LL_INFO100, "EECM::FixContextForEnC: Old SP=%p, FP=%p\n", (void*)oldStackBase, (void*)GetFP(&oldCtx))); + pContext->PCTAddr = (TADDR)ESP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); -#if defined(TARGET_AMD64) - // Note: we cannot assert anything about the relationship between oldFixedStackSize - // and newFixedStackSize. It's possible the edited frame grows (new locals) or - // shrinks (less temporaries). - DWORD oldFixedStackSize = pOldCodeInfo->GetFixedStackSize(); - DWORD newFixedStackSize = pNewCodeInfo->GetFixedStackSize(); + pContext->SP = ESP; +} - // This verifies no localallocs were used in the old method. - // JIT is required to emit frame register for EnC-compliant code - _ASSERTE(pOldCodeInfo->HasFrameRegister()); - _ASSERTE(pNewCodeInfo->HasFrameRegister()); +inline SIZE_T GetStackParameterSize(hdrInfo * info) +{ + SUPPORTS_DAC; + return (info->varargs ? 0 : info->argSize); // Note varargs is caller-popped +} -#elif defined(TARGET_ARM64) - DWORD oldFixedStackSize = oldGcDecoder.GetSizeOfEditAndContinueFixedStackFrame(); - DWORD newFixedStackSize = newGcDecoder.GetSizeOfEditAndContinueFixedStackFrame(); -#else - PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); -#endif +//**************************************************************************** +// This is the value ESP is incremented by on doing a "return" - LOG((LF_CORDB, LL_INFO100, "EECM::FixContextForEnC: Old and new fixed stack sizes are %u and %u\n", oldFixedStackSize, newFixedStackSize)); +inline SIZE_T ESPIncrOnReturn(hdrInfo * info) +{ + SUPPORTS_DAC; + return sizeof(void *) + // pop off the return address + GetStackParameterSize(info); +} -#if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) - // win-x64: SP == FP before localloc - if (oldStackBase != GetFP(&oldCtx)) +/*****************************************************************************/ + +void UnwindEpilog( + PREGDISPLAY pContext, + hdrInfo * info, + PTR_CBYTE epilogBase, + unsigned flags) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + _ASSERTE(info->epilogOffs != hdrInfo::NOT_IN_EPILOG); + // _ASSERTE(flags & ActiveStackFrame); // Wont work for thread death + _ASSERTE(info->epilogOffs > 0); + + if (info->ebpFrame || info->doubleAlign) { - return E_FAIL; + UnwindEbpDoubleAlignFrameEpilog(pContext, info, epilogBase, flags); } -#else - // All other 64-bit targets use frame chaining with the FP stored right below the - // return address (LR is always pushed on arm64). FP + 16 == SP + oldFixedStackSize - // gives the caller's SP before stack alloc. - if (GetFP(&oldCtx) + 16 != oldStackBase + oldFixedStackSize) + else { - return E_FAIL; + UnwindEspFrameEpilog(pContext, info, epilogBase, flags); } + +#ifdef _DEBUG + if (flags & UpdateAllRegs) + TRASH_CALLEE_UNSAVED_REGS(pContext); #endif - // EnC remap inside handlers is not supported - if (pOldCodeInfo->IsFunclet() || pNewCodeInfo->IsFunclet()) - return CORDBG_E_ENC_IN_FUNCLET; + /* Now adjust stack pointer */ - if (oldSizeOfPreservedArea != newSizeOfPreservedArea) - { - _ASSERTE(!"FixContextForEnC called with method whose frame header size changed from old to new version."); - return E_FAIL; - } + pContext->SP += ESPIncrOnReturn(info); +} - TADDR callerSP = oldStackBase + oldFixedStackSize; +/*****************************************************************************/ + +void UnwindEspFrameProlog( + PREGDISPLAY pContext, + hdrInfo * info, + PTR_CBYTE methodStart, + unsigned flags) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + /* we are in the middle of the prolog */ + _ASSERTE(info->prologOffs != hdrInfo::NOT_IN_PROLOG); + _ASSERTE(!info->ebpFrame && !info->doubleAlign); + + unsigned offset = 0; #ifdef _DEBUG - // If the old method has a PSPSym, then its value should == initial-SP (i.e. - // oldStackBase) for x64 and callerSP for arm64 - INT32 nOldPspSymStackSlot = oldGcDecoder.GetPSPSymStackSlot(); - if (nOldPspSymStackSlot != NO_PSP_SYM) + // If the first two instructions are 'nop, int3', then we will + // assume that is from a JitHalt operation and skip past it + if (methodStart[0] == X86_INSTR_NOP && methodStart[1] == X86_INSTR_INT3) { -#if defined(TARGET_AMD64) - TADDR oldPSP = *PTR_TADDR(oldStackBase + nOldPspSymStackSlot); - _ASSERTE(oldPSP == oldStackBase); -#else - TADDR oldPSP = *PTR_TADDR(callerSP + nOldPspSymStackSlot); - _ASSERTE(oldPSP == callerSP); -#endif + offset += 2; } -#endif // _DEBUG - -#else - PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); #endif - // 2) Get all the info about current variables, registers, etc - - const ICorDebugInfo::NativeVarInfo * pOldVar; + const DWORD curOffs = info->prologOffs; + unsigned ESP = pContext->SP; - // sorted by varNumber - ICorDebugInfo::NativeVarInfo * oldMethodVarsSorted = NULL; - ICorDebugInfo::NativeVarInfo * oldMethodVarsSortedBase = NULL; - ICorDebugInfo::NativeVarInfo *newMethodVarsSorted = NULL; - ICorDebugInfo::NativeVarInfo *newMethodVarsSortedBase = NULL; + // Find out how many callee-saved regs have already been pushed - SIZE_T *rgVal1 = NULL; - SIZE_T *rgVal2 = NULL; + unsigned regsMask = RM_NONE; + PTR_DWORD savedRegPtr = PTR_DWORD((TADDR)ESP); + for (unsigned i = 0; i < ARRAY_SIZE(CALLEE_SAVED_REGISTERS_MASK); i++) { - SIZE_T local; + RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i]; - // We'll need to sort the old native var info by variable number, since the - // order of them isn't necc. the same. We'll use the number as the key. - // We will assume we may have hidden arguments (which have negative values as the index) + if (!(info->savedRegMask & regMask)) + continue; - unsigned oldNumVars = unsigned(-ICorDebugInfo::UNKNOWN_ILNUM); - for (pOldVar = oldMethodVars, local = 0; - local < oldMethodVarsCount; - local++, pOldVar++) + if (InstructionAlreadyExecuted(offset, curOffs)) { - DWORD varNumber = pOldVar->varNumber; - if (signed(varNumber) >= 0) - { - // This is an explicit (not special) var, so add its varNumber + 1 to our - // max count ("+1" because varNumber is zero-based). - oldNumVars = max(oldNumVars, unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) + varNumber + 1); - } + ESP += sizeof(void*); + regsMask |= regMask; } - oldMethodVarsSortedBase = new (nothrow) ICorDebugInfo::NativeVarInfo[oldNumVars]; - if (!oldMethodVarsSortedBase) + offset = SKIP_PUSH_REG(methodStart, offset); + } + + if (info->rawStkSize) + { + offset = SKIP_ALLOC_FRAME(info->rawStkSize, methodStart, offset); + + // Note that this assumes that only the last instruction in SKIP_ALLOC_FRAME + // actually updates ESP + if (InstructionAlreadyExecuted(offset, curOffs + 1)) { - hr = E_FAIL; - goto ErrExit; + savedRegPtr += (info->rawStkSize / sizeof(DWORD)); + ESP += info->rawStkSize; } - oldMethodVarsSorted = oldMethodVarsSortedBase + (-ICorDebugInfo::UNKNOWN_ILNUM); + } - memset((void *)oldMethodVarsSortedBase, 0, oldNumVars * sizeof(ICorDebugInfo::NativeVarInfo)); + // + // Stack probe checks here + // - for (local = 0; local < oldNumVars;local++) - oldMethodVarsSortedBase[local].loc.vlType = ICorDebugInfo::VLT_INVALID; + // Poison the value, we don't set it properly at the end of the prolog + INDEBUG(offset = 0xCCCCCCCC); - BYTE **rgVCs = NULL; - DWORD oldMethodOffset = pOldCodeInfo->GetRelOffset(); - for (pOldVar = oldMethodVars, local = 0; - local < oldMethodVarsCount; - local++, pOldVar++) - { - DWORD varNumber = pOldVar->varNumber; + // Always restore EBP + if (regsMask & RM_EBP) + pContext->SetEbpLocation(savedRegPtr++); - _ASSERTE(varNumber + unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) < oldNumVars); + if (flags & UpdateAllRegs) + { + if (regsMask & RM_EBX) + pContext->SetEbxLocation(savedRegPtr++); + if (regsMask & RM_ESI) + pContext->SetEsiLocation(savedRegPtr++); + if (regsMask & RM_EDI) + pContext->SetEdiLocation(savedRegPtr++); + + TRASH_CALLEE_UNSAVED_REGS(pContext); + } - // Only care about old local variables alive at oldMethodOffset - if (pOldVar->startOffset <= oldMethodOffset && - pOldVar->endOffset > oldMethodOffset) - { - // Indexing should be performed with a signed value - could be negative. - oldMethodVarsSorted[(int32_t)varNumber] = *pOldVar; - } - } +#if 0 +// NOTE: +// THIS IS ONLY TRUE IF PROLOGSIZE DOES NOT INCLUDE REG-VAR INITIALIZATION !!!! +// + /* there is (potentially) only one additional + instruction in the prolog, (push ebp) + but if we would have been passed that instruction, + info->prologOffs would be hdrInfo::NOT_IN_PROLOG! + */ + _ASSERTE(offset == info->prologOffs); +#endif - // 3) Next sort the new var info by varNumber. We want to do this here, since - // we're allocating memory (which may fail) - do this before going to step 2 + pContext->SP = ESP; +} - // First, count the new vars the same way we did the old vars above. +/*****************************************************************************/ - const ICorDebugInfo::NativeVarInfo * pNewVar; +void UnwindEspFrame( + PREGDISPLAY pContext, + hdrInfo * info, + PTR_CBYTE table, + PTR_CBYTE methodStart, + DWORD curOffs, + unsigned flags) +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; - unsigned newNumVars = unsigned(-ICorDebugInfo::UNKNOWN_ILNUM); - for (pNewVar = newMethodVars, local = 0; - local < newMethodVarsCount; - local++, pNewVar++) + _ASSERTE(!info->ebpFrame && !info->doubleAlign); + _ASSERTE(info->epilogOffs == hdrInfo::NOT_IN_EPILOG); + + unsigned ESP = pContext->SP; + + + if (info->prologOffs != hdrInfo::NOT_IN_PROLOG) + { + if (info->prologOffs != 0) // Do nothing for the very start of the method { - DWORD varNumber = pNewVar->varNumber; - if (signed(varNumber) >= 0) - { - // This is an explicit (not special) var, so add its varNumber + 1 to our - // max count ("+1" because varNumber is zero-based). - newNumVars = max(newNumVars, unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) + varNumber + 1); - } + UnwindEspFrameProlog(pContext, info, methodStart, flags); + ESP = pContext->SP; } + } + else + { + /* we are past the prolog, ESP has been set above */ - // sorted by varNumber - newMethodVarsSortedBase = new (nothrow) ICorDebugInfo::NativeVarInfo[newNumVars]; - if (!newMethodVarsSortedBase) + // Are there any arguments pushed on the stack? + + ESP += GetPushedArgSize(info, table, curOffs); + + ESP += info->rawStkSize; + + const RegMask regsMask = info->savedRegMask; + + for (unsigned i = ARRAY_SIZE(CALLEE_SAVED_REGISTERS_MASK); i > 0; i--) { - hr = E_FAIL; - goto ErrExit; + RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i - 1]; + + if ((regMask & regsMask) == 0) + continue; + + SetLocation(pContext, i - 1, PTR_DWORD((TADDR)ESP)); + + ESP += sizeof(unsigned); } - newMethodVarsSorted = newMethodVarsSortedBase + (-ICorDebugInfo::UNKNOWN_ILNUM); + } - memset(newMethodVarsSortedBase, 0, newNumVars * sizeof(ICorDebugInfo::NativeVarInfo)); - for (local = 0; local < newNumVars;local++) - newMethodVarsSortedBase[local].loc.vlType = ICorDebugInfo::VLT_INVALID; + /* we can now set the (address of the) return address */ - DWORD newMethodOffset = pNewCodeInfo->GetRelOffset(); + pContext->PCTAddr = (TADDR)ESP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); + + /* Now adjust stack pointer */ + + pContext->SP = ESP + ESPIncrOnReturn(info); +} + + +/*****************************************************************************/ + +void UnwindEbpDoubleAlignFrameProlog( + PREGDISPLAY pContext, + hdrInfo * info, + PTR_CBYTE methodStart, + unsigned flags) +{ + LIMITED_METHOD_DAC_CONTRACT; + + _ASSERTE(info->prologOffs != hdrInfo::NOT_IN_PROLOG); + _ASSERTE(info->ebpFrame || info->doubleAlign); + + DWORD offset = 0; + +#ifdef _DEBUG + // If the first two instructions are 'nop, int3', then we will + // assume that is from a JitHalt operation and skip past it + if (methodStart[0] == X86_INSTR_NOP && methodStart[1] == X86_INSTR_INT3) + { + offset += 2; + } +#endif + + /* Check for the case where EBP has not been updated yet. */ + + const DWORD curOffs = info->prologOffs; + + // If we have still not excecuted "push ebp; mov ebp, esp", then we need to + // report the frame relative to ESP + + if (!InstructionAlreadyExecuted(offset + 1, curOffs)) + { + _ASSERTE(CheckInstrByte(methodStart [offset], X86_INSTR_PUSH_EBP) || + CheckInstrWord(*PTR_WORD(methodStart + offset), X86_INSTR_W_MOV_EBP_ESP) || + CheckInstrByte(methodStart [offset], X86_INSTR_JMP_NEAR_REL32)); // a rejit jmp-stamp + + /* If we're past the "push ebp", adjust ESP to pop EBP off */ + + if (curOffs == (offset + 1)) + pContext->SP += sizeof(TADDR); + + /* Stack pointer points to return address */ + + pContext->PCTAddr = (TADDR)pContext->SP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); + + /* EBP and callee-saved registers still have the correct value */ + + return; + } + + // We are atleast after the "push ebp; mov ebp, esp" + + offset = SKIP_MOV_REG_REG(methodStart, + SKIP_PUSH_REG(methodStart, offset)); + + /* At this point, EBP has been set up. The caller's ESP and the return value + can be determined using EBP. Since we are still in the prolog, + we need to know our exact location to determine the callee-saved registers */ + + const unsigned curEBP = GetRegdisplayFP(pContext); + + if (flags & UpdateAllRegs) + { + PTR_DWORD pSavedRegs = PTR_DWORD((TADDR)curEBP); + + /* make sure that we align ESP just like the method's prolog did */ + if (info->doubleAlign) + { + // "and esp,-8" + offset = SKIP_ARITH_REG(-8, methodStart, offset); + if (curEBP & 0x04) + { + pSavedRegs--; +#ifdef _DEBUG + if (dspPtr) printf("EnumRef: dblalign ebp: %08X\n", curEBP); +#endif + } + } + + /* Increment "offset" in steps to see which callee-saved + registers have been pushed already */ - for (pNewVar = newMethodVars, local = 0; - local < newMethodVarsCount; - local++, pNewVar++) + for (unsigned i = 0; i < STRING_LENGTH(CALLEE_SAVED_REGISTERS_MASK); i++) { - DWORD varNumber = pNewVar->varNumber; + RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i]; + _ASSERTE(regMask != RM_EBP); - _ASSERTE(varNumber + unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) < newNumVars); + if ((info->savedRegMask & regMask) == 0) + continue; - // Only care about new local variables alive at newMethodOffset - if (pNewVar->startOffset <= newMethodOffset && - pNewVar->endOffset > newMethodOffset) + if (InstructionAlreadyExecuted(offset, curOffs)) { - // Indexing should be performed with a signed valued - could be negative. - newMethodVarsSorted[(int32_t)varNumber] = *pNewVar; + SetLocation(pContext, i, PTR_DWORD(--pSavedRegs)); } + + // "push reg" + offset = SKIP_PUSH_REG(methodStart, offset) ; } - _ASSERTE(newNumVars >= oldNumVars || - !"Not allowed to reduce the number of locals between versions!"); + TRASH_CALLEE_UNSAVED_REGS(pContext); + } - LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: gathered info!\n")); + /* The caller's saved EBP is pointed to by our EBP */ - rgVal1 = new (nothrow) SIZE_T[newNumVars]; - if (rgVal1 == NULL) + pContext->SetEbpLocation(PTR_DWORD((TADDR)curEBP)); + pContext->SP = DWORD((TADDR)(curEBP + sizeof(void *))); + + /* Stack pointer points to return address */ + + pContext->PCTAddr = (TADDR)pContext->SP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); +} + +/*****************************************************************************/ + +bool UnwindEbpDoubleAlignFrame( + PREGDISPLAY pContext, + EECodeInfo *pCodeInfo, + hdrInfo *info, + PTR_CBYTE table, + PTR_CBYTE methodStart, + DWORD curOffs, + unsigned flags, + StackwalkCacheUnwindInfo *pUnwindInfo) // out-only, perf improvement +{ + LIMITED_METHOD_CONTRACT; + SUPPORTS_DAC; + + _ASSERTE(info->ebpFrame || info->doubleAlign); + + const unsigned curESP = pContext->SP; + const unsigned curEBP = GetRegdisplayFP(pContext); + + /* First check if we are in a filter (which is obviously after the prolog) */ + + if (info->handlers && info->prologOffs == hdrInfo::NOT_IN_PROLOG) + { + TADDR baseSP; + +#ifdef FEATURE_EH_FUNCLETS + // Funclets' frame pointers(EBP) are always restored so they can access to main function's local variables. + // Therefore the value of EBP is invalid for unwinder so we should use ESP instead. + // TODO If funclet frame layout is changed from CodeGen::genFuncletProlog() and genFuncletEpilog(), + // we need to change here accordingly. It is likely to have changes when introducing PSPSym. + // TODO Currently we assume that ESP of funclet frames is always fixed but actually it could change. + if (pCodeInfo->IsFunclet()) { - hr = E_FAIL; - goto ErrExit; + baseSP = curESP; + // Set baseSP as initial SP + baseSP += GetPushedArgSize(info, table, curOffs); + + // 16-byte stack alignment padding (allocated in genFuncletProlog) + // Current funclet frame layout (see CodeGen::genFuncletProlog() and genFuncletEpilog()): + // prolog: sub esp, 12 + // epilog: add esp, 12 + // ret + // SP alignment padding should be added for all instructions except the first one and the last one. + // Epilog may not exist (unreachable), so we need to check the instruction code. + const TADDR funcletStart = pCodeInfo->GetJitManager()->GetFuncletStartAddress(pCodeInfo); + if (funcletStart != pCodeInfo->GetCodeAddress() && methodStart[pCodeInfo->GetRelOffset()] != X86_INSTR_RETN) + baseSP += 12; + + pContext->PCTAddr = baseSP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); + + pContext->SP = (DWORD)(baseSP + sizeof(TADDR)); + + return true; } +#else // FEATURE_EH_FUNCLETS - rgVal2 = new (nothrow) SIZE_T[newNumVars]; - if (rgVal2 == NULL) + FrameType frameType = GetHandlerFrameInfo(info, curEBP, + curESP, (DWORD) IGNORE_VAL, + &baseSP); + + /* If we are in a filter, we only need to unwind the funclet stack. + For catches/finallies, the normal handling will + cause the frame to be unwound all the way up to ebp skipping + other frames above it. This is OK, as those frames will be + dead. Also, the EE will detect that this has happened and it + will handle any EE frames correctly. + */ + + if (frameType == FR_INVALID) { - hr = E_FAIL; - goto ErrExit; + return false; } - // 4) Next we'll zero them out, so any variables that aren't in scope - // in the old method, but are in scope in the new, will have the - // default, zero, value. + if (frameType == FR_FILTER) + { + pContext->PCTAddr = baseSP; + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); - memset(rgVal1, 0, sizeof(SIZE_T) * newNumVars); - memset(rgVal2, 0, sizeof(SIZE_T) * newNumVars); + pContext->SP = (DWORD)(baseSP + sizeof(TADDR)); - unsigned varsToGet = (oldNumVars > newNumVars) - ? newNumVars - : oldNumVars; + // pContext->pEbp = same as before; - // 2) Get all the info about current variables, registers, etc. +#ifdef _DEBUG + /* The filter has to be called by the VM. So we dont need to + update callee-saved registers. + */ - hr = g_pDebugInterface->GetVariablesFromOffset(pOldCodeInfo->GetMethodDesc(), - varsToGet, - oldMethodVarsSortedBase, - oldMethodOffset, - &oldCtx, - rgVal1, - rgVal2, - newNumVars, - &rgVCs); - if (FAILED(hr)) - { - goto ErrExit; + if (flags & UpdateAllRegs) + { + static DWORD s_badData = 0xDEADBEEF; + + pContext->SetEaxLocation(&s_badData); + pContext->SetEcxLocation(&s_badData); + pContext->SetEdxLocation(&s_badData); + + pContext->SetEbxLocation(&s_badData); + pContext->SetEsiLocation(&s_badData); + pContext->SetEdiLocation(&s_badData); + } +#endif + + if (pUnwindInfo) + { + // The filter funclet is like an ESP-framed-method. + pUnwindInfo->fUseEbp = FALSE; + pUnwindInfo->fUseEbpAsFrameReg = FALSE; + } + + return true; } +#endif // !FEATURE_EH_FUNCLETS + } + // + // Prolog of an EBP method + // - LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: got vars!\n")); + if (info->prologOffs != hdrInfo::NOT_IN_PROLOG) + { + UnwindEbpDoubleAlignFrameProlog(pContext, info, methodStart, flags); - /*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* - * IMPORTANT : Once we start munging on the context, we cannot return - * EnC_FAIL, as this should be a transacted commit, - **=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*/ + /* Now adjust stack pointer. */ -#if defined(TARGET_X86) - // Zero out all the registers as some may hold new variables. - pCtx->Eax = pCtx->Ecx = pCtx->Edx = pCtx->Ebx = pCtx->Esi = pCtx->Edi = 0; + pContext->SP += ESPIncrOnReturn(info); + return true; + } - // 3) zero out the stack frame - this'll initialize _all_ variables + if (flags & UpdateAllRegs) + { + // Get to the first callee-saved register + PTR_DWORD pSavedRegs = PTR_DWORD((TADDR)curEBP); - /*------------------------------------------------------------------------- - * Adjust the stack height - */ - pCtx->Esp -= (newInfo.stackSize - oldInfo.stackSize); + if (info->doubleAlign && (curEBP & 0x04)) + pSavedRegs--; - // Zero-init the local and tempory section of new stack frame being careful to avoid - // touching anything in the frame header. - // This is necessary to ensure that any JIT temporaries in the old version can't be mistaken - // for ObjRefs now. - size_t frameHeaderSize = GetSizeOfFrameHeaderForEnC( &newInfo ); - _ASSERTE( frameHeaderSize <= oldInfo.stackSize ); - _ASSERTE( GetSizeOfFrameHeaderForEnC( &oldInfo ) == frameHeaderSize ); + for (unsigned i = 0; i < STRING_LENGTH(CALLEE_SAVED_REGISTERS_MASK); i++) + { + RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i]; + if ((info->savedRegMask & regMask) == 0) + continue; -#elif defined(TARGET_AMD64) && !defined(UNIX_AMD64_ABI) + SetLocation(pContext, i, --pSavedRegs); + } + } - // Next few statements zero out all registers that may end up holding new variables. + /* The caller's ESP will be equal to EBP + retAddrSize + argSize. */ - // volatile int registers (JIT may use these to enregister variables) - pCtx->Rax = pCtx->Rcx = pCtx->Rdx = pCtx->R8 = pCtx->R9 = pCtx->R10 = pCtx->R11 = 0; + pContext->SP = (DWORD)(curEBP + sizeof(curEBP) + ESPIncrOnReturn(info)); - // volatile float registers - pCtx->Xmm1.High = pCtx->Xmm1.Low = 0; - pCtx->Xmm2.High = pCtx->Xmm2.Low = 0; - pCtx->Xmm3.High = pCtx->Xmm3.Low = 0; - pCtx->Xmm4.High = pCtx->Xmm4.Low = 0; - pCtx->Xmm5.High = pCtx->Xmm5.Low = 0; + /* The caller's saved EIP is right after our EBP */ - // 3) zero out the stack frame - this'll initialize _all_ variables + pContext->PCTAddr = (TADDR)curEBP + RETURN_ADDR_OFFS * sizeof(TADDR); + pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); - /*------------------------------------------------------------------------- - * Adjust the stack height - */ + /* The caller's saved EBP is pointed to by our EBP */ - TADDR newStackBase = callerSP - newFixedStackSize; + pContext->SetEbpLocation(PTR_DWORD((TADDR)curEBP)); + return true; +} - SetSP(pCtx, newStackBase); +bool UnwindStackFrame(PREGDISPLAY pContext, + EECodeInfo *pCodeInfo, + unsigned flags, + CodeManState *pState, + StackwalkCacheUnwindInfo *pUnwindInfo /* out-only, perf improvement */) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + HOST_NOCALLS; + SUPPORTS_DAC; + } CONTRACTL_END; - // We want to zero-out everything pushed after the frame header. This way we'll zero - // out locals (both old & new) and temporaries. This is necessary to ensure that any - // JIT temporaries in the old version can't be mistaken for ObjRefs now. (I am told - // this last point is less of an issue on x64 as it is on x86, but zeroing out the - // temporaries is still the cleanest, most robust way to go.) - size_t frameHeaderSize = newSizeOfPreservedArea; - _ASSERTE(frameHeaderSize <= oldFixedStackSize); - _ASSERTE(frameHeaderSize <= newFixedStackSize); + // Address where the method has been interrupted + PCODE breakPC = pContext->ControlPC; + _ASSERTE(PCODEToPINSTR(breakPC) == pCodeInfo->GetCodeAddress()); - // For EnC-compliant x64 code, FP == SP. Since SP changed above, update FP now - pCtx->Rbp = newStackBase; + PTR_CBYTE methodStart = PTR_CBYTE(pCodeInfo->GetSavedMethodCode()); -#else -#if defined(TARGET_ARM64) - // Zero out volatile part of stack frame - // x0-x17 - memset(&pCtx->X[0], 0, sizeof(pCtx->X[0]) * 18); - // v0-v7 - memset(&pCtx->V[0], 0, sizeof(pCtx->V[0]) * 8); - // v16-v31 - memset(&pCtx->V[16], 0, sizeof(pCtx->V[0]) * 16); -#elif defined(TARGET_AMD64) - // SysV ABI - pCtx->Rax = pCtx->Rdi = pCtx->Rsi = pCtx->Rdx = pCtx->Rcx = pCtx->R8 = pCtx->R9 = 0; + GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); + PTR_VOID methodInfoPtr = gcInfoToken.Info; + DWORD curOffs = pCodeInfo->GetRelOffset(); - // volatile float registers - memset(&pCtx->Xmm0, 0, sizeof(pCtx->Xmm0) * 16); -#else - PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); -#endif + _ASSERTE(sizeof(CodeManStateBuf) <= sizeof(pState->stateBuf)); + CodeManStateBuf * stateBuf = (CodeManStateBuf*)pState->stateBuf; - TADDR newStackBase = callerSP - newFixedStackSize; + if (pState->dwIsSet == 0) + { + /* Extract the necessary information from the info block header */ - SetSP(pCtx, newStackBase); + stateBuf->hdrInfoSize = (DWORD)DecodeGCHdrInfo(gcInfoToken, + curOffs, + &stateBuf->hdrInfoBody); + } - size_t frameHeaderSize = newSizeOfPreservedArea; - _ASSERTE(frameHeaderSize <= oldFixedStackSize); - _ASSERTE(frameHeaderSize <= newFixedStackSize); + PTR_CBYTE table = dac_cast(methodInfoPtr) + stateBuf->hdrInfoSize; - // EnC prolog saves only FP (and LR on arm64), and FP points to saved FP for frame chaining. - // These should already be set up from previous version. - _ASSERTE(GetFP(pCtx) == callerSP - 16); -#endif + hdrInfo * info = &stateBuf->hdrInfoBody; - // Perform some debug-only sanity checks on stack variables. Some checks are - // performed differently between X86/AMD64. + info->isSpeculativeStackWalk = ((flags & SpeculativeStackwalk) != 0); -#ifdef _DEBUG - for( unsigned i = 0; i < newNumVars; i++ ) - { - // Make sure that stack variables existing in both old and new methods did not - // move. This matters if the address of a local is used in the remapped method. - // For example: - // - // static unsafe void Main(string[] args) - // { - // int x; - // int* p = &x; - // <- Edit made here - cannot move address of x - // *p = 5; - // } - // - if ((i + unsigned(-ICorDebugInfo::UNKNOWN_ILNUM) < oldNumVars) && // Does variable exist in old method? - (oldMethodVarsSorted[i].loc.vlType == ICorDebugInfo::VLT_STK) && // Is the variable on the stack? - (newMethodVarsSorted[i].loc.vlType == ICorDebugInfo::VLT_STK)) - { - SIZE_T * pOldVarStackLocation = NativeVarStackAddr(oldMethodVarsSorted[i].loc, &oldCtx); - SIZE_T * pNewVarStackLocation = NativeVarStackAddr(newMethodVarsSorted[i].loc, pCtx); - _ASSERTE(pOldVarStackLocation == pNewVarStackLocation); - } + if (pUnwindInfo != NULL) + { + pUnwindInfo->fUseEbpAsFrameReg = info->ebpFrame; + pUnwindInfo->fUseEbp = ((info->savedRegMask & RM_EBP) != 0); + } - // Sanity-check that the range we're clearing contains all of the stack variables + if (info->epilogOffs != hdrInfo::NOT_IN_EPILOG) + { + /*--------------------------------------------------------------------- + * First, handle the epilog + */ -#if defined(TARGET_X86) - const ICorDebugInfo::VarLoc &varLoc = newMethodVarsSortedBase[i].loc; - if( varLoc.vlType == ICorDebugInfo::VLT_STK ) - { - // This is an EBP frame, all stack variables should be EBP relative - _ASSERTE( varLoc.vlStk.vlsBaseReg == ICorDebugInfo::REGNUM_EBP ); - // Generic special args may show up as locals with positive offset from EBP, so skip them - if( varLoc.vlStk.vlsOffset <= 0 ) - { - // Normal locals must occur after the header on the stack - _ASSERTE( unsigned(-varLoc.vlStk.vlsOffset) >= frameHeaderSize ); - // Value must occur before the top of the stack - _ASSERTE( unsigned(-varLoc.vlStk.vlsOffset) < newInfo.stackSize ); - } + PTR_CBYTE epilogBase = methodStart + (curOffs - info->epilogOffs); + UnwindEpilog(pContext, info, epilogBase, flags); + } + else if (!info->ebpFrame && !info->doubleAlign) + { + /*--------------------------------------------------------------------- + * Now handle ESP frames + */ - // Ideally we'd like to verify that the stack locals (if any) start at exactly the end - // of the header. However, we can't easily determine the size of value classes here, - // and so (since the stack grows towards 0) can't easily determine where the end of - // the local lies. - } -#elif defined(TARGET_AMD64) || defined(TARGET_ARM64) - switch(newMethodVarsSortedBase[i].loc.vlType) - { - default: - // No validation here for non-stack locals - break; + UnwindEspFrame(pContext, info, table, methodStart, curOffs, flags); + return true; + } + else + { + /*--------------------------------------------------------------------- + * Now we know that have an EBP frame + */ + + if (!UnwindEbpDoubleAlignFrame(pContext, pCodeInfo, info, table, methodStart, curOffs, flags, pUnwindInfo)) + return false; + } + + // TODO [DAVBR]: For the full fix for VsWhidbey 450273, all the below + // may be uncommented once isLegalManagedCodeCaller works properly + // with non-return address inputs, and with non-DEBUG builds + /* + // Ensure isLegalManagedCodeCaller succeeds for speculative stackwalks. + // (We just assert this below for non-speculative stackwalks.) + // + FAIL_IF_SPECULATIVE_WALK(isLegalManagedCodeCaller(GetControlPC(pContext))); + */ - case ICorDebugInfo::VLT_STK_BYREF: - { - // For byrefs, verify that the ptr will be zeroed out + return true; +} - SIZE_T regOffs = GetRegOffsInCONTEXT(newMethodVarsSortedBase[i].loc.vlStk.vlsBaseReg); - TADDR baseReg = *(TADDR *)(regOffs + (BYTE*)pCtx); - TADDR addrOfPtr = baseReg + newMethodVarsSortedBase[i].loc.vlStk.vlsOffset; +#endif // TARGET_X86 - _ASSERTE( - // The ref must exist in the portion we'll zero-out - ( - (newStackBase <= addrOfPtr) && - (addrOfPtr < newStackBase + (newFixedStackSize - frameHeaderSize)) - ) || - // OR in the caller's frame (for parameters) - (addrOfPtr >= newStackBase + newFixedStackSize)); +#ifdef FEATURE_EH_FUNCLETS +#ifdef TARGET_X86 +size_t EECodeManager::GetResumeSp( PCONTEXT pContext ) +{ + PCODE currentPc = PCODE(pContext->Eip); - // Deliberately fall through, so that we also verify that the value that the ptr - // points to will be zeroed out - // ... - } - __fallthrough; + _ASSERTE(ExecutionManager::IsManagedCode(currentPc)); - case ICorDebugInfo::VLT_STK: - case ICorDebugInfo::VLT_STK2: - case ICorDebugInfo::VLT_REG_STK: - case ICorDebugInfo::VLT_STK_REG: - SIZE_T * pVarStackLocation = NativeVarStackAddr(newMethodVarsSortedBase[i].loc, pCtx); - _ASSERTE (pVarStackLocation != NULL); - _ASSERTE( - // The value must exist in the portion we'll zero-out - ( - (newStackBase <= (TADDR) pVarStackLocation) && - ((TADDR) pVarStackLocation < newStackBase + (newFixedStackSize - frameHeaderSize)) - ) || - // OR in the caller's frame (for parameters) - ((TADDR) pVarStackLocation >= newStackBase + newFixedStackSize)); - break; - } -#else // !X86, !X64, !ARM64 - PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); -#endif - } + EECodeInfo codeInfo(currentPc); -#endif // _DEBUG + PTR_CBYTE methodStart = PTR_CBYTE(codeInfo.GetSavedMethodCode()); - // Clear the local and temporary stack space + GCInfoToken gcInfoToken = codeInfo.GetGCInfoToken(); + PTR_VOID methodInfoPtr = gcInfoToken.Info; + DWORD curOffs = codeInfo.GetRelOffset(); -#if defined(TARGET_X86) - memset((void*)(size_t)(pCtx->Esp), 0, newInfo.stackSize - frameHeaderSize ); -#elif defined(TARGET_AMD64) || defined(TARGET_ARM64) - memset((void*)newStackBase, 0, newFixedStackSize - frameHeaderSize); + CodeManStateBuf stateBuf; - // Restore PSPSym for the new function. Its value should be set to our new FP. But - // first, we gotta find PSPSym's location on the stack - INT32 nNewPspSymStackSlot = newGcDecoder.GetPSPSymStackSlot(); - if (nNewPspSymStackSlot != NO_PSP_SYM) - { -#if defined(TARGET_AMD64) - *PTR_TADDR(newStackBase + nNewPspSymStackSlot) = newStackBase; -#elif defined(TARGET_ARM64) - *PTR_TADDR(callerSP + nNewPspSymStackSlot) = callerSP; -#else - PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); -#endif - } -#else // !X86, !X64, !ARM64 - PORTABILITY_ASSERT("Edit-and-continue not enabled on this platform."); -#endif + stateBuf.hdrInfoSize = (DWORD)DecodeGCHdrInfo(gcInfoToken, + curOffs, + &stateBuf.hdrInfoBody); - // 4) Put the variables from step 3 into their new locations. + PTR_CBYTE table = dac_cast(methodInfoPtr) + stateBuf.hdrInfoSize; - LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: set vars!\n")); + hdrInfo *info = &stateBuf.hdrInfoBody; - // Move the old variables into their new places. + _ASSERTE(info->epilogOffs == hdrInfo::NOT_IN_EPILOG && info->prologOffs == hdrInfo::NOT_IN_PROLOG); - hr = g_pDebugInterface->SetVariablesAtOffset(pNewCodeInfo->GetMethodDesc(), - newNumVars, - newMethodVarsSortedBase, - newMethodOffset, - pCtx, // place them into the new context - rgVal1, - rgVal2, - rgVCs); + bool isESPFrame = !info->ebpFrame && !info->doubleAlign; - /*-----------------------------------------------------------------------*/ + if (codeInfo.IsFunclet()) + { + // Treat funclet's frame as ESP frame + isESPFrame = true; } -ErrExit: - if (oldMethodVarsSortedBase) - delete[] oldMethodVarsSortedBase; - if (newMethodVarsSortedBase) - delete[] newMethodVarsSortedBase; - if (rgVal1 != NULL) - delete[] rgVal1; - if (rgVal2 != NULL) - delete[] rgVal2; - LOG((LF_ENC, LL_INFO100, "EECM::FixContextForEnC: exiting!\n")); + if (isESPFrame) + { + const size_t curESP = (size_t)(pContext->Esp); + return curESP + GetPushedArgSize(info, table, curOffs); + } - return hr; + const size_t curEBP = (size_t)(pContext->Ebp); + return GetOutermostBaseFP(curEBP, info); } -#endif // !FEATURE_METADATA_UPDATER +#endif // TARGET_X86 +#endif // FEATURE_EH_FUNCLETS -#endif // #ifndef DACCESS_COMPILE +#ifndef FEATURE_EH_FUNCLETS -#ifdef USE_GC_INFO_DECODER /***************************************************************************** * - * Is the function currently at a "GC safe point" ? + * Unwind the current stack frame, i.e. update the virtual register + * set in pContext. This will be similar to the state after the function + * returns back to caller (IP points to after the call, Frame and Stack + * pointer has been reset, callee-saved registers restored (if UpdateAllRegs), + * callee-unsaved registers are trashed. + * Returns success of operation. */ -bool EECodeManager::IsGcSafe( EECodeInfo *pCodeInfo, - DWORD dwRelOffset) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - } CONTRACTL_END; - - GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); - - GcInfoDecoder gcInfoDecoder( - gcInfoToken, - DECODE_INTERRUPTIBILITY, - dwRelOffset - ); - return gcInfoDecoder.IsInterruptible(); +bool EECodeManager::UnwindStackFrame(PREGDISPLAY pContext, + EECodeInfo *pCodeInfo, + unsigned flags, + CodeManState *pState, + StackwalkCacheUnwindInfo *pUnwindInfo /* out-only, perf improvement */) +{ +#ifdef TARGET_X86 + return ::UnwindStackFrame(pContext, pCodeInfo, flags, pState, pUnwindInfo); +#else // TARGET_X86 + PORTABILITY_ASSERT("EECodeManager::UnwindStackFrame"); + return false; +#endif // _TARGET_???_ } -#if defined(TARGET_ARM) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64) -bool EECodeManager::HasTailCalls( EECodeInfo *pCodeInfo) +/*****************************************************************************/ +#else // !FEATURE_EH_FUNCLETS +/*****************************************************************************/ + +bool EECodeManager::UnwindStackFrame(PREGDISPLAY pContext, + EECodeInfo *pCodeInfo, + unsigned flags, + CodeManState *pState, + StackwalkCacheUnwindInfo *pUnwindInfo /* out-only, perf improvement */) { CONTRACTL { NOTHROW; GC_NOTRIGGER; } CONTRACTL_END; - GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); - - GcInfoDecoder gcInfoDecoder( - gcInfoToken, - DECODE_HAS_TAILCALLS, - 0 - ); +#if defined(TARGET_AMD64) + // To avoid unnecessary computation, we only crack the unwind info if pUnwindInfo is not NULL, which only happens + // if the LIGHTUNWIND flag is passed to StackWalkFramesEx(). + if (pUnwindInfo != NULL) + { + pCodeInfo->GetOffsetsFromUnwindInfo(&(pUnwindInfo->RSPOffsetFromUnwindInfo), + &(pUnwindInfo->RBPOffset)); + } +#endif // TARGET_AMD64 - return gcInfoDecoder.HasTailCalls(); + _ASSERTE(pCodeInfo != NULL); + Thread::VirtualUnwindCallFrame(pContext, pCodeInfo); + return true; } -#endif // TARGET_ARM || TARGET_ARM64 || TARGET_LOONGARCH64 || TARGET_RISCV64 -#if defined(TARGET_AMD64) && defined(_DEBUG) +/*****************************************************************************/ +#endif // FEATURE_EH_FUNCLETS -struct FindEndOfLastInterruptibleRegionState -{ - unsigned curOffset; - unsigned endOffset; - unsigned lastRangeOffset; -}; +/*****************************************************************************/ -bool FindEndOfLastInterruptibleRegionCB ( - UINT32 startOffset, - UINT32 stopOffset, - LPVOID hCallback) +/* report args in 'msig' to the GC. + 'argsStart' is start of the stack-based arguments + 'varArgSig' describes the arguments + 'ctx' has the GC reporting info +*/ +void promoteVarArgs(PTR_BYTE argsStart, PTR_VASigCookie varArgSig, GCCONTEXT* ctx) { - FindEndOfLastInterruptibleRegionState *pState = (FindEndOfLastInterruptibleRegionState*)hCallback; + WRAPPER_NO_CONTRACT; - // - // If the current range doesn't overlap the given range, keep searching. - // - if ( startOffset >= pState->endOffset - || stopOffset < pState->curOffset) - { - return false; - } + //Note: no instantiations needed for varargs + MetaSig msig(varArgSig->signature, + varArgSig->pModule, + NULL); - // - // If the range overlaps the end, then the last point is the end. - // - if ( stopOffset > pState->endOffset - /*&& startOffset < pState->endOffset*/) - { - // The ranges should be sorted in increasing order. - CONSISTENCY_CHECK(startOffset >= pState->lastRangeOffset); + PTR_BYTE pFrameBase = argsStart - TransitionBlock::GetOffsetOfArgs(); - pState->lastRangeOffset = pState->endOffset; - return true; - } + ArgIterator argit(&msig); - // - // See if the end of this range is the closet to the end that we've found - // so far. - // - if (stopOffset > pState->lastRangeOffset) - pState->lastRangeOffset = stopOffset; +#ifdef TARGET_X86 + // For the X86 target the JIT does not report any of the fixed args for a varargs method + // So we report the fixed args via the promoteArgs call below + bool skipFixedArgs = false; +#else + // For other platforms the JITs do report the fixed args of a varargs method + // So we must tell promoteArgs to skip to the end of the fixed args + bool skipFixedArgs = true; +#endif - return false; + bool inVarArgs = false; + + int argOffset; + while ((argOffset = argit.GetNextOffset()) != TransitionBlock::InvalidOffset) + { + if (msig.GetArgProps().AtSentinel()) + inVarArgs = true; + + // if skipFixedArgs is false we report all arguments + // otherwise we just report the varargs. + if (!skipFixedArgs || inVarArgs) + { + ArgDestination argDest(pFrameBase, argOffset, argit.GetArgLocDescForStructInRegs()); + msig.GcScanRoots(&argDest, ctx->f, ctx->sc); + } + } } -/* - Locates the end of the last interruptible region in the given code range. - Returns 0 if the entire range is uninterruptible. Returns the end point - if the entire range is interruptible. -*/ -unsigned EECodeManager::FindEndOfLastInterruptibleRegion(unsigned curOffset, - unsigned endOffset, - GCInfoToken gcInfoToken) -{ #ifndef DACCESS_COMPILE - GcInfoDecoder gcInfoDecoder( - gcInfoToken, - DECODE_FOR_RANGES_CALLBACK - ); - - FindEndOfLastInterruptibleRegionState state; - state.curOffset = curOffset; - state.endOffset = endOffset; - state.lastRangeOffset = 0; - - gcInfoDecoder.EnumerateInterruptibleRanges(&FindEndOfLastInterruptibleRegionCB, &state); +FCIMPL1(void, GCReporting::Register, GCFrame* frame) +{ + FCALL_CONTRACT; - return state.lastRangeOffset; -#else - DacNotImpl(); - return NULL; -#endif // #ifndef DACCESS_COMPILE + // Construct a GCFrame. + _ASSERTE(frame != NULL); + frame->Push(GetThread()); } +FCIMPLEND -#endif // TARGET_AMD64 && _DEBUG +FCIMPL1(void, GCReporting::Unregister, GCFrame* frame) +{ + FCALL_CONTRACT; + // Destroy the GCFrame. + _ASSERTE(frame != NULL); + frame->Remove(); +} +FCIMPLEND +#endif // !DACCESS_COMPILE -#else // !USE_GC_INFO_DECODER +#ifndef USE_GC_INFO_DECODER /***************************************************************************** * - * Is the function currently at a "GC safe point" ? + * Enumerate all live object references in that function using + * the virtual register set. + * Returns success of operation. */ -bool EECodeManager::IsGcSafe( EECodeInfo *pCodeInfo, - DWORD dwRelOffset) + +bool EECodeManager::EnumGcRefs( PREGDISPLAY pContext, + EECodeInfo *pCodeInfo, + unsigned flags, + GCEnumCallback pCallBack, + LPVOID hCallBack, + DWORD relOffsetOverride) { CONTRACTL { NOTHROW; GC_NOTRIGGER; - SUPPORTS_DAC; } CONTRACTL_END; - hdrInfo info; - BYTE * table; +#ifdef FEATURE_EH_FUNCLETS + if (flags & ParentOfFuncletStackFrame) + { + LOG((LF_GCROOTS, LL_INFO100000, "Not reporting this frame because it was already reported via another funclet.\n")); + return true; + } +#endif // FEATURE_EH_FUNCLETS - /* Extract the necessary information from the info block header */ + GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); + unsigned curOffs = pCodeInfo->GetRelOffset(); - table = (BYTE *)DecodeGCHdrInfo(pCodeInfo->GetGCInfoToken(), - dwRelOffset, - &info); + unsigned EBP = GetRegdisplayFP(pContext); + unsigned ESP = pContext->SP; - /* workaround: prevent interruption within prolog/epilog */ + unsigned ptrOffs; - if (info.prologOffs != hdrInfo::NOT_IN_PROLOG || info.epilogOffs != hdrInfo::NOT_IN_EPILOG) - return false; + unsigned count; -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xBEEF); + hdrInfo info; + PTR_CBYTE table = PTR_CBYTE(gcInfoToken.Info); +#if 0 + printf("EECodeManager::EnumGcRefs - EIP = %08x ESP = %08x offset = %x GC Info is at %08x\n", *pContext->pPC, ESP, curOffs, table); #endif - return (info.interruptible); -} -#endif // !USE_GC_INFO_DECODER + /* Extract the necessary information from the info block header */ + table += DecodeGCHdrInfo(gcInfoToken, + curOffs, + &info); -#if defined(FEATURE_EH_FUNCLETS) + _ASSERTE( curOffs <= info.methodSize); -void EECodeManager::EnsureCallerContextIsValid( PREGDISPLAY pRD, EECodeInfo * pCodeInfo /*= NULL*/, unsigned flags /*= 0*/) -{ - CONTRACTL - { - NOTHROW; - GC_NOTRIGGER; - SUPPORTS_DAC; +#ifdef _DEBUG +// if ((gcInfoToken.Info == (void*)0x37760d0) && (curOffs == 0x264)) +// __asm int 3; + + if (trEnumGCRefs) { + static unsigned lastESP = 0; + unsigned diffESP = ESP - lastESP; + if (diffESP > 0xFFFF) { + printf("------------------------------------------------------\n"); + } + lastESP = ESP; + printf("EnumGCRefs [%s][%s] at %s.%s + 0x%03X:\n", + info.ebpFrame?"ebp":" ", + info.interruptible?"int":" ", + "UnknownClass","UnknownMethod", curOffs); + fflush(stdout); } - CONTRACTL_END; +#endif - if( !pRD->IsCallerContextValid ) + /* Are we in the prolog or epilog of the method? */ + + if (info.prologOffs != hdrInfo::NOT_IN_PROLOG || + info.epilogOffs != hdrInfo::NOT_IN_EPILOG) { - if ((flags & LightUnwind) && (pCodeInfo != NULL)) - { -#if !defined(DACCESS_COMPILE) && defined(HAS_LIGHTUNWIND) - LightUnwindStackFrame(pRD, pCodeInfo, EnsureCallerStackFrameIsValid); -#else - // We need to make a copy here (instead of switching the pointers), in order to preserve the current context - *(pRD->pCallerContext) = *(pRD->pCurrentContext); - // Skip updating context registers for light unwind - Thread::VirtualUnwindCallFrame(pRD->pCallerContext, NULL, pCodeInfo); + +#if !DUMP_PTR_REFS + // Under normal circumstances the system will not suspend a thread + // if it is in the prolog or epilog of the function. However ThreadAbort + // exception or stack overflows can cause EH to happen in a prolog. + // Once in the handler, a GC can happen, so we can get to this code path. + // However since we are tearing down this frame, we don't need to report + // anything and we can simply return. + + _ASSERTE(flags & ExecutionAborted); #endif + return true; + } + +#ifdef _DEBUG +#define CHK_AND_REPORT_REG(reg, doIt, iptr, regName) \ + if (doIt) \ + { \ + if (dspPtr) \ + printf(" Live pointer register %s: ", #regName); \ + pCallBack(hCallBack, \ + (OBJECTREF*)(pContext->Get##regName##Location()), \ + (iptr ? GC_CALL_INTERIOR : 0) \ + | CHECK_APP_DOMAIN \ + DAC_ARG(DacSlotLocation(reg, 0, false))); \ } - else - { - // We need to make a copy here (instead of switching the pointers), in order to preserve the current context - *(pRD->pCallerContext) = *(pRD->pCurrentContext); - *(pRD->pCallerContextPointers) = *(pRD->pCurrentContextPointers); - Thread::VirtualUnwindCallFrame(pRD->pCallerContext, pRD->pCallerContextPointers, pCodeInfo); - } +#else // !_DEBUG +#define CHK_AND_REPORT_REG(reg, doIt, iptr, regName) \ + if (doIt) \ + pCallBack(hCallBack, \ + (OBJECTREF*)(pContext->Get##regName##Location()), \ + (iptr ? GC_CALL_INTERIOR : 0) \ + | CHECK_APP_DOMAIN \ + DAC_ARG(DacSlotLocation(reg, 0, false))); - pRD->IsCallerContextValid = TRUE; - } +#endif // _DEBUG - _ASSERTE( pRD->IsCallerContextValid ); -} + /* What kind of a frame is this ? */ -size_t EECodeManager::GetCallerSp( PREGDISPLAY pRD ) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - SUPPORTS_DAC; - } CONTRACTL_END; + FrameType frameType = FR_NORMAL; + TADDR baseSP = 0; - // Don't add usage of this field. This is only temporary. - // See ExceptionTracker::InitializeCrawlFrame() for more information. - if (!pRD->IsCallerSPValid) + if (info.handlers) { - EnsureCallerContextIsValid(pRD, NULL); - } + _ASSERTE(info.ebpFrame); - return GetSP(pRD->pCallerContext); -} + bool hasInnerFilter, hadInnerFilter; + frameType = GetHandlerFrameInfo(&info, EBP, + ESP, (DWORD) IGNORE_VAL, + &baseSP, NULL, + &hasInnerFilter, &hadInnerFilter); + _ASSERTE(frameType != FR_INVALID); -#endif // FEATURE_EH_FUNCLETS + /* If this is the parent frame of a filter which is currently + executing, then the filter would have enumerated the frame using + the filter PC. + */ -#ifdef HAS_LIGHTUNWIND -/* - * Light unwind the current stack frame, using provided cache entry. - * pPC, Esp and pEbp of pContext are updated. - */ + if (hasInnerFilter) + return true; -// static -void EECodeManager::LightUnwindStackFrame(PREGDISPLAY pRD, EECodeInfo* pCodeInfo, LightUnwindFlag flag) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - } CONTRACTL_END; + /* If are in a try and we had a filter execute, we may have reported + GC refs from the filter (and not using the try's offset). So + we had better use the filter's end offset, as the try is + effectively dead and its GC ref's would be stale */ -#ifdef TARGET_AMD64 - ULONG RBPOffset, RSPOffset; - pCodeInfo->GetOffsetsFromUnwindInfo(&RSPOffset, &RBPOffset); + if (hadInnerFilter) + { + PTR_TADDR pFirstBaseSPslot = GetFirstBaseSPslotPtr(EBP, &info); + curOffs = (unsigned)pFirstBaseSPslot[1] - 1; + _ASSERTE(curOffs < info.methodSize); - if (pRD->IsCallerContextValid) - { - pRD->pCurrentContext->Rbp = pRD->pCallerContext->Rbp; - pRD->pCurrentContext->Rsp = pRD->pCallerContext->Rsp; - pRD->pCurrentContext->Rip = pRD->pCallerContext->Rip; + /* Extract the necessary information from the info block header */ + + table = PTR_CBYTE(gcInfoToken.Info); + + table += DecodeGCHdrInfo(gcInfoToken, + curOffs, + &info); + } } - else + + bool willContinueExecution = !(flags & ExecutionAborted); + unsigned pushedSize = 0; + + /* if we have been interrupted we don't have to report registers/arguments + * because we are about to lose this context anyway. + * Alas, if we are in a ebp-less method we have to parse the table + * in order to adjust ESP. + * + * Note that we report "this" for all methods, even if + * noncontinuable, because of the off chance they may be + * synchronized and we have to release the monitor on unwind. This + * could conceivably be optimized, but it turns out to be more + * expensive to check whether we're synchronized (which involves + * consulting metadata) than to just report "this" all the time in + * our most important scenarios. + */ + + if (info.interruptible) { - PCONTEXT pSourceCtx = NULL; - PCONTEXT pTargetCtx = NULL; - if (flag == UnwindCurrentStackFrame) - { - pTargetCtx = pRD->pCurrentContext; - pSourceCtx = pRD->pCurrentContext; - } - else + unsigned curOffsRegs = curOffs; + + // Don't decrement curOffsRegs when it is 0, as it is an unsigned and will wrap to MAX_UINT + // + if (curOffsRegs > 0) { - pTargetCtx = pRD->pCallerContext; - pSourceCtx = pRD->pCurrentContext; + // If we are not on the active stack frame, we need to report gc registers + // that are live before the call. The reason is that the liveness of gc registers + // may change across a call to a method that does not return. In this case the instruction + // after the call may be a jump target and a register that didn't have a live gc pointer + // before the call may have a live gc pointer after the jump. To make sure we report the + // registers that have live gc pointers before the call we subtract 1 from curOffs. + if ((flags & ActiveStackFrame) == 0) + { + // We are not the top most stack frame (i.e. the ActiveStackFrame) + curOffsRegs--; // decrement curOffsRegs + } } - // Unwind RBP. The offset is relative to the current sp. - if (RBPOffset == 0) + pushedSize = scanArgRegTableI(skipToArgReg(info, table), curOffsRegs, curOffs, &info); + + RegMask regs = info.regMaskResult; + RegMask iregs = info.iregMaskResult; + ptrArgTP args = info.argMaskResult; + ptrArgTP iargs = info.iargMaskResult; + + _ASSERTE((isZero(args) || pushedSize != 0) || info.ebpFrame); + _ASSERTE((args & iargs) == iargs); + // Only synchronized methods and generic code that accesses + // the type context via "this" need to report "this". + // If its reported for other methods, its probably + // done incorrectly. So flag such cases. + _ASSERTE(info.thisPtrResult == REGI_NA || + pCodeInfo->GetMethodDesc()->IsSynchronized() || + pCodeInfo->GetMethodDesc()->AcquiresInstMethodTableFromThis()); + + /* now report registers and arguments if we are not interrupted */ + + if (willContinueExecution) { - pTargetCtx->Rbp = pSourceCtx->Rbp; + + /* Propagate unsafed registers only in "current" method */ + /* If this is not the active method, then the callee wil + * trash these registers, and so we wont need to report them */ + + if (flags & ActiveStackFrame) + { + CHK_AND_REPORT_REG(REGI_EAX, regs & RM_EAX, iregs & RM_EAX, Eax); + CHK_AND_REPORT_REG(REGI_ECX, regs & RM_ECX, iregs & RM_ECX, Ecx); + CHK_AND_REPORT_REG(REGI_EDX, regs & RM_EDX, iregs & RM_EDX, Edx); + } + + CHK_AND_REPORT_REG(REGI_EBX, regs & RM_EBX, iregs & RM_EBX, Ebx); + CHK_AND_REPORT_REG(REGI_EBP, regs & RM_EBP, iregs & RM_EBP, Ebp); + CHK_AND_REPORT_REG(REGI_ESI, regs & RM_ESI, iregs & RM_ESI, Esi); + CHK_AND_REPORT_REG(REGI_EDI, regs & RM_EDI, iregs & RM_EDI, Edi); + _ASSERTE(!(regs & RM_ESP)); + + /* Report any pending pointer arguments */ + + DWORD * pPendingArgFirst; // points **AT** first parameter + if (!info.ebpFrame) + { + // -sizeof(void*) because we want to point *AT* first parameter + pPendingArgFirst = (DWORD *)(size_t)(ESP + pushedSize - sizeof(void*)); + } + else + { + _ASSERTE(willContinueExecution); + + if (info.handlers) + { + // -sizeof(void*) because we want to point *AT* first parameter + pPendingArgFirst = (DWORD *)(size_t)(baseSP - sizeof(void*)); + } + else if (info.localloc) + { + baseSP = *(DWORD *)(size_t)(EBP - GetLocallocSPOffset(&info)); + // -sizeof(void*) because we want to point *AT* first parameter + pPendingArgFirst = (DWORD *)(size_t) (baseSP - sizeof(void*)); + } + else + { + // Note that 'info.stackSize includes the size for pushing EBP, but EBP is pushed + // BEFORE EBP is set from ESP, thus (EBP - info.stackSize) actually points past + // the frame by one DWORD, and thus points *AT* the first parameter + + pPendingArgFirst = (DWORD *)(size_t)(EBP - info.stackSize); + } + } + + if (!isZero(args)) + { + unsigned i = 0; + ptrArgTP b(1); + for (; !isZero(args) && (i < MAX_PTRARG_OFS); i += 1, b <<= 1) + { + if (intersect(args,b)) + { + unsigned argAddr = (unsigned)(size_t)(pPendingArgFirst - i); + bool iptr = false; + + setDiff(args, b); + if (intersect(iargs,b)) + { + setDiff(iargs, b); + iptr = true; + } + +#ifdef _DEBUG + if (dspPtr) + { + printf(" Pushed ptr arg [E"); + if (info.ebpFrame) + printf("BP-%02XH]: ", EBP - argAddr); + else + printf("SP+%02XH]: ", argAddr - ESP); + } +#endif + _ASSERTE(true == GC_CALL_INTERIOR); + pCallBack(hCallBack, (OBJECTREF *)(size_t)argAddr, (int)iptr | CHECK_APP_DOMAIN + DAC_ARG(DacSlotLocation(info.ebpFrame ? REGI_EBP : REGI_ESP, + info.ebpFrame ? EBP - argAddr : argAddr - ESP, + true))); + } + } + } } else { - pTargetCtx->Rbp = *(UINT_PTR*)(pSourceCtx->Rsp + RBPOffset); + // Is "this" enregistered. If so, report it as we might need to + // release the monitor for synchronized methods. + // Else, it is on the stack and will be reported below. + + if (info.thisPtrResult != REGI_NA) + { + // Synchronized methods and methods satisfying + // MethodDesc::AcquiresInstMethodTableFromThis (i.e. those + // where "this" is reported in thisPtrResult) are + // not supported on value types. + _ASSERTE((regNumToMask(info.thisPtrResult) & info.iregMaskResult)== 0); + + void * thisReg = getCalleeSavedReg(pContext, info.thisPtrResult); + pCallBack(hCallBack, (OBJECTREF *)thisReg, CHECK_APP_DOMAIN + DAC_ARG(DacSlotLocation(info.thisPtrResult, 0, false))); + } } + } + else /* not interruptible */ + { + pushedSize = scanArgRegTable(skipToArgReg(info, table), curOffs, &info); - // Adjust the sp. From this pointer onwards pCurrentContext->Rsp is the caller sp. - pTargetCtx->Rsp = pSourceCtx->Rsp + RSPOffset; + RegMask regMask = info.regMaskResult; + RegMask iregMask = info.iregMaskResult; + ptrArgTP argMask = info.argMaskResult; + ptrArgTP iargMask = info.iargMaskResult; + unsigned argHnum = info.argHnumResult; + PTR_CBYTE argTab = info.argTabResult; - // Retrieve the return address. - pTargetCtx->Rip = *(UINT_PTR*)((pTargetCtx->Rsp) - sizeof(UINT_PTR)); - } + // Only synchronized methods and generic code that accesses + // the type context via "this" need to report "this". + // If its reported for other methods, its probably + // done incorrectly. So flag such cases. + _ASSERTE(info.thisPtrResult == REGI_NA || + pCodeInfo->GetMethodDesc()->IsSynchronized() || + pCodeInfo->GetMethodDesc()->AcquiresInstMethodTableFromThis()); - if (flag == UnwindCurrentStackFrame) - { - SyncRegDisplayToCurrentContext(pRD); - pRD->IsCallerContextValid = FALSE; - pRD->IsCallerSPValid = FALSE; // Don't add usage of this field. This is only temporary. - } -#else - PORTABILITY_ASSERT("EECodeManager::LightUnwindStackFrame is not implemented on this platform."); + + /* now report registers and arguments if we are not interrupted */ + + if (willContinueExecution) + { + + /* Report all live pointer registers */ + + CHK_AND_REPORT_REG(REGI_EDI, regMask & RM_EDI, iregMask & RM_EDI, Edi); + CHK_AND_REPORT_REG(REGI_ESI, regMask & RM_ESI, iregMask & RM_ESI, Esi); + CHK_AND_REPORT_REG(REGI_EBX, regMask & RM_EBX, iregMask & RM_EBX, Ebx); + CHK_AND_REPORT_REG(REGI_EBP, regMask & RM_EBP, iregMask & RM_EBP, Ebp); + + /* Esp cant be reported */ + _ASSERTE(!(regMask & RM_ESP)); + /* No callee-trashed registers */ + _ASSERTE(!(regMask & RM_CALLEE_TRASHED)); + /* EBP can't be reported unless we have an EBP-less frame */ + _ASSERTE(!(regMask & RM_EBP) || !(info.ebpFrame)); + + /* Report any pending pointer arguments */ + + if (argTab != 0) + { + unsigned lowBits, stkOffs, argAddr, val; + + // argMask does not fit in 32-bits + // thus arguments are reported via a table + // Both of these are very rare cases + + do + { + val = fastDecodeUnsigned(argTab); + + lowBits = val & OFFSET_MASK; + stkOffs = val & ~OFFSET_MASK; + _ASSERTE((lowBits == 0) || (lowBits == byref_OFFSET_FLAG)); + + argAddr = ESP + stkOffs; +#ifdef _DEBUG + if (dspPtr) + printf(" Pushed %sptr arg at [ESP+%02XH]", + lowBits ? "iptr " : "", stkOffs); #endif -} -#endif // HAS_LIGHTUNWIND + _ASSERTE(byref_OFFSET_FLAG == GC_CALL_INTERIOR); + pCallBack(hCallBack, (OBJECTREF *)(size_t)argAddr, lowBits | CHECK_APP_DOMAIN + DAC_ARG(DacSlotLocation(REGI_ESP, stkOffs, true))); + } + while(--argHnum); -#ifdef FEATURE_EH_FUNCLETS -#ifdef TARGET_X86 -size_t EECodeManager::GetResumeSp( PCONTEXT pContext ) -{ - PCODE currentPc = PCODE(pContext->Eip); + _ASSERTE(info.argTabResult + info.argTabBytes == argTab); + } + else + { + unsigned argAddr = ESP; + + while (!isZero(argMask)) + { + _ASSERTE(argHnum-- > 0); - _ASSERTE(ExecutionManager::IsManagedCode(currentPc)); + if (toUnsigned(argMask) & 1) + { + bool iptr = false; - EECodeInfo codeInfo(currentPc); + if (toUnsigned(iargMask) & 1) + iptr = true; +#ifdef _DEBUG + if (dspPtr) + printf(" Pushed ptr arg at [ESP+%02XH]", + argAddr - ESP); +#endif + _ASSERTE(true == GC_CALL_INTERIOR); + pCallBack(hCallBack, (OBJECTREF *)(size_t)argAddr, (int)iptr | CHECK_APP_DOMAIN + DAC_ARG(DacSlotLocation(REGI_ESP, argAddr - ESP, true))); + } + + argMask >>= 1; + iargMask >>= 1; + argAddr += 4; + } - PTR_CBYTE methodStart = PTR_CBYTE(codeInfo.GetSavedMethodCode()); + } - GCInfoToken gcInfoToken = codeInfo.GetGCInfoToken(); - PTR_VOID methodInfoPtr = gcInfoToken.Info; - DWORD curOffs = codeInfo.GetRelOffset(); + } + else + { + // Is "this" enregistered. If so, report it as we will need to + // release the monitor. Else, it is on the stack and will be + // reported below. - CodeManStateBuf stateBuf; + // For partially interruptible code, info.thisPtrResult will be + // the last known location of "this". So the compiler needs to + // generate information which is correct at every point in the code, + // not just at call sites. - stateBuf.hdrInfoSize = (DWORD)DecodeGCHdrInfo(gcInfoToken, - curOffs, - &stateBuf.hdrInfoBody); + if (info.thisPtrResult != REGI_NA) + { + // Synchronized methods on value types are not supported + _ASSERTE((regNumToMask(info.thisPtrResult) & info.iregMaskResult)== 0); - PTR_CBYTE table = dac_cast(methodInfoPtr) + stateBuf.hdrInfoSize; + void * thisReg = getCalleeSavedReg(pContext, info.thisPtrResult); + pCallBack(hCallBack, (OBJECTREF *)thisReg, CHECK_APP_DOMAIN + DAC_ARG(DacSlotLocation(info.thisPtrResult, 0, false))); + } + } - hdrInfo *info = &stateBuf.hdrInfoBody; + } //info.interruptible - _ASSERTE(info->epilogOffs == hdrInfo::NOT_IN_EPILOG && info->prologOffs == hdrInfo::NOT_IN_PROLOG); + /* compute the argument base (reference point) */ - bool isESPFrame = !info->ebpFrame && !info->doubleAlign; + unsigned argBase; - if (codeInfo.IsFunclet()) - { - // Treat funclet's frame as ESP frame - isESPFrame = true; - } + if (info.ebpFrame) + argBase = EBP; + else + argBase = ESP + pushedSize; - if (isESPFrame) - { - const size_t curESP = (size_t)(pContext->Esp); - return curESP + GetPushedArgSize(info, table, curOffs); - } +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xBEEF); +#endif - const size_t curEBP = (size_t)(pContext->Ebp); - return GetOutermostBaseFP(curEBP, info); -} -#endif // TARGET_X86 -#endif // FEATURE_EH_FUNCLETS + unsigned ptrAddr; + unsigned lowBits; -#ifndef FEATURE_EH_FUNCLETS -/***************************************************************************** - * - * Unwind the current stack frame, i.e. update the virtual register - * set in pContext. This will be similar to the state after the function - * returns back to caller (IP points to after the call, Frame and Stack - * pointer has been reset, callee-saved registers restored (if UpdateAllRegs), - * callee-unsaved registers are trashed. - * Returns success of operation. - */ + /* Process the untracked frame variable table */ -bool EECodeManager::UnwindStackFrame(PREGDISPLAY pContext, - EECodeInfo *pCodeInfo, - unsigned flags, - CodeManState *pState) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - HOST_NOCALLS; - SUPPORTS_DAC; - } CONTRACTL_END; +#if defined(FEATURE_EH_FUNCLETS) // funclets + // Filters are the only funclet that run during the 1st pass, and must have + // both the leaf and the parent frame reported. In order to avoid double + // reporting of the untracked variables, do not report them for the filter. + if (!pCodeInfo->GetJitManager()->IsFilterFunclet(pCodeInfo)) +#endif // FEATURE_EH_FUNCLETS + { + count = info.untrackedCnt; + int lastStkOffs = 0; + while (count-- > 0) + { + int stkOffs = fastDecodeSigned(table); + stkOffs = lastStkOffs - stkOffs; + lastStkOffs = stkOffs; + + _ASSERTE(0 == ~OFFSET_MASK % sizeof(void*)); + + lowBits = OFFSET_MASK & stkOffs; + stkOffs &= ~OFFSET_MASK; + + ptrAddr = argBase + stkOffs; + if (info.doubleAlign && stkOffs >= int(info.stackSize - sizeof(void*))) { + // We encode the arguments as if they were ESP based variables even though they aren't + // If this frame would have ben an ESP based frame, This fake frame is one DWORD + // smaller than the real frame because it did not push EBP but the real frame did. + // Thus to get the correct EBP relative offset we have to adjust by info.stackSize-sizeof(void*) + ptrAddr = EBP + (stkOffs-(info.stackSize - sizeof(void*))); + } -#ifdef TARGET_X86 - bool updateAllRegs = flags & UpdateAllRegs; +#ifdef _DEBUG + if (dspPtr) + { + printf(" Untracked %s%s local at [E", + (lowBits & pinned_OFFSET_FLAG) ? "pinned " : "", + (lowBits & byref_OFFSET_FLAG) ? "byref" : ""); - // Address where the method has been interrupted - PCODE breakPC = pContext->ControlPC; - _ASSERTE(PCODEToPINSTR(breakPC) == pCodeInfo->GetCodeAddress()); + int dspOffs = ptrAddr; + char frameType; - GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); - PTR_VOID methodInfoPtr = gcInfoToken.Info; - DWORD curOffs = pCodeInfo->GetRelOffset(); + if (info.ebpFrame) { + dspOffs -= EBP; + frameType = 'B'; + } + else { + dspOffs -= ESP; + frameType = 'S'; + } - _ASSERTE(sizeof(CodeManStateBuf) <= sizeof(pState->stateBuf)); - CodeManStateBuf * stateBuf = (CodeManStateBuf*)pState->stateBuf; + if (dspOffs < 0) + printf("%cP-%02XH]: ", frameType, -dspOffs); + else + printf("%cP+%02XH]: ", frameType, +dspOffs); + } +#endif - if (pState->dwIsSet == 0) - { - /* Extract the necessary information from the info block header */ + _ASSERTE((pinned_OFFSET_FLAG == GC_CALL_PINNED) && + (byref_OFFSET_FLAG == GC_CALL_INTERIOR)); + pCallBack(hCallBack, (OBJECTREF*)(size_t)ptrAddr, lowBits | CHECK_APP_DOMAIN + DAC_ARG(DacSlotLocation(info.ebpFrame ? REGI_EBP : REGI_ESP, + info.ebpFrame ? EBP - ptrAddr : ptrAddr - ESP, + true))); + } - stateBuf->hdrInfoSize = (DWORD)DecodeGCHdrInfo(gcInfoToken, - curOffs, - &stateBuf->hdrInfoBody); } - PTR_CBYTE table = dac_cast(methodInfoPtr) + stateBuf->hdrInfoSize; +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xCAFE); +#endif - hdrInfo * info = &stateBuf->hdrInfoBody; + /* Process the frame variable lifetime table */ + count = info.varPtrTableSize; - info->isSpeculativeStackWalk = ((flags & SpeculativeStackwalk) != 0); + /* If we are not in the active method, we are currently pointing + * to the return address; at the return address stack variables + * can become dead if the call the last instruction of a try block + * and the return address is the jump around the catch block. Therefore + * we simply assume an offset inside of call instruction. + */ - return UnwindStackFrameX86(pContext, - PTR_CBYTE(pCodeInfo->GetSavedMethodCode()), - curOffs, - info, - table, - IN_EH_FUNCLETS_COMMA(PTR_CBYTE(pCodeInfo->GetJitManager()->GetFuncletStartAddress(pCodeInfo))) - IN_EH_FUNCLETS_COMMA(pCodeInfo->IsFunclet()) - updateAllRegs); -#else // TARGET_X86 - PORTABILITY_ASSERT("EECodeManager::UnwindStackFrame"); - return false; -#endif // _TARGET_???_ -} + unsigned newCurOffs; -/*****************************************************************************/ -#else // !FEATURE_EH_FUNCLETS -/*****************************************************************************/ + if (willContinueExecution) + { + newCurOffs = (flags & ActiveStackFrame) ? curOffs // after "call" + : curOffs-1; // inside "call" + } + else + { + /* However if ExecutionAborted, then this must be one of the + * ExceptionFrames. Handle accordingly + */ + _ASSERTE(!(flags & AbortingCall) || !(flags & ActiveStackFrame)); -bool EECodeManager::UnwindStackFrame(PREGDISPLAY pContext, - EECodeInfo *pCodeInfo, - unsigned flags, - CodeManState *pState) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - } CONTRACTL_END; + newCurOffs = (flags & AbortingCall) ? curOffs-1 // inside "call" + : curOffs; // at faulting instr, or start of "try" + } - _ASSERTE(pCodeInfo != NULL); + ptrOffs = 0; -#ifdef HAS_LIGHTUNWIND - if (flags & LightUnwind) + while (count-- > 0) { - LightUnwindStackFrame(pContext, pCodeInfo, UnwindCurrentStackFrame); - return true; - } -#endif - - Thread::VirtualUnwindCallFrame(pContext, pCodeInfo); - return true; -} + int stkOffs; + unsigned begOffs; + unsigned endOffs; -/*****************************************************************************/ -#endif // FEATURE_EH_FUNCLETS + stkOffs = fastDecodeUnsigned(table); + begOffs = ptrOffs + fastDecodeUnsigned(table); + endOffs = begOffs + fastDecodeUnsigned(table); -/*****************************************************************************/ + _ASSERTE(0 == ~OFFSET_MASK % sizeof(void*)); -/* report args in 'msig' to the GC. - 'argsStart' is start of the stack-based arguments - 'varArgSig' describes the arguments - 'ctx' has the GC reporting info -*/ -void promoteVarArgs(PTR_BYTE argsStart, PTR_VASigCookie varArgSig, GCCONTEXT* ctx) -{ - WRAPPER_NO_CONTRACT; + lowBits = OFFSET_MASK & stkOffs; + stkOffs &= ~OFFSET_MASK; - SigTypeContext typeContext(varArgSig->classInst, varArgSig->methodInst); - MetaSig msig(varArgSig->signature, - varArgSig->pModule, - &typeContext); + if (info.ebpFrame) { + stkOffs = -stkOffs; + _ASSERTE(stkOffs < 0); + } + else { + _ASSERTE(stkOffs >= 0); + } - PTR_BYTE pFrameBase = argsStart - TransitionBlock::GetOffsetOfArgs(); + ptrAddr = argBase + stkOffs; - ArgIterator argit(&msig); + /* Is this variable live right now? */ -#ifdef TARGET_X86 - // For the X86 target the JIT does not report any of the fixed args for a varargs method - // So we report the fixed args via the promoteArgs call below - bool skipFixedArgs = false; + if (newCurOffs >= begOffs) + { + if (newCurOffs < endOffs) + { +#ifdef _DEBUG + if (dspPtr) { + printf(" Frame %s%s local at [E", + (lowBits & byref_OFFSET_FLAG) ? "byref " : "", +#ifndef FEATURE_EH_FUNCLETS + (lowBits & this_OFFSET_FLAG) ? "this-ptr" : ""); #else - // For other platforms the JITs do report the fixed args of a varargs method - // So we must tell promoteArgs to skip to the end of the fixed args - bool skipFixedArgs = true; + (lowBits & pinned_OFFSET_FLAG) ? "pinned" : ""); #endif - bool inVarArgs = false; - int argOffset; - while ((argOffset = argit.GetNextOffset()) != TransitionBlock::InvalidOffset) - { - if (msig.GetArgProps().AtSentinel()) - inVarArgs = true; + int dspOffs = ptrAddr; + char frameType; - // if skipFixedArgs is false we report all arguments - // otherwise we just report the varargs. - if (!skipFixedArgs || inVarArgs) - { - ArgDestination argDest(pFrameBase, argOffset, argit.GetArgLocDescForStructInRegs()); - msig.GcScanRoots(&argDest, ctx->f, ctx->sc); + if (info.ebpFrame) { + dspOffs -= EBP; + frameType = 'B'; + } + else { + dspOffs -= ESP; + frameType = 'S'; + } + + if (dspOffs < 0) + printf("%cP-%02XH]: ", frameType, -dspOffs); + else + printf("%cP+%02XH]: ", frameType, +dspOffs); + } +#endif + + unsigned flags = CHECK_APP_DOMAIN; +#ifndef FEATURE_EH_FUNCLETS + // First Bit : byref + // Second Bit : this + // The second bit means `this` not `pinned`. So we ignore it. + flags |= lowBits & byref_OFFSET_FLAG; +#else + // First Bit : byref + // Second Bit : pinned + // Both bits are valid + flags |= lowBits; +#endif + + _ASSERTE(byref_OFFSET_FLAG == GC_CALL_INTERIOR); + pCallBack(hCallBack, (OBJECTREF*)(size_t)ptrAddr, flags + DAC_ARG(DacSlotLocation(info.ebpFrame ? REGI_EBP : REGI_ESP, + info.ebpFrame ? EBP - ptrAddr : ptrAddr - ESP, + true))); + } } + // exit loop early if start of live range is beyond PC, as ranges are sorted by lower bound + else break; + + ptrOffs = begOffs; } -} -#ifndef DACCESS_COMPILE -FCIMPL1(void, GCReporting::Register, GCFrame* frame) -{ - FCALL_CONTRACT; - // Construct a GCFrame. - _ASSERTE(frame != NULL); - frame->Push(GetThread()); -} -FCIMPLEND +#if VERIFY_GC_TABLES + _ASSERTE(*castto(table, unsigned short *)++ == 0xBABE); +#endif -FCIMPL1(void, GCReporting::Unregister, GCFrame* frame) -{ - FCALL_CONTRACT; +#ifdef FEATURE_EH_FUNCLETS // funclets + // + // If we're in a funclet, we do not want to report the incoming varargs. This is + // taken care of by the parent method and the funclet should access those arguments + // by way of the parent method's stack frame. + // + if(pCodeInfo->IsFunclet()) + { + return true; + } +#endif // FEATURE_EH_FUNCLETS - // Destroy the GCFrame. - _ASSERTE(frame != NULL); - frame->Remove(); -} -FCIMPLEND -#endif // !DACCESS_COMPILE + /* Are we a varargs function, if so we have to report all args + except 'this' (note that the GC tables created by the x86 jit + do not contain ANY arguments except 'this' (even if they + were statically declared */ -#ifndef USE_GC_INFO_DECODER + if (info.varargs) { + LOG((LF_GCINFO, LL_INFO100, "Reporting incoming vararg GC refs\n")); -/***************************************************************************** - * - * Enumerate all live object references in that function using - * the virtual register set. - * Returns success of operation. - */ + PTR_BYTE argsStart; -bool EECodeManager::EnumGcRefs( PREGDISPLAY pContext, - EECodeInfo *pCodeInfo, - unsigned flags, - GCEnumCallback pCallBack, - LPVOID hCallBack, - DWORD relOffsetOverride) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - } CONTRACTL_END; + if (info.ebpFrame || info.doubleAlign) + argsStart = PTR_BYTE((size_t)EBP) + 2* sizeof(void*); // pushed EBP and retAddr + else + argsStart = PTR_BYTE((size_t)argBase) + info.stackSize + sizeof(void*); // ESP + locals + retAddr - PTR_CBYTE methodStart = PTR_CBYTE(pCodeInfo->GetSavedMethodCode()); - unsigned curOffs = pCodeInfo->GetRelOffset(); - GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); +#if defined(_DEBUG) && !defined(DACCESS_COMPILE) + // Note that I really want to say hCallBack is a GCCONTEXT, but this is pretty close + extern void GcEnumObject(LPVOID pData, OBJECTREF *pObj, uint32_t flags); + _ASSERTE((void*) GcEnumObject == pCallBack); +#endif + GCCONTEXT *pCtx = (GCCONTEXT *) hCallBack; - if (relOffsetOverride != NO_OVERRIDE_OFFSET) - { - curOffs = relOffsetOverride; + // For varargs, look up the signature using the varArgSig token passed on the stack + PTR_VASigCookie varArgSig = *PTR_PTR_VASigCookie(argsStart); + + promoteVarArgs(argsStart, varArgSig, pCtx); } - return ::EnumGcRefsX86(pContext, - methodStart, - curOffs, - gcInfoToken, - IN_EH_FUNCLETS_COMMA(PTR_CBYTE(pCodeInfo->GetJitManager()->GetFuncletStartAddress(pCodeInfo))) - IN_EH_FUNCLETS_COMMA(pCodeInfo->IsFunclet()) - IN_EH_FUNCLETS_COMMA(pCodeInfo->GetJitManager()->IsFilterFunclet(pCodeInfo)) - flags, - pCallBack, - hCallBack); + return true; } #else // !USE_GC_INFO_DECODER @@ -1650,6 +5450,7 @@ OBJECTREF EECodeManager::GetInstance( PREGDISPLAY pContext, hdrInfo info; unsigned stackDepth; TADDR taArgBase; + unsigned count; /* Extract the necessary information from the info block header */ @@ -1708,7 +5509,7 @@ OBJECTREF EECodeManager::GetInstance( PREGDISPLAY pContext, /* The 'this' pointer can never be located in the untracked table */ /* as we only allow pinned and byrefs in the untracked table */ - unsigned count = info.untrackedCnt; + count = info.untrackedCnt; while (count-- > 0) { fastSkipSigned(table); @@ -1953,6 +5754,8 @@ void * EECodeManager::GetGSCookieAddr(PREGDISPLAY pContext, GC_NOTRIGGER; } CONTRACTL_END; + _ASSERTE(sizeof(CodeManStateBuf) <= sizeof(pState->stateBuf)); + GCInfoToken gcInfoToken = pCodeInfo->GetGCInfoToken(); unsigned relOffset = pCodeInfo->GetRelOffset(); @@ -1964,8 +5767,6 @@ void * EECodeManager::GetGSCookieAddr(PREGDISPLAY pContext, #endif #ifndef USE_GC_INFO_DECODER - _ASSERTE(sizeof(CodeManStateBuf) <= sizeof(pState->stateBuf)); - CodeManStateBuf * stateBuf = (CodeManStateBuf*)pState->stateBuf; /* Extract the necessary information from the info block header */ diff --git a/src/coreclr/vm/excep.cpp b/src/coreclr/vm/excep.cpp index 0eec20394de4ae..e723e087d57f4c 100644 --- a/src/coreclr/vm/excep.cpp +++ b/src/coreclr/vm/excep.cpp @@ -2889,7 +2889,7 @@ VOID DECLSPEC_NORETURN RealCOMPlusThrow(Object *exceptionObj) CONTRACTL_END; OBJECTREF throwable = ObjectToOBJECTREF(exceptionObj); - RealCOMPlusThrowWorker(throwable, FALSE); + RealCOMPlusThrow(throwable, FALSE); } #endif // USE_CHECKED_OBJECTREFS @@ -6292,6 +6292,9 @@ EXTERN_C void JIT_StackProbe_End(); #ifdef FEATURE_EH_FUNCLETS #ifndef TARGET_X86 +EXTERN_C void JIT_MemSet_End(); +EXTERN_C void JIT_MemCpy_End(); + EXTERN_C void JIT_WriteBarrier_End(); EXTERN_C void JIT_CheckedWriteBarrier_End(); EXTERN_C void JIT_ByRefWriteBarrier_End(); @@ -6342,6 +6345,9 @@ bool IsIPInMarkedJitHelper(UINT_PTR uControlPc) if (GetEEFuncEntryPoint(name) <= uControlPc && uControlPc < GetEEFuncEntryPoint(name##_End)) return true; #ifndef TARGET_X86 + CHECK_RANGE(JIT_MemSet) + CHECK_RANGE(JIT_MemCpy) + CHECK_RANGE(JIT_WriteBarrier) CHECK_RANGE(JIT_CheckedWriteBarrier) CHECK_RANGE(JIT_ByRefWriteBarrier) @@ -7755,42 +7761,6 @@ void UnwindAndContinueRethrowHelperInsideCatch(Frame* pEntryFrame, Exception* pE #endif } -#ifdef FEATURE_EH_FUNCLETS -// -// This function continues exception interception unwind after it crossed native frames using -// standard EH / SEH. -// -VOID DECLSPEC_NORETURN ContinueExceptionInterceptionUnwind() -{ - STATIC_CONTRACT_THROWS; - STATIC_CONTRACT_GC_TRIGGERS; - STATIC_CONTRACT_MODE_ANY; - - GCX_COOP(); - - Thread *pThread = GetThread(); - ThreadExceptionState* pExState = pThread->GetExceptionState(); - UINT_PTR uInterceptStackFrame = 0; - - pExState->GetDebuggerState()->GetDebuggerInterceptInfo(NULL, NULL, - (PBYTE*)&uInterceptStackFrame, - NULL, NULL); - - PREPARE_NONVIRTUAL_CALLSITE(METHOD__EH__UNWIND_AND_INTERCEPT); - DECLARE_ARGHOLDER_ARRAY(args, 2); - args[ARGNUM_0] = PTR_TO_ARGHOLDER((ExInfo*)pExState->GetCurrentExceptionTracker()); - args[ARGNUM_1] = PTR_TO_ARGHOLDER(uInterceptStackFrame); - pThread->IncPreventAbort(); - - //Ex.RhUnwindAndIntercept(throwable, &exInfo) - CRITICAL_CALLSITE; - CALL_MANAGED_METHOD_NORET(args) - - UNREACHABLE(); -} - -#endif // FEATURE_EH_FUNCLETS - // // This does the work of the Unwind and Continue Hanlder after the catch clause of that handler. The stack has been // unwound by the time this is called. Keep that in mind when deciding where to put new code :) @@ -7814,18 +7784,7 @@ VOID DECLSPEC_NORETURN UnwindAndContinueRethrowHelperAfterCatch(Frame* pEntryFra #ifdef FEATURE_EH_FUNCLETS if (g_isNewExceptionHandlingEnabled && !nativeRethrow) { - Thread *pThread = GetThread(); - ThreadExceptionState* pExState = pThread->GetExceptionState(); - ExInfo *pPrevExInfo = (ExInfo*)pExState->GetCurrentExceptionTracker(); - if (pPrevExInfo != NULL && pPrevExInfo->m_DebuggerExState.GetDebuggerInterceptContext() != NULL) - { - ContinueExceptionInterceptionUnwind(); - UNREACHABLE(); - } - else - { - DispatchManagedException(orThrowable, /* preserveStackTrace */ false); - } + DispatchManagedException(orThrowable); } else #endif // FEATURE_EH_FUNCLETS @@ -8894,9 +8853,11 @@ BOOL IsThrowableThreadAbortException(OBJECTREF oThrowable) #if defined(FEATURE_EH_FUNCLETS) PTR_ExceptionTrackerBase GetEHTrackerForPreallocatedException(OBJECTREF oPreAllocThrowable, PTR_ExceptionTrackerBase pStartingEHTracker) -#else +#elif TARGET_X86 PTR_ExInfo GetEHTrackerForPreallocatedException(OBJECTREF oPreAllocThrowable, PTR_ExInfo pStartingEHTracker) +#else +#error Unsupported platform #endif { CONTRACTL @@ -8914,9 +8875,11 @@ PTR_ExInfo GetEHTrackerForPreallocatedException(OBJECTREF oPreAllocThrowable, // Get the reference to the current exception tracker #if defined(FEATURE_EH_FUNCLETS) PTR_ExceptionTrackerBase pEHTracker = (pStartingEHTracker != NULL) ? pStartingEHTracker : GetThread()->GetExceptionState()->GetCurrentExceptionTracker(); -#else +#elif TARGET_X86 PTR_ExInfo pEHTracker = (pStartingEHTracker != NULL) ? pStartingEHTracker : GetThread()->GetExceptionState()->GetCurrentExceptionTracker(); -#endif +#else // !(HOST_64BIT || TARGET_X86) +#error Unsupported platform +#endif // HOST_64BIT BOOL fFoundTracker = FALSE; @@ -8991,10 +8954,13 @@ PTR_EHWatsonBucketTracker GetWatsonBucketTrackerForPreallocatedException(OBJECTR #if defined(FEATURE_EH_FUNCLETS) PTR_ExceptionTrackerBase pEHTracker = NULL; PTR_ExceptionTrackerBase pPreviousEHTracker = NULL; -#else + +#elif TARGET_X86 PTR_ExInfo pEHTracker = NULL; PTR_ExInfo pPreviousEHTracker = NULL; -#endif +#else // !(HOST_64BIT || TARGET_X86) +#error Unsupported platform +#endif // HOST_64BIT if (fStartSearchFromPreviousTracker) { diff --git a/src/coreclr/vm/excep.h b/src/coreclr/vm/excep.h index 0d0217607d9d83..9acd2b945c4066 100644 --- a/src/coreclr/vm/excep.h +++ b/src/coreclr/vm/excep.h @@ -520,9 +520,9 @@ EXCEPTION_HANDLER_DECL(COMPlusFrameHandlerRevCom); // Pop off any SEH handlers we have registered below pTargetSP VOID __cdecl PopSEHRecords(LPVOID pTargetSP); -#ifdef DEBUGGING_SUPPORTED +#if defined(TARGET_X86) && defined(DEBUGGING_SUPPORTED) VOID UnwindExceptionTrackerAndResumeInInterceptionFrame(ExInfo* pExInfo, EHContext* context); -#endif // DEBUGGING_SUPPORTED +#endif // TARGET_X86 && DEBUGGING_SUPPORTED BOOL PopNestedExceptionRecords(LPVOID pTargetSP, BOOL bCheckForUnknownHandlers = FALSE); VOID PopNestedExceptionRecords(LPVOID pTargetSP, T_CONTEXT *pCtx, void *pSEH); @@ -846,10 +846,6 @@ void ResetThreadAbortState(PTR_Thread pThread, CrawlFrame *pCf, StackFrame sfCur X86_ONLY(EXCEPTION_REGISTRATION_RECORD* GetNextCOMPlusSEHRecord(EXCEPTION_REGISTRATION_RECORD* pRec);) -#ifdef FEATURE_EH_FUNCLETS -VOID DECLSPEC_NORETURN ContinueExceptionInterceptionUnwind(); -#endif // FEATURE_EH_FUNCLETS - #endif // !DACCESS_COMPILE #endif // __excep_h__ diff --git a/src/coreclr/vm/exceptionhandling.cpp b/src/coreclr/vm/exceptionhandling.cpp index a6118ef56bca14..8c10895a4f12ea 100644 --- a/src/coreclr/vm/exceptionhandling.cpp +++ b/src/coreclr/vm/exceptionhandling.cpp @@ -19,7 +19,6 @@ #include "corinfo.h" #include "exceptionhandlingqcalls.h" #include "exinfo.h" -#include "configuration.h" #if defined(TARGET_X86) #define USE_CURRENT_CONTEXT_IN_FILTER @@ -237,7 +236,7 @@ void InitializeExceptionHandling() // Initialize the lock used for synchronizing access to the stacktrace in the exception object g_StackTraceArrayLock.Init(LOCK_TYPE_DEFAULT, TRUE); - g_isNewExceptionHandlingEnabled = Configuration::GetKnobBooleanValue(W("System.Runtime.LegacyExceptionHandling"), CLRConfig::EXTERNAL_LegacyExceptionHandling ) == 0; + g_isNewExceptionHandlingEnabled = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_EnableNewExceptionHandling) != 0; #ifdef TARGET_UNIX // Register handler of hardware exceptions like null reference in PAL @@ -930,18 +929,8 @@ ProcessCLRExceptionNew(IN PEXCEPTION_RECORD pExceptionRecord, else { GCX_COOP(); - ThreadExceptionState* pExState = pThread->GetExceptionState(); - ExInfo *pPrevExInfo = (ExInfo*)pExState->GetCurrentExceptionTracker(); - if (pPrevExInfo != NULL && pPrevExInfo->m_DebuggerExState.GetDebuggerInterceptContext() != NULL) - { - ContinueExceptionInterceptionUnwind(); - UNREACHABLE(); - } - else - { - OBJECTREF oref = ExceptionTracker::CreateThrowable(pExceptionRecord, FALSE); - DispatchManagedException(oref, pContextRecord, /* preserveStackTrace */ false); - } + OBJECTREF oref = ExceptionTracker::CreateThrowable(pExceptionRecord, FALSE); + DispatchManagedException(oref, pContextRecord); } #endif // !HOST_UNIX EEPOLICY_HANDLE_FATAL_ERROR_WITH_MESSAGE(COR_E_EXECUTIONENGINE, _T("SEH exception leaked into managed code")); @@ -4310,12 +4299,9 @@ EXCEPTION_DISPOSITION ClrDebuggerDoUnwindAndIntercept(X86_FIRST_ARG(EXCEPTION_RE { GCX_COOP(); - ExInfo* pExInfo = (ExInfo*)pExState->GetCurrentExceptionTracker(); - _ASSERTE(pExInfo != NULL); - PREPARE_NONVIRTUAL_CALLSITE(METHOD__EH__UNWIND_AND_INTERCEPT); DECLARE_ARGHOLDER_ARRAY(args, 2); - args[ARGNUM_0] = PTR_TO_ARGHOLDER(pExInfo); + args[ARGNUM_0] = PTR_TO_ARGHOLDER(pExState->GetCurrentExceptionTracker()); args[ARGNUM_1] = PTR_TO_ARGHOLDER(uInterceptStackFrame); pThread->IncPreventAbort(); @@ -5452,7 +5438,7 @@ BOOL HandleHardwareException(PAL_SEHException* ex) if (ex->GetExceptionRecord()->ExceptionCode != STATUS_BREAKPOINT && ex->GetExceptionRecord()->ExceptionCode != STATUS_SINGLE_STEP) { // A hardware exception is handled only if it happened in a jitted code or - // in one of the JIT helper functions + // in one of the JIT helper functions (JIT_MemSet, ...) PCODE controlPc = GetIP(ex->GetContextRecord()); if (ExecutionManager::IsManagedCode(controlPc) && IsGcMarker(ex->GetContextRecord(), ex->GetExceptionRecord())) { @@ -6624,13 +6610,6 @@ bool ExceptionTracker::IsInStackRegionUnwoundBySpecifiedException(CrawlFrame * p // Remember that sfLowerBound and sfUpperBound are in the "OS format". // Refer to the comment for CallerStackFrame for more information. - - if (g_isNewExceptionHandlingEnabled) - { - // The new exception handling sets the ranges always to the SP of the unwound frame - return (sfLowerBound < csfToCheck) && (csfToCheck <= sfUpperBound); - } - #ifndef STACK_RANGE_BOUNDS_ARE_CALLER_SP if ((sfLowerBound < csfToCheck) && (csfToCheck <= sfUpperBound)) #else // !STACK_RANGE_BOUNDS_ARE_CALLER_SP @@ -7553,48 +7532,29 @@ void MarkInlinedCallFrameAsEHHelperCall(Frame* pFrame) pInlinedCallFrame->m_Datum = (PTR_NDirectMethodDesc)((TADDR)pInlinedCallFrame->m_Datum | (TADDR)InlinedCallFrameMarker::ExceptionHandlingHelper); } -static TADDR GetSpForDiagnosticReporting(REGDISPLAY *pRD) -{ -#ifdef ESTABLISHER_FRAME_ADDRESS_IS_CALLER_SP - return CallerStackFrame::FromRegDisplay(pRD).SP; -#else - return GetSP(pRD->pCurrentContext); -#endif -} - extern "C" void QCALLTYPE AppendExceptionStackFrame(QCall::ObjectHandleOnStack exceptionObj, SIZE_T ip, SIZE_T sp, int flags, ExInfo *pExInfo) { QCALL_CONTRACT; BEGIN_QCALL; + GCX_COOP(); Thread* pThread = GET_THREAD(); + Frame* pFrame = pThread->GetFrame(); + MarkInlinedCallFrameAsFuncletCall(pFrame); - { - GCX_COOP(); - - Frame* pFrame = pThread->GetFrame(); - MarkInlinedCallFrameAsEHHelperCall(pFrame); - - bool canAllocateMemory = !(exceptionObj.Get() == CLRException::GetPreallocatedOutOfMemoryException()) && - !(exceptionObj.Get() == CLRException::GetPreallocatedStackOverflowException()); + bool canAllocateMemory = !(exceptionObj.Get() == CLRException::GetPreallocatedOutOfMemoryException()) && + !(exceptionObj.Get() == CLRException::GetPreallocatedStackOverflowException()); - MethodDesc *pMD = pExInfo->m_frameIter.m_crawl.GetFunction(); + MethodDesc *pMD = pExInfo->m_frameIter.m_crawl.GetFunction(); #if _DEBUG - EECodeInfo codeInfo(ip); - _ASSERTE(codeInfo.IsValid()); - _ASSERTE(pMD == codeInfo.GetMethodDesc()); + EECodeInfo codeInfo(ip); + _ASSERTE(codeInfo.IsValid()); + _ASSERTE(pMD == codeInfo.GetMethodDesc()); #endif // _DEBUG - pExInfo->m_StackTraceInfo.AppendElement(canAllocateMemory, ip, sp, pMD, &pExInfo->m_frameIter.m_crawl); - pExInfo->m_StackTraceInfo.SaveStackTrace(canAllocateMemory, pExInfo->m_hThrowable, /*bReplaceStack*/FALSE, /*bSkipLastElement*/FALSE); - } - - // Notify the debugger that we are on the first pass for a managed exception. - // Note that this callback is made for every managed frame. - TADDR spForDebugger = GetSpForDiagnosticReporting(pExInfo->m_frameIter.m_crawl.GetRegisterSet()); - EEToDebuggerExceptionInterfaceWrapper::FirstChanceManagedException(pThread, ip, spForDebugger); - + pExInfo->m_StackTraceInfo.AppendElement(canAllocateMemory, ip, sp, pMD, &pExInfo->m_frameIter.m_crawl); + pExInfo->m_StackTraceInfo.SaveStackTrace(canAllocateMemory, pExInfo->m_hThrowable, /*bReplaceStack*/FALSE, /*bSkipLastElement*/FALSE); if (!pExInfo->DeliveredFirstChanceNotification()) { ExceptionNotifications::DeliverFirstChanceNotification(); @@ -7629,6 +7589,15 @@ UINT_PTR GetEstablisherFrame(REGDISPLAY* pvRegDisplay, ExInfo* exInfo) #endif } +static TADDR GetSpForDiagnosticReporting(REGDISPLAY *pRD) +{ +#ifdef ESTABLISHER_FRAME_ADDRESS_IS_CALLER_SP + return CallerStackFrame::FromRegDisplay(pRD).SP; +#else + return GetSP(pRD->pCurrentContext); +#endif +} + extern "C" void * QCALLTYPE CallCatchFunclet(QCall::ObjectHandleOnStack exceptionObj, BYTE* pHandlerIP, REGDISPLAY* pvRegDisplay, ExInfo* exInfo) { QCALL_CONTRACT; @@ -7694,7 +7663,6 @@ extern "C" void * QCALLTYPE CallCatchFunclet(QCall::ObjectHandleOnStack exceptio BOOL fIntercepted = pThread->GetExceptionState()->GetFlags()->DebuggerInterceptInfo(); if (fIntercepted) { - _ASSERTE(pHandlerIP == NULL); // retrieve the interception information MethodDesc *pInterceptMD = NULL; StackFrame sfInterceptStackFrame; @@ -7702,24 +7670,18 @@ extern "C" void * QCALLTYPE CallCatchFunclet(QCall::ObjectHandleOnStack exceptio ULONG_PTR ulRelOffset; pThread->GetExceptionState()->GetDebuggerState()->GetDebuggerInterceptInfo(&pInterceptMD, NULL, (PBYTE*)&(sfInterceptStackFrame.SP), &ulRelOffset, NULL); - if (sfInterceptStackFrame.SP == GetSP(pvRegDisplay->pCurrentContext)) - { - PCODE pStartAddress = pInterceptMD->GetNativeCode(); - EECodeInfo codeInfo(pStartAddress); - _ASSERTE(codeInfo.IsValid()); + PCODE pStartAddress = pInterceptMD->GetNativeCode(); - // Note that the value returned for ulRelOffset is actually the offset, - // so we need to adjust it to get the actual IP. - _ASSERTE(FitsIn(ulRelOffset)); - uResumePC = codeInfo.GetJitManager()->GetCodeAddressForRelOffset(codeInfo.GetMethodToken(), static_cast(ulRelOffset)); + EECodeInfo codeInfo(pStartAddress); + _ASSERTE(codeInfo.IsValid()); - SetIP(pvRegDisplay->pCurrentContext, uResumePC); - } - else - { - fIntercepted = FALSE; - } + // Note that the value returned for ulRelOffset is actually the offset, + // so we need to adjust it to get the actual IP. + _ASSERTE(FitsIn(ulRelOffset)); + uResumePC = codeInfo.GetJitManager()->GetCodeAddressForRelOffset(codeInfo.GetMethodToken(), static_cast(ulRelOffset)); + + SetIP(pvRegDisplay->pCurrentContext, uResumePC); } #endif // DEBUGGING_SUPPORTED @@ -7835,10 +7797,6 @@ extern "C" void QCALLTYPE ResumeAtInterceptionLocation(REGDISPLAY* pvRegDisplay) MarkInlinedCallFrameAsFuncletCall(pFrame); UINT_PTR targetSp = GetSP(pvRegDisplay->pCurrentContext); - ExInfo *pExInfo = (PTR_ExInfo)pThread->GetExceptionState()->GetCurrentExceptionTracker(); - - pExInfo->m_ScannedStackRange.ExtendUpperBound(targetSp); - PopExplicitFrames(pThread, (void*)targetSp); // This must be done before we pop the ExInfos. @@ -7977,7 +7935,7 @@ struct ExtendedEHClauseEnumerator : EH_CLAUSE_ENUMERATOR unsigned EHCount; }; -extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, IJitManager::MethodRegionInfo* pMethodRegionInfo, EH_CLAUSE_ENUMERATOR * pEHEnum) +extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, BYTE** pMethodStartAddress, EH_CLAUSE_ENUMERATOR * pEHEnum) { QCALL_CONTRACT; @@ -7991,7 +7949,7 @@ extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *p IJitManager* pJitMan = pFrameIter->m_crawl.GetJitManager(); const METHODTOKEN& MethToken = pFrameIter->m_crawl.GetMethodToken(); - pJitMan->JitTokenToMethodRegionInfo(MethToken, pMethodRegionInfo); + *pMethodStartAddress = (BYTE*)pJitMan->JitTokenToStartAddress(MethToken); pExtendedEHEnum->EHCount = pJitMan->InitializeEHEnumeration(MethToken, pEHEnum); END_QCALL; @@ -8177,6 +8135,10 @@ static void NotifyFunctionEnter(StackFrameIterator *pThis, Thread *pThread, ExIn EEToProfilerExceptionInterfaceWrapper::ExceptionSearchFunctionLeave(pExInfo->m_pMDToReportFunctionLeave); } EEToProfilerExceptionInterfaceWrapper::ExceptionSearchFunctionEnter(pMD); + // Notify the debugger that we are on the first pass for a managed exception. + // Note that this callback is made for every managed frame. + TADDR spForDebugger = GetSpForDiagnosticReporting(pThis->m_crawl.GetRegisterSet()); + EEToDebuggerExceptionInterfaceWrapper::FirstChanceManagedException(pThread, GetControlPC(pThis->m_crawl.GetRegisterSet()), spForDebugger); } else { @@ -8207,10 +8169,23 @@ extern "C" bool QCALLTYPE SfiInit(StackFrameIterator* pThis, CONTEXT* pStackwalk // just clear the thread state. pThread->ResetThrowControlForThread(); - pFrame = pExInfo->m_pInitialFrame; + // Skip the SfiInit pinvoke frame + pFrame = pThread->GetFrame()->PtrNextFrame(); NotifyExceptionPassStarted(pThis, pThread, pExInfo); + if (pFrame == FRAME_TOP) + { + // There are no managed frames on the stack, fail fast and report unhandled exception + LONG disposition = InternalUnhandledExceptionFilter_Worker((EXCEPTION_POINTERS *)&pExInfo->m_ptrs); +#ifdef HOST_WINDOWS + CreateCrashDumpIfEnabled(/* fSOException */ FALSE); + RaiseFailFastException(pExInfo->m_ptrs.ExceptionRecord, NULL, 0); +#else + CrashDumpAndTerminateProcess(pExInfo->m_ExceptionCode); +#endif + } + REGDISPLAY* pRD = &pExInfo->m_regDisplay; pThread->FillRegDisplay(pRD, pStackwalkCtx); @@ -8240,7 +8215,6 @@ extern "C" bool QCALLTYPE SfiInit(StackFrameIterator* pThis, CONTEXT* pStackwalk !(pExInfo->m_exception == CLRException::GetPreallocatedStackOverflowException()); pExInfo->m_StackTraceInfo.AppendElement(canAllocateMemory, NULL, GetRegdisplaySP(pExInfo->m_frameIter.m_crawl.GetRegisterSet()), pMD, &pExInfo->m_frameIter.m_crawl); - pExInfo->m_StackTraceInfo.SaveStackTrace(canAllocateMemory, pExInfo->m_hThrowable, /*bReplaceStack*/FALSE, /*bSkipLastElement*/FALSE); #if defined(DEBUGGING_SUPPORTED) if (NotifyDebuggerOfStub(pThread, pFrame)) @@ -8280,18 +8254,6 @@ extern "C" bool QCALLTYPE SfiInit(StackFrameIterator* pThis, CONTEXT* pStackwalk *pfIsExceptionIntercepted = CheckExceptionInterception(pThis, pExInfo); } - else - { - // There are no managed frames on the stack, fail fast and report unhandled exception - LONG disposition = InternalUnhandledExceptionFilter_Worker((EXCEPTION_POINTERS *)&pExInfo->m_ptrs); -#ifdef HOST_WINDOWS - CreateCrashDumpIfEnabled(/* fSOException */ FALSE); - GetThread()->SetThreadStateNC(Thread::TSNC_ProcessedUnhandledException); - RaiseException(pExInfo->m_ExceptionCode, EXCEPTION_NONCONTINUABLE_EXCEPTION, pExInfo->m_ptrs.ExceptionRecord->NumberParameters, pExInfo->m_ptrs.ExceptionRecord->ExceptionInformation); -#else - CrashDumpAndTerminateProcess(pExInfo->m_ExceptionCode); -#endif - } return result; } @@ -8499,8 +8461,6 @@ extern "C" bool QCALLTYPE SfiNext(StackFrameIterator* pThis, uint* uExCollideCla !(pTopExInfo->m_exception == CLRException::GetPreallocatedStackOverflowException()); pTopExInfo->m_StackTraceInfo.AppendElement(canAllocateMemory, NULL, GetRegdisplaySP(pTopExInfo->m_frameIter.m_crawl.GetRegisterSet()), pMD, &pTopExInfo->m_frameIter.m_crawl); - pTopExInfo->m_StackTraceInfo.SaveStackTrace(canAllocateMemory, pTopExInfo->m_hThrowable, /*bReplaceStack*/FALSE, /*bSkipLastElement*/FALSE); - #if defined(DEBUGGING_SUPPORTED) if (NotifyDebuggerOfStub(pThread, pFrame)) { diff --git a/src/coreclr/vm/exceptionhandlingqcalls.h b/src/coreclr/vm/exceptionhandlingqcalls.h index 7054080cef3c00..7747c14f531d2f 100644 --- a/src/coreclr/vm/exceptionhandlingqcalls.h +++ b/src/coreclr/vm/exceptionhandlingqcalls.h @@ -17,7 +17,7 @@ extern "C" void QCALLTYPE CallFinallyFunclet(BYTE* pHandlerIP, REGDISPLAY* pvReg extern "C" BOOL QCALLTYPE CallFilterFunclet(QCall::ObjectHandleOnStack exceptionObj, BYTE* pFilterP, REGDISPLAY* pvRegDisplay); extern "C" void QCALLTYPE ResumeAtInterceptionLocation(REGDISPLAY* pvRegDisplay); extern "C" void QCALLTYPE AppendExceptionStackFrame(QCall::ObjectHandleOnStack exceptionObj, SIZE_T ip, SIZE_T sp, int flags, ExInfo *pExInfo); -extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, IJitManager::MethodRegionInfo *pMethodRegionInfo, EH_CLAUSE_ENUMERATOR * pEHEnum); +extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, BYTE** pMethodStartAddress, EH_CLAUSE_ENUMERATOR * pEHEnum); extern "C" BOOL QCALLTYPE EHEnumNext(EH_CLAUSE_ENUMERATOR* pEHEnum, RhEHClause* pEHClause); extern "C" bool QCALLTYPE SfiInit(StackFrameIterator* pThis, CONTEXT* pStackwalkCtx, bool instructionFault, bool* pIsExceptionIntercepted); extern "C" bool QCALLTYPE SfiNext(StackFrameIterator* pThis, unsigned int* uExCollideClauseIdx, bool* fUnwoundReversePInvoke, bool* pIsExceptionIntercepted); diff --git a/src/coreclr/vm/exinfo.cpp b/src/coreclr/vm/exinfo.cpp index 7728d9bed3fef0..093f3436fb1b9d 100644 --- a/src/coreclr/vm/exinfo.cpp +++ b/src/coreclr/vm/exinfo.cpp @@ -121,10 +121,10 @@ void ExInfo::Init() m_pTopMostHandlerDuringSO = NULL; -#ifdef DEBUGGING_SUPPORTED +#if defined(TARGET_X86) && defined(DEBUGGING_SUPPORTED) m_InterceptionContext.Init(); m_ValidInterceptionContext = FALSE; -#endif // DEBUGGING_SUPPORTED +#endif //TARGET_X86 && DEBUGGING_SUPPORTED } ExInfo::ExInfo() @@ -330,7 +330,6 @@ ExInfo::ExInfo(Thread *pThread, EXCEPTION_RECORD *pExceptionRecord, CONTEXT *pEx { m_StackTraceInfo.AllocateStackTrace(); pThread->GetExceptionState()->m_pCurrentTracker = this; - m_pInitialFrame = pThread->GetFrame(); if (exceptionKind == ExKind::HardwareFault) { // Hardware exception handling needs to start on the FaultingExceptionFrame, so we are @@ -384,36 +383,8 @@ void ExInfo::ReleaseResources() void ExInfo::PopExInfos(Thread *pThread, void *targetSp) { ExInfo *pExInfo = (PTR_ExInfo)pThread->GetExceptionState()->GetCurrentExceptionTracker(); -#if defined(DEBUGGING_SUPPORTED) - DWORD_PTR dwInterceptStackFrame = 0; - - // This method may be called on an unmanaged thread, in which case no interception can be done. - if (pExInfo) - { - ThreadExceptionState* pExState = pThread->GetExceptionState(); - - // If the exception is intercepted, then pop trackers according to the stack frame at which - // the exception is intercepted. We must retrieve the frame pointer before we start popping trackers. - if (pExState->GetFlags()->DebuggerInterceptInfo()) - { - pExState->GetDebuggerState()->GetDebuggerInterceptInfo(NULL, NULL, (PBYTE*)&dwInterceptStackFrame, - NULL, NULL); - } - } -#endif // DEBUGGING_SUPPORTED - while (pExInfo && pExInfo < (void*)targetSp) { -#if defined(DEBUGGING_SUPPORTED) - if (g_pDebugInterface != NULL) - { - if (pExInfo->m_ScannedStackRange.GetUpperBound().SP < dwInterceptStackFrame) - { - g_pDebugInterface->DeleteInterceptContext(pExInfo->m_DebuggerExState.GetDebuggerInterceptContext()); - } - } -#endif // DEBUGGING_SUPPORTED - pExInfo->ReleaseResources(); pExInfo = (PTR_ExInfo)pExInfo->m_pPrevNestedInfo; } diff --git a/src/coreclr/vm/exinfo.h b/src/coreclr/vm/exinfo.h index db6851fc4d5a16..ce612621e86973 100644 --- a/src/coreclr/vm/exinfo.h +++ b/src/coreclr/vm/exinfo.h @@ -136,7 +136,7 @@ class ExInfo EHClauseInfo m_EHClauseInfo; ExceptionFlags m_ExceptionFlags; -#ifdef DEBUGGING_SUPPORTED +#if defined(TARGET_X86) && defined(DEBUGGING_SUPPORTED) EHContext m_InterceptionContext; BOOL m_ValidInterceptionContext; #endif @@ -155,7 +155,9 @@ class ExInfo ExInfo& operator=(const ExInfo &from); }; +#if defined(TARGET_X86) PTR_ExInfo GetEHTrackerForPreallocatedException(OBJECTREF oPreAllocThrowable, PTR_ExInfo pStartingEHTracker); +#endif // TARGET_X86 #else // !FEATURE_EH_FUNCLETS @@ -267,8 +269,6 @@ struct ExInfo : public ExceptionTrackerBase // CONTEXT and REGDISPLAY used by the StackFrameIterator for stack walking CONTEXT m_exContext; REGDISPLAY m_regDisplay; - // Initial explicit frame for stack walking - Frame *m_pInitialFrame; #if defined(TARGET_UNIX) void TakeExceptionPointersOwnership(PAL_SEHException* ex); diff --git a/src/coreclr/vm/frames.cpp b/src/coreclr/vm/frames.cpp index c78b4ea6c1fa76..a429ee4ad422b5 100644 --- a/src/coreclr/vm/frames.cpp +++ b/src/coreclr/vm/frames.cpp @@ -1364,10 +1364,10 @@ void TransitionFrame::PromoteCallerStack(promote_func* fn, ScanContext* sc) { VASigCookie *varArgSig = GetVASigCookie(); - SigTypeContext typeContext(varArgSig->classInst, varArgSig->methodInst); + //Note: no instantiations needed for varargs MetaSig msig(varArgSig->signature, varArgSig->pModule, - &typeContext); + NULL); PromoteCallerStackHelper (fn, sc, pFunction, &msig); } } @@ -1498,10 +1498,10 @@ void TransitionFrame::PromoteCallerStackUsingGCRefMap(promote_func* fn, ScanCont { VASigCookie *varArgSig = dac_cast(*ppObj); - SigTypeContext typeContext(varArgSig->classInst, varArgSig->methodInst); + //Note: no instantiations needed for varargs MetaSig msig(varArgSig->signature, varArgSig->pModule, - &typeContext); + NULL); PromoteCallerStackHelper (fn, sc, NULL, &msig); } break; @@ -1525,10 +1525,10 @@ void PInvokeCalliFrame::PromoteCallerStack(promote_func* fn, ScanContext* sc) return; } - SigTypeContext typeContext(varArgSig->classInst, varArgSig->methodInst); + // no instantiations needed for varargs MetaSig msig(varArgSig->signature, varArgSig->pModule, - &typeContext); + NULL); PromoteCallerStackHelper(fn, sc, NULL, &msig); } diff --git a/src/coreclr/vm/gc_unwind_x86.inl b/src/coreclr/vm/gc_unwind_x86.inl deleted file mode 100644 index 8f981ed84cd609..00000000000000 --- a/src/coreclr/vm/gc_unwind_x86.inl +++ /dev/null @@ -1,3837 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -// This file is shared between CoreCLR and NativeAOT. Some of the differences are handled -// with the FEATURE_NATIVEAOT and FEATURE_EH_FUNCLETS defines. There are three main methods -// that are used by both runtimes - DecodeGCHdrInfo, UnwindStackFrameX86, and EnumGcRefsX86. - -#define RETURN_ADDR_OFFS 1 // in DWORDS - -#define X86_INSTR_TEST_ESP_SIB 0x24 -#define X86_INSTR_PUSH_0 0x6A // push 00, entire instruction is 0x6A00 -#define X86_INSTR_PUSH_IMM 0x68 // push NNNN, -#define X86_INSTR_W_PUSH_IND_IMM 0x35FF // push [NNNN] -#define X86_INSTR_CALL_REL32 0xE8 // call rel32 -#define X86_INSTR_W_CALL_IND_IMM 0x15FF // call [addr32] -#define X86_INSTR_NOP 0x90 // nop -#define X86_INSTR_NOP2 0x9090 // 2-byte nop -#define X86_INSTR_NOP3_1 0x9090 // 1st word of 3-byte nop -#define X86_INSTR_NOP3_3 0x90 // 3rd byte of 3-byte nop -#define X86_INSTR_NOP4 0x90909090 // 4-byte nop -#define X86_INSTR_NOP5_1 0x90909090 // 1st dword of 5-byte nop -#define X86_INSTR_NOP5_5 0x90 // 5th byte of 5-byte nop -#define X86_INSTR_INT3 0xCC // int3 -#define X86_INSTR_HLT 0xF4 // hlt -#define X86_INSTR_PUSH_EAX 0x50 // push eax -#define X86_INSTR_PUSH_EBP 0x55 // push ebp -#define X86_INSTR_W_MOV_EBP_ESP 0xEC8B // mov ebp, esp -#define X86_INSTR_POP_ECX 0x59 // pop ecx -#define X86_INSTR_RET 0xC2 // ret imm16 -#define X86_INSTR_RETN 0xC3 // ret -#define X86_INSTR_XOR 0x33 // xor -#define X86_INSTR_w_TEST_ESP_EAX 0x0485 // test [esp], eax -#define X86_INSTR_w_TEST_ESP_DWORD_OFFSET_EAX 0x8485 // test [esp-dwOffset], eax -#define X86_INSTR_w_LEA_ESP_EBP_BYTE_OFFSET 0x658d // lea esp, [ebp-bOffset] -#define X86_INSTR_w_LEA_ESP_EBP_DWORD_OFFSET 0xa58d // lea esp, [ebp-dwOffset] -#define X86_INSTR_w_LEA_EAX_ESP_BYTE_OFFSET 0x448d // lea eax, [esp-bOffset] -#define X86_INSTR_w_LEA_EAX_ESP_DWORD_OFFSET 0x848d // lea eax, [esp-dwOffset] -#define X86_INSTR_JMP_NEAR_REL32 0xE9 // near jmp rel32 -#define X86_INSTR_w_JMP_FAR_IND_IMM 0x25FF // far jmp [addr32] - -#ifdef _DEBUG -// For dumping of verbose info. -#ifndef DACCESS_COMPILE -static bool trFixContext = false; -#endif -static bool trEnumGCRefs = false; -static bool dspPtr = false; // prints the live ptrs as reported -#endif - -__forceinline unsigned decodeUnsigned(PTR_CBYTE& src) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - -#ifdef DACCESS_COMPILE - PTR_CBYTE begin = src; -#endif - - BYTE byte = *src++; - unsigned value = byte & 0x7f; - while (byte & 0x80) - { -#ifdef DACCESS_COMPILE - // In DAC builds, the target data may be corrupt. Rather than return incorrect data - // and risk wasting time in a potentially long loop, we want to fail early and gracefully. - // The data is encoded with 7 value-bits per byte, and so we may need to read a maximum - // of 5 bytes (7*5=35) to read a full 32-bit integer. - if ((src - begin) > 5) - { - DacError(CORDBG_E_TARGET_INCONSISTENT); - } -#endif - - byte = *src++; - value <<= 7; - value += byte & 0x7f; - } - return value; -} - -__forceinline int decodeSigned(PTR_CBYTE& src) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - -#ifdef DACCESS_COMPILE - PTR_CBYTE begin = src; -#endif - - BYTE byte = *src++; - BYTE first = byte; - int value = byte & 0x3f; - while (byte & 0x80) - { -#ifdef DACCESS_COMPILE - // In DAC builds, the target data may be corrupt. Rather than return incorrect data - // and risk wasting time in a potentially long loop, we want to fail early and gracefully. - // The data is encoded with 7 value-bits per byte, and so we may need to read a maximum - // of 5 bytes (7*5=35) to read a full 32-bit integer. - if ((src - begin) > 5) - { - DacError(CORDBG_E_TARGET_INCONSISTENT); - } -#endif - - byte = *src++; - value <<= 7; - value += byte & 0x7f; - } - if (first & 0x40) - value = -value; - return value; -} - -// Fast versions of the above, with one iteration of the loop unrolled -#define fastDecodeUnsigned(src) (((*(src) & 0x80) == 0) ? (unsigned) (*(src)++) : decodeUnsigned((src))) -#define fastDecodeSigned(src) (((*(src) & 0xC0) == 0) ? (unsigned) (*(src)++) : decodeSigned((src))) - -// Fast skipping past encoded integers -#ifndef DACCESS_COMPILE -#define fastSkipUnsigned(src) { while ((*(src)++) & 0x80) { } } -#define fastSkipSigned(src) { while ((*(src)++) & 0x80) { } } -#else -// In DAC builds we want to trade-off a little perf in the common case for reliaiblity against corrupt data. -#define fastSkipUnsigned(src) (decodeUnsigned(src)) -#define fastSkipSigned(src) (decodeSigned(src)) -#endif - - -/***************************************************************************** - * - * Decodes the X86 GcInfo header and returns the decoded information - * in the hdrInfo struct. - * curOffset is the code offset within the active method used in the - * computation of PrologOffs/EpilogOffs. - * Returns the size of the header (number of bytes decoded). - */ -size_t DecodeGCHdrInfo(GCInfoToken gcInfoToken, - unsigned curOffset, - hdrInfo * infoPtr) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - HOST_NOCALLS; - SUPPORTS_DAC; - } CONTRACTL_END; - - PTR_CBYTE table = (PTR_CBYTE) gcInfoToken.Info; -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xFEEF); -#endif - - infoPtr->methodSize = fastDecodeUnsigned(table); - - _ASSERTE(curOffset >= 0); - _ASSERTE(curOffset <= infoPtr->methodSize); - - /* Decode the InfoHdr */ - - InfoHdr header; - table = decodeHeader(table, gcInfoToken.Version, &header); - - BOOL hasArgTabOffset = FALSE; - if (header.untrackedCnt == HAS_UNTRACKED) - { - hasArgTabOffset = TRUE; - header.untrackedCnt = fastDecodeUnsigned(table); - } - - if (header.varPtrTableSize == HAS_VARPTR) - { - hasArgTabOffset = TRUE; - header.varPtrTableSize = fastDecodeUnsigned(table); - } - - if (header.gsCookieOffset == HAS_GS_COOKIE_OFFSET) - { - header.gsCookieOffset = fastDecodeUnsigned(table); - } - - if (header.syncStartOffset == HAS_SYNC_OFFSET) - { - header.syncStartOffset = decodeUnsigned(table); - header.syncEndOffset = decodeUnsigned(table); - - _ASSERTE(header.syncStartOffset != INVALID_SYNC_OFFSET && header.syncEndOffset != INVALID_SYNC_OFFSET); - _ASSERTE(header.syncStartOffset < header.syncEndOffset); - } - - if (header.revPInvokeOffset == HAS_REV_PINVOKE_FRAME_OFFSET) - { - header.revPInvokeOffset = fastDecodeUnsigned(table); - } - - /* Some sanity checks on header */ - - _ASSERTE( header.prologSize + - (size_t)(header.epilogCount*header.epilogSize) <= infoPtr->methodSize); - _ASSERTE( header.epilogCount == 1 || !header.epilogAtEnd); - - _ASSERTE( header.untrackedCnt <= header.argCount+header.frameSize); - - _ASSERTE( header.ebpSaved || !(header.ebpFrame || header.doubleAlign)); - _ASSERTE(!header.ebpFrame || !header.doubleAlign ); - _ASSERTE( header.ebpFrame || !header.security ); - _ASSERTE( header.ebpFrame || !header.handlers ); - _ASSERTE( header.ebpFrame || !header.localloc ); - _ASSERTE( header.ebpFrame || !header.editNcontinue); // : Esp frames NYI for EnC - - /* Initialize the infoPtr struct */ - - infoPtr->argSize = header.argCount * 4; - infoPtr->ebpFrame = header.ebpFrame; - infoPtr->interruptible = header.interruptible; - infoPtr->returnKind = (ReturnKind) header.returnKind; - - infoPtr->prologSize = header.prologSize; - infoPtr->epilogSize = header.epilogSize; - infoPtr->epilogCnt = header.epilogCount; - infoPtr->epilogEnd = header.epilogAtEnd; - - infoPtr->untrackedCnt = header.untrackedCnt; - infoPtr->varPtrTableSize = header.varPtrTableSize; - infoPtr->gsCookieOffset = header.gsCookieOffset; - - infoPtr->syncStartOffset = header.syncStartOffset; - infoPtr->syncEndOffset = header.syncEndOffset; - infoPtr->revPInvokeOffset = header.revPInvokeOffset; - - infoPtr->doubleAlign = header.doubleAlign; - infoPtr->handlers = header.handlers; - infoPtr->localloc = header.localloc; - infoPtr->editNcontinue = header.editNcontinue; - infoPtr->varargs = header.varargs; - infoPtr->profCallbacks = header.profCallbacks; - infoPtr->genericsContext = header.genericsContext; - infoPtr->genericsContextIsMethodDesc = header.genericsContextIsMethodDesc; - infoPtr->isSpeculativeStackWalk = false; - - /* Are we within the prolog of the method? */ - - if (curOffset < infoPtr->prologSize) - { - infoPtr->prologOffs = curOffset; - } - else - { - infoPtr->prologOffs = hdrInfo::NOT_IN_PROLOG; - } - - /* Assume we're not in the epilog of the method */ - - infoPtr->epilogOffs = hdrInfo::NOT_IN_EPILOG; - - /* Are we within an epilog of the method? */ - - if (infoPtr->epilogCnt) - { - unsigned epilogStart; - - if (infoPtr->epilogCnt > 1 || !infoPtr->epilogEnd) - { -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xFACE); -#endif - epilogStart = 0; - for (unsigned i = 0; i < infoPtr->epilogCnt; i++) - { - epilogStart += fastDecodeUnsigned(table); - if (curOffset > epilogStart && - curOffset < epilogStart + infoPtr->epilogSize) - { - infoPtr->epilogOffs = curOffset - epilogStart; - } - } - } - else - { - epilogStart = infoPtr->methodSize - infoPtr->epilogSize; - - if (curOffset > epilogStart && - curOffset < epilogStart + infoPtr->epilogSize) - { - infoPtr->epilogOffs = curOffset - epilogStart; - } - } - - infoPtr->syncEpilogStart = epilogStart; - } - - unsigned argTabOffset = INVALID_ARGTAB_OFFSET; - if (hasArgTabOffset) - { - argTabOffset = fastDecodeUnsigned(table); - } - infoPtr->argTabOffset = argTabOffset; - - size_t frameDwordCount = header.frameSize; - - /* Set the rawStackSize to the number of bytes that it bumps ESP */ - - infoPtr->rawStkSize = (UINT)(frameDwordCount * sizeof(size_t)); - - /* Calculate the callee saves regMask and adjust stackSize to */ - /* include the callee saves register spills */ - - unsigned savedRegs = RM_NONE; - unsigned savedRegsCount = 0; - - if (header.ediSaved) - { - savedRegsCount++; - savedRegs |= RM_EDI; - } - if (header.esiSaved) - { - savedRegsCount++; - savedRegs |= RM_ESI; - } - if (header.ebxSaved) - { - savedRegsCount++; - savedRegs |= RM_EBX; - } - if (header.ebpSaved) - { - savedRegsCount++; - savedRegs |= RM_EBP; - } - - infoPtr->savedRegMask = (RegMask)savedRegs; - - infoPtr->savedRegsCountExclFP = savedRegsCount; - if (header.ebpFrame || header.doubleAlign) - { - _ASSERTE(header.ebpSaved); - infoPtr->savedRegsCountExclFP = savedRegsCount - 1; - } - - frameDwordCount += savedRegsCount; - - infoPtr->stackSize = (UINT)(frameDwordCount * sizeof(size_t)); - - _ASSERTE(infoPtr->gsCookieOffset == INVALID_GS_COOKIE_OFFSET || - (infoPtr->gsCookieOffset < infoPtr->stackSize) && - ((header.gsCookieOffset % sizeof(void*)) == 0)); - - return table - PTR_CBYTE(gcInfoToken.Info); -} - -/*****************************************************************************/ - -// We do a "pop eax; jmp eax" to return from a fault or finally handler -const size_t END_FIN_POP_STACK = sizeof(TADDR); - -inline -size_t GetLocallocSPOffset(hdrInfo * info) -{ - LIMITED_METHOD_DAC_CONTRACT; - - _ASSERTE(info->localloc && info->ebpFrame); - - unsigned position = info->savedRegsCountExclFP + - 1; - return position * sizeof(TADDR); -} - -#ifndef FEATURE_NATIVEAOT -inline -size_t GetParamTypeArgOffset(hdrInfo * info) -{ - LIMITED_METHOD_DAC_CONTRACT; - - _ASSERTE((info->genericsContext || info->handlers) && info->ebpFrame); - - unsigned position = info->savedRegsCountExclFP + - info->localloc + - 1; // For CORINFO_GENERICS_CTXT_FROM_PARAMTYPEARG - return position * sizeof(TADDR); -} - -inline size_t GetStartShadowSPSlotsOffset(hdrInfo * info) -{ - LIMITED_METHOD_DAC_CONTRACT; - - _ASSERTE(info->handlers && info->ebpFrame); - - return GetParamTypeArgOffset(info) + - sizeof(TADDR); // Slot for end-of-last-executed-filter -} - -/***************************************************************************** - * Returns the start of the hidden slots for the shadowSP for functions - * with exception handlers. There is one slot per nesting level starting - * near Ebp and is zero-terminated after the active slots. - */ - -inline -PTR_TADDR GetFirstBaseSPslotPtr(TADDR ebp, hdrInfo * info) -{ - LIMITED_METHOD_DAC_CONTRACT; - - _ASSERTE(info->handlers && info->ebpFrame); - - size_t offsetFromEBP = GetStartShadowSPSlotsOffset(info) - + sizeof(TADDR); // to get to the *start* of the next slot - - return PTR_TADDR(ebp - offsetFromEBP); -} - -inline size_t GetEndShadowSPSlotsOffset(hdrInfo * info, unsigned maxHandlerNestingLevel) -{ - LIMITED_METHOD_DAC_CONTRACT; - - _ASSERTE(info->handlers && info->ebpFrame); - - unsigned numberOfShadowSPSlots = maxHandlerNestingLevel + - 1 + // For zero-termination - 1; // For a filter (which can be active at the same time as a catch/finally handler - - return GetStartShadowSPSlotsOffset(info) + - (numberOfShadowSPSlots * sizeof(TADDR)); -} - -/***************************************************************************** - * returns the base frame pointer corresponding to the target nesting level. - */ - -inline -TADDR GetOutermostBaseFP(TADDR ebp, hdrInfo * info) -{ - LIMITED_METHOD_DAC_CONTRACT; - - // we are not taking into account double alignment. We are - // safe because the jit currently bails on double alignment if there - // are handles or localalloc - _ASSERTE(!info->doubleAlign); - if (info->localloc) - { - // If the function uses localloc we will fetch the ESP from the localloc - // slot. - PTR_TADDR pLocalloc = PTR_TADDR(ebp - GetLocallocSPOffset(info)); - - return (*pLocalloc); - } - else - { - // Default, go back all the method's local stack size - return ebp - info->stackSize + sizeof(int); - } -} - -/***************************************************************************** - * - * For functions with handlers, checks if it is currently in a handler. - * Either of unwindESP or unwindLevel will specify the target nesting level. - * If unwindLevel is specified, info about the funclet at that nesting level - * will be returned. (Use if you are interested in a specific nesting level.) - * If unwindESP is specified, info for nesting level invoked before the stack - * reached unwindESP will be returned. (Use if you have a specific ESP value - * during stack walking.) - * - * *pBaseSP is set to the base SP (base of the stack on entry to - * the current funclet) corresponding to the target nesting level. - * *pNestLevel is set to the nesting level of the target nesting level (useful - * if unwindESP!=IGNORE_VAL - * *pHasInnerFilter will be set to true (only when unwindESP!=IGNORE_VAL) if a filter - * is currently active, but the target nesting level is an outer nesting level. - * *pHadInnerFilter - was the last use of the frame to execute a filter. - * This mainly affects GC lifetime reporting. - */ - -enum FrameType -{ - FR_NORMAL, // Normal method frame - no exceptions currently active - FR_FILTER, // Frame-let of a filter - FR_HANDLER, // Frame-let of a callable catch/fault/finally - - FR_INVALID, // Invalid frame (for speculative stackwalks) -}; - -enum { IGNORE_VAL = -1 }; - -FrameType GetHandlerFrameInfo(hdrInfo * info, - TADDR frameEBP, - TADDR unwindESP, - DWORD unwindLevel, - TADDR * pBaseSP = NULL, /* OUT */ - DWORD * pNestLevel = NULL, /* OUT */ - bool * pHasInnerFilter = NULL, /* OUT */ - bool * pHadInnerFilter = NULL) /* OUT */ -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - HOST_NOCALLS; - SUPPORTS_DAC; - } CONTRACTL_END; - - _ASSERTE(info->ebpFrame && info->handlers); - // One and only one of them should be IGNORE_VAL - _ASSERTE((unwindESP == (TADDR) IGNORE_VAL) != - (unwindLevel == (DWORD) IGNORE_VAL)); - _ASSERTE(pHasInnerFilter == NULL || unwindESP != (TADDR) IGNORE_VAL); - - // Many of the conditions that we'd like to assert cannot be asserted in the case that we're - // in the middle of a stackwalk seeded by a profiler, since such seeds can't be trusted - // (profilers are external, untrusted sources). So during profiler walks, we test the condition - // and throw an exception if it's not met. Otherwise, we just assert the condition. - #define FAIL_IF_SPECULATIVE_WALK(condition) \ - if (info->isSpeculativeStackWalk) \ - { \ - if (!(condition)) \ - { \ - return FR_INVALID; \ - } \ - } \ - else \ - { \ - _ASSERTE(condition); \ - } - - PTR_TADDR pFirstBaseSPslot = GetFirstBaseSPslotPtr(frameEBP, info); - TADDR baseSP = GetOutermostBaseFP(frameEBP, info); - bool nonLocalHandlers = false; // Are the funclets invoked by EE (instead of managed code itself) - bool hasInnerFilter = false; - bool hadInnerFilter = false; - - /* Get the last non-zero slot >= unwindESP, or lvl curSlotVal || - (baseSP == curSlotVal && pSlot == pFirstBaseSPslot)); - - if (curSlotVal == LCL_FINALLY_MARK) - { - // Locally called finally - baseSP -= sizeof(TADDR); - } - else - { - // Is this a funclet we unwound before (can only happen with filters) ? - // If unwindESP is specified, normally we expect it to be the last entry in the shadow slot array. - // Or, if there is a filter, we expect unwindESP to be the second last entry. However, this may - // not be the case in DAC builds. For example, the user can use .cxr in an EH clause to set a - // CONTEXT captured in the try clause. In this case, unwindESP will be the ESP of the parent - // function, but the shadow slot array will contain the SP of the EH clause, which is closer to - // the leaf than the parent method. - - if (unwindESP != (TADDR) IGNORE_VAL && - unwindESP > END_FIN_POP_STACK + - (curSlotVal & ~ICodeManager::SHADOW_SP_BITS)) - { - // In non-DAC builds, the only time unwindESP is closer to the root than entries in the shadow - // slot array is when the last entry in the array is for a filter. Also, filters can't have - // nested handlers. - if ((pSlot[0] & ICodeManager::SHADOW_SP_IN_FILTER) && - (pSlot[-1] == 0) && - !(baseSP & ICodeManager::SHADOW_SP_IN_FILTER)) - { - if (pSlot[0] & ICodeManager::SHADOW_SP_FILTER_DONE) - hadInnerFilter = true; - else - hasInnerFilter = true; - break; - } - else - { -#if defined(DACCESS_COMPILE) - // In DAC builds, this could happen. We just need to bail out of this loop early. - break; -#else // !DACCESS_COMPILE - // In non-DAC builds, this is an error. - FAIL_IF_SPECULATIVE_WALK(FALSE); -#endif // DACCESS_COMPILE - } - } - - nonLocalHandlers = true; - baseSP = curSlotVal; - } - } -#endif // FEATURE_EH_FUNCLETS - - if (unwindESP != (TADDR) IGNORE_VAL) - { - FAIL_IF_SPECULATIVE_WALK(baseSP >= unwindESP || - baseSP == unwindESP - sizeof(TADDR)); // About to locally call a finally - - if (baseSP < unwindESP) // About to locally call a finally - baseSP = unwindESP; - } - else - { - FAIL_IF_SPECULATIVE_WALK(lvl == unwindLevel); // unwindLevel must be currently active on stack - } - - if (pBaseSP) - *pBaseSP = baseSP & ~ICodeManager::SHADOW_SP_BITS; - - if (pNestLevel) - { - *pNestLevel = (DWORD)lvl; - } - - if (pHasInnerFilter) - *pHasInnerFilter = hasInnerFilter; - - if (pHadInnerFilter) - *pHadInnerFilter = hadInnerFilter; - - if (baseSP & ICodeManager::SHADOW_SP_IN_FILTER) - { - FAIL_IF_SPECULATIVE_WALK(!hasInnerFilter); // nested filters not allowed - return FR_FILTER; - } - else if (nonLocalHandlers) - { - return FR_HANDLER; - } - else - { - return FR_NORMAL; - } - - #undef FAIL_IF_SPECULATIVE_WALK -} - -// Returns the number of bytes at the beginning of the stack frame that shouldn't be -// modified by an EnC. This is everything except the space for locals and temporaries. -inline size_t GetSizeOfFrameHeaderForEnC(hdrInfo * info) -{ - WRAPPER_NO_CONTRACT; - - // See comment above Compiler::lvaAssignFrameOffsets() in src\jit\il\lclVars.cpp - // for frame layout - - // EnC supports increasing the maximum handler nesting level by always - // assuming that the max is MAX_EnC_HANDLER_NESTING_LEVEL. Methods with - // a higher max cannot be updated by EnC - - // Take the offset (from EBP) of the last slot of the header, plus one for the EBP slot itself - // to get the total size of the header. - return sizeof(TADDR) + - GetEndShadowSPSlotsOffset(info, MAX_EnC_HANDLER_NESTING_LEVEL); -} -#endif - -/*****************************************************************************/ -static -PTR_CBYTE skipToArgReg(const hdrInfo& info, PTR_CBYTE table) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - SUPPORTS_DAC; - } CONTRACTL_END; - -#ifdef _DEBUG - PTR_CBYTE tableStart = table; -#else - if (info.argTabOffset != INVALID_ARGTAB_OFFSET) - { - return table + info.argTabOffset; - } -#endif - - unsigned count; - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xBEEF); -#endif - - /* Skip over the untracked frame variable table */ - - count = info.untrackedCnt; - while (count-- > 0) { - fastSkipSigned(table); - } - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xCAFE); -#endif - - /* Skip over the frame variable lifetime table */ - - count = info.varPtrTableSize; - while (count-- > 0) { - fastSkipUnsigned(table); fastSkipUnsigned(table); fastSkipUnsigned(table); - } - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *) == 0xBABE); -#endif - -#if defined(_DEBUG) && defined(CONSISTENCY_CHECK_MSGF) - if (info.argTabOffset != INVALID_ARGTAB_OFFSET) - { - CONSISTENCY_CHECK_MSGF((info.argTabOffset == (unsigned) (table - tableStart)), - ("table = %p, tableStart = %p, info.argTabOffset = %d", table, tableStart, info.argTabOffset)); - } -#endif - - return table; -} - -/*****************************************************************************/ - -#define regNumToMask(regNum) RegMask(1<<(regNum)) - -/***************************************************************************** - Helper for scanArgRegTable() and scanArgRegTableI() for regMasks - */ - -void * getCalleeSavedReg(PREGDISPLAY pContext, regNum reg) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - switch (reg) - { - case REGI_EBP: return pContext->GetEbpLocation(); - case REGI_EBX: return pContext->GetEbxLocation(); - case REGI_ESI: return pContext->GetEsiLocation(); - case REGI_EDI: return pContext->GetEdiLocation(); - - default: _ASSERTE(!"bad info.thisPtrResult"); return NULL; - } -} - -/***************************************************************************** - These functions converts the bits in the GC encoding to RegMask - */ - -inline -RegMask convertCalleeSavedRegsMask(unsigned inMask) // EBP,EBX,ESI,EDI -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - _ASSERTE((inMask & 0x0F) == inMask); - - unsigned outMask = RM_NONE; - if (inMask & 0x1) outMask |= RM_EDI; - if (inMask & 0x2) outMask |= RM_ESI; - if (inMask & 0x4) outMask |= RM_EBX; - if (inMask & 0x8) outMask |= RM_EBP; - - return (RegMask) outMask; -} - -inline -RegMask convertAllRegsMask(unsigned inMask) // EAX,ECX,EDX,EBX, EBP,ESI,EDI -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - _ASSERTE((inMask & 0xEF) == inMask); - - unsigned outMask = RM_NONE; - if (inMask & 0x01) outMask |= RM_EAX; - if (inMask & 0x02) outMask |= RM_ECX; - if (inMask & 0x04) outMask |= RM_EDX; - if (inMask & 0x08) outMask |= RM_EBX; - if (inMask & 0x20) outMask |= RM_EBP; - if (inMask & 0x40) outMask |= RM_ESI; - if (inMask & 0x80) outMask |= RM_EDI; - - return (RegMask)outMask; -} - -/***************************************************************************** - * scan the register argument table for the not fully interruptible case. - this function is called to find all live objects (pushed arguments) - and to get the stack base for EBP-less methods. - - NOTE: If info->argTabResult is NULL, info->argHnumResult indicates - how many bits in argMask are valid - If info->argTabResult is non-NULL, then the argMask field does - not fit in 32-bits and the value in argMask meaningless. - Instead argHnum specifies the number of (variable-length) elements - in the array, and argTabBytes specifies the total byte size of the - array. [ Note this is an extremely rare case ] - */ - -#ifdef _PREFAST_ -#pragma warning(push) -#pragma warning(disable:21000) // Suppress PREFast warning about overly large function -#endif -static -unsigned scanArgRegTable(PTR_CBYTE table, - unsigned curOffs, - hdrInfo * info) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - SUPPORTS_DAC; - } CONTRACTL_END; - - regNum thisPtrReg = REGI_NA; -#ifdef _DEBUG - bool isCall = false; -#endif - unsigned regMask = 0; // EBP,EBX,ESI,EDI - unsigned argMask = 0; - unsigned argHnum = 0; - PTR_CBYTE argTab = 0; - unsigned argTabBytes = 0; - unsigned stackDepth = 0; - - unsigned iregMask = 0; // EBP,EBX,ESI,EDI - unsigned iargMask = 0; - unsigned iptrMask = 0; - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xBABE); -#endif - - unsigned scanOffs = 0; - - _ASSERTE(scanOffs <= info->methodSize); - - if (info->ebpFrame) { - /* - Encoding table for methods with an EBP frame and - that are not fully interruptible - - The encoding used is as follows: - - this pointer encodings: - - 01000000 this pointer in EBX - 00100000 this pointer in ESI - 00010000 this pointer in EDI - - tiny encoding: - - 0bsdDDDD - requires code delta < 16 (4-bits) - requires pushed argmask == 0 - - where DDDD is code delta - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - d indicates that register EDI is a live pointer - - small encoding: - - 1DDDDDDD bsdAAAAA - - requires code delta < 120 (7-bits) - requires pushed argmask < 64 (5-bits) - - where DDDDDDD is code delta - AAAAA is the pushed args mask - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - d indicates that register EDI is a live pointer - - medium encoding - - 0xFD aaaaaaaa AAAAdddd bseDDDDD - - requires code delta < 0x1000000000 (9-bits) - requires pushed argmask < 0x1000000000000 (12-bits) - - where DDDDD is the upper 5-bits of the code delta - dddd is the low 4-bits of the code delta - AAAA is the upper 4-bits of the pushed arg mask - aaaaaaaa is the low 8-bits of the pushed arg mask - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - e indicates that register EDI is a live pointer - - medium encoding with interior pointers - - 0xF9 DDDDDDDD bsdAAAAAA iiiIIIII - - requires code delta < (8-bits) - requires pushed argmask < (5-bits) - - where DDDDDDD is the code delta - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - d indicates that register EDI is a live pointer - AAAAA is the pushed arg mask - iii indicates that EBX,EDI,ESI are interior pointers - IIIII indicates that bits is the arg mask are interior - pointers - - large encoding - - 0xFE [0BSD0bsd][32-bit code delta][32-bit argMask] - - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - d indicates that register EDI is a live pointer - B indicates that register EBX is an interior pointer - S indicates that register ESI is an interior pointer - D indicates that register EDI is an interior pointer - requires pushed argmask < 32-bits - - large encoding with interior pointers - - 0xFA [0BSD0bsd][32-bit code delta][32-bit argMask][32-bit interior pointer mask] - - - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - d indicates that register EDI is a live pointer - B indicates that register EBX is an interior pointer - S indicates that register ESI is an interior pointer - D indicates that register EDI is an interior pointer - requires pushed argmask < 32-bits - requires pushed iArgmask < 32-bits - - huge encoding This is the only encoding that supports - a pushed argmask which is greater than - 32-bits. - - 0xFB [0BSD0bsd][32-bit code delta] - [32-bit table count][32-bit table size] - [pushed ptr offsets table...] - - b indicates that register EBX is a live pointer - s indicates that register ESI is a live pointer - d indicates that register EDI is a live pointer - B indicates that register EBX is an interior pointer - S indicates that register ESI is an interior pointer - D indicates that register EDI is an interior pointer - the list count is the number of entries in the list - the list size gives the byte-length of the list - the offsets in the list are variable-length - */ - while (scanOffs < curOffs) - { - iregMask = 0; - iargMask = 0; - argTab = NULL; -#ifdef _DEBUG - isCall = true; -#endif - - /* Get the next byte and check for a 'special' entry */ - - unsigned encType = *table++; -#if defined(DACCESS_COMPILE) - // In this scenario, it is invalid to have a zero byte in the GC info encoding (refer to the - // comments above). At least one bit has to be set. For example, a byte can represent which - // register is the "this" pointer, and this byte has to be 0x10, 0x20, or 0x40. Having a zero - // byte indicates there is most likely some sort of DAC error, and it may lead to problems such as - // infinite loops. So we bail out early instead. - if (encType == 0) - { - DacError(CORDBG_E_TARGET_INCONSISTENT); - UNREACHABLE(); - } -#endif // DACCESS_COMPILE - - switch (encType) - { - unsigned val, nxt; - - default: - - /* A tiny or small call entry */ - val = encType; - if ((val & 0x80) == 0x00) { - if (val & 0x0F) { - /* A tiny call entry */ - scanOffs += (val & 0x0F); - regMask = (val & 0x70) >> 4; - argMask = 0; - argHnum = 0; - } - else { - /* This pointer liveness encoding */ - regMask = (val & 0x70) >> 4; - if (regMask == 0x1) - thisPtrReg = REGI_EDI; - else if (regMask == 0x2) - thisPtrReg = REGI_ESI; - else if (regMask == 0x4) - thisPtrReg = REGI_EBX; - else - _ASSERTE(!"illegal encoding for 'this' pointer liveness"); - } - } - else { - /* A small call entry */ - scanOffs += (val & 0x7F); - val = *table++; - regMask = val >> 5; - argMask = val & 0x1F; - argHnum = 5; - } - break; - - case 0xFD: // medium encoding - - argMask = *table++; - val = *table++; - argMask |= ((val & 0xF0) << 4); - argHnum = 12; - nxt = *table++; - scanOffs += (val & 0x0F) + ((nxt & 0x1F) << 4); - regMask = nxt >> 5; // EBX,ESI,EDI - - break; - - case 0xF9: // medium encoding with interior pointers - - scanOffs += *table++; - val = *table++; - argMask = val & 0x1F; - argHnum = 5; - regMask = val >> 5; - val = *table++; - iargMask = val & 0x1F; - iregMask = val >> 5; - - break; - - case 0xFE: // large encoding - case 0xFA: // large encoding with interior pointers - - val = *table++; - regMask = val & 0x7; - iregMask = val >> 4; - scanOffs += *dac_cast(table); table += sizeof(DWORD); - argMask = *dac_cast(table); table += sizeof(DWORD); - argHnum = 31; - if (encType == 0xFA) // read iargMask - { - iargMask = *dac_cast(table); table += sizeof(DWORD); - } - break; - - case 0xFB: // huge encoding This is the only partially interruptible - // encoding that supports a pushed ArgMask - // which is greater than 32-bits. - // The ArgMask is encoded using the argTab - val = *table++; - regMask = val & 0x7; - iregMask = val >> 4; - scanOffs += *dac_cast(table); table += sizeof(DWORD); - argHnum = *dac_cast(table); table += sizeof(DWORD); - argTabBytes = *dac_cast(table); table += sizeof(DWORD); - argTab = table; table += argTabBytes; - - argMask = 0; - break; - - case 0xFF: - scanOffs = curOffs + 1; - break; - - } // end case - - // iregMask & iargMask are subsets of regMask & argMask respectively - - _ASSERTE((iregMask & regMask) == iregMask); - _ASSERTE((iargMask & argMask) == iargMask); - - } // end while - - } - else { - -/* - * Encoding table for methods with an ESP frame and are not fully interruptible - * This encoding does not support a pushed ArgMask greater than 32 - * - * The encoding used is as follows: - * - * push 000DDDDD ESP push one item with 5-bit delta - * push 00100000 [pushCount] ESP push multiple items - * reserved 0011xxxx - * skip 01000000 [Delta] Skip Delta, arbitrary sized delta - * skip 0100DDDD Skip small Delta, for call (DDDD != 0) - * pop 01CCDDDD ESP pop CC items with 4-bit delta (CC != 00) - * call 1PPPPPPP Call Pattern, P=[0..79] - * call 1101pbsd DDCCCMMM Call RegMask=pbsd,ArgCnt=CCC, - * ArgMask=MMM Delta=commonDelta[DD] - * call 1110pbsd [ArgCnt] [ArgMask] Call ArgCnt,RegMask=pbsd,[32-bit ArgMask] - * call 11111000 [PBSDpbsd][32-bit delta][32-bit ArgCnt] - * [32-bit PndCnt][32-bit PndSize][PndOffs...] - * iptr 11110000 [IPtrMask] Arbitrary 32-bit Interior Pointer Mask - * thisptr 111101RR This pointer is in Register RR - * 00=EDI,01=ESI,10=EBX,11=EBP - * reserved 111100xx xx != 00 - * reserved 111110xx xx != 00 - * reserved 11111xxx xxx != 000 && xxx != 111(EOT) - * - * The value 11111111 [0xFF] indicates the end of the table. - * - * An offset (at which stack-walking is performed) without an explicit encoding - * is assumed to be a trivial call-site (no GC registers, stack empty before and - * after) to avoid having to encode all trivial calls. - * - * Note on the encoding used for interior pointers - * - * The iptr encoding must immediately precede a call encoding. It is used to - * transform a normal GC pointer addresses into an interior pointers for GC purposes. - * The mask supplied to the iptr encoding is read from the least signicant bit - * to the most signicant bit. (i.e the lowest bit is read first) - * - * p indicates that register EBP is a live pointer - * b indicates that register EBX is a live pointer - * s indicates that register ESI is a live pointer - * d indicates that register EDI is a live pointer - * P indicates that register EBP is an interior pointer - * B indicates that register EBX is an interior pointer - * S indicates that register ESI is an interior pointer - * D indicates that register EDI is an interior pointer - * - * As an example the following sequence indicates that EDI.ESI and the 2nd pushed pointer - * in ArgMask are really interior pointers. The pointer in ESI in a normal pointer: - * - * iptr 11110000 00010011 => read Interior Ptr, Interior Ptr, Normal Ptr, Normal Ptr, Interior Ptr - * call 11010011 DDCCC011 RRRR=1011 => read EDI is a GC-pointer, ESI is a GC-pointer. EBP is a GC-pointer - * MMM=0011 => read two GC-pointers arguments on the stack (nested call) - * - * Since the call instruction mentions 5 GC-pointers we list them in the required order: - * EDI, ESI, EBP, 1st-pushed pointer, 2nd-pushed pointer - * - * And we apply the Interior Pointer mask mmmm=10011 to the above five ordered GC-pointers - * we learn that EDI and ESI are interior GC-pointers and that the second push arg is an - * interior GC-pointer. - */ - -#if defined(DACCESS_COMPILE) - DWORD cbZeroBytes = 0; -#endif // DACCESS_COMPILE - - while (scanOffs <= curOffs) - { - unsigned callArgCnt; - unsigned skip; - unsigned newRegMask, inewRegMask; - unsigned newArgMask, inewArgMask; - unsigned oldScanOffs = scanOffs; - - if (iptrMask) - { - // We found this iptrMask in the previous iteration. - // This iteration must be for a call. Set these variables - // so that they are available at the end of the loop - - inewRegMask = iptrMask & 0x0F; // EBP,EBX,ESI,EDI - inewArgMask = iptrMask >> 4; - - iptrMask = 0; - } - else - { - // Zero out any stale values. - - inewRegMask = 0; - inewArgMask = 0; - } - - /* Get the next byte and decode it */ - - unsigned val = *table++; -#if defined(DACCESS_COMPILE) - // In this scenario, a 0 means that there is a push at the current offset. For a struct with - // two double fields, the JIT may use two movq instructions to push the struct onto the stack, and - // the JIT will encode 4 pushes at the same code offset. This means that we can have up to 4 - // consecutive bytes of 0 without changing the code offset. Having more than 4 consecutive bytes - // of zero indicates that there is most likely some sort of DAC error, and it may lead to problems - // such as infinite loops. So we bail out early instead. - if (val == 0) - { - cbZeroBytes += 1; - if (cbZeroBytes > 4) - { - DacError(CORDBG_E_TARGET_INCONSISTENT); - UNREACHABLE(); - } - } - else - { - cbZeroBytes = 0; - } -#endif // DACCESS_COMPILE - -#ifdef _DEBUG - if (scanOffs != curOffs) - isCall = false; -#endif - - /* Check pushes, pops, and skips */ - - if (!(val & 0x80)) { - - // iptrMask can immediately precede only calls - - _ASSERTE(inewRegMask == 0); - _ASSERTE(inewArgMask == 0); - - if (!(val & 0x40)) { - - unsigned pushCount; - - if (!(val & 0x20)) - { - // - // push 000DDDDD ESP push one item, 5-bit delta - // - pushCount = 1; - scanOffs += val & 0x1f; - } - else - { - // - // push 00100000 [pushCount] ESP push multiple items - // - _ASSERTE(val == 0x20); - pushCount = fastDecodeUnsigned(table); - } - - if (scanOffs > curOffs) - { - scanOffs = oldScanOffs; - goto FINISHED; - } - - stackDepth += pushCount; - } - else if ((val & 0x3f) != 0) { - // - // pop 01CCDDDD pop CC items, 4-bit delta - // - scanOffs += val & 0x0f; - if (scanOffs > curOffs) - { - scanOffs = oldScanOffs; - goto FINISHED; - } - stackDepth -= (val & 0x30) >> 4; - - } else if (scanOffs < curOffs) { - // - // skip 01000000 [Delta] Skip arbitrary sized delta - // - skip = fastDecodeUnsigned(table); - scanOffs += skip; - } - else // don't process a skip if we are already at curOffs - goto FINISHED; - - /* reset regs and args state since we advance past last call site */ - - regMask = 0; - iregMask = 0; - argMask = 0; - iargMask = 0; - argHnum = 0; - - } - else /* It must be a call, thisptr, or iptr */ - { - switch ((val & 0x70) >> 4) { - default: // case 0-4, 1000xxxx through 1100xxxx - // - // call 1PPPPPPP Call Pattern, P=[0..79] - // - decodeCallPattern((val & 0x7f), &callArgCnt, - &newRegMask, &newArgMask, &skip); - // If we've already reached curOffs and the skip amount - // is non-zero then we are done - if ((scanOffs == curOffs) && (skip > 0)) - goto FINISHED; - // otherwise process this call pattern - scanOffs += skip; - if (scanOffs > curOffs) - goto FINISHED; -#ifdef _DEBUG - isCall = true; -#endif - regMask = newRegMask; - argMask = newArgMask; argTab = NULL; - iregMask = inewRegMask; - iargMask = inewArgMask; - stackDepth -= callArgCnt; - argHnum = 2; // argMask is known to be <= 3 - break; - - case 5: - // - // call 1101RRRR DDCCCMMM Call RegMask=RRRR,ArgCnt=CCC, - // ArgMask=MMM Delta=commonDelta[DD] - // - newRegMask = val & 0xf; // EBP,EBX,ESI,EDI - val = *table++; // read next byte - skip = callCommonDelta[val>>6]; - // If we've already reached curOffs and the skip amount - // is non-zero then we are done - if ((scanOffs == curOffs) && (skip > 0)) - goto FINISHED; - // otherwise process this call encoding - scanOffs += skip; - if (scanOffs > curOffs) - goto FINISHED; -#ifdef _DEBUG - isCall = true; -#endif - regMask = newRegMask; - iregMask = inewRegMask; - callArgCnt = (val >> 3) & 0x7; - stackDepth -= callArgCnt; - argMask = (val & 0x7); argTab = NULL; - iargMask = inewArgMask; - argHnum = 3; - break; - - case 6: - // - // call 1110RRRR [ArgCnt] [ArgMask] - // Call ArgCnt,RegMask=RRR,ArgMask - // -#ifdef _DEBUG - isCall = true; -#endif - regMask = val & 0xf; // EBP,EBX,ESI,EDI - iregMask = inewRegMask; - callArgCnt = fastDecodeUnsigned(table); - stackDepth -= callArgCnt; - argMask = fastDecodeUnsigned(table); argTab = NULL; - iargMask = inewArgMask; - argHnum = sizeof(argMask) * 8; // The size of argMask in bits - break; - - case 7: - switch (val & 0x0C) - { - case 0x00: - // - // 0xF0 iptr 11110000 [IPtrMask] Arbitrary Interior Pointer Mask - // - iptrMask = fastDecodeUnsigned(table); - break; - - case 0x04: - // - // 0xF4 thisptr 111101RR This pointer is in Register RR - // 00=EDI,01=ESI,10=EBX,11=EBP - // - { - static const regNum calleeSavedRegs[] = - { REGI_EDI, REGI_ESI, REGI_EBX, REGI_EBP }; - thisPtrReg = calleeSavedRegs[val&0x3]; - } - break; - - case 0x08: - // - // 0xF8 call 11111000 [PBSDpbsd][32-bit delta][32-bit ArgCnt] - // [32-bit PndCnt][32-bit PndSize][PndOffs...] - // - val = *table++; - skip = *dac_cast(table); table += sizeof(DWORD); -// [VSUQFE 4670] - // If we've already reached curOffs and the skip amount - // is non-zero then we are done - if ((scanOffs == curOffs) && (skip > 0)) - goto FINISHED; -// [VSUQFE 4670] - scanOffs += skip; - if (scanOffs > curOffs) - goto FINISHED; -#ifdef _DEBUG - isCall = true; -#endif - regMask = val & 0xF; - iregMask = val >> 4; - callArgCnt = *dac_cast(table); table += sizeof(DWORD); - stackDepth -= callArgCnt; - argHnum = *dac_cast(table); table += sizeof(DWORD); - argTabBytes = *dac_cast(table); table += sizeof(DWORD); - argTab = table; - table += argTabBytes; - break; - - case 0x0C: - // - // 0xFF end 11111111 End of table marker - // - _ASSERTE(val==0xff); - goto FINISHED; - - default: - _ASSERTE(!"reserved GC encoding"); - break; - } - break; - - } // end switch - - } // end else (!(val & 0x80)) - - // iregMask & iargMask are subsets of regMask & argMask respectively - - _ASSERTE((iregMask & regMask) == iregMask); - _ASSERTE((iargMask & argMask) == iargMask); - - } // end while - - } // end else ebp-less frame - -FINISHED: - - // iregMask & iargMask are subsets of regMask & argMask respectively - - _ASSERTE((iregMask & regMask) == iregMask); - _ASSERTE((iargMask & argMask) == iargMask); - - if (scanOffs != curOffs) - { - /* must have been a boring call */ - info->regMaskResult = RM_NONE; - info->argMaskResult = ptrArgTP(0); - info->iregMaskResult = RM_NONE; - info->iargMaskResult = ptrArgTP(0); - info->argHnumResult = 0; - info->argTabResult = NULL; - info->argTabBytes = 0; - } - else - { - info->regMaskResult = convertCalleeSavedRegsMask(regMask); - info->argMaskResult = ptrArgTP(argMask); - info->argHnumResult = argHnum; - info->iregMaskResult = convertCalleeSavedRegsMask(iregMask); - info->iargMaskResult = ptrArgTP(iargMask); - info->argTabResult = argTab; - info->argTabBytes = argTabBytes; - } - -#ifdef _DEBUG - if (scanOffs != curOffs) { - isCall = false; - } - _ASSERTE(thisPtrReg == REGI_NA || (!isCall || (regNumToMask(thisPtrReg) & info->regMaskResult))); -#endif - info->thisPtrResult = thisPtrReg; - - _ASSERTE(int(stackDepth) < INT_MAX); // check that it did not underflow - return (stackDepth * sizeof(unsigned)); -} -#ifdef _PREFAST_ -#pragma warning(pop) -#endif - - -/***************************************************************************** - * scan the register argument table for the fully interruptible case. - this function is called to find all live objects (pushed arguments) - and to get the stack base for fully interruptible methods. - Returns size of things pushed on the stack for ESP frames - - Arguments: - table - The pointer table - curOffsRegs - The current code offset that should be used for reporting registers - curOffsArgs - The current code offset that should be used for reporting args - info - Incoming arg used to determine if there's a frame, and to save results - */ - -static -unsigned scanArgRegTableI(PTR_CBYTE table, - unsigned curOffsRegs, - unsigned curOffsArgs, - hdrInfo * info) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - SUPPORTS_DAC; - } CONTRACTL_END; - - regNum thisPtrReg = REGI_NA; - unsigned ptrRegs = 0; // The mask of registers that contain pointers - unsigned iptrRegs = 0; // The subset of ptrRegs that are interior pointers - unsigned ptrOffs = 0; // The code offset of the table entry we are currently looking at - unsigned argCnt = 0; // The number of args that have been pushed - - ptrArgTP ptrArgs(0); // The mask of stack values that contain pointers. - ptrArgTP iptrArgs(0); // The subset of ptrArgs that are interior pointers. - ptrArgTP argHigh(0); // The current mask position that corresponds to the top of the stack. - - bool isThis = false; - bool iptr = false; - - // The comment before the call to scanArgRegTableI in EnumGCRefs - // describes why curOffsRegs can be smaller than curOffsArgs. - _ASSERTE(curOffsRegs <= curOffsArgs); - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xBABE); -#endif - - bool hasPartialArgInfo; - -#ifndef UNIX_X86_ABI - hasPartialArgInfo = info->ebpFrame; -#else - // For x86/Linux, interruptible code always has full arg info - // - // This should be aligned with emitFullArgInfo setting at - // emitter::emitEndCodeGen (in JIT) - hasPartialArgInfo = false; -#endif - - /* - Encoding table for methods that are fully interruptible - - The encoding used is as follows: - - ptr reg dead 00RRRDDD [RRR != 100] - ptr reg live 01RRRDDD [RRR != 100] - - non-ptr arg push 10110DDD [SSS == 110] - ptr arg push 10SSSDDD [SSS != 110] && [SSS != 111] - ptr arg pop 11CCCDDD [CCC != 000] && [CCC != 110] && [CCC != 111] - little delta skip 11000DDD [CCC == 000] - bigger delta skip 11110BBB [CCC == 110] - - The values used in the encodings are as follows: - - DDD code offset delta from previous entry (0-7) - BBB bigger delta 000=8,001=16,010=24,...,111=64 - RRR register number (EAX=000,ECX=001,EDX=010,EBX=011, - EBP=101,ESI=110,EDI=111), ESP=100 is reserved - SSS argument offset from base of stack. This is - redundant for frameless methods as we can - infer it from the previous pushes+pops. However, - for EBP-methods, we only report GC pushes, and - so we need SSS - CCC argument count being popped (includes only ptrs for EBP methods) - - The following are the 'large' versions: - - large delta skip 10111000 [0xB8] , encodeUnsigned(delta) - - large ptr arg push 11111000 [0xF8] , encodeUnsigned(pushCount) - large non-ptr arg push 11111001 [0xF9] , encodeUnsigned(pushCount) - large ptr arg pop 11111100 [0xFC] , encodeUnsigned(popCount) - large arg dead 11111101 [0xFD] , encodeUnsigned(popCount) for caller-pop args. - Any GC args go dead after the call, - but are still sitting on the stack - - this pointer prefix 10111100 [0xBC] the next encoding is a ptr live - or a ptr arg push - and contains the this pointer - - interior or by-ref 10111111 [0xBF] the next encoding is a ptr live - pointer prefix or a ptr arg push - and contains an interior - or by-ref pointer - - - The value 11111111 [0xFF] indicates the end of the table. - */ - -#if defined(DACCESS_COMPILE) - bool fLastByteIsZero = false; -#endif // DACCESS_COMPILE - - /* Have we reached the instruction we're looking for? */ - - while (ptrOffs <= curOffsArgs) - { - unsigned val; - - int isPop; - unsigned argOfs; - - unsigned regMask; - - // iptrRegs & iptrArgs are subsets of ptrRegs & ptrArgs respectively - - _ASSERTE((iptrRegs & ptrRegs) == iptrRegs); - _ASSERTE((iptrArgs & ptrArgs) == iptrArgs); - - /* Now find the next 'life' transition */ - - val = *table++; -#if defined(DACCESS_COMPILE) - // In this scenario, a zero byte means that EAX is going dead at the current offset. Since EAX - // can't go dead more than once at any given offset, it's invalid to have two consecutive bytes - // of zero. If this were to happen, then it means that there is most likely some sort of DAC - // error, and it may lead to problems such as infinite loops. So we bail out early instead. - if ((val == 0) && fLastByteIsZero) - { - DacError(CORDBG_E_TARGET_INCONSISTENT); - UNREACHABLE(); - } - fLastByteIsZero = (val == 0); -#endif // DACCESS_COMPILE - - if (!(val & 0x80)) - { - /* A small 'regPtr' encoding */ - - regNum reg; - - ptrOffs += (val ) & 0x7; - if (ptrOffs > curOffsArgs) { - iptr = isThis = false; - goto REPORT_REFS; - } - else if (ptrOffs > curOffsRegs) { - iptr = isThis = false; - continue; - } - - reg = (regNum)((val >> 3) & 0x7); - regMask = 1 << reg; // EAX,ECX,EDX,EBX,---,EBP,ESI,EDI - -#if 0 - printf("regMask = %04X -> %04X\n", ptrRegs, - (val & 0x40) ? (ptrRegs | regMask) - : (ptrRegs & ~regMask)); -#endif - - /* The register is becoming live/dead here */ - - if (val & 0x40) - { - /* Becomes Live */ - _ASSERTE((ptrRegs & regMask) == 0); - - ptrRegs |= regMask; - - if (isThis) - { - thisPtrReg = reg; - } - if (iptr) - { - iptrRegs |= regMask; - } - } - else - { - /* Becomes Dead */ - _ASSERTE((ptrRegs & regMask) != 0); - - ptrRegs &= ~regMask; - - if (reg == thisPtrReg) - { - thisPtrReg = REGI_NA; - } - if (iptrRegs & regMask) - { - iptrRegs &= ~regMask; - } - } - iptr = isThis = false; - continue; - } - - /* This is probably an argument push/pop */ - - argOfs = (val & 0x38) >> 3; - - /* 6 [110] and 7 [111] are reserved for other encodings */ - if (argOfs < 6) - { - - /* A small argument encoding */ - - ptrOffs += (val & 0x07); - if (ptrOffs > curOffsArgs) { - iptr = isThis = false; - goto REPORT_REFS; - } - isPop = (val & 0x40); - - ARG: - - if (isPop) - { - if (argOfs == 0) - continue; // little skip encoding - - /* We remove (pop) the top 'argOfs' entries */ - - _ASSERTE(argOfs || argOfs <= argCnt); - - /* adjust # of arguments */ - - argCnt -= argOfs; - _ASSERTE(argCnt < MAX_PTRARG_OFS); - -// printf("[%04X] popping %u args: mask = %04X\n", ptrOffs, argOfs, (int)ptrArgs); - - do - { - _ASSERTE(!isZero(argHigh)); - - /* Do we have an argument bit that's on? */ - - if (intersect(ptrArgs, argHigh)) - { - /* Turn off the bit */ - - setDiff(ptrArgs, argHigh); - setDiff(iptrArgs, argHigh); - - /* We've removed one more argument bit */ - - argOfs--; - } - else if (hasPartialArgInfo) - argCnt--; - else /* full arg info && not a ref */ - argOfs--; - - /* Continue with the next lower bit */ - - argHigh >>= 1; - } - while (argOfs); - - _ASSERTE(!hasPartialArgInfo || - isZero(argHigh) || - (argHigh == CONSTRUCT_ptrArgTP(1, (argCnt-1)))); - - if (hasPartialArgInfo) - { - // We always leave argHigh pointing to the next ptr arg. - // So, while argHigh is non-zero, and not a ptrArg, we shift right (and subtract - // one arg from our argCnt) until it is a ptrArg. - while (!intersect(argHigh, ptrArgs) && (!isZero(argHigh))) - { - argHigh >>= 1; - argCnt--; - } - } - - } - else - { - /* Add a new ptr arg entry at stack offset 'argOfs' */ - - if (argOfs >= MAX_PTRARG_OFS) - { - _ASSERTE_ALL_BUILDS(!"scanArgRegTableI: args pushed 'too deep'"); - } - else - { - /* Full arg info reports all pushes, and thus - argOffs has to be consistent with argCnt */ - - _ASSERTE(hasPartialArgInfo || argCnt == argOfs); - - /* store arg count */ - - argCnt = argOfs + 1; - _ASSERTE((argCnt < MAX_PTRARG_OFS)); - - /* Compute the appropriate argument offset bit */ - - ptrArgTP argMask = CONSTRUCT_ptrArgTP(1, argOfs); - -// printf("push arg at offset %02u --> mask = %04X\n", argOfs, (int)argMask); - - /* We should never push twice at the same offset */ - - _ASSERTE(!intersect( ptrArgs, argMask)); - _ASSERTE(!intersect(iptrArgs, argMask)); - - /* We should never push within the current highest offset */ - - // _ASSERTE(argHigh < argMask); - - /* This is now the highest bit we've set */ - - argHigh = argMask; - - /* Set the appropriate bit in the argument mask */ - - ptrArgs |= argMask; - - if (iptr) - iptrArgs |= argMask; - } - - iptr = isThis = false; - } - continue; - } - else if (argOfs == 6) - { - if (val & 0x40) { - /* Bigger delta 000=8,001=16,010=24,...,111=64 */ - ptrOffs += (((val & 0x07) + 1) << 3); - } - else { - /* non-ptr arg push */ - _ASSERTE(!hasPartialArgInfo); - ptrOffs += (val & 0x07); - if (ptrOffs > curOffsArgs) { - iptr = isThis = false; - goto REPORT_REFS; - } - argHigh = CONSTRUCT_ptrArgTP(1, argCnt); - argCnt++; - _ASSERTE(argCnt < MAX_PTRARG_OFS); - } - continue; - } - - /* argOfs was 7 [111] which is reserved for the larger encodings */ - - _ASSERTE(argOfs==7); - - switch (val) - { - case 0xFF: - iptr = isThis = false; - goto REPORT_REFS; // the method might loop !!! - - case 0xB8: - val = fastDecodeUnsigned(table); - ptrOffs += val; - continue; - - case 0xBC: - isThis = true; - break; - - case 0xBF: - iptr = true; - break; - - case 0xF8: - case 0xFC: - isPop = val & 0x04; - argOfs = fastDecodeUnsigned(table); - goto ARG; - - case 0xFD: { - argOfs = fastDecodeUnsigned(table); - _ASSERTE(argOfs && argOfs <= argCnt); - - // Kill the top "argOfs" pointers. - - ptrArgTP argMask; - for(argMask = CONSTRUCT_ptrArgTP(1, argCnt); (argOfs != 0); argMask >>= 1) - { - _ASSERTE(!isZero(argMask) && !isZero(ptrArgs)); // there should be remaining pointers - - if (intersect(ptrArgs, argMask)) - { - setDiff(ptrArgs, argMask); - setDiff(iptrArgs, argMask); - argOfs--; - } - } - - // For partial arg info, need to find the next highest pointer for argHigh - - if (hasPartialArgInfo) - { - for(argHigh = ptrArgTP(0); !isZero(argMask); argMask >>= 1) - { - if (intersect(ptrArgs, argMask)) { - argHigh = argMask; - break; - } - } - } - } break; - - case 0xF9: - argOfs = fastDecodeUnsigned(table); - argCnt += argOfs; - break; - - default: - _ASSERTE(!"Unexpected special code %04X"); - } - } - - /* Report all live pointer registers */ -REPORT_REFS: - - _ASSERTE((iptrRegs & ptrRegs) == iptrRegs); // iptrRegs is a subset of ptrRegs - _ASSERTE((iptrArgs & ptrArgs) == iptrArgs); // iptrArgs is a subset of ptrArgs - - /* Save the current live register, argument set, and argCnt */ - - info->regMaskResult = convertAllRegsMask(ptrRegs); - info->argMaskResult = ptrArgs; - info->argHnumResult = 0; - info->iregMaskResult = convertAllRegsMask(iptrRegs); - info->iargMaskResult = iptrArgs; - - info->thisPtrResult = thisPtrReg; - _ASSERTE(thisPtrReg == REGI_NA || (regNumToMask(thisPtrReg) & info->regMaskResult)); - - if (hasPartialArgInfo) - { - return 0; - } - else - { - _ASSERTE(int(argCnt) < INT_MAX); // check that it did not underflow - return (argCnt * sizeof(unsigned)); - } -} - -/*****************************************************************************/ - -unsigned GetPushedArgSize(hdrInfo * info, PTR_CBYTE table, DWORD curOffs) -{ - SUPPORTS_DAC; - - unsigned sz; - - if (info->interruptible) - { - sz = scanArgRegTableI(skipToArgReg(*info, table), - curOffs, - curOffs, - info); - } - else - { - sz = scanArgRegTable(skipToArgReg(*info, table), - curOffs, - info); - } - - return sz; -} - -/*****************************************************************************/ - -inline -void TRASH_CALLEE_UNSAVED_REGS(PREGDISPLAY pContext) -{ - LIMITED_METHOD_DAC_CONTRACT; - -#ifdef _DEBUG - /* This is not completely correct as we lose the current value, but - it should not really be useful to anyone. */ - static DWORD s_badData = 0xDEADBEEF; - pContext->SetEaxLocation(&s_badData); - pContext->SetEcxLocation(&s_badData); - pContext->SetEdxLocation(&s_badData); -#endif //_DEBUG -} - -/***************************************************************************** - * Sizes of certain i386 instructions which are used in the prolog/epilog - */ - -// Can we use sign-extended byte to encode the imm value, or do we need a dword -#define CAN_COMPRESS(val) ((INT8)(val) == (INT32)(val)) - -#define SZ_ADD_REG(val) ( 2 + (CAN_COMPRESS(val) ? 1 : 4)) -#define SZ_AND_REG(val) SZ_ADD_REG(val) -#define SZ_POP_REG 1 -#define SZ_LEA(offset) SZ_ADD_REG(offset) -#define SZ_MOV_REG_REG 2 - -bool IsMarkerInstr(BYTE val) -{ - SUPPORTS_DAC; - -#ifdef _DEBUG - if (val == X86_INSTR_INT3) - { - return true; - } -#ifdef HAVE_GCCOVER - else // GcCover might have stomped on the instruction - { - if (GCStress::IsEnabled()) - { - if (IsGcCoverageInterruptInstructionVal(val)) - { - return true; - } - } - } -#endif // HAVE_GCCOVER -#endif // _DEBUG - - return false; -} - -/* Check if the given instruction opcode is the one we expect. - This is a "necessary" but not "sufficient" check as it ignores the check - if the instruction is one of our special markers (for debugging and GcStress) */ - -bool CheckInstrByte(BYTE val, BYTE expectedValue) -{ - SUPPORTS_DAC; - return ((val == expectedValue) || IsMarkerInstr(val)); -} - -/* Similar to CheckInstrByte(). Use this to check a masked opcode (ignoring - optional bits in the opcode encoding). - valPattern is the masked out value. - expectedPattern is the mask value we expect. - val is the actual instruction opcode - */ -bool CheckInstrBytePattern(BYTE valPattern, BYTE expectedPattern, BYTE val) -{ - SUPPORTS_DAC; - - _ASSERTE((valPattern & val) == valPattern); - - return ((valPattern == expectedPattern) || IsMarkerInstr(val)); -} - -/* Similar to CheckInstrByte() */ - -bool CheckInstrWord(WORD val, WORD expectedValue) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - return ((val == expectedValue) || IsMarkerInstr(val & 0xFF)); -} - -// Use this to check if the instruction at offset "walkOffset" has already -// been executed -// "actualHaltOffset" is the offset when the code was suspended -// It is assumed that there is linear control flow from offset 0 to "actualHaltOffset". -// -// This has been factored out just so that the intent of the comparison -// is clear (compared to the opposite intent) - -bool InstructionAlreadyExecuted(unsigned walkOffset, unsigned actualHaltOffset) -{ - SUPPORTS_DAC; - return (walkOffset < actualHaltOffset); -} - -// skips past a "arith REG, IMM" -inline unsigned SKIP_ARITH_REG(int val, PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - - unsigned delta = 0; - if (val != 0) - { -#ifdef _DEBUG - // Confirm that arith instruction is at the correct place - _ASSERTE(CheckInstrBytePattern(base[offset ] & 0xFD, 0x81, base[offset]) && - CheckInstrBytePattern(base[offset+1] & 0xC0, 0xC0, base[offset+1])); - // only use DWORD form if needed - _ASSERTE(((base[offset] & 2) != 0) == CAN_COMPRESS(val) || - IsMarkerInstr(base[offset])); -#endif - delta = 2 + (CAN_COMPRESS(val) ? 1 : 4); - } - return(offset + delta); -} - -inline unsigned SKIP_PUSH_REG(PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - - // Confirm it is a push instruction - _ASSERTE(CheckInstrBytePattern(base[offset] & 0xF8, 0x50, base[offset])); - return(offset + 1); -} - -inline unsigned SKIP_POP_REG(PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - - // Confirm it is a pop instruction - _ASSERTE(CheckInstrBytePattern(base[offset] & 0xF8, 0x58, base[offset])); - return(offset + 1); -} - -inline unsigned SKIP_MOV_REG_REG(PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - - // Confirm it is a move instruction - // Note that only the first byte may have been stomped on by IsMarkerInstr() - // So we can check the second byte directly - _ASSERTE(CheckInstrBytePattern(base[offset] & 0xFD, 0x89, base[offset]) && - (base[offset+1] & 0xC0) == 0xC0); - return(offset + 2); -} - -inline unsigned SKIP_LEA_ESP_EBP(int val, PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - -#ifdef _DEBUG - // Confirm it is the right instruction - // Note that only the first byte may have been stomped on by IsMarkerInstr() - // So we can check the second byte directly - WORD wOpcode = *(PTR_WORD)base; - _ASSERTE((CheckInstrWord(wOpcode, X86_INSTR_w_LEA_ESP_EBP_BYTE_OFFSET) && - (val == *(PTR_SBYTE)(base+2)) && - CAN_COMPRESS(val)) || - (CheckInstrWord(wOpcode, X86_INSTR_w_LEA_ESP_EBP_DWORD_OFFSET) && - (val == *(PTR_INT32)(base+2)) && - !CAN_COMPRESS(val))); -#endif - - unsigned delta = 2 + (CAN_COMPRESS(val) ? 1 : 4); - return(offset + delta); -} - -inline unsigned SKIP_LEA_EAX_ESP(int val, PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - -#ifdef _DEBUG - WORD wOpcode = *(PTR_WORD)(base + offset); - if (CheckInstrWord(wOpcode, X86_INSTR_w_LEA_EAX_ESP_BYTE_OFFSET)) - { - _ASSERTE(val == *(PTR_SBYTE)(base + offset + 3)); - _ASSERTE(CAN_COMPRESS(val)); - } - else - { - _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_w_LEA_EAX_ESP_DWORD_OFFSET)); - _ASSERTE(val == *(PTR_INT32)(base + offset + 3)); - _ASSERTE(!CAN_COMPRESS(val)); - } -#endif - - unsigned delta = 3 + (CAN_COMPRESS(-val) ? 1 : 4); - return(offset + delta); -} - -inline unsigned SKIP_HELPER_CALL(PTR_CBYTE base, unsigned offset) -{ - LIMITED_METHOD_DAC_CONTRACT; - - unsigned delta; - - if (CheckInstrByte(base[offset], X86_INSTR_CALL_REL32)) - { - delta = 5; - } - else - { -#ifdef _DEBUG - WORD wOpcode = *(PTR_WORD)(base+offset); - _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_W_CALL_IND_IMM)); -#endif - delta = 6; - } - - return(offset+delta); -} - -unsigned SKIP_ALLOC_FRAME(int size, PTR_CBYTE base, unsigned offset) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - SUPPORTS_DAC; - } CONTRACTL_END; - - _ASSERTE(size != 0); - - if (size == sizeof(void*)) - { - // JIT emits "push eax" instead of "sub esp,4" - return SKIP_PUSH_REG(base, offset); - } - - const int STACK_PROBE_PAGE_SIZE_BYTES = 4096; - const int STACK_PROBE_BOUNDARY_THRESHOLD_BYTES = 1024; - - int lastProbedLocToFinalSp = size; - - if (size < STACK_PROBE_PAGE_SIZE_BYTES) - { - // sub esp, size - offset = SKIP_ARITH_REG(size, base, offset); - } - else - { - WORD wOpcode = *(PTR_WORD)(base + offset); - - if (CheckInstrWord(wOpcode, X86_INSTR_w_TEST_ESP_DWORD_OFFSET_EAX)) - { - // In .NET 5.0 and earlier for frames that have size smaller than 0x3000 bytes - // JIT emits one or two 'test eax, [esp-dwOffset]' instructions before adjusting the stack pointer. - _ASSERTE(size < 0x3000); - - // test eax, [esp-0x1000] - offset += 7; - lastProbedLocToFinalSp -= 0x1000; - - if (size >= 0x2000) - { -#ifdef _DEBUG - wOpcode = *(PTR_WORD)(base + offset); - _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_w_TEST_ESP_DWORD_OFFSET_EAX)); -#endif - //test eax, [esp-0x2000] - offset += 7; - lastProbedLocToFinalSp -= 0x1000; - } - - // sub esp, size - offset = SKIP_ARITH_REG(size, base, offset); - } - else - { - bool pushedStubParam = false; - - if (CheckInstrByte(base[offset], X86_INSTR_PUSH_EAX)) - { - // push eax - offset = SKIP_PUSH_REG(base, offset); - pushedStubParam = true; - } - - if (CheckInstrByte(base[offset], X86_INSTR_XOR)) - { - // In .NET Core 3.1 and earlier for frames that have size greater than or equal to 0x3000 bytes - // JIT emits the following loop. - _ASSERTE(size >= 0x3000); - - offset += 2; - // xor eax, eax 2 - // [nop] 0-3 - // loop: - // test [esp + eax], eax 3 - // sub eax, 0x1000 5 - // cmp eax, -size 5 - // jge loop 2 - - // R2R images that support ReJIT may have extra nops we need to skip over. - while (offset < 5) - { - if (CheckInstrByte(base[offset], X86_INSTR_NOP)) - { - offset++; - } - else - { - break; - } - } - - offset += 15; - - if (pushedStubParam) - { - // pop eax - offset = SKIP_POP_REG(base, offset); - } - - // sub esp, size - return SKIP_ARITH_REG(size, base, offset); - } - else - { - // In .NET 5.0 and later JIT emits a call to JIT_StackProbe helper. - - if (pushedStubParam) - { - // lea eax, [esp-size+4] - offset = SKIP_LEA_EAX_ESP(-size + 4, base, offset); - // call JIT_StackProbe - offset = SKIP_HELPER_CALL(base, offset); - // pop eax - offset = SKIP_POP_REG(base, offset); - // sub esp, size - return SKIP_ARITH_REG(size, base, offset); - } - else - { - // lea eax, [esp-size] - offset = SKIP_LEA_EAX_ESP(-size, base, offset); - // call JIT_StackProbe - offset = SKIP_HELPER_CALL(base, offset); - // mov esp, eax - return SKIP_MOV_REG_REG(base, offset); - } - } - } - } - - if (lastProbedLocToFinalSp + STACK_PROBE_BOUNDARY_THRESHOLD_BYTES > STACK_PROBE_PAGE_SIZE_BYTES) - { -#ifdef _DEBUG - WORD wOpcode = *(PTR_WORD)(base + offset); - _ASSERTE(CheckInstrWord(wOpcode, X86_INSTR_w_TEST_ESP_EAX)); -#endif - // test [esp], eax - offset += 3; - } - - return offset; -} - -/*****************************************************************************/ - -const RegMask CALLEE_SAVED_REGISTERS_MASK[] = -{ - RM_EDI, // first register to be pushed - RM_ESI, - RM_EBX, - RM_EBP // last register to be pushed -}; - -static void SetLocation(PREGDISPLAY pRD, int ind, PDWORD loc) -{ -#if defined(FEATURE_NATIVEAOT) - static const SIZE_T OFFSET_OF_CALLEE_SAVED_REGISTERS[] = - { - offsetof(REGDISPLAY, pRdi), // first register to be pushed - offsetof(REGDISPLAY, pRsi), - offsetof(REGDISPLAY, pRbx), - offsetof(REGDISPLAY, pRbp), // last register to be pushed - }; - - SIZE_T offsetOfRegPtr = OFFSET_OF_CALLEE_SAVED_REGISTERS[ind]; - *(LPVOID*)(PBYTE(pRD) + offsetOfRegPtr) = loc; -#elif defined(FEATURE_EH_FUNCLETS) - static const SIZE_T OFFSET_OF_CALLEE_SAVED_REGISTERS[] = - { - offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Edi), // first register to be pushed - offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Esi), - offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Ebx), - offsetof(T_KNONVOLATILE_CONTEXT_POINTERS, Ebp), // last register to be pushed - }; - - SIZE_T offsetOfRegPtr = OFFSET_OF_CALLEE_SAVED_REGISTERS[ind]; - *(LPVOID*)(PBYTE(pRD->pCurrentContextPointers) + offsetOfRegPtr) = loc; -#else - static const SIZE_T OFFSET_OF_CALLEE_SAVED_REGISTERS[] = - { - offsetof(REGDISPLAY, pEdi), // first register to be pushed - offsetof(REGDISPLAY, pEsi), - offsetof(REGDISPLAY, pEbx), - offsetof(REGDISPLAY, pEbp), // last register to be pushed - }; - - SIZE_T offsetOfRegPtr = OFFSET_OF_CALLEE_SAVED_REGISTERS[ind]; - *(LPVOID*)(PBYTE(pRD) + offsetOfRegPtr) = loc; -#endif -} - -/*****************************************************************************/ - -void UnwindEspFrameEpilog( - PREGDISPLAY pContext, - hdrInfo * info, - PTR_CBYTE epilogBase, - bool updateAllRegs) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - _ASSERTE(info->epilogOffs != hdrInfo::NOT_IN_EPILOG); - _ASSERTE(!info->ebpFrame && !info->doubleAlign); - _ASSERTE(info->epilogOffs > 0); - - int offset = 0; - unsigned ESP = pContext->SP; - - if (info->rawStkSize) - { - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - { - /* We have NOT executed the "ADD ESP, FrameSize", - so manually adjust stack pointer */ - ESP += info->rawStkSize; - } - - // We have already popped off the frame (excluding the callee-saved registers) - - if (epilogBase[0] == X86_INSTR_POP_ECX) - { - // We may use "POP ecx" for doing "ADD ESP, 4", - // or we may not (in the case of JMP epilogs) - _ASSERTE(info->rawStkSize == sizeof(void*)); - offset = SKIP_POP_REG(epilogBase, offset); - } - else - { - // "add esp, rawStkSize" - offset = SKIP_ARITH_REG(info->rawStkSize, epilogBase, offset); - } - } - - /* Remaining callee-saved regs are at ESP. Need to update - regsMask as well to exclude registers which have already been popped. */ - - const RegMask regsMask = info->savedRegMask; - - /* Increment "offset" in steps to see which callee-saved - registers have already been popped */ - - for (unsigned i = ARRAY_SIZE(CALLEE_SAVED_REGISTERS_MASK); i > 0; i--) - { - RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i - 1]; - - if (!(regMask & regsMask)) - continue; - - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - { - /* We have NOT yet popped off the register. - Get the value from the stack if needed */ - if (updateAllRegs || (regMask == RM_EBP)) - { - SetLocation(pContext, i - 1, PTR_DWORD((TADDR)ESP)); - } - - /* Adjust ESP */ - ESP += sizeof(void*); - } - - offset = SKIP_POP_REG(epilogBase, offset); - } - - //CEE_JMP generates an epilog similar to a normal CEE_RET epilog except for the last instruction - _ASSERTE(CheckInstrBytePattern(epilogBase[offset] & X86_INSTR_RET, X86_INSTR_RET, epilogBase[offset]) //ret - || CheckInstrBytePattern(epilogBase[offset], X86_INSTR_JMP_NEAR_REL32, epilogBase[offset]) //jmp ret32 - || CheckInstrWord(*PTR_WORD(epilogBase + offset), X86_INSTR_w_JMP_FAR_IND_IMM)); //jmp [addr32] - - /* Finally we can set pPC */ - SetRegdisplayPCTAddr(pContext, (TADDR)ESP); - - pContext->SP = ESP; -} - -/*****************************************************************************/ - -void UnwindEbpDoubleAlignFrameEpilog( - PREGDISPLAY pContext, - hdrInfo * info, - PTR_CBYTE epilogBase, - bool updateAllRegs) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - _ASSERTE(info->epilogOffs != hdrInfo::NOT_IN_EPILOG); - _ASSERTE(info->ebpFrame || info->doubleAlign); - - _ASSERTE(info->argSize < 0x10000); // "ret" only has a 2 byte operand - - /* See how many instructions we have executed in the - epilog to determine which callee-saved registers - have already been popped */ - int offset = 0; - - unsigned ESP = pContext->SP; - - bool needMovEspEbp = false; - - if (info->doubleAlign) - { - // add esp, rawStkSize - - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - ESP += info->rawStkSize; - _ASSERTE(info->rawStkSize != 0); - offset = SKIP_ARITH_REG(info->rawStkSize, epilogBase, offset); - - // We also need "mov esp, ebp" after popping the callee-saved registers - needMovEspEbp = true; - } - else - { - bool needLea = false; - - if (info->localloc) - { - // ESP may be variable if a localloc was actually executed. We will reset it. - // lea esp, [ebp-calleeSavedRegs] - - needLea = true; - } - else if (info->savedRegsCountExclFP == 0) - { - // We will just generate "mov esp, ebp" and be done with it. - - if (info->rawStkSize != 0) - { - needMovEspEbp = true; - } - } - else if (info->rawStkSize == 0) - { - // do nothing before popping the callee-saved registers - } - else if (info->rawStkSize == sizeof(void*)) - { - // "pop ecx" will make ESP point to the callee-saved registers - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - ESP += sizeof(void*); - offset = SKIP_POP_REG(epilogBase, offset); - } - else - { - // We need to make ESP point to the callee-saved registers - // lea esp, [ebp-calleeSavedRegs] - - needLea = true; - } - - if (needLea) - { - // lea esp, [ebp-calleeSavedRegs] - - unsigned calleeSavedRegsSize = info->savedRegsCountExclFP * sizeof(void*); - - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - ESP = GetRegdisplayFP(pContext) - calleeSavedRegsSize; - - offset = SKIP_LEA_ESP_EBP(-int(calleeSavedRegsSize), epilogBase, offset); - } - } - - for (unsigned i = STRING_LENGTH(CALLEE_SAVED_REGISTERS_MASK); i > 0; i--) - { - RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i - 1]; - _ASSERTE(regMask != RM_EBP); - - if ((info->savedRegMask & regMask) == 0) - continue; - - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - { - if (updateAllRegs) - { - SetLocation(pContext, i - 1, PTR_DWORD((TADDR)ESP)); - } - ESP += sizeof(void*); - } - - offset = SKIP_POP_REG(epilogBase, offset); - } - - if (needMovEspEbp) - { - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - ESP = GetRegdisplayFP(pContext); - - offset = SKIP_MOV_REG_REG(epilogBase, offset); - } - - // Have we executed the pop EBP? - if (!InstructionAlreadyExecuted(offset, info->epilogOffs)) - { - pContext->SetEbpLocation(PTR_DWORD(TADDR(ESP))); - ESP += sizeof(void*); - } - offset = SKIP_POP_REG(epilogBase, offset); - - SetRegdisplayPCTAddr(pContext, (TADDR)ESP); - - pContext->SP = ESP; -} - -inline SIZE_T GetStackParameterSize(hdrInfo * info) -{ - SUPPORTS_DAC; - return (info->varargs ? 0 : info->argSize); // Note varargs is caller-popped -} - -//**************************************************************************** -// This is the value ESP is incremented by on doing a "return" - -inline SIZE_T ESPIncrOnReturn(hdrInfo * info) -{ - SUPPORTS_DAC; - return sizeof(void *) + // pop off the return address - GetStackParameterSize(info); -} - -/*****************************************************************************/ - -void UnwindEpilog( - PREGDISPLAY pContext, - hdrInfo * info, - PTR_CBYTE epilogBase, - bool updateAllRegs) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - _ASSERTE(info->epilogOffs != hdrInfo::NOT_IN_EPILOG); - // _ASSERTE(flags & ActiveStackFrame); // Wont work for thread death - _ASSERTE(info->epilogOffs > 0); - - if (info->ebpFrame || info->doubleAlign) - { - UnwindEbpDoubleAlignFrameEpilog(pContext, info, epilogBase, updateAllRegs); - } - else - { - UnwindEspFrameEpilog(pContext, info, epilogBase, updateAllRegs); - } - -#ifdef _DEBUG - if (updateAllRegs) - TRASH_CALLEE_UNSAVED_REGS(pContext); -#endif - - /* Now adjust stack pointer */ - - pContext->SP += ESPIncrOnReturn(info); -} - -/*****************************************************************************/ - -void UnwindEspFrameProlog( - PREGDISPLAY pContext, - hdrInfo * info, - PTR_CBYTE methodStart, - bool updateAllRegs) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - /* we are in the middle of the prolog */ - _ASSERTE(info->prologOffs != hdrInfo::NOT_IN_PROLOG); - _ASSERTE(!info->ebpFrame && !info->doubleAlign); - - unsigned offset = 0; - -#ifdef _DEBUG - // If the first two instructions are 'nop, int3', then we will - // assume that is from a JitHalt operation and skip past it - if (methodStart[0] == X86_INSTR_NOP && methodStart[1] == X86_INSTR_INT3) - { - offset += 2; - } -#endif - - const DWORD curOffs = info->prologOffs; - unsigned ESP = pContext->SP; - - // Find out how many callee-saved regs have already been pushed - - unsigned regsMask = RM_NONE; - PTR_DWORD savedRegPtr = PTR_DWORD((TADDR)ESP); - - for (unsigned i = 0; i < ARRAY_SIZE(CALLEE_SAVED_REGISTERS_MASK); i++) - { - RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i]; - - if (!(info->savedRegMask & regMask)) - continue; - - if (InstructionAlreadyExecuted(offset, curOffs)) - { - ESP += sizeof(void*); - regsMask |= regMask; - } - - offset = SKIP_PUSH_REG(methodStart, offset); - } - - if (info->rawStkSize) - { - offset = SKIP_ALLOC_FRAME(info->rawStkSize, methodStart, offset); - - // Note that this assumes that only the last instruction in SKIP_ALLOC_FRAME - // actually updates ESP - if (InstructionAlreadyExecuted(offset, curOffs + 1)) - { - savedRegPtr += (info->rawStkSize / sizeof(DWORD)); - ESP += info->rawStkSize; - } - } - - // - // Stack probe checks here - // - - // Poison the value, we don't set it properly at the end of the prolog -#ifdef _DEBUG - offset = 0xCCCCCCCC; -#endif - - // Always restore EBP - if (regsMask & RM_EBP) - pContext->SetEbpLocation(savedRegPtr++); - - if (updateAllRegs) - { - if (regsMask & RM_EBX) - pContext->SetEbxLocation(savedRegPtr++); - if (regsMask & RM_ESI) - pContext->SetEsiLocation(savedRegPtr++); - if (regsMask & RM_EDI) - pContext->SetEdiLocation(savedRegPtr++); - - TRASH_CALLEE_UNSAVED_REGS(pContext); - } - -#if 0 -// NOTE: -// THIS IS ONLY TRUE IF PROLOGSIZE DOES NOT INCLUDE REG-VAR INITIALIZATION !!!! -// - /* there is (potentially) only one additional - instruction in the prolog, (push ebp) - but if we would have been passed that instruction, - info->prologOffs would be hdrInfo::NOT_IN_PROLOG! - */ - _ASSERTE(offset == info->prologOffs); -#endif - - pContext->SP = ESP; -} - -/*****************************************************************************/ - -void UnwindEspFrame( - PREGDISPLAY pContext, - hdrInfo * info, - PTR_CBYTE table, - PTR_CBYTE methodStart, - DWORD curOffs, - unsigned flags) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - _ASSERTE(!info->ebpFrame && !info->doubleAlign); - _ASSERTE(info->epilogOffs == hdrInfo::NOT_IN_EPILOG); - - unsigned ESP = pContext->SP; - - - if (info->prologOffs != hdrInfo::NOT_IN_PROLOG) - { - if (info->prologOffs != 0) // Do nothing for the very start of the method - { - UnwindEspFrameProlog(pContext, info, methodStart, flags); - ESP = pContext->SP; - } - } - else - { - /* we are past the prolog, ESP has been set above */ - - // Are there any arguments pushed on the stack? - - ESP += GetPushedArgSize(info, table, curOffs); - - ESP += info->rawStkSize; - - const RegMask regsMask = info->savedRegMask; - - for (unsigned i = ARRAY_SIZE(CALLEE_SAVED_REGISTERS_MASK); i > 0; i--) - { - RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i - 1]; - - if ((regMask & regsMask) == 0) - continue; - - SetLocation(pContext, i - 1, PTR_DWORD((TADDR)ESP)); - - ESP += sizeof(unsigned); - } - } - - /* we can now set the (address of the) return address */ - - SetRegdisplayPCTAddr(pContext, (TADDR)ESP); - - /* Now adjust stack pointer */ - - pContext->SP = ESP + ESPIncrOnReturn(info); -} - - -/*****************************************************************************/ - -void UnwindEbpDoubleAlignFrameProlog( - PREGDISPLAY pContext, - hdrInfo * info, - PTR_CBYTE methodStart, - bool updateAllRegs) -{ - LIMITED_METHOD_DAC_CONTRACT; - - _ASSERTE(info->prologOffs != hdrInfo::NOT_IN_PROLOG); - _ASSERTE(info->ebpFrame || info->doubleAlign); - - DWORD offset = 0; - -#ifdef _DEBUG - // If the first two instructions are 'nop, int3', then we will - // assume that is from a JitHalt operation and skip past it - if (methodStart[0] == X86_INSTR_NOP && methodStart[1] == X86_INSTR_INT3) - { - offset += 2; - } -#endif - - /* Check for the case where EBP has not been updated yet. */ - - const DWORD curOffs = info->prologOffs; - - // If we have still not excecuted "push ebp; mov ebp, esp", then we need to - // report the frame relative to ESP - - if (!InstructionAlreadyExecuted(offset + 1, curOffs)) - { - _ASSERTE(CheckInstrByte(methodStart [offset], X86_INSTR_PUSH_EBP) || - CheckInstrWord(*PTR_WORD(methodStart + offset), X86_INSTR_W_MOV_EBP_ESP) || - CheckInstrByte(methodStart [offset], X86_INSTR_JMP_NEAR_REL32)); // a rejit jmp-stamp - - /* If we're past the "push ebp", adjust ESP to pop EBP off */ - - if (curOffs == (offset + 1)) - pContext->SP += sizeof(TADDR); - - /* Stack pointer points to return address */ - - SetRegdisplayPCTAddr(pContext, (TADDR)pContext->SP); - - /* EBP and callee-saved registers still have the correct value */ - - return; - } - - // We are atleast after the "push ebp; mov ebp, esp" - - offset = SKIP_MOV_REG_REG(methodStart, - SKIP_PUSH_REG(methodStart, offset)); - - /* At this point, EBP has been set up. The caller's ESP and the return value - can be determined using EBP. Since we are still in the prolog, - we need to know our exact location to determine the callee-saved registers */ - - const unsigned curEBP = GetRegdisplayFP(pContext); - - if (updateAllRegs) - { - PTR_DWORD pSavedRegs = PTR_DWORD((TADDR)curEBP); - - /* make sure that we align ESP just like the method's prolog did */ - if (info->doubleAlign) - { - // "and esp,-8" - offset = SKIP_ARITH_REG(-8, methodStart, offset); - if (curEBP & 0x04) - { - pSavedRegs--; -#ifdef _DEBUG - if (dspPtr) printf("EnumRef: dblalign ebp: %08X\n", curEBP); -#endif - } - } - - /* Increment "offset" in steps to see which callee-saved - registers have been pushed already */ - - for (unsigned i = 0; i < STRING_LENGTH(CALLEE_SAVED_REGISTERS_MASK); i++) - { - RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i]; - _ASSERTE(regMask != RM_EBP); - - if ((info->savedRegMask & regMask) == 0) - continue; - - if (InstructionAlreadyExecuted(offset, curOffs)) - { - SetLocation(pContext, i, PTR_DWORD(--pSavedRegs)); - } - - // "push reg" - offset = SKIP_PUSH_REG(methodStart, offset) ; - } - - TRASH_CALLEE_UNSAVED_REGS(pContext); - } - - /* The caller's saved EBP is pointed to by our EBP */ - - pContext->SetEbpLocation(PTR_DWORD((TADDR)curEBP)); - pContext->SP = DWORD((TADDR)(curEBP + sizeof(void *))); - - /* Stack pointer points to return address */ - - SetRegdisplayPCTAddr(pContext, (TADDR)pContext->SP); -} - -/*****************************************************************************/ - -bool UnwindEbpDoubleAlignFrame( - PREGDISPLAY pContext, - hdrInfo *info, - PTR_CBYTE table, - PTR_CBYTE methodStart, - DWORD curOffs, - IN_EH_FUNCLETS_COMMA(PTR_CBYTE funcletStart) - IN_EH_FUNCLETS_COMMA(bool isFunclet) - bool updateAllRegs) -{ - LIMITED_METHOD_CONTRACT; - SUPPORTS_DAC; - - _ASSERTE(info->ebpFrame || info->doubleAlign); - - const unsigned curESP = pContext->SP; - const unsigned curEBP = GetRegdisplayFP(pContext); - - /* First check if we are in a filter (which is obviously after the prolog) */ - - if (info->handlers && info->prologOffs == hdrInfo::NOT_IN_PROLOG) - { - TADDR baseSP; - -#ifdef FEATURE_EH_FUNCLETS - // Funclets' frame pointers(EBP) are always restored so they can access to main function's local variables. - // Therefore the value of EBP is invalid for unwinder so we should use ESP instead. - // TODO If funclet frame layout is changed from CodeGen::genFuncletProlog() and genFuncletEpilog(), - // we need to change here accordingly. It is likely to have changes when introducing PSPSym. - // TODO Currently we assume that ESP of funclet frames is always fixed but actually it could change. - if (isFunclet) - { - baseSP = curESP; - // Set baseSP as initial SP - baseSP += GetPushedArgSize(info, table, curOffs); - -#ifdef UNIX_X86_ABI - // 16-byte stack alignment padding (allocated in genFuncletProlog) - // Current funclet frame layout (see CodeGen::genFuncletProlog() and genFuncletEpilog()): - // prolog: sub esp, 12 - // epilog: add esp, 12 - // ret - // SP alignment padding should be added for all instructions except the first one and the last one. - // Epilog may not exist (unreachable), so we need to check the instruction code. - if (funcletStart != methodStart + curOffs && methodStart[curOffs] != X86_INSTR_RETN) - baseSP += 12; -#endif - - SetRegdisplayPCTAddr(pContext, (TADDR)baseSP); - - pContext->SP = (DWORD)(baseSP + sizeof(TADDR)); - - return true; - } -#else // FEATURE_EH_FUNCLETS - - FrameType frameType = GetHandlerFrameInfo(info, curEBP, - curESP, (DWORD) IGNORE_VAL, - &baseSP); - - /* If we are in a filter, we only need to unwind the funclet stack. - For catches/finallies, the normal handling will - cause the frame to be unwound all the way up to ebp skipping - other frames above it. This is OK, as those frames will be - dead. Also, the EE will detect that this has happened and it - will handle any EE frames correctly. - */ - - if (frameType == FR_INVALID) - { - return false; - } - - if (frameType == FR_FILTER) - { - SetRegdisplayPCTAddr(pContext, (TADDR)baseSP); - - pContext->SP = (DWORD)(baseSP + sizeof(TADDR)); - - // pContext->pEbp = same as before; - -#ifdef _DEBUG - /* The filter has to be called by the VM. So we dont need to - update callee-saved registers. - */ - - if (updateAllRegs) - { - static DWORD s_badData = 0xDEADBEEF; - - pContext->SetEaxLocation(&s_badData); - pContext->SetEcxLocation(&s_badData); - pContext->SetEdxLocation(&s_badData); - - pContext->SetEbxLocation(&s_badData); - pContext->SetEsiLocation(&s_badData); - pContext->SetEdiLocation(&s_badData); - } -#endif - - return true; - } -#endif // !FEATURE_EH_FUNCLETS - } - - // - // Prolog of an EBP method - // - - if (info->prologOffs != hdrInfo::NOT_IN_PROLOG) - { - UnwindEbpDoubleAlignFrameProlog(pContext, info, methodStart, updateAllRegs); - - /* Now adjust stack pointer. */ - - pContext->SP += ESPIncrOnReturn(info); - return true; - } - - if (updateAllRegs) - { - // Get to the first callee-saved register - PTR_DWORD pSavedRegs = PTR_DWORD((TADDR)curEBP); - - if (info->doubleAlign && (curEBP & 0x04)) - pSavedRegs--; - - for (unsigned i = 0; i < STRING_LENGTH(CALLEE_SAVED_REGISTERS_MASK); i++) - { - RegMask regMask = CALLEE_SAVED_REGISTERS_MASK[i]; - if ((info->savedRegMask & regMask) == 0) - continue; - - SetLocation(pContext, i, --pSavedRegs); - } - } - - /* The caller's ESP will be equal to EBP + retAddrSize + argSize. */ - - pContext->SP = (DWORD)(curEBP + sizeof(curEBP) + ESPIncrOnReturn(info)); - - /* The caller's saved EIP is right after our EBP */ - - SetRegdisplayPCTAddr(pContext, (TADDR)curEBP + RETURN_ADDR_OFFS * sizeof(TADDR)); - - /* The caller's saved EBP is pointed to by our EBP */ - - pContext->SetEbpLocation(PTR_DWORD((TADDR)curEBP)); - return true; -} - -bool UnwindStackFrameX86(PREGDISPLAY pContext, - PTR_CBYTE methodStart, - DWORD curOffs, - hdrInfo * info, - PTR_CBYTE table, - IN_EH_FUNCLETS_COMMA(PTR_CBYTE funcletStart) - IN_EH_FUNCLETS_COMMA(bool isFunclet) - bool updateAllRegs) -{ - if (info->epilogOffs != hdrInfo::NOT_IN_EPILOG) - { - /*--------------------------------------------------------------------- - * First, handle the epilog - */ - - PTR_CBYTE epilogBase = methodStart + (curOffs - info->epilogOffs); - UnwindEpilog(pContext, info, epilogBase, updateAllRegs); - } - else if (!info->ebpFrame && !info->doubleAlign) - { - /*--------------------------------------------------------------------- - * Now handle ESP frames - */ - - UnwindEspFrame(pContext, info, table, methodStart, curOffs, updateAllRegs); - return true; - } - else - { - /*--------------------------------------------------------------------- - * Now we know that have an EBP frame - */ - - if (!UnwindEbpDoubleAlignFrame(pContext, - info, - table, - methodStart, - curOffs, - IN_EH_FUNCLETS_COMMA(funcletStart) - IN_EH_FUNCLETS_COMMA(isFunclet) - updateAllRegs)) - return false; - } - - // TODO [DAVBR]: For the full fix for VsWhidbey 450273, all the below - // may be uncommented once isLegalManagedCodeCaller works properly - // with non-return address inputs, and with non-DEBUG builds - /* - // Ensure isLegalManagedCodeCaller succeeds for speculative stackwalks. - // (We just assert this below for non-speculative stackwalks.) - // - FAIL_IF_SPECULATIVE_WALK(isLegalManagedCodeCaller(GetControlPC(pContext))); - */ - - return true; -} - -bool EnumGcRefsX86(PREGDISPLAY pContext, - PTR_CBYTE methodStart, - DWORD curOffs, - GCInfoToken gcInfoToken, - IN_EH_FUNCLETS_COMMA(PTR_CBYTE funcletStart) - IN_EH_FUNCLETS_COMMA(bool isFunclet) - IN_EH_FUNCLETS_COMMA(bool isFilterFunclet) - unsigned flags, - GCEnumCallback pCallBack, - LPVOID hCallBack) -{ -#ifdef FEATURE_EH_FUNCLETS - if (flags & ParentOfFuncletStackFrame) - { - LOG((LF_GCROOTS, LL_INFO100000, "Not reporting this frame because it was already reported via another funclet.\n")); - return true; - } -#endif // FEATURE_EH_FUNCLETS - - unsigned EBP = GetRegdisplayFP(pContext); - unsigned ESP = pContext->SP; - - unsigned ptrOffs; - - unsigned count; - - hdrInfo info; - PTR_CBYTE table = PTR_CBYTE(gcInfoToken.Info); -#if 0 - printf("EECodeManager::EnumGcRefs - EIP = %08x ESP = %08x offset = %x GC Info is at %08x\n", *pContext->pPC, ESP, curOffs, table); -#endif - - - /* Extract the necessary information from the info block header */ - - table += DecodeGCHdrInfo(gcInfoToken, - curOffs, - &info); - - _ASSERTE( curOffs <= info.methodSize); - -#ifdef _DEBUG -// if ((gcInfoToken.Info == (void*)0x37760d0) && (curOffs == 0x264)) -// __asm int 3; - - if (trEnumGCRefs) { - static unsigned lastESP = 0; - unsigned diffESP = ESP - lastESP; - if (diffESP > 0xFFFF) { - printf("------------------------------------------------------\n"); - } - lastESP = ESP; - printf("EnumGCRefs [%s][%s] at %s.%s + 0x%03X:\n", - info.ebpFrame?"ebp":" ", - info.interruptible?"int":" ", - "UnknownClass","UnknownMethod", curOffs); - fflush(stdout); - } -#endif - - /* Are we in the prolog or epilog of the method? */ - - if (info.prologOffs != hdrInfo::NOT_IN_PROLOG || - info.epilogOffs != hdrInfo::NOT_IN_EPILOG) - { - -#if !DUMP_PTR_REFS - // Under normal circumstances the system will not suspend a thread - // if it is in the prolog or epilog of the function. However ThreadAbort - // exception or stack overflows can cause EH to happen in a prolog. - // Once in the handler, a GC can happen, so we can get to this code path. - // However since we are tearing down this frame, we don't need to report - // anything and we can simply return. - - _ASSERTE(flags & ExecutionAborted); -#endif - return true; - } - -#ifdef _DEBUG -#define CHK_AND_REPORT_REG(reg, doIt, iptr, regName) \ - if (doIt) \ - { \ - if (dspPtr) \ - printf(" Live pointer register %s: ", #regName); \ - pCallBack(hCallBack, \ - (OBJECTREF*)(pContext->Get##regName##Location()), \ - (iptr ? GC_CALL_INTERIOR : 0) \ - | CHECK_APP_DOMAIN \ - DAC_ARG(DacSlotLocation(reg, 0, false))); \ - } -#else // !_DEBUG -#define CHK_AND_REPORT_REG(reg, doIt, iptr, regName) \ - if (doIt) \ - pCallBack(hCallBack, \ - (OBJECTREF*)(pContext->Get##regName##Location()), \ - (iptr ? GC_CALL_INTERIOR : 0) \ - | CHECK_APP_DOMAIN \ - DAC_ARG(DacSlotLocation(reg, 0, false))); - -#endif // _DEBUG - - /* What kind of a frame is this ? */ - -#ifndef FEATURE_EH_FUNCLETS - FrameType frameType = FR_NORMAL; - TADDR baseSP = 0; - - if (info.handlers) - { - _ASSERTE(info.ebpFrame); - - bool hasInnerFilter, hadInnerFilter; - frameType = GetHandlerFrameInfo(&info, EBP, - ESP, (DWORD) IGNORE_VAL, - &baseSP, NULL, - &hasInnerFilter, &hadInnerFilter); - _ASSERTE(frameType != FR_INVALID); - - /* If this is the parent frame of a filter which is currently - executing, then the filter would have enumerated the frame using - the filter PC. - */ - - if (hasInnerFilter) - return true; - - /* If are in a try and we had a filter execute, we may have reported - GC refs from the filter (and not using the try's offset). So - we had better use the filter's end offset, as the try is - effectively dead and its GC ref's would be stale */ - - if (hadInnerFilter) - { - PTR_TADDR pFirstBaseSPslot = GetFirstBaseSPslotPtr(EBP, &info); - curOffs = (unsigned)pFirstBaseSPslot[1] - 1; - _ASSERTE(curOffs < info.methodSize); - - /* Extract the necessary information from the info block header */ - - table = PTR_CBYTE(gcInfoToken.Info); - - table += DecodeGCHdrInfo(gcInfoToken, - curOffs, - &info); - } - } -#endif - - bool willContinueExecution = !(flags & ExecutionAborted); - unsigned pushedSize = 0; - - /* if we have been interrupted we don't have to report registers/arguments - * because we are about to lose this context anyway. - * Alas, if we are in a ebp-less method we have to parse the table - * in order to adjust ESP. - * - * Note that we report "this" for all methods, even if - * noncontinuable, because of the off chance they may be - * synchronized and we have to release the monitor on unwind. This - * could conceivably be optimized, but it turns out to be more - * expensive to check whether we're synchronized (which involves - * consulting metadata) than to just report "this" all the time in - * our most important scenarios. - */ - - if (info.interruptible) - { - unsigned curOffsRegs = curOffs; - - // Don't decrement curOffsRegs when it is 0, as it is an unsigned and will wrap to MAX_UINT - // - if (curOffsRegs > 0) - { - // If we are not on the active stack frame, we need to report gc registers - // that are live before the call. The reason is that the liveness of gc registers - // may change across a call to a method that does not return. In this case the instruction - // after the call may be a jump target and a register that didn't have a live gc pointer - // before the call may have a live gc pointer after the jump. To make sure we report the - // registers that have live gc pointers before the call we subtract 1 from curOffs. - if ((flags & ActiveStackFrame) == 0) - { - // We are not the top most stack frame (i.e. the ActiveStackFrame) - curOffsRegs--; // decrement curOffsRegs - } - } - - pushedSize = scanArgRegTableI(skipToArgReg(info, table), curOffsRegs, curOffs, &info); - - RegMask regs = info.regMaskResult; - RegMask iregs = info.iregMaskResult; - ptrArgTP args = info.argMaskResult; - ptrArgTP iargs = info.iargMaskResult; - - _ASSERTE((isZero(args) || pushedSize != 0) || info.ebpFrame); - _ASSERTE((args & iargs) == iargs); - - /* now report registers and arguments if we are not interrupted */ - - if (willContinueExecution) - { - - /* Propagate unsafed registers only in "current" method */ - /* If this is not the active method, then the callee wil - * trash these registers, and so we wont need to report them */ - - if (flags & ActiveStackFrame) - { - CHK_AND_REPORT_REG(REGI_EAX, regs & RM_EAX, iregs & RM_EAX, Eax); - CHK_AND_REPORT_REG(REGI_ECX, regs & RM_ECX, iregs & RM_ECX, Ecx); - CHK_AND_REPORT_REG(REGI_EDX, regs & RM_EDX, iregs & RM_EDX, Edx); - } - - CHK_AND_REPORT_REG(REGI_EBX, regs & RM_EBX, iregs & RM_EBX, Ebx); - CHK_AND_REPORT_REG(REGI_EBP, regs & RM_EBP, iregs & RM_EBP, Ebp); - CHK_AND_REPORT_REG(REGI_ESI, regs & RM_ESI, iregs & RM_ESI, Esi); - CHK_AND_REPORT_REG(REGI_EDI, regs & RM_EDI, iregs & RM_EDI, Edi); - _ASSERTE(!(regs & RM_ESP)); - - /* Report any pending pointer arguments */ - - DWORD * pPendingArgFirst; // points **AT** first parameter - if (!info.ebpFrame) - { - // -sizeof(void*) because we want to point *AT* first parameter - pPendingArgFirst = (DWORD *)(size_t)(ESP + pushedSize - sizeof(void*)); - } - else - { - _ASSERTE(willContinueExecution); - -#ifdef FEATURE_EH_FUNCLETS - // Funclets' frame pointers(EBP) are always restored so they can access to main function's local variables. - // Therefore the value of EBP is invalid for unwinder so we should use ESP instead. - // See UnwindStackFrame for details. - if (isFunclet) - { - TADDR baseSP = ESP; - // Set baseSP as initial SP - baseSP += GetPushedArgSize(&info, table, curOffs); - -#ifdef UNIX_X86_ABI - // 16-byte stack alignment padding (allocated in genFuncletProlog) - // Current funclet frame layout (see CodeGen::genFuncletProlog() and genFuncletEpilog()): - // prolog: sub esp, 12 - // epilog: add esp, 12 - // ret - // SP alignment padding should be added for all instructions except the first one and the last one. - // Epilog may not exist (unreachable), so we need to check the instruction code. - if (funcletStart != methodStart + curOffs && methodStart[curOffs] != X86_INSTR_RETN) - baseSP += 12; -#endif - - // -sizeof(void*) because we want to point *AT* first parameter - pPendingArgFirst = (DWORD *)(size_t)(baseSP - sizeof(void*)); - } -#else // FEATURE_EH_FUNCLETS - if (info.handlers) - { - // -sizeof(void*) because we want to point *AT* first parameter - pPendingArgFirst = (DWORD *)(size_t)(baseSP - sizeof(void*)); - } -#endif - else if (info.localloc) - { - TADDR locallocBaseSP = *(DWORD *)(size_t)(EBP - GetLocallocSPOffset(&info)); - // -sizeof(void*) because we want to point *AT* first parameter - pPendingArgFirst = (DWORD *)(size_t) (locallocBaseSP - sizeof(void*)); - } - else - { - // Note that 'info.stackSize includes the size for pushing EBP, but EBP is pushed - // BEFORE EBP is set from ESP, thus (EBP - info.stackSize) actually points past - // the frame by one DWORD, and thus points *AT* the first parameter - - pPendingArgFirst = (DWORD *)(size_t)(EBP - info.stackSize); - } - } - - if (!isZero(args)) - { - unsigned i = 0; - ptrArgTP b(1); - for (; !isZero(args) && (i < MAX_PTRARG_OFS); i += 1, b <<= 1) - { - if (intersect(args,b)) - { - unsigned argAddr = (unsigned)(size_t)(pPendingArgFirst - i); - bool iptr = false; - - setDiff(args, b); - if (intersect(iargs,b)) - { - setDiff(iargs, b); - iptr = true; - } - -#ifdef _DEBUG - if (dspPtr) - { - printf(" Pushed ptr arg [E"); - if (info.ebpFrame) - printf("BP-%02XH]: ", EBP - argAddr); - else - printf("SP+%02XH]: ", argAddr - ESP); - } -#endif - _ASSERTE(true == GC_CALL_INTERIOR); - pCallBack(hCallBack, (OBJECTREF *)(size_t)argAddr, (int)iptr | CHECK_APP_DOMAIN - DAC_ARG(DacSlotLocation(info.ebpFrame ? REGI_EBP : REGI_ESP, - info.ebpFrame ? EBP - argAddr : argAddr - ESP, - true))); - } - } - } - } - else - { - // Is "this" enregistered. If so, report it as we might need to - // release the monitor for synchronized methods. - // Else, it is on the stack and will be reported below. - - if (info.thisPtrResult != REGI_NA) - { - // Synchronized methods and methods satisfying - // MethodDesc::AcquiresInstMethodTableFromThis (i.e. those - // where "this" is reported in thisPtrResult) are - // not supported on value types. - _ASSERTE((regNumToMask(info.thisPtrResult) & info.iregMaskResult)== 0); - - void * thisReg = getCalleeSavedReg(pContext, info.thisPtrResult); - pCallBack(hCallBack, (OBJECTREF *)thisReg, CHECK_APP_DOMAIN - DAC_ARG(DacSlotLocation(info.thisPtrResult, 0, false))); - } - } - } - else /* not interruptible */ - { - pushedSize = scanArgRegTable(skipToArgReg(info, table), curOffs, &info); - - RegMask regMask = info.regMaskResult; - RegMask iregMask = info.iregMaskResult; - ptrArgTP argMask = info.argMaskResult; - ptrArgTP iargMask = info.iargMaskResult; - unsigned argHnum = info.argHnumResult; - PTR_CBYTE argTab = info.argTabResult; - - /* now report registers and arguments if we are not interrupted */ - - if (willContinueExecution) - { - - /* Report all live pointer registers */ - - CHK_AND_REPORT_REG(REGI_EDI, regMask & RM_EDI, iregMask & RM_EDI, Edi); - CHK_AND_REPORT_REG(REGI_ESI, regMask & RM_ESI, iregMask & RM_ESI, Esi); - CHK_AND_REPORT_REG(REGI_EBX, regMask & RM_EBX, iregMask & RM_EBX, Ebx); - CHK_AND_REPORT_REG(REGI_EBP, regMask & RM_EBP, iregMask & RM_EBP, Ebp); - - /* Esp cant be reported */ - _ASSERTE(!(regMask & RM_ESP)); - /* No callee-trashed registers */ - _ASSERTE(!(regMask & RM_CALLEE_TRASHED)); - /* EBP can't be reported unless we have an EBP-less frame */ - _ASSERTE(!(regMask & RM_EBP) || !(info.ebpFrame)); - - /* Report any pending pointer arguments */ - - if (argTab != 0) - { - unsigned lowBits, stkOffs, argAddr, val; - - // argMask does not fit in 32-bits - // thus arguments are reported via a table - // Both of these are very rare cases - - do - { - val = fastDecodeUnsigned(argTab); - - lowBits = val & OFFSET_MASK; - stkOffs = val & ~OFFSET_MASK; - _ASSERTE((lowBits == 0) || (lowBits == byref_OFFSET_FLAG)); - - argAddr = ESP + stkOffs; -#ifdef _DEBUG - if (dspPtr) - printf(" Pushed %sptr arg at [ESP+%02XH]", - lowBits ? "iptr " : "", stkOffs); -#endif - _ASSERTE(byref_OFFSET_FLAG == GC_CALL_INTERIOR); - pCallBack(hCallBack, (OBJECTREF *)(size_t)argAddr, lowBits | CHECK_APP_DOMAIN - DAC_ARG(DacSlotLocation(REGI_ESP, stkOffs, true))); - } - while(--argHnum); - - _ASSERTE(info.argTabResult + info.argTabBytes == argTab); - } - else - { - unsigned argAddr = ESP; - - while (!isZero(argMask)) - { - _ASSERTE(argHnum-- > 0); - - if (toUnsigned(argMask) & 1) - { - bool iptr = false; - - if (toUnsigned(iargMask) & 1) - iptr = true; -#ifdef _DEBUG - if (dspPtr) - printf(" Pushed ptr arg at [ESP+%02XH]", - argAddr - ESP); -#endif - _ASSERTE(true == GC_CALL_INTERIOR); - pCallBack(hCallBack, (OBJECTREF *)(size_t)argAddr, (int)iptr | CHECK_APP_DOMAIN - DAC_ARG(DacSlotLocation(REGI_ESP, argAddr - ESP, true))); - } - - argMask >>= 1; - iargMask >>= 1; - argAddr += 4; - } - - } - - } - else - { - // Is "this" enregistered. If so, report it as we will need to - // release the monitor. Else, it is on the stack and will be - // reported below. - - // For partially interruptible code, info.thisPtrResult will be - // the last known location of "this". So the compiler needs to - // generate information which is correct at every point in the code, - // not just at call sites. - - if (info.thisPtrResult != REGI_NA) - { - // Synchronized methods on value types are not supported - _ASSERTE((regNumToMask(info.thisPtrResult) & info.iregMaskResult)== 0); - - void * thisReg = getCalleeSavedReg(pContext, info.thisPtrResult); - pCallBack(hCallBack, (OBJECTREF *)thisReg, CHECK_APP_DOMAIN - DAC_ARG(DacSlotLocation(info.thisPtrResult, 0, false))); - } - } - - } //info.interruptible - - /* compute the argument base (reference point) */ - - unsigned argBase; - - if (info.ebpFrame) - argBase = EBP; - else - argBase = ESP + pushedSize; - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xBEEF); -#endif - - unsigned ptrAddr; - unsigned lowBits; - - - /* Process the untracked frame variable table */ - -#if defined(FEATURE_EH_FUNCLETS) // funclets - // Filters are the only funclet that run during the 1st pass, and must have - // both the leaf and the parent frame reported. In order to avoid double - // reporting of the untracked variables, do not report them for the filter. - if (!isFilterFunclet) -#endif // FEATURE_EH_FUNCLETS - { - count = info.untrackedCnt; - int lastStkOffs = 0; - while (count-- > 0) - { - int stkOffs = fastDecodeSigned(table); - stkOffs = lastStkOffs - stkOffs; - lastStkOffs = stkOffs; - - _ASSERTE(0 == ~OFFSET_MASK % sizeof(void*)); - - lowBits = OFFSET_MASK & stkOffs; - stkOffs &= ~OFFSET_MASK; - - ptrAddr = argBase + stkOffs; - if (info.doubleAlign && stkOffs >= int(info.stackSize - sizeof(void*))) { - // We encode the arguments as if they were ESP based variables even though they aren't - // If this frame would have ben an ESP based frame, This fake frame is one DWORD - // smaller than the real frame because it did not push EBP but the real frame did. - // Thus to get the correct EBP relative offset we have to adjust by info.stackSize-sizeof(void*) - ptrAddr = EBP + (stkOffs-(info.stackSize - sizeof(void*))); - } - -#ifdef _DEBUG - if (dspPtr) - { - printf(" Untracked %s%s local at [E", - (lowBits & pinned_OFFSET_FLAG) ? "pinned " : "", - (lowBits & byref_OFFSET_FLAG) ? "byref" : ""); - - int dspOffs = ptrAddr; - char frameType; - - if (info.ebpFrame) { - dspOffs -= EBP; - frameType = 'B'; - } - else { - dspOffs -= ESP; - frameType = 'S'; - } - - if (dspOffs < 0) - printf("%cP-%02XH]: ", frameType, -dspOffs); - else - printf("%cP+%02XH]: ", frameType, +dspOffs); - } -#endif - - _ASSERTE((pinned_OFFSET_FLAG == GC_CALL_PINNED) && - (byref_OFFSET_FLAG == GC_CALL_INTERIOR)); - pCallBack(hCallBack, (OBJECTREF*)(size_t)ptrAddr, lowBits | CHECK_APP_DOMAIN - DAC_ARG(DacSlotLocation(info.ebpFrame ? REGI_EBP : REGI_ESP, - info.ebpFrame ? EBP - ptrAddr : ptrAddr - ESP, - true))); - } - - } - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xCAFE); -#endif - - /* Process the frame variable lifetime table */ - count = info.varPtrTableSize; - - /* If we are not in the active method, we are currently pointing - * to the return address; at the return address stack variables - * can become dead if the call the last instruction of a try block - * and the return address is the jump around the catch block. Therefore - * we simply assume an offset inside of call instruction. - */ - - unsigned newCurOffs; - - if (willContinueExecution) - { - newCurOffs = (flags & ActiveStackFrame) ? curOffs // after "call" - : curOffs-1; // inside "call" - } - else - { - /* However if ExecutionAborted, then this must be one of the - * ExceptionFrames. Handle accordingly - */ - _ASSERTE(!(flags & AbortingCall) || !(flags & ActiveStackFrame)); - - newCurOffs = (flags & AbortingCall) ? curOffs-1 // inside "call" - : curOffs; // at faulting instr, or start of "try" - } - - ptrOffs = 0; - - while (count-- > 0) - { - int stkOffs; - unsigned begOffs; - unsigned endOffs; - - stkOffs = fastDecodeUnsigned(table); - begOffs = ptrOffs + fastDecodeUnsigned(table); - endOffs = begOffs + fastDecodeUnsigned(table); - - _ASSERTE(0 == ~OFFSET_MASK % sizeof(void*)); - - lowBits = OFFSET_MASK & stkOffs; - stkOffs &= ~OFFSET_MASK; - - if (info.ebpFrame) { - stkOffs = -stkOffs; - _ASSERTE(stkOffs < 0); - } - else { - _ASSERTE(stkOffs >= 0); - } - - ptrAddr = argBase + stkOffs; - - /* Is this variable live right now? */ - - if (newCurOffs >= begOffs) - { - if (newCurOffs < endOffs) - { -#ifdef _DEBUG - if (dspPtr) { - printf(" Frame %s%s local at [E", - (lowBits & byref_OFFSET_FLAG) ? "byref " : "", -#ifndef FEATURE_EH_FUNCLETS - (lowBits & this_OFFSET_FLAG) ? "this-ptr" : ""); -#else - (lowBits & pinned_OFFSET_FLAG) ? "pinned" : ""); -#endif - - - int dspOffs = ptrAddr; - char frameType; - - if (info.ebpFrame) { - dspOffs -= EBP; - frameType = 'B'; - } - else { - dspOffs -= ESP; - frameType = 'S'; - } - - if (dspOffs < 0) - printf("%cP-%02XH]: ", frameType, -dspOffs); - else - printf("%cP+%02XH]: ", frameType, +dspOffs); - } -#endif - - unsigned flags = CHECK_APP_DOMAIN; -#ifndef FEATURE_EH_FUNCLETS - // First Bit : byref - // Second Bit : this - // The second bit means `this` not `pinned`. So we ignore it. - flags |= lowBits & byref_OFFSET_FLAG; -#else - // First Bit : byref - // Second Bit : pinned - // Both bits are valid - flags |= lowBits; -#endif - - _ASSERTE(byref_OFFSET_FLAG == GC_CALL_INTERIOR); - pCallBack(hCallBack, (OBJECTREF*)(size_t)ptrAddr, flags - DAC_ARG(DacSlotLocation(info.ebpFrame ? REGI_EBP : REGI_ESP, - info.ebpFrame ? EBP - ptrAddr : ptrAddr - ESP, - true))); - } - } - // exit loop early if start of live range is beyond PC, as ranges are sorted by lower bound - else break; - - ptrOffs = begOffs; - } - - -#if VERIFY_GC_TABLES - _ASSERTE(*castto(table, unsigned short *)++ == 0xBABE); -#endif - -#ifdef FEATURE_EH_FUNCLETS // funclets - // - // If we're in a funclet, we do not want to report the incoming varargs. This is - // taken care of by the parent method and the funclet should access those arguments - // by way of the parent method's stack frame. - // - if (isFunclet) - { - return true; - } -#endif // FEATURE_EH_FUNCLETS - - /* Are we a varargs function, if so we have to report all args - except 'this' (note that the GC tables created by the x86 jit - do not contain ANY arguments except 'this' (even if they - were statically declared */ - - if (info.varargs) { -#ifdef FEATURE_NATIVEAOT - PORTABILITY_ASSERT("EnumGCRefs: VarArgs"); -#else - LOG((LF_GCINFO, LL_INFO100, "Reporting incoming vararg GC refs\n")); - - PTR_BYTE argsStart; - - if (info.ebpFrame || info.doubleAlign) - argsStart = PTR_BYTE((size_t)EBP) + 2* sizeof(void*); // pushed EBP and retAddr - else - argsStart = PTR_BYTE((size_t)argBase) + info.stackSize + sizeof(void*); // ESP + locals + retAddr - -#if defined(_DEBUG) && !defined(DACCESS_COMPILE) - // Note that I really want to say hCallBack is a GCCONTEXT, but this is pretty close - extern void GcEnumObject(LPVOID pData, OBJECTREF *pObj, uint32_t flags); - _ASSERTE((void*) GcEnumObject == pCallBack); -#endif - GCCONTEXT *pCtx = (GCCONTEXT *) hCallBack; - - // For varargs, look up the signature using the varArgSig token passed on the stack - PTR_VASigCookie varArgSig = *PTR_PTR_VASigCookie(argsStart); - - promoteVarArgs(argsStart, varArgSig, pCtx); -#endif - } - - return true; -} diff --git a/src/coreclr/vm/gccover.cpp b/src/coreclr/vm/gccover.cpp index 44ccfd4c46bec7..278e6a84a91f58 100644 --- a/src/coreclr/vm/gccover.cpp +++ b/src/coreclr/vm/gccover.cpp @@ -1635,7 +1635,7 @@ void DoGcStress (PCONTEXT regs, NativeCodeVersion nativeCodeVersion) // unwind out of the prolog or epilog gcCover->codeMan->UnwindStackFrame(®Disp, - &codeInfo, UpdateAllRegs, &codeManState); + &codeInfo, UpdateAllRegs, &codeManState, NULL); // Note we always doing the unwind, since that at does some checking (that we // unwind to a valid return address), but we only do the precise checking when diff --git a/src/coreclr/vm/i386/asmconstants.h b/src/coreclr/vm/i386/asmconstants.h index 876925dbb94413..7de14a6c063184 100644 --- a/src/coreclr/vm/i386/asmconstants.h +++ b/src/coreclr/vm/i386/asmconstants.h @@ -231,6 +231,8 @@ ASMCONSTANTS_C_ASSERT(OFFSETOF__FrameHandlerExRecord__m_pEntryFrame == offsetof( #define STACK_OVERWRITE_BARRIER_VALUE 0xabcdefab #endif +#define SIZEOF_FrameHandlerExRecordWithBarrier 0x5c +ASMCONSTANTS_C_ASSERT(SIZEOF_FrameHandlerExRecordWithBarrier == sizeof(FrameHandlerExRecordWithBarrier)) #endif diff --git a/src/coreclr/vm/i386/cgenx86.cpp b/src/coreclr/vm/i386/cgenx86.cpp index b50ff163b2a24b..9bbc1046e806a2 100644 --- a/src/coreclr/vm/i386/cgenx86.cpp +++ b/src/coreclr/vm/i386/cgenx86.cpp @@ -766,7 +766,7 @@ void PInvokeCalliFrame::UpdateRegDisplay(const PREGDISPLAY pRD, bool updateFloat CONTRACT_END; VASigCookie *pVASigCookie = GetVASigCookie(); - UpdateRegDisplayHelper(pRD, pVASigCookie->sizeOfArgs); + UpdateRegDisplayHelper(pRD, pVASigCookie->sizeOfArgs+sizeof(int)); LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK PInvokeCalliFrame::UpdateRegDisplay(ip:%p, sp:%p)\n", pRD->ControlPC, pRD->SP)); diff --git a/src/coreclr/vm/i386/excepx86.cpp b/src/coreclr/vm/i386/excepx86.cpp index ed782c7a37ffd8..f52766d7f51e50 100644 --- a/src/coreclr/vm/i386/excepx86.cpp +++ b/src/coreclr/vm/i386/excepx86.cpp @@ -2970,7 +2970,8 @@ void ResumeAtJitEH(CrawlFrame* pCf, bool unwindSuccess = pCf->GetCodeManager()->UnwindStackFrame(pCf->GetRegisterSet(), pCf->GetCodeInfo(), pCf->GetCodeManagerFlags(), - pCf->GetCodeManState()); + pCf->GetCodeManState(), + NULL /* StackwalkCacheUnwindInfo* */); _ASSERTE(unwindSuccess); if (((TADDR)pThread->m_pFrame < pCf->GetRegisterSet()->SP)) diff --git a/src/coreclr/vm/i386/jithelp.asm b/src/coreclr/vm/i386/jithelp.asm index 0faf7cde0e0b26..5f6890b8312e03 100644 --- a/src/coreclr/vm/i386/jithelp.asm +++ b/src/coreclr/vm/i386/jithelp.asm @@ -75,9 +75,7 @@ EXTERN g_GCShadowEnd:DWORD INVALIDGCVALUE equ 0CCCCCCCDh endif -ifndef FEATURE_EH_FUNCLETS EXTERN _COMPlusEndCatch@20:PROC -endif .686P .XMM @@ -1300,7 +1298,6 @@ ret _JIT_PatchedCodeEnd@0 endp -ifndef FEATURE_EH_FUNCLETS ; Note that the debugger skips this entirely when doing SetIP, ; since COMPlusCheckForAbort should always return 0. Excep.cpp:LeaveCatch ; asserts that to be true. If this ends up doing more work, then the @@ -1328,7 +1325,6 @@ JIT_EndCatch PROC stdcall public jmp edx ; eip = new eip JIT_EndCatch ENDP -endif ; The following helper will access ("probe") a word on each page of the stack ; starting with the page right beneath esp down to the one pointed to by eax. diff --git a/src/coreclr/vm/ilstubcache.cpp b/src/coreclr/vm/ilstubcache.cpp index 1d8d14456a1f0b..d0f55495c82904 100644 --- a/src/coreclr/vm/ilstubcache.cpp +++ b/src/coreclr/vm/ilstubcache.cpp @@ -500,10 +500,8 @@ MethodDesc* ILStubCache::GetStubMethodDesc( ILStubHashBlob* pHashBlob, DWORD dwStubFlags, Module* pSigModule, - Module* pSigLoaderModule, PCCOR_SIGNATURE pSig, DWORD cbSig, - SigTypeContext* pTypeContext, AllocMemTracker* pamTracker, bool& bILStubCreator, MethodDesc *pLastMD) @@ -540,23 +538,22 @@ MethodDesc* ILStubCache::GetStubMethodDesc( // Couldn't find it, let's make a new one. // - if (pSigLoaderModule == NULL) + Module *pContainingModule = pSigModule; + if (pTargetMD != NULL) { - pSigLoaderModule = (pTargetMD != NULL) ? pTargetMD->GetLoaderModule() : pSigModule; + // loader module may be different from signature module for generic targets + pContainingModule = pTargetMD->GetLoaderModule(); } + MethodTable *pStubMT = GetOrCreateStubMethodTable(pContainingModule); + SigTypeContext typeContext; - if (pTypeContext == NULL) + if (pTargetMD != NULL) { - if (pTargetMD != NULL) - { - SigTypeContext::InitTypeContext(pTargetMD, &typeContext); - } - pTypeContext = &typeContext; + SigTypeContext::InitTypeContext(pTargetMD, &typeContext); } - MethodTable *pStubMT = GetOrCreateStubMethodTable(pSigLoaderModule); - pMD = ILStubCache::CreateNewMethodDesc(m_pAllocator->GetHighFrequencyHeap(), pStubMT, dwStubFlags, pSigModule, pSig, cbSig, pTypeContext, pamTracker); + pMD = ILStubCache::CreateNewMethodDesc(m_pAllocator->GetHighFrequencyHeap(), pStubMT, dwStubFlags, pSigModule, pSig, cbSig, &typeContext, pamTracker); if (SF_IsSharedStub(dwStubFlags)) { diff --git a/src/coreclr/vm/ilstubcache.h b/src/coreclr/vm/ilstubcache.h index c53fd7a1878c28..6324bad28eebf7 100644 --- a/src/coreclr/vm/ilstubcache.h +++ b/src/coreclr/vm/ilstubcache.h @@ -53,10 +53,8 @@ class ILStubCache final ILStubHashBlob* pHashBlob, DWORD dwStubFlags, // bitmask of NDirectStubFlags Module* pSigModule, - Module* pSigLoaderModule, PCCOR_SIGNATURE pSig, DWORD cbSig, - SigTypeContext* pTypeContext, AllocMemTracker* pamTracker, bool& bILStubCreator, MethodDesc* pLastMD); diff --git a/src/coreclr/vm/interoputil.cpp b/src/coreclr/vm/interoputil.cpp index 7a91cd41f7c14b..d9e5d43759626a 100644 --- a/src/coreclr/vm/interoputil.cpp +++ b/src/coreclr/vm/interoputil.cpp @@ -2558,7 +2558,7 @@ BOOL IsMethodVisibleFromCom(MethodDesc *pMD) mdMethodDef md = pMD->GetMemberDef(); // See if there is property information for this member. - hr = pMD->GetMDImport()->GetPropertyInfoForMethodDef(md, &pd, &pPropName, &uSemantic); + hr = pMD->GetModule()->GetPropertyInfoForMethodDef(md, &pd, &pPropName, &uSemantic); IfFailThrow(hr); if (hr == S_OK) diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 1da02114960293..450752ae367789 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -3596,14 +3596,6 @@ NOINLINE HCIMPL3(CORINFO_MethodPtr, JIT_VirtualFunctionPointer_Framed, Object * } HCIMPLEND -HCIMPL3(void, Jit_NativeMemSet, void* pDest, int value, size_t length) -{ - _ASSERTE(pDest != nullptr); - FCALL_CONTRACT; - memset(pDest, value, length); -} -HCIMPLEND - HCIMPL1(Object*, JIT_GetRuntimeFieldStub, CORINFO_FIELD_HANDLE field) { FCALL_CONTRACT; diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 05d9365e967964..62d69c0fe4ef98 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -2627,37 +2627,6 @@ bool CEEInfo::getSystemVAmd64PassStructInRegisterDescriptor( #endif // !defined(UNIX_AMD64_ABI_ITF) } -void CEEInfo::getSwiftLowering(CORINFO_CLASS_HANDLE structHnd, CORINFO_SWIFT_LOWERING* pLowering) -{ - CONTRACTL{ - THROWS; - GC_TRIGGERS; - MODE_PREEMPTIVE; - } CONTRACTL_END; - - JIT_TO_EE_TRANSITION(); - - TypeHandle th(structHnd); - - bool useNativeLayout = false; - MethodTable* methodTablePtr = nullptr; - if (!th.IsTypeDesc()) - { - methodTablePtr = th.AsMethodTable(); - } - else - { - _ASSERTE(th.IsNativeValueType()); - - useNativeLayout = true; - methodTablePtr = th.AsNativeValueType(); - } - - methodTablePtr->GetNativeSwiftPhysicalLowering(pLowering, useNativeLayout); - - EE_TO_JIT_TRANSITION(); -} - /*********************************************************************/ unsigned CEEInfo::getClassNumInstanceFields (CORINFO_CLASS_HANDLE clsHnd) { @@ -2823,7 +2792,6 @@ void CEEInfo::MethodCompileComplete(CORINFO_METHOD_HANDLE methHnd) void CEEInfo::embedGenericHandle( CORINFO_RESOLVED_TOKEN * pResolvedToken, bool fEmbedParent, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT *pResult) { CONTRACTL { @@ -2926,7 +2894,6 @@ void CEEInfo::embedGenericHandle( pResolvedToken, NULL, pTemplateMD, - GetMethod(callerHandle), &pResult->lookup); } else @@ -3024,7 +2991,7 @@ MethodDesc * CEEInfo::GetMethodForSecurity(CORINFO_METHOD_HANDLE callerHandle) return m_pMethodForSecurity_Value; } - MethodDesc * pCallerMethod = GetMethod(callerHandle); + MethodDesc * pCallerMethod = (MethodDesc *)callerHandle; //If the caller is generic, load the open type and then load the field again, This allows us to //differentiate between BadGeneric containing a memberRef for a field of type InaccessibleClass and @@ -3097,7 +3064,6 @@ void CEEInfo::ComputeRuntimeLookupForSharedGenericToken(DictionaryEntryKind entr CORINFO_RESOLVED_TOKEN * pResolvedToken, CORINFO_RESOLVED_TOKEN * pConstrainedResolvedToken, MethodDesc * pTemplateMD /* for method-based slots */, - MethodDesc * pCallerMD, CORINFO_LOOKUP *pResultLookup) { CONTRACTL{ @@ -3105,8 +3071,6 @@ void CEEInfo::ComputeRuntimeLookupForSharedGenericToken(DictionaryEntryKind entr PRECONDITION(CheckPointer(pResultLookup)); } CONTRACTL_END; - _ASSERT(pCallerMD != nullptr); - pResultLookup->lookupKind.needsRuntimeLookup = true; pResultLookup->lookupKind.runtimeLookupFlags = 0; @@ -3122,8 +3086,16 @@ void CEEInfo::ComputeRuntimeLookupForSharedGenericToken(DictionaryEntryKind entr // Unless we decide otherwise, just do the lookup via a helper function pResult->indirections = CORINFO_USEHELPER; - MethodDesc* pContextMD = pCallerMD; + // Runtime lookups in inlined contexts are not supported by the runtime for now + if (pResolvedToken->tokenContext != METHOD_BEING_COMPILED_CONTEXT()) + { + pResultLookup->lookupKind.runtimeLookupKind = CORINFO_LOOKUP_NOT_SUPPORTED; + return; + } + + MethodDesc* pContextMD = GetMethodFromContext(pResolvedToken->tokenContext); MethodTable* pContextMT = pContextMD->GetMethodTable(); + bool isStaticVirtual = (pConstrainedResolvedToken != nullptr && pContextMD != nullptr && pContextMD->IsStatic()); // There is a pathological case where invalid IL refereces __Canon type directly, but there is no dictionary availabled to store the lookup. if (!pContextMD->IsSharedByGenericInstantiations()) @@ -5286,6 +5258,19 @@ void CEEInfo::getCallInfo( { pResult->exactContextNeedsRuntimeLookup = TRUE; } + + // Use main method as the context as long as the methods are called on the same type + if (pResult->exactContextNeedsRuntimeLookup && + pResolvedToken->tokenContext == METHOD_BEING_COMPILED_CONTEXT() && + constrainedType.IsNull() && + exactType == m_pMethodBeingCompiled->GetMethodTable() && + ((pResolvedToken->cbTypeSpec == 0) || IsTypeSpecForTypicalInstantiation(SigPointer(pResolvedToken->pTypeSpec, pResolvedToken->cbTypeSpec)))) + { + // The typespec signature should be only missing for dynamic methods + _ASSERTE((pResolvedToken->cbTypeSpec != 0) || m_pMethodBeingCompiled->IsDynamicMethod()); + + pResult->contextHandle = METHOD_BEING_COMPILED_CONTEXT(); + } } // @@ -5437,7 +5422,6 @@ void CEEInfo::getCallInfo( pResolvedToken, pConstrainedResolvedToken, pMD, - GetMethod(callerHandle), &pResult->codePointerLookup); } else @@ -5489,7 +5473,6 @@ void CEEInfo::getCallInfo( pResolvedToken, pConstrainedResolvedToken, pMD, - GetMethod(callerHandle), &pResult->stubLookup); } else @@ -5531,7 +5514,7 @@ void CEEInfo::getCallInfo( pResult->hMethod = CORINFO_METHOD_HANDLE(pTargetMD); pResult->accessAllowed = CORINFO_ACCESS_ALLOWED; - MethodDesc* callerMethod = GetMethod(callerHandle); + MethodDesc* callerMethod = (MethodDesc*)callerHandle; if ((flags & CORINFO_CALLINFO_SECURITYCHECKS) && RequiresAccessCheck(pResolvedToken->tokenScope)) { @@ -6305,7 +6288,6 @@ bool CEEInfo::getReadyToRunHelper( CORINFO_RESOLVED_TOKEN * pResolvedToken, CORINFO_LOOKUP_KIND * pGenericLookupKind, CorInfoHelpFunc id, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_CONST_LOOKUP * pLookup ) { @@ -6318,8 +6300,7 @@ void CEEInfo::getReadyToRunDelegateCtorHelper( CORINFO_RESOLVED_TOKEN * pTargetMethod, mdToken targetConstraint, CORINFO_CLASS_HANDLE delegateType, - CORINFO_METHOD_HANDLE callerHandle, - CORINFO_LOOKUP * pLookup + CORINFO_LOOKUP * pLookup ) { LIMITED_METHOD_CONTRACT; @@ -6398,11 +6379,7 @@ CORINFO_VARARGS_HANDLE CEEInfo::getVarArgsHandle(CORINFO_SIG_INFO *sig, Module* module = GetModule(sig->scope); - Instantiation classInst = Instantiation((TypeHandle*) sig->sigInst.classInst, sig->sigInst.classInstCount); - Instantiation methodInst = Instantiation((TypeHandle*) sig->sigInst.methInst, sig->sigInst.methInstCount); - SigTypeContext typeContext = SigTypeContext(classInst, methodInst); - - result = CORINFO_VARARGS_HANDLE(module->GetVASigCookie(Signature(sig->pSig, sig->cbSig), &typeContext)); + result = CORINFO_VARARGS_HANDLE(module->GetVASigCookie(Signature(sig->pSig, sig->cbSig))); EE_TO_JIT_TRANSITION(); @@ -8952,7 +8929,6 @@ CORINFO_METHOD_HANDLE CEEInfo::getUnboxedEntry( /*********************************************************************/ void CEEInfo::expandRawHandleIntrinsic( CORINFO_RESOLVED_TOKEN * pResolvedToken, - CORINFO_METHOD_HANDLE callerHandle, CORINFO_GENERICHANDLE_RESULT * pResult) { LIMITED_METHOD_CONTRACT; @@ -9894,13 +9870,10 @@ bool CEEInfo::pInvokeMarshalingRequired(CORINFO_METHOD_HANDLE method, CORINFO_SI if (method == NULL) { // check the call site signature - SigTypeContext typeContext; - GetTypeContext(&callSiteSig->sigInst, &typeContext); result = NDirect::MarshalingRequired( NULL, callSiteSig->pSig, - GetModule(callSiteSig->scope), - &typeContext); + GetModule(callSiteSig->scope)); } else { @@ -10709,10 +10682,7 @@ void* CEEJitInfo::getHelperFtn(CorInfoHelpFunc ftnNum, /* IN */ dynamicFtnNum == DYNAMIC_CORINFO_HELP_CHKCASTCLASS_SPECIAL || dynamicFtnNum == DYNAMIC_CORINFO_HELP_UNBOX || dynamicFtnNum == DYNAMIC_CORINFO_HELP_ARRADDR_ST || - dynamicFtnNum == DYNAMIC_CORINFO_HELP_LDELEMA_REF || - dynamicFtnNum == DYNAMIC_CORINFO_HELP_MEMSET || - dynamicFtnNum == DYNAMIC_CORINFO_HELP_MEMZERO || - dynamicFtnNum == DYNAMIC_CORINFO_HELP_MEMCPY) + dynamicFtnNum == DYNAMIC_CORINFO_HELP_LDELEMA_REF) { Precode* pPrecode = Precode::GetPrecodeFromEntryPoint((PCODE)hlpDynamicFuncTable[dynamicFtnNum].pfnHelper); _ASSERTE(pPrecode->GetType() == PRECODE_FIXUP); @@ -10986,22 +10956,6 @@ void CEEJitInfo::reportRichMappings( EE_TO_JIT_TRANSITION(); } -void CEEJitInfo::reportMetadata( - const char* key, - const void* value, - size_t length) -{ - CONTRACTL { - NOTHROW; - GC_NOTRIGGER; - MODE_PREEMPTIVE; - } CONTRACTL_END; - - JIT_TO_EE_TRANSITION_LEAF(); - - EE_TO_JIT_TRANSITION_LEAF(); -} - void CEEJitInfo::setPatchpointInfo(PatchpointInfo* patchpointInfo) { CONTRACTL { @@ -13557,8 +13511,7 @@ BOOL LoadDynamicInfoEntry(Module *currentModule, } { VarArgs: - SigTypeContext typeContext = SigTypeContext(); - result = (size_t) CORINFO_VARARGS_HANDLE(currentModule->GetVASigCookie(Signature(pSig, cSig), &typeContext)); + result = (size_t) CORINFO_VARARGS_HANDLE(currentModule->GetVASigCookie(Signature(pSig, cSig))); } break; @@ -14468,12 +14421,6 @@ void CEEInfo::reportRichMappings( UNREACHABLE(); // only called on derived class. } -void CEEInfo::reportMetadata(const char* key, const void* value, size_t length) -{ - LIMITED_METHOD_CONTRACT; - UNREACHABLE(); // only called on derived class. -} - void CEEInfo::setPatchpointInfo(PatchpointInfo* patchpointInfo) { LIMITED_METHOD_CONTRACT; diff --git a/src/coreclr/vm/jitinterface.h b/src/coreclr/vm/jitinterface.h index 9b307983a24d0c..1708a05df5e314 100644 --- a/src/coreclr/vm/jitinterface.h +++ b/src/coreclr/vm/jitinterface.h @@ -400,6 +400,9 @@ extern "C" #endif // TARGET_AMD64 || TARGET_ARM + void STDCALL JIT_MemSet(void *dest, int c, SIZE_T count); + void STDCALL JIT_MemCpy(void *dest, const void *src, SIZE_T count); + void STDMETHODCALLTYPE JIT_ProfilerEnterLeaveTailcallStub(UINT_PTR ProfilerHandle); #if !defined(TARGET_ARM64) && !defined(TARGET_LOONGARCH64) && !defined(TARGET_RISCV64) void STDCALL JIT_StackProbe(); @@ -558,7 +561,6 @@ class CEEInfo : public ICorJitInfo CORINFO_RESOLVED_TOKEN * pResolvedToken, CORINFO_RESOLVED_TOKEN * pConstrainedResolvedToken /* for ConstrainedMethodEntrySlot */, MethodDesc * pTemplateMD /* for method-based slots */, - MethodDesc * pCallerMD, CORINFO_LOOKUP *pResultLookup); #if defined(FEATURE_GDBJIT) @@ -895,8 +897,6 @@ class CEEJitInfo : public CEEInfo ICorDebugInfo::RichOffsetMapping* mappings, uint32_t numMappings) override final; - void reportMetadata(const char* key, const void* value, size_t length) override final; - void* getHelperFtn(CorInfoHelpFunc ftnNum, /* IN */ void ** ppIndirection) override final; /* OUT */ static PCODE getHelperFtnStatic(CorInfoHelpFunc ftnNum); diff --git a/src/coreclr/vm/loongarch64/crthelpers.S b/src/coreclr/vm/loongarch64/crthelpers.S new file mode 100644 index 00000000000000..88fd21938fdaa2 --- /dev/null +++ b/src/coreclr/vm/loongarch64/crthelpers.S @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "unixasmmacros.inc" + +// JIT_MemSet/JIT_MemCpy +// +// It is IMPORTANT that the exception handling code is able to find these guys +// on the stack, but on non-windows platforms we can just defer to the platform +// implementation. +// +LEAF_ENTRY JIT_MemSet, _TEXT + beq $a2, $zero, LOCAL_LABEL(JIT_MemSet_ret) + + ld.b $zero, $a0, 0 //Is this really needed ? + + b memset + +LOCAL_LABEL(JIT_MemSet_ret): + jirl $r0, $ra, 0 + +////NOTO: Here must use LEAF_END_MARKED! not LEAF_END !!! +LEAF_END_MARKED JIT_MemSet, _TEXT + +LEAF_ENTRY JIT_MemCpy, _TEXT + beq $a2, $zero, LOCAL_LABEL(JIT_MemCpy_ret) + + ld.b $zero, $a0, 0 + ld.b $zero, $a1, 0 //Is this really needed ? + + b memcpy + +LOCAL_LABEL(JIT_MemCpy_ret): + jirl $r0, $ra, 0 + +////NOTO: Here must use LEAF_END_MARKED! not LEAF_END !!! +LEAF_END_MARKED JIT_MemCpy, _TEXT diff --git a/src/coreclr/vm/loongarch64/stubs.cpp b/src/coreclr/vm/loongarch64/stubs.cpp index 052d71ebc1e44e..da93bd587ed344 100644 --- a/src/coreclr/vm/loongarch64/stubs.cpp +++ b/src/coreclr/vm/loongarch64/stubs.cpp @@ -1928,7 +1928,7 @@ PCODE DynamicHelpers::CreateDictionaryLookupHelper(LoaderAllocator * pAllocator, } } - _ASSERTE((indirectionsDataSize ? indirectionsDataSize : codeSize) == dataOffset); + _ASSERTE(indirectionsDataSize == dataOffset); // No null test required if (!pLookup->testForNull) diff --git a/src/coreclr/vm/metasig.h b/src/coreclr/vm/metasig.h index 182acc55e643fe..45cb5700db5293 100644 --- a/src/coreclr/vm/metasig.h +++ b/src/coreclr/vm/metasig.h @@ -237,9 +237,6 @@ DEFINE_METASIG(SM(PtrSByt_RetInt, P(B), i)) DEFINE_METASIG(SM(IntPtr_RetIntPtr, I, I)) DEFINE_METASIG(SM(UIntPtr_RetIntPtr, U, I)) DEFINE_METASIG(SM(PtrByte_PtrByte_Int_RetVoid, P(b) P(b) i, v)) -DEFINE_METASIG(SM(RefByte_RefByte_UIntPtr_RetVoid, r(b) r(b) U, v)) -DEFINE_METASIG(SM(RefByte_Byte_UIntPtr_RetVoid, r(b) b U, v)) -DEFINE_METASIG(SM(RefByte_UIntPtr_RetVoid, r(b) U, v)) DEFINE_METASIG(SM(PtrVoid_Byte_UInt_RetVoid, P(v) b K, v)) DEFINE_METASIG(SM(RefObj_IntPtr_RetVoid, r(j) I, v)) DEFINE_METASIG(SM(RefObj_RefIntPtr_RetVoid, r(j) r(I), v)) diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp index 2c445d9054e188..41307b3d1a8f23 100644 --- a/src/coreclr/vm/methodtable.cpp +++ b/src/coreclr/vm/methodtable.cpp @@ -4026,400 +4026,6 @@ int MethodTable::GetRiscV64PassStructInRegisterFlags(CORINFO_CLASS_HANDLE cls) } #endif -#if !defined(DACCESS_COMPILE) -namespace -{ - // Underlying type specified so we can use memset in the algorithm below - // to set a range of values to a particular tag. - enum class SwiftPhysicalLoweringTag : uint8_t - { - Empty, - Opaque, - Int64, - Float, - Double - }; - - uint32_t GetAlignment(SwiftPhysicalLoweringTag tag) - { - LIMITED_METHOD_CONTRACT; - - switch (tag) - { - case SwiftPhysicalLoweringTag::Int64: - return 8; - case SwiftPhysicalLoweringTag::Float: - return 4; - case SwiftPhysicalLoweringTag::Double: - return 8; - default: - return 1; - } - } - - void SetLoweringRange(CQuickArray& intervals, uint32_t start, uint32_t size, SwiftPhysicalLoweringTag tag) - { - STANDARD_VM_CONTRACT; - - bool forceOpaque = false; - - if (!IS_ALIGNED(start, GetAlignment(tag))) - { - // If the start of the range is not aligned, we need to force the entire range to be opaque. - forceOpaque = true; - } - - // Check if any of the range is non-empty. - // If so, we need to force this range to be opaque - // and extend the range mark the existing tag's range as opaque. - for (uint32_t i = 0; i < size; i++) - { - SwiftPhysicalLoweringTag currentTag = intervals[start + i]; - if (currentTag != SwiftPhysicalLoweringTag::Empty - && currentTag != tag) - { - forceOpaque = true; - - // Extend out start to the beginning of the existing tag's range - // and extend size to the end of the existing tag's range (if non-opaque/empty). - start = (uint32_t)ALIGN_DOWN(start, GetAlignment(currentTag)); - size = (uint32_t)ALIGN_UP(size + start, GetAlignment(currentTag)) - start; - break; - } - } - - if (forceOpaque) - { - tag = SwiftPhysicalLoweringTag::Opaque; - } - - memset(&intervals[start], (uint8_t)tag, sizeof(SwiftPhysicalLoweringTag) * size); - } - - void GetNativeSwiftPhysicalLowering(CQuickArray& intervals, PTR_MethodTable pMT, uint32_t offset = 0); - void GetNativeSwiftPhysicalLoweringForInlineArray(CQuickArray& intervals, PTR_MethodTable pMT, uint32_t offset = 0); - - void GetNativeSwiftPhysicalLoweringForField(CQuickArray& intervals, FieldDesc* pFieldDesc, uint32_t offset = 0) - { - STANDARD_VM_CONTRACT; - - PTR_MethodTable fieldType = pFieldDesc->GetFieldTypeHandleThrowing().GetMethodTable(); - CorElementType corType = fieldType->GetVerifierCorElementType(); - - if (corType == ELEMENT_TYPE_VALUETYPE) - { - if (fieldType->GetClass()->IsInlineArray()) - { - GetNativeSwiftPhysicalLoweringForInlineArray(intervals, fieldType, offset); - } - else - { - GetNativeSwiftPhysicalLowering(intervals, fieldType, offset); - } - } - else if (corType == ELEMENT_TYPE_R4) - { - SetLoweringRange(intervals, offset, 4, SwiftPhysicalLoweringTag::Float); - } - else if (corType == ELEMENT_TYPE_R8) - { - SetLoweringRange(intervals, offset, 8, SwiftPhysicalLoweringTag::Double); - } - else if (corType == ELEMENT_TYPE_I8 || corType == ELEMENT_TYPE_U8) - { - SetLoweringRange(intervals, offset, 8, SwiftPhysicalLoweringTag::Int64); - } - else - { - SetLoweringRange(intervals, offset, fieldType->GetNumInstanceFieldBytes(), SwiftPhysicalLoweringTag::Opaque); - } - } - - void GetNativeSwiftPhysicalLoweringForInlineArray(CQuickArray& intervals, PTR_MethodTable pMT, uint32_t offset) - { - STANDARD_VM_CONTRACT; - _ASSERTE(pMT->GetClass()->IsInlineArray()); - FieldDesc* pElementField = pMT->GetApproxFieldDescListRaw(); - - // If the type is an inline array, we need to calculate the size based on the number of elements. - const void* pVal; // The custom value. - ULONG cbVal; // Size of the custom value. - HRESULT hr = pMT->GetCustomAttribute( - WellKnownAttribute::InlineArrayAttribute, - &pVal, &cbVal); - - _ASSERTE(hr == S_OK); - if (hr != S_OK) - { - ThrowHR(hr); - } - - // Validity of the InlineArray attribute is checked at type-load time, - // so we only assert here as we should have already checked this and failed - // type load if this condition is false. - _ASSERTE(cbVal >= (sizeof(INT32) + 2)); - if (cbVal <= (sizeof(INT32) + 2)) - { - return; - } - - INT32 repeat = GET_UNALIGNED_VAL32((byte*)pVal + 2); - - // Use the one FieldDesc to calculate the Swift intervals - // Use FieldDescs to calculate the Swift intervals - PTR_FieldDesc pFieldDesc = pMT->GetApproxFieldDescListRaw(); - for (int32_t i = 0; i < repeat; i++) - { - GetNativeSwiftPhysicalLoweringForField(intervals, pFieldDesc, offset + pFieldDesc->GetOffset() + pFieldDesc->GetSize() * i); - } - } - - void GetNativeSwiftPhysicalLowering(CQuickArray& intervals, PTR_MethodTable pMT, uint32_t offset) - { - STANDARD_VM_CONTRACT; - // Use FieldDescs to calculate the Swift intervals - PTR_FieldDesc pFieldDescList = pMT->GetApproxFieldDescListRaw(); - for (uint32_t i = 0; i < pMT->GetNumIntroducedInstanceFields(); i++) - { - PTR_FieldDesc pFieldDesc = pFieldDescList + i; - GetNativeSwiftPhysicalLoweringForField(intervals, pFieldDesc, offset + pFieldDesc->GetOffset()); - } - } - - void GetNativeSwiftPhysicalLowering(CQuickArray& intervals, EEClassNativeLayoutInfo const* pNativeLayoutInfo, uint32_t offset = 0) - { - STANDARD_VM_CONTRACT; - // Use NativeLayout to calculate the Swift intervals - NativeFieldDescriptor const* pNativeFieldDescs = pNativeLayoutInfo->GetNativeFieldDescriptors(); - for (uint32_t i = 0; i < pNativeLayoutInfo->GetNumFields(); i++) - { - NativeFieldDescriptor const& nfd = pNativeFieldDescs[i]; - if (nfd.GetCategory() == NativeFieldCategory::NESTED) - { - PTR_MethodTable fieldType = nfd.GetNestedNativeMethodTable(); - for (uint32_t i = 0; i < nfd.GetNumElements(); i++) - { - if (fieldType->IsBlittable()) - { - GetNativeSwiftPhysicalLowering(intervals, fieldType, offset + nfd.GetExternalOffset() + fieldType->GetNativeSize() * i); - } - else - { - GetNativeSwiftPhysicalLowering(intervals, fieldType->GetNativeLayoutInfo(), offset + nfd.GetExternalOffset() + fieldType->GetNativeSize() * i); - } - } - } - else if (nfd.GetCategory() == NativeFieldCategory::FLOAT) - { - _ASSERTE(nfd.NativeSize() == 4 || nfd.NativeSize() == 8); - SetLoweringRange(intervals, offset + nfd.GetExternalOffset(), nfd.NativeSize(), nfd.NativeSize() == 4 ? SwiftPhysicalLoweringTag::Float : SwiftPhysicalLoweringTag::Double); - } - else if (nfd.GetCategory() == NativeFieldCategory::INTEGER && nfd.NativeSize() == 8) - { - SetLoweringRange(intervals, offset + nfd.GetExternalOffset(), nfd.NativeSize(), SwiftPhysicalLoweringTag::Int64); - } - else - { - SetLoweringRange(intervals, offset + nfd.GetExternalOffset(), nfd.NativeSize(), SwiftPhysicalLoweringTag::Opaque); - } - } - } -} - -void MethodTable::GetNativeSwiftPhysicalLowering(CORINFO_SWIFT_LOWERING* pSwiftLowering, bool useNativeLayout) -{ - STANDARD_VM_CONTRACT; - - // We'll build the intervals by scanning the fields byte-by-byte and then calculate the lowering intervals - // from that information. - CQuickArray loweredBytes; - loweredBytes.AllocThrows(GetNumInstanceFieldBytes()); - memset(loweredBytes.Ptr(), (uint8_t)SwiftPhysicalLoweringTag::Empty, sizeof(SwiftPhysicalLoweringTag) * loweredBytes.Size()); - - if (useNativeLayout && !IsBlittable()) - { - // Use NativeLayout to calculate the layout - ::GetNativeSwiftPhysicalLowering(loweredBytes, GetNativeLayoutInfo()); - } - else if (GetClass()->IsInlineArray()) - { - // Use InlineArray to calculate the layout - ::GetNativeSwiftPhysicalLoweringForInlineArray(loweredBytes, PTR_MethodTable(this)); - } - else - { - ::GetNativeSwiftPhysicalLowering(loweredBytes, PTR_MethodTable(this)); - } - - struct SwiftLoweringInterval - { - uint32_t offset; - uint32_t size; - SwiftPhysicalLoweringTag tag; - }; - - // Build intervals from the byte sequences - CQuickArrayList intervals; - for (uint32_t i = 0; i < loweredBytes.Size(); ++i) - { - // Don't create an interval for empty bytes - if (loweredBytes[i] == SwiftPhysicalLoweringTag::Empty) - { - continue; - } - - bool startNewInterval = - // We're at the start of the type - i == 0 - // We're starting a new float (as we're aligned) - || (IS_ALIGNED(i, 4) && loweredBytes[i] == SwiftPhysicalLoweringTag::Float) - // We're starting a new double or int64_t (as we're aligned) - || (IS_ALIGNED(i, 8) && (loweredBytes[i] == SwiftPhysicalLoweringTag::Double || loweredBytes[i] == SwiftPhysicalLoweringTag::Int64)) - // We've changed interval types - || loweredBytes[i] != loweredBytes[i - 1]; - - if (startNewInterval) - { - SwiftLoweringInterval interval; - interval.offset = i; - interval.size = 1; - interval.tag = loweredBytes[i]; - intervals.Push(interval); - } - else - { - intervals[intervals.Size() - 1].size++; - } - } - - // Merge opaque intervals that are in the same pointer-sized block. - CQuickArrayList mergedIntervals; - - for (uint32_t i = 0; i < intervals.Size(); ++i) - { - SwiftLoweringInterval interval = intervals[i]; - - if (i != 0 && interval.tag == SwiftPhysicalLoweringTag::Opaque) - { - // Merge two opaque intervals when the previous interval ends in the same pointer-sized block - SwiftLoweringInterval prevInterval = intervals[i - 1]; - if (prevInterval.tag == SwiftPhysicalLoweringTag::Opaque && - (prevInterval.offset + prevInterval.size) / TARGET_POINTER_SIZE == interval.offset / TARGET_POINTER_SIZE) - { - SwiftLoweringInterval& lastInterval = mergedIntervals[mergedIntervals.Size() - 1]; - lastInterval.size = interval.offset + interval.size - lastInterval.offset; - continue; - } - } - - // Otherwise keep all intervals - mergedIntervals.Push(interval); - } - - // Now we have the intervals, we can calculate the lowering. - CorInfoType loweredTypes[MAX_SWIFT_LOWERED_ELEMENTS]; - uint32_t offsets[MAX_SWIFT_LOWERED_ELEMENTS]; - uint32_t numLoweredTypes = 0; - - for (uint32_t i = 0; i < mergedIntervals.Size(); i++) - { - SwiftLoweringInterval interval = mergedIntervals[i]; - - if (numLoweredTypes == ARRAY_SIZE(loweredTypes)) - { - // If we have more than four intervals, this type is passed by-reference in Swift. - pSwiftLowering->byReference = true; - return; - } - - offsets[numLoweredTypes] = interval.offset; - - switch (interval.tag) - { - case SwiftPhysicalLoweringTag::Empty: - _ASSERTE(!"Empty intervals should have been dropped during interval construction"); - break; - - case SwiftPhysicalLoweringTag::Int64: - loweredTypes[numLoweredTypes++] = CORINFO_TYPE_LONG; - break; - case SwiftPhysicalLoweringTag::Float: - loweredTypes[numLoweredTypes++] = CORINFO_TYPE_FLOAT; - break; - case SwiftPhysicalLoweringTag::Double: - loweredTypes[numLoweredTypes++] = CORINFO_TYPE_DOUBLE; - break; - case SwiftPhysicalLoweringTag::Opaque: - { - // We need to split the opaque ranges into integer parameters. - // As part of this splitting, we must ensure that we don't introduce alignment padding. - // This lowering algorithm should produce a lowered type sequence that would have the same padding for - // a naturally-aligned struct with the lowered fields as the original type has. - // This algorithm intends to split the opaque range into the least number of lowered elements that covers the entire range. - // The lowered range is allowed to extend past the end of the opaque range (including past the end of the struct), - // but not into the next non-empty interval. - // However, due to the properties of the lowering (the only non-8 byte elements of the lowering are 4-byte floats), - // we'll never encounter a scneario where we need would need to account for a correctly-aligned - // opaque range of > 4 bytes that we must not pad to 8 bytes. - - - // As long as we need to fill more than 4 bytes and the sequence is currently 8-byte aligned, we'll split into 8-byte integers. - // If we have more than 2 bytes but less than 4 and the sequence is 4-byte aligned, we'll use a 4-byte integer to represent the rest of the parameters. - // If we have 2 bytes and the sequence is 2-byte aligned, we'll use a 2-byte integer to represent the rest of the parameters. - // If we have 1 byte, we'll use a 1-byte integer to represent the rest of the parameters. - uint32_t opaqueIntervalStart = interval.offset; - // The remaining size here may become negative, so use a signed type. - int32_t remainingIntervalSize = static_cast(interval.size); - while (remainingIntervalSize > 0) - { - if (numLoweredTypes == ARRAY_SIZE(loweredTypes)) - { - // If we have more than four intervals and we still need to add another interval, this type is passed by-reference in Swift. - pSwiftLowering->byReference = true; - return; - } - - offsets[numLoweredTypes] = opaqueIntervalStart; - - if (remainingIntervalSize > 4 && IS_ALIGNED(opaqueIntervalStart, 8)) - { - loweredTypes[numLoweredTypes] = CORINFO_TYPE_LONG; - opaqueIntervalStart += 8; - remainingIntervalSize -= 8; - } - else if (remainingIntervalSize > 2 && IS_ALIGNED(opaqueIntervalStart, 4)) - { - loweredTypes[numLoweredTypes] = CORINFO_TYPE_INT; - opaqueIntervalStart += 4; - remainingIntervalSize -= 4; - } - else if (remainingIntervalSize > 1 && IS_ALIGNED(opaqueIntervalStart, 2)) - { - loweredTypes[numLoweredTypes] = CORINFO_TYPE_SHORT; - opaqueIntervalStart += 2; - remainingIntervalSize -= 2; - } - else - { - loweredTypes[numLoweredTypes] = CORINFO_TYPE_BYTE; - opaqueIntervalStart += 1; - remainingIntervalSize -= 1; - } - - numLoweredTypes++; - } - } - } - } - - memcpy(pSwiftLowering->loweredElements, loweredTypes, numLoweredTypes * sizeof(CorInfoType)); - memcpy(pSwiftLowering->offsets, offsets, numLoweredTypes * sizeof(uint32_t)); - pSwiftLowering->numLoweredElements = numLoweredTypes; - pSwiftLowering->byReference = false; -} - -#endif // !DACCESS_COMPILE - #if !defined(DACCESS_COMPILE) //========================================================================================== void MethodTable::AllocateRegularStaticBoxes() diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h index 83057c623fba98..4984d010a71bba 100644 --- a/src/coreclr/vm/methodtable.h +++ b/src/coreclr/vm/methodtable.h @@ -840,10 +840,6 @@ class MethodTable bool ClassifyEightBytesWithNativeLayout(SystemVStructRegisterPassingHelperPtr helperPtr, unsigned int nestingLevel, unsigned int startOffsetOfStruct, EEClassNativeLayoutInfo const* nativeLayoutInfo); #endif // defined(UNIX_AMD64_ABI_ITF) -#if !defined(DACCESS_COMPILE) - void GetNativeSwiftPhysicalLowering(CORINFO_SWIFT_LOWERING* pSwiftLowering, bool useNativeLayout); -#endif - // Copy m_dwFlags from another method table void CopyFlags(MethodTable * pOldMT) { diff --git a/src/coreclr/vm/methodtablebuilder.cpp b/src/coreclr/vm/methodtablebuilder.cpp index e52b0ced06aef0..b6edb7fee7e546 100644 --- a/src/coreclr/vm/methodtablebuilder.cpp +++ b/src/coreclr/vm/methodtablebuilder.cpp @@ -5988,17 +5988,17 @@ MethodTableBuilder::bmtMethodHandle MethodTableBuilder::FindDeclMethodOnClassInH } } - if (pDeclType == NULL) - { // Method's type is not a parent. - BuildMethodTableThrowException(IDS_CLASSLOAD_MI_DECLARATIONNOTFOUND, it.Token()); - } - // Instead of using the Substitution chain that reaches back to the type being loaded, instead // use a substitution chain that points back to the open type associated with the memberref of the declsig. Substitution emptySubstitution; Substitution* pDeclTypeSubstitution = &emptySubstitution; DWORD lengthOfSubstitutionChainHandled = pDeclType->GetSubstitution().GetLength(); + if (pDeclType == NULL) + { // Method's type is not a parent. + BuildMethodTableThrowException(IDS_CLASSLOAD_MI_DECLARATIONNOTFOUND, it.Token()); + } + // 3. Find the matching method. bmtRTType *pCurDeclType = pDeclType; do @@ -10051,11 +10051,6 @@ void MethodTableBuilder::CheckForSystemTypes() // The Procedure Call Standard for ARM 64-bit (with SVE support) defaults to // 16-byte alignment for __m256. - pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16; - #elif defined(TARGET_RISCV64) - // TODO-RISCV64: Update alignment to proper value when we implement RISC-V intrinsic. - // RISC-V Vector Extenstion Intrinsic Document - // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/vector_type_infos.adoc pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16; #else pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 32; // sizeof(__m256) @@ -10073,12 +10068,6 @@ void MethodTableBuilder::CheckForSystemTypes() // 16-byte alignment for __m256. pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16; - - #elif defined(TARGET_RISCV64) - // TODO-RISCV64: Update alignment to proper value when we implement RISC-V intrinsic. - // RISC-V Vector Extenstion Intrinsic Document - // https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/vector_type_infos.adoc - pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 16; #else pLayout->m_ManagedLargestAlignmentRequirementOfAllMembers = 64; // sizeof(__m512) #endif // TARGET_ARM elif TARGET_ARM64 diff --git a/src/coreclr/vm/nativeeventsource.cpp b/src/coreclr/vm/nativeeventsource.cpp index a8a160eaf620d5..b4bca0355e6c7a 100644 --- a/src/coreclr/vm/nativeeventsource.cpp +++ b/src/coreclr/vm/nativeeventsource.cpp @@ -34,22 +34,6 @@ extern "C" BOOL QCALLTYPE IsEventSourceLoggingEnabled() return retVal; } -extern "C" LPWSTR QCALLTYPE EventSource_GetClrConfig(LPCWSTR configName) -{ - QCALL_CONTRACT; - - LPWSTR ret = NULL; - - BEGIN_QCALL; - CLRConfig::ConfigStringInfo info; - info.name = configName; - info.options = CLRConfig::LookupOptions::Default; - ret = CLRConfig::GetConfigValue(info); - END_QCALL; - - return ret; -} - #endif //defined(FEATURE_EVENTSOURCE_XPLAT) #ifdef FEATURE_PERFTRACING @@ -60,7 +44,7 @@ extern "C" LPWSTR QCALLTYPE EventSource_GetClrConfig(LPCWSTR configName) // change genRuntimeEventSources.py script to not emit the body that throws NotImplementedException for the event that // want to be fired from managed code. // See https://github.com/dotnet/runtime/pull/47829 for an example of how to do this. -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadStart(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -70,7 +54,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStar END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadStop(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -80,7 +64,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadWait(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -90,7 +74,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(_In_z_ short minWorkerThreads, _In_z_ short maxWorkerThreads, _In_z_ short minIOCompletionThreads, _In_z_ short maxIOCompletionThreads, _In_z_ short clrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolMinMaxThreads(_In_z_ short minWorkerThreads, _In_z_ short maxWorkerThreads, _In_z_ short minIOCompletionThreads, _In_z_ short maxIOCompletionThreads, _In_z_ short clrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -100,7 +84,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(_I END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(_In_z_ double throughput, _In_z_ short clrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentSample(_In_z_ double throughput, _In_z_ short clrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -110,7 +94,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdju END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(_In_z_ double averageThroughput, _In_z_ uint newWorkerThreadCount, _In_z_ uint reason, _In_z_ short clrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentAdjustment(_In_z_ double averageThroughput, _In_z_ uint newWorkerThreadCount, _In_z_ uint reason, _In_z_ short clrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -120,7 +104,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdju END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats(_In_z_ double duration, _In_z_ double throughput, _In_z_ double threadWave, _In_z_ double throughputWave, _In_z_ double throughputErrorEstimate, _In_z_ double AverageThroughputErrorEstimate, _In_z_ double ThroughputRatio, _In_z_ double confidence, _In_z_ double newControlSetting, _In_z_ short newThreadWaveMagnitude, _In_z_ short ClrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentStats(_In_z_ double duration, _In_z_ double throughput, _In_z_ double threadWave, _In_z_ double throughputWave, _In_z_ double throughputErrorEstimate, _In_z_ double AverageThroughputErrorEstimate, _In_z_ double ThroughputRatio, _In_z_ double confidence, _In_z_ double newControlSetting, _In_z_ short newThreadWaveMagnitude, _In_z_ short ClrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -130,7 +114,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdju END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ bool multiDequeues, _In_z_ short ClrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolIOEnqueue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ bool multiDequeues, _In_z_ short ClrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -140,7 +124,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue(_In_z_ END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIODequeue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolIODequeue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -150,7 +134,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIODequeue(_In_z_ END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(_In_z_ uint count, _In_z_ short ClrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolWorkingThreadCount(_In_z_ uint count, _In_z_ short ClrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -160,7 +144,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkingThreadCou END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOPack(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID) +extern "C" void QCALLTYPE LogThreadPoolIOPack(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -170,7 +154,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOPack(_In_z_ vo END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionLockCreated(void* LockID, void* AssociatedObjectID, uint16_t ClrInstanceID) +extern "C" void QCALLTYPE LogContentionLockCreated(void* LockID, void* AssociatedObjectID, uint16_t ClrInstanceID) { QCALL_CONTRACT; BEGIN_QCALL; @@ -180,7 +164,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionLockCreated(void END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionStart( +extern "C" void QCALLTYPE LogContentionStart( uint8_t ContentionFlags, uint16_t ClrInstanceID, void* LockID, @@ -195,7 +179,7 @@ extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionStart( END_QCALL; } -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs) +extern "C" void QCALLTYPE LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs) { QCALL_CONTRACT; BEGIN_QCALL; diff --git a/src/coreclr/vm/nativeeventsource.h b/src/coreclr/vm/nativeeventsource.h index 9f98001d46433e..a407c4b5ebb10b 100644 --- a/src/coreclr/vm/nativeeventsource.h +++ b/src/coreclr/vm/nativeeventsource.h @@ -14,27 +14,23 @@ #include "qcall.h" -#if defined(FEATURE_EVENTSOURCE_XPLAT) +#if defined(FEATURE_PERFTRACING) extern "C" void QCALLTYPE LogEventSource(_In_z_ int eventID, _In_z_ LPCWSTR eventName, _In_z_ LPCWSTR eventSourceName, _In_z_ LPCWSTR payload); extern "C" BOOL QCALLTYPE IsEventSourceLoggingEnabled(); -extern "C" LPWSTR QCALLTYPE EventSource_GetClrConfig(LPCWSTR configName); -#endif //defined(FEATURE_EVENTSOURCE_XPLAT) - -#if defined(FEATURE_PERFTRACING) -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolMinMaxThreads(_In_z_ short minWorkerThreads, _In_z_ short maxWorkerThreads, _In_z_ short minIOCompletionThreads, _In_z_ short maxIOCompletionThreads, _In_z_ short clrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample(_In_z_ double throughput, _In_z_ short clrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment(_In_z_ double averageThroughput, _In_z_ uint newWorkerThreadCount, _In_z_ uint reason, _In_z_ short clrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats(_In_z_ double duration, _In_z_ double throughput, _In_z_ double threadWave, _In_z_ double throughputWave, _In_z_ double throughputErrorEstimate, _In_z_ double AverageThroughputErrorEstimate, _In_z_ double ThroughputRatio, _In_z_ double confidence, _In_z_ double newControlSetting, _In_z_ short newThreadWaveMagnitude, _In_z_ short ClrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOEnqueue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ bool multiDequeues, _In_z_ short ClrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIODequeue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount(_In_z_ uint count, _In_z_ short ClrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogThreadPoolIOPack(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionLockCreated(void* LockID, void* AssociatedObjectID, uint16_t ClrInstanceID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionStart(uint8_t ContentionFlags, uint16_t ClrInstanceID, void* LockID, void* AssociatedObjectID, uint64_t LockOwnerThreadID); -extern "C" void QCALLTYPE NativeRuntimeEventSource_LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs); +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadStart(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadStop(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadWait(_In_z_ uint activeWorkerThreadCount, _In_z_ uint retiredWorkerThreadCount, _In_z_ short clrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolMinMaxThreads(_In_z_ short minWorkerThreads, _In_z_ short maxWorkerThreads, _In_z_ short minIOCompletionThreads, _In_z_ short maxIOCompletionThreads, _In_z_ short clrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentSample(_In_z_ double throughput, _In_z_ short clrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentAdjustment(_In_z_ double averageThroughput, _In_z_ uint newWorkerThreadCount, _In_z_ uint reason, _In_z_ short clrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolWorkerThreadAdjustmentStats(_In_z_ double duration, _In_z_ double throughput, _In_z_ double threadWave, _In_z_ double throughputWave, _In_z_ double throughputErrorEstimate, _In_z_ double AverageThroughputErrorEstimate, _In_z_ double ThroughputRatio, _In_z_ double confidence, _In_z_ double newControlSetting, _In_z_ short newThreadWaveMagnitude, _In_z_ short ClrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolIOEnqueue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ bool multiDequeues, _In_z_ short ClrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolIODequeue(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolWorkingThreadCount(_In_z_ uint count, _In_z_ short ClrInstanceID); +extern "C" void QCALLTYPE LogThreadPoolIOPack(_In_z_ void* nativeOverlapped, _In_z_ void* overlapped, _In_z_ short ClrInstanceID); +extern "C" void QCALLTYPE LogContentionLockCreated(void* LockID, void* AssociatedObjectID, uint16_t ClrInstanceID); +extern "C" void QCALLTYPE LogContentionStart(uint8_t ContentionFlags, uint16_t ClrInstanceID, void* LockID, void* AssociatedObjectID, uint64_t LockOwnerThreadID); +extern "C" void QCALLTYPE LogContentionStop(uint8_t ContentionFlags, uint16_t ClrInstanceID, double DurationNs); #endif // defined(FEATURE_PERFTRACING) #endif //_NATIVEEVENTSOURCE_H_ diff --git a/src/coreclr/vm/profdetach.cpp b/src/coreclr/vm/profdetach.cpp index 09f11458c38871..bf138209ce6ad6 100644 --- a/src/coreclr/vm/profdetach.cpp +++ b/src/coreclr/vm/profdetach.cpp @@ -326,7 +326,7 @@ void ProfilingAPIDetach::ExecuteEvacuationLoop() { CRITSEC_Holder csh(ProfilingAPIUtility::GetStatusCrst()); - while (s_profilerDetachInfos.Size() > 0) + for (SIZE_T pos = 0; pos < s_profilerDetachInfos.Size(); ++pos) { ProfilerDetachInfo current = s_profilerDetachInfos.Pop(); diff --git a/src/coreclr/vm/proftoeeinterfaceimpl.cpp b/src/coreclr/vm/proftoeeinterfaceimpl.cpp index d85dc93f950921..f2ce0f1b21595d 100644 --- a/src/coreclr/vm/proftoeeinterfaceimpl.cpp +++ b/src/coreclr/vm/proftoeeinterfaceimpl.cpp @@ -8411,7 +8411,8 @@ HRESULT ProfToEEInterfaceImpl::ProfilerEbpWalker( &rd, &codeInfo, SpeculativeStackwalk, - &codeManState); + &codeManState, + NULL); ctxCur.Ebp = *rd.GetEbpLocation(); ctxCur.Esp = rd.SP; diff --git a/src/coreclr/vm/qcallentrypoints.cpp b/src/coreclr/vm/qcallentrypoints.cpp index cddd522914f6a0..ec72d3112f5f86 100644 --- a/src/coreclr/vm/qcallentrypoints.cpp +++ b/src/coreclr/vm/qcallentrypoints.cpp @@ -24,6 +24,7 @@ #include "floatdouble.h" #include "floatsingle.h" #include "comdatetime.h" +#include "compatibilityswitch.h" #include "debugdebugger.h" #include "assemblynative.hpp" #include "comthreadpool.h" @@ -93,15 +94,12 @@ static const Entry s_QCall[] = DllImportEntry(Delegate_FindMethodHandle) DllImportEntry(Delegate_InternalEqualMethodHandles) DllImportEntry(Environment_Exit) - DllImportEntry(Environment_FailFast) DllImportEntry(Environment_GetProcessorCount) DllImportEntry(ExceptionNative_GetMessageFromNativeResources) DllImportEntry(RuntimeTypeHandle_CreateInstanceForAnotherGenericParameter) DllImportEntry(QCall_GetGCHandleForTypeHandle) DllImportEntry(QCall_FreeGCHandleForTypeHandle) DllImportEntry(MethodTable_AreTypesEquivalent) - DllImportEntry(MethodTable_CanCompareBitsOrUseFastGetHashCode) - DllImportEntry(ValueType_GetHashCodeStrategy) DllImportEntry(RuntimeTypeHandle_MakePointer) DllImportEntry(RuntimeTypeHandle_MakeByRef) DllImportEntry(RuntimeTypeHandle_MakeSZArray) @@ -324,7 +322,6 @@ static const Entry s_QCall[] = DllImportEntry(GetFileLoadExceptionMessage) DllImportEntry(FileLoadException_GetMessageForHR) DllImportEntry(Interlocked_MemoryBarrierProcessWide) - DllImportEntry(ObjectNative_AllocateUninitializedClone) DllImportEntry(Monitor_Wait) DllImportEntry(Monitor_Pulse) DllImportEntry(Monitor_PulseAll) @@ -351,23 +348,22 @@ static const Entry s_QCall[] = #if defined(FEATURE_EVENTSOURCE_XPLAT) DllImportEntry(IsEventSourceLoggingEnabled) DllImportEntry(LogEventSource) - DllImportEntry(EventSource_GetClrConfig) #endif #if defined(FEATURE_PERFTRACING) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkerThreadStart) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkerThreadStop) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkerThreadWait) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolMinMaxThreads) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentSample) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentAdjustment) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkerThreadAdjustmentStats) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolIOEnqueue) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolIODequeue) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolIOPack) - DllImportEntry(NativeRuntimeEventSource_LogThreadPoolWorkingThreadCount) - DllImportEntry(NativeRuntimeEventSource_LogContentionLockCreated) - DllImportEntry(NativeRuntimeEventSource_LogContentionStart) - DllImportEntry(NativeRuntimeEventSource_LogContentionStop) + DllImportEntry(LogThreadPoolWorkerThreadStart) + DllImportEntry(LogThreadPoolWorkerThreadStop) + DllImportEntry(LogThreadPoolWorkerThreadWait) + DllImportEntry(LogThreadPoolMinMaxThreads) + DllImportEntry(LogThreadPoolWorkerThreadAdjustmentSample) + DllImportEntry(LogThreadPoolWorkerThreadAdjustmentAdjustment) + DllImportEntry(LogThreadPoolWorkerThreadAdjustmentStats) + DllImportEntry(LogThreadPoolIOEnqueue) + DllImportEntry(LogThreadPoolIODequeue) + DllImportEntry(LogThreadPoolIOPack) + DllImportEntry(LogThreadPoolWorkingThreadCount) + DllImportEntry(LogContentionLockCreated) + DllImportEntry(LogContentionStart) + DllImportEntry(LogContentionStop) DllImportEntry(EventPipeInternal_Enable) DllImportEntry(EventPipeInternal_Disable) DllImportEntry(EventPipeInternal_GetSessionInfo) diff --git a/src/coreclr/vm/reflectioninvocation.cpp b/src/coreclr/vm/reflectioninvocation.cpp index 54e226bb0189a3..fb5cd978b85fdf 100644 --- a/src/coreclr/vm/reflectioninvocation.cpp +++ b/src/coreclr/vm/reflectioninvocation.cpp @@ -885,7 +885,14 @@ static StackWalkAction SkipMethods(CrawlFrame* frame, VOID* data) { if (!frame->IsInCalleesFrames(pSkip->pStackMark)) return SWA_CONTINUE; - pSkip->pMeth = pFunc; + if (pFunc->RequiresInstMethodDescArg()) + { + pSkip->pMeth = (MethodDesc *) frame->GetParamTypeArg(); + if (pSkip->pMeth == NULL) + pSkip->pMeth = pFunc; + } + else + pSkip->pMeth = pFunc; return SWA_ABORT; } @@ -898,9 +905,9 @@ FCIMPL1(ReflectMethodObject*, RuntimeMethodHandle::GetCurrentMethod, StackCrawlM SkipStruct skip; skip.pStackMark = stackMark; skip.pMeth = 0; - GetThread()->StackWalkFrames(SkipMethods, &skip, FUNCTIONSONLY | LIGHTUNWIND); + StackWalkFunctions(GetThread(), SkipMethods, &skip); - // If C.m was called, the stack walker returns C<__Canon>.m<__Canon>. We cannot + // If C.m was called, the stack walker returns C.m. We cannot // get know that the instantiation used Foo or Bar at that point. So the next best thing // is to return C.m

and that's what LoadTypicalMethodDefinition will do for us. diff --git a/src/coreclr/vm/riscv64/crthelpers.S b/src/coreclr/vm/riscv64/crthelpers.S new file mode 100644 index 00000000000000..3151387b3cafd3 --- /dev/null +++ b/src/coreclr/vm/riscv64/crthelpers.S @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "unixasmmacros.inc" + +// JIT_MemSet/JIT_MemCpy +// +// It is IMPORTANT that the exception handling code is able to find these guys +// on the stack, but on non-windows platforms we can just defer to the platform +// implementation. +// +LEAF_ENTRY JIT_MemSet, _TEXT + beq a2, zero, LOCAL_LABEL(JIT_MemSet_ret) + + lb zero, 0(a0) // Is this really needed ? + + tail memset + +LOCAL_LABEL(JIT_MemSet_ret): + ret +LEAF_END_MARKED JIT_MemSet, _TEXT + +////NOTE: Here must use LEAF_END_MARKED! not LEAF_END !!! +LEAF_ENTRY JIT_MemCpy, _TEXT + beq a2, zero, LOCAL_LABEL(JIT_MemCpy_ret) + + lb zero, 0(a0) + lb zero, 0(a1) // Is this really needed ? + + tail memcpy + +LOCAL_LABEL(JIT_MemCpy_ret): + ret + +////NOTE: Here must use LEAF_END_MARKED! not LEAF_END !!! +LEAF_END_MARKED JIT_MemCpy, _TEXT diff --git a/src/coreclr/vm/riscv64/stubs.cpp b/src/coreclr/vm/riscv64/stubs.cpp index 4ce55a3849e12b..40c945a810b674 100644 --- a/src/coreclr/vm/riscv64/stubs.cpp +++ b/src/coreclr/vm/riscv64/stubs.cpp @@ -1932,7 +1932,7 @@ PCODE DynamicHelpers::CreateDictionaryLookupHelper(LoaderAllocator * pAllocator, p += 4; } - BYTE* pBLECall = NULL; + BYTE* pBLTCall = NULL; for (WORD i = 0; i < pLookup->indirections; i++) { @@ -1962,8 +1962,8 @@ PCODE DynamicHelpers::CreateDictionaryLookupHelper(LoaderAllocator * pAllocator, p += 4; *(DWORD*)p = ITypeInstr(0x13, 0, RegT4, RegT4, slotOffset & 0xfff);// addi t4, t4, (slotOffset&0xfff) p += 4; - // bge t4, t5, CALL HELPER - pBLECall = p; // Offset filled later + // blt t4, t5, CALL HELPER + pBLTCall = p; // Offset filled later p += 4; } @@ -2005,8 +2005,8 @@ PCODE DynamicHelpers::CreateDictionaryLookupHelper(LoaderAllocator * pAllocator, p += 4; // CALL HELPER: - if (pBLECall != NULL) - *(DWORD*)pBLECall = BTypeInstr(0x63, 0x5, RegT4, RegT5, (UINT32)(p - pBLECall)); + if (pBLTCall != NULL) + *(DWORD*)pBLTCall = BTypeInstr(0x63, 0x4, RegT4, RegT5, (UINT32)(p - pBLTCall)); *(DWORD*)p = ITypeInstr(0x13, 0, RegA0, RegT2, 0);// addi a0, t2, 0 p += 4; diff --git a/src/coreclr/vm/rtlfunctions.cpp b/src/coreclr/vm/rtlfunctions.cpp index 0e77e31a02d162..f3f80338f3f8ec 100644 --- a/src/coreclr/vm/rtlfunctions.cpp +++ b/src/coreclr/vm/rtlfunctions.cpp @@ -16,7 +16,7 @@ #include "rtlfunctions.h" -#ifdef HOST_AMD64 +#ifdef TARGET_AMD64 RtlVirtualUnwindFn* RtlVirtualUnwind_Unsafe = NULL; @@ -45,7 +45,7 @@ HRESULT EnsureRtlFunctions() return S_OK; } -#else // HOST_AMD64 +#else // TARGET_AMD64 HRESULT EnsureRtlFunctions() { @@ -53,9 +53,9 @@ HRESULT EnsureRtlFunctions() return S_OK; } -#endif // HOST_AMD64 +#endif // TARGET_AMD64 -#ifndef HOST_X86 +#if defined(FEATURE_EH_FUNCLETS) VOID InstallEEFunctionTable ( PVOID pvTableID, @@ -127,4 +127,5 @@ VOID InstallEEFunctionTable ( } } -#endif // HOST_X86 +#endif // FEATURE_EH_FUNCLETS + diff --git a/src/coreclr/vm/rtlfunctions.h b/src/coreclr/vm/rtlfunctions.h index ef0f64cba9dbdf..6d9ff9689ca4fd 100644 --- a/src/coreclr/vm/rtlfunctions.h +++ b/src/coreclr/vm/rtlfunctions.h @@ -49,7 +49,7 @@ PVOID DecodeDynamicFunctionTableContext (PVOID pvContext) #endif // FEATURE_EH_FUNCLETS -#if !defined(DACCESS_COMPILE) && defined(HOST_WINDOWS) && !defined(HOST_X86) +#if defined(FEATURE_EH_FUNCLETS) && !defined(DACCESS_COMPILE) && !defined(TARGET_UNIX) // Wrapper for RtlInstallFunctionTableCallback. VOID InstallEEFunctionTable( @@ -67,12 +67,12 @@ VOID DeleteEEFunctionTable( RtlDeleteFunctionTable((PT_RUNTIME_FUNCTION)((ULONG64)pvTableID | 3)); } -#else +#else // FEATURE_EH_FUNCLETS && !DACCESS_COMPILE && !TARGET_UNIX #define InstallEEFunctionTable(pvTableID, pvStartRange, cbRange, pfnGetRuntimeFunctionCallback, pvContext, TableType) do { } while (0) #define DeleteEEFunctionTable(pvTableID) do { } while (0) -#endif +#endif // FEATURE_EH_FUNCLETS && !DACCESS_COMPILE && !TARGET_UNIX #endif // !__RTLFUNCTIONS_H__ diff --git a/src/coreclr/vm/siginfo.cpp b/src/coreclr/vm/siginfo.cpp index 531f6d95f68670..115b5454bf5943 100644 --- a/src/coreclr/vm/siginfo.cpp +++ b/src/coreclr/vm/siginfo.cpp @@ -4813,11 +4813,9 @@ BOOL MetaSig::CompareVariableConstraints(const Substitution *pSubst1, if ((specialConstraints2 & (gpDefaultConstructorConstraint | gpNotNullableValueTypeConstraint)) == 0) return FALSE; } - - // Constraints that 'allow' must check the overridden first - if ((specialConstraints2 & gpAllowByRefLike) != 0) + if ((specialConstraints1 & gpAcceptByRefLike) != 0) { - if ((specialConstraints1 & gpAllowByRefLike) == 0) + if ((specialConstraints2 & gpAcceptByRefLike) == 0) return FALSE; } } diff --git a/src/coreclr/vm/spinlock.cpp b/src/coreclr/vm/spinlock.cpp index e750ebab5d527b..e135dcd7945f61 100644 --- a/src/coreclr/vm/spinlock.cpp +++ b/src/coreclr/vm/spinlock.cpp @@ -33,11 +33,7 @@ SpinLock::SpinLock() STATIC_CONTRACT_NOTHROW; STATIC_CONTRACT_GC_NOTRIGGER; - m_lock = 0; - -#ifdef _DEBUG m_Initialized = UnInitialized; -#endif } void SpinLock::Init(LOCK_TYPE type, bool RequireCoopGC) @@ -49,7 +45,6 @@ void SpinLock::Init(LOCK_TYPE type, bool RequireCoopGC) } CONTRACTL_END; -#ifdef _DEBUG if (m_Initialized == Initialized) { _ASSERTE (type == m_LockType); @@ -77,12 +72,17 @@ void SpinLock::Init(LOCK_TYPE type, bool RequireCoopGC) } } + { + m_lock = 0; + } + +#ifdef _DEBUG m_LockType = type; m_requireCoopGCMode = RequireCoopGC; +#endif _ASSERTE (m_Initialized == BeingInitialized); m_Initialized = Initialized; -#endif } #ifdef _DEBUG diff --git a/src/coreclr/vm/spinlock.h b/src/coreclr/vm/spinlock.h index b407d1c135569f..7601d2341d3b7b 100644 --- a/src/coreclr/vm/spinlock.h +++ b/src/coreclr/vm/spinlock.h @@ -153,7 +153,6 @@ class SpinLock LONG m_lock; // LONG used in interlocked exchange }; -#ifdef _DEBUG enum SpinLockState { UnInitialized, @@ -164,6 +163,7 @@ class SpinLock Volatile m_Initialized; // To verify initialized // And initialize once +#ifdef _DEBUG LOCK_TYPE m_LockType; // lock type to track statistics // Check for dead lock situation. diff --git a/src/coreclr/vm/stackwalk.cpp b/src/coreclr/vm/stackwalk.cpp index 0103bb709a475b..cd971db4236f28 100644 --- a/src/coreclr/vm/stackwalk.cpp +++ b/src/coreclr/vm/stackwalk.cpp @@ -30,6 +30,7 @@ CrawlFrame::CrawlFrame() LIMITED_METHOD_DAC_CONTRACT; pCurGSCookie = NULL; pFirstGSCookie = NULL; + isCachedMethod = FALSE; } Assembly* CrawlFrame::GetAssembly() @@ -994,8 +995,11 @@ StackWalkAction Thread::StackWalkFrames(PSTACKWALKFRAMESCALLBACK pCallback, { // Initialize the context memset(&ctx, 0x00, sizeof(T_CONTEXT)); + SetIP(&ctx, 0); + SetSP(&ctx, 0); + SetFP(&ctx, 0); LOG((LF_GCROOTS, LL_INFO100000, "STACKWALK starting with partial context\n")); - FillRegDisplay(&rd, &ctx, !!(flags & LIGHTUNWIND)); + FillRegDisplay(&rd, &ctx); } #ifdef STACKWALKER_MAY_POP_FRAMES @@ -1006,6 +1010,18 @@ StackWalkAction Thread::StackWalkFrames(PSTACKWALKFRAMESCALLBACK pCallback, return StackWalkFramesEx(&rd, pCallback, pData, flags, pStartFrame); } +StackWalkAction StackWalkFunctions(Thread * thread, + PSTACKWALKFRAMESCALLBACK pCallback, + VOID * pData) +{ + // Note: there are cases (i.e., exception handling) where we may never return from this function. This means + // that any C++ destructors pushed in this function will never execute, and it means that this function can + // never have a dynamic contract. + STATIC_CONTRACT_WRAPPER; + + return thread->StackWalkFrames(pCallback, pData, FUNCTIONSONLY); +} + // ---------------------------------------------------------------------------- // StackFrameIterator::StackFrameIterator // @@ -1187,8 +1203,7 @@ BOOL StackFrameIterator::Init(Thread * pThread, m_crawl.pRD = pRegDisp; - m_codeManFlags = (ICodeManagerFlags) - (((flags & (QUICKUNWIND | LIGHTUNWIND)) ? 0 : UpdateAllRegs) | ((flags & LIGHTUNWIND) ? LightUnwind : 0)); + m_codeManFlags = (ICodeManagerFlags)((flags & QUICKUNWIND) ? 0 : UpdateAllRegs); m_scanFlag = ExecutionManager::GetScanFlags(); #if defined(ELIMINATE_FEF) @@ -1293,8 +1308,7 @@ BOOL StackFrameIterator::ResetRegDisp(PREGDISPLAY pRegDisp, m_crawl.pRD = pRegDisp; - m_codeManFlags = (ICodeManagerFlags) - (((m_flags & (QUICKUNWIND | LIGHTUNWIND)) ? 0 : UpdateAllRegs) | ((m_flags & LIGHTUNWIND) ? LightUnwind : 0)); + m_codeManFlags = (ICodeManagerFlags)((m_flags & QUICKUNWIND) ? 0 : UpdateAllRegs); // make sure the REGDISPLAY is synchronized with the CONTEXT UpdateRegDisp(); @@ -1313,7 +1327,7 @@ BOOL StackFrameIterator::ResetRegDisp(PREGDISPLAY pRegDisp, { // On 64-bit and ARM, we stop at the explicit frames contained in a managed stack frame // before the managed stack frame itself. - EECodeManager::EnsureCallerContextIsValid(m_crawl.pRD, NULL, m_codeManFlags); + EECodeManager::EnsureCallerContextIsValid(m_crawl.pRD, NULL); curSP = GetSP(m_crawl.pRD->pCallerContext); } #endif // PROCESS_EXPLICIT_FRAME_BEFORE_MANAGED_FRAME @@ -1464,6 +1478,9 @@ void StackFrameIterator::ResetCrawlFrame() m_crawl.pThread = this->m_pThread; + m_crawl.isCachedMethod = false; + m_crawl.stackWalkCache.ClearEntry(); + m_crawl.pCurGSCookie = NULL; m_crawl.pFirstGSCookie = NULL; } @@ -2545,17 +2562,76 @@ StackWalkAction StackFrameIterator::NextRaw(void) DBG_ADDR(GetRegdisplaySP(m_crawl.pRD)), DBG_ADDR(GetControlPC(m_crawl.pRD)))); - if (!m_crawl.GetCodeManager()->UnwindStackFrame( - m_crawl.pRD, - &m_cachedCodeInfo, - m_codeManFlags - | m_crawl.GetCodeManagerFlags() - | ((m_flags & PROFILER_DO_STACK_SNAPSHOT) ? SpeculativeStackwalk : 0), - &m_crawl.codeManState)) +#if !defined(DACCESS_COMPILE) && defined(HAS_QUICKUNWIND) + StackwalkCacheEntry *pCacheEntry = m_crawl.GetStackwalkCacheEntry(); + if (pCacheEntry != NULL) { - LOG((LF_CORPROF, LL_INFO100, "**PROF: m_crawl.GetCodeManager()->UnwindStackFrame failure leads to SWA_FAILED.\n")); - retVal = SWA_FAILED; - goto Cleanup; + _ASSERTE(m_crawl.stackWalkCache.Enabled() && (m_flags & LIGHTUNWIND)); + + // lightened schema: take stack unwind info from stackwalk cache + EECodeManager::QuickUnwindStackFrame(m_crawl.pRD, pCacheEntry, EECodeManager::UnwindCurrentStackFrame); + } + else +#endif // !DACCESS_COMPILE && HAS_QUICKUNWIND + { +#if !defined(DACCESS_COMPILE) + // non-optimized stack unwind schema, doesn't use StackwalkCache + UINT_PTR curSP = (UINT_PTR)GetRegdisplaySP(m_crawl.pRD); + UINT_PTR curIP = (UINT_PTR)GetControlPC(m_crawl.pRD); +#endif // !DACCESS_COMPILE + + bool fInsertCacheEntry = m_crawl.stackWalkCache.Enabled() && + (m_flags & LIGHTUNWIND) && + (m_pCachedGSCookie == NULL); + + // Is this a dynamic method. Dynamic methods can be GC collected and so IP to method mapping + // is not persistent. Therefore do not cache information for this frame. + BOOL isCollectableMethod = ExecutionManager::IsCollectibleMethod(m_crawl.GetMethodToken()); + if(isCollectableMethod) + fInsertCacheEntry = FALSE; + + StackwalkCacheUnwindInfo unwindInfo; + + if (!m_crawl.GetCodeManager()->UnwindStackFrame( + m_crawl.pRD, + &m_cachedCodeInfo, + m_codeManFlags + | m_crawl.GetCodeManagerFlags() + | ((m_flags & PROFILER_DO_STACK_SNAPSHOT) ? SpeculativeStackwalk : 0), + &m_crawl.codeManState, + (fInsertCacheEntry ? &unwindInfo : NULL))) + { + LOG((LF_CORPROF, LL_INFO100, "**PROF: m_crawl.GetCodeManager()->UnwindStackFrame failure leads to SWA_FAILED.\n")); + retVal = SWA_FAILED; + goto Cleanup; + } + +#if !defined(DACCESS_COMPILE) + // store into hashtable if fits, otherwise just use old schema + if (fInsertCacheEntry) + { + // + // information we add to cache, consists of two parts: + // 1. SPOffset - locals, etc. of current method, adding which to current ESP we get to retAddr ptr + // 2. argSize - size of pushed function arguments, the rest we need to add to get new ESP + // we have to store two parts of ESP delta, since we need to update pPC also, and so require retAddr ptr + // + // newSP = oldSP + SPOffset + sizeof(PTR) + argSize + // + UINT_PTR SPOffset = (UINT_PTR)GetRegdisplayStackMark(m_crawl.pRD) - curSP; + UINT_PTR argSize = (UINT_PTR)GetRegdisplaySP(m_crawl.pRD) - curSP - SPOffset - sizeof(void*); + + StackwalkCacheEntry cacheEntry = {0}; + if (cacheEntry.Init( + curIP, + SPOffset, + &unwindInfo, + argSize)) + { + m_crawl.stackWalkCache.Insert(&cacheEntry); + } + } +#endif // !DACCESS_COMPILE } #define FAIL_IF_SPECULATIVE_WALK(condition) \ @@ -2670,7 +2746,14 @@ StackWalkAction StackFrameIterator::NextRaw(void) // better not be suspended. CONSISTENCY_CHECK(!(m_flags & THREAD_IS_SUSPENDED)); - EECodeManager::EnsureCallerContextIsValid(m_crawl.pRD, NULL, m_codeManFlags); +#if !defined(DACCESS_COMPILE) + if (m_crawl.stackWalkCache.Enabled() && (m_flags & LIGHTUNWIND)) + { + m_crawl.isCachedMethod = m_crawl.stackWalkCache.Lookup((UINT_PTR)adr); + } +#endif // DACCESS_COMPILE + + EECodeManager::EnsureCallerContextIsValid(m_crawl.pRD, m_crawl.GetStackwalkCacheEntry()); m_pvResumableFrameTargetSP = (LPVOID)GetSP(m_crawl.pRD->pCallerContext); } #endif // RECORD_RESUMABLE_FRAME_SP @@ -2941,6 +3024,16 @@ void StackFrameIterator::ProcessCurrentFrame(void) // This must be a JITed/managed native method. There is no explicit frame. //------------------------------------------------------------------------ +#if !defined(DACCESS_COMPILE) + m_crawl.isCachedMethod = FALSE; + if (m_crawl.stackWalkCache.Enabled() && (m_flags & LIGHTUNWIND)) + { + m_crawl.isCachedMethod = m_crawl.stackWalkCache.Lookup((UINT_PTR)GetControlPC(m_crawl.pRD)); + _ASSERTE (m_crawl.isCachedMethod != m_crawl.stackWalkCache.IsEmpty()); + } +#endif // DACCESS_COMPILE + + #if defined(FEATURE_EH_FUNCLETS) m_crawl.isFilterFuncletCached = false; #endif // FEATURE_EH_FUNCLETS @@ -3013,7 +3106,7 @@ BOOL StackFrameIterator::CheckForSkippedFrames(void) // frame will be reported before its containing method. // This should always succeed! If it doesn't, it's a bug somewhere else! - EECodeManager::EnsureCallerContextIsValid(m_crawl.pRD, &m_cachedCodeInfo, m_codeManFlags); + EECodeManager::EnsureCallerContextIsValid(m_crawl.pRD, m_crawl.GetStackwalkCacheEntry(), &m_cachedCodeInfo); pvReferenceSP = GetSP(m_crawl.pRD->pCallerContext); #endif // PROCESS_EXPLICIT_FRAME_BEFORE_MANAGED_FRAME @@ -3283,6 +3376,180 @@ void StackFrameIterator::ResetNextExInfoForSP(TADDR SP) } #endif // FEATURE_EH_FUNCLETS +#if defined(TARGET_AMD64) && !defined(DACCESS_COMPILE) +static CrstStatic g_StackwalkCacheLock; // Global StackwalkCache lock; only used on AMD64 +EXTERN_C void moveOWord(LPVOID src, LPVOID target); +#endif // TARGET_AMD64 + +/* + copies 64-bit *src to *target, atomically accessing the data + requires 64-bit alignment for atomic load/store +*/ +inline static void atomicMoveCacheEntry(UINT64* src, UINT64* target) +{ + LIMITED_METHOD_CONTRACT; + +#ifdef TARGET_X86 + // the most negative value is used a sort of integer infinity + // value, so it have to be avoided + _ASSERTE(*src != 0x8000000000000000); + __asm + { + mov eax, src + fild qword ptr [eax] + mov eax, target + fistp qword ptr [eax] + } +#elif defined(TARGET_AMD64) && !defined(DACCESS_COMPILE) + // On AMD64 there's no way to move 16 bytes atomically, so we need to take a lock before calling moveOWord(). + CrstHolder ch(&g_StackwalkCacheLock); + moveOWord(src, target); +#endif +} + +/* +============================================================ +Here is an implementation of StackwalkCache class, used to optimize performance +of stack walking. Currently each CrawlFrame has a StackwalkCache member, which implements +functionality for caching already walked methods (see Thread::StackWalkFramesEx). +See class and corresponding types declaration at stackwalktypes.h +We do use global cache g_StackwalkCache[] with InterlockCompareExchange, fitting +each cache entry into 8 bytes. +============================================================ +*/ + +#ifndef DACCESS_COMPILE +#define LOG_NUM_OF_CACHE_ENTRIES 10 +#else +// Stack walk cache is disabled in DAC - save space +#define LOG_NUM_OF_CACHE_ENTRIES 0 +#endif +#define NUM_OF_CACHE_ENTRIES (1 << LOG_NUM_OF_CACHE_ENTRIES) + +static StackwalkCacheEntry g_StackwalkCache[NUM_OF_CACHE_ENTRIES] = {}; // Global StackwalkCache + +#ifdef DACCESS_COMPILE +const BOOL StackwalkCache::s_Enabled = FALSE; +#else +BOOL StackwalkCache::s_Enabled = FALSE; + +/* + StackwalkCache class constructor. + Set "enable/disable optimization" flag according to registry key. +*/ +StackwalkCache::StackwalkCache() +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + ClearEntry(); + + static BOOL stackwalkCacheEnableChecked = FALSE; + if (!stackwalkCacheEnableChecked) + { + // We can enter this block on multiple threads because of racing. + // However, that is OK since this operation is idempotent + + s_Enabled = ((g_pConfig->DisableStackwalkCache() == 0) && + // disable cache if for some reason it is not aligned + IS_ALIGNED((void*)&g_StackwalkCache[0], STACKWALK_CACHE_ENTRY_ALIGN_BOUNDARY)); + stackwalkCacheEnableChecked = TRUE; + } +} + +#endif // #ifndef DACCESS_COMPILE + +// static +void StackwalkCache::Init() +{ +#if defined(TARGET_AMD64) && !defined(DACCESS_COMPILE) + g_StackwalkCacheLock.Init(CrstSecurityStackwalkCache, CRST_UNSAFE_ANYMODE); +#endif // TARGET_AMD64 +} + +/* + Returns efficient hash table key based on provided IP. + CPU architecture dependent. +*/ +inline unsigned StackwalkCache::GetKey(UINT_PTR IP) +{ + LIMITED_METHOD_CONTRACT; + return (unsigned)(((IP >> LOG_NUM_OF_CACHE_ENTRIES) ^ IP) & (NUM_OF_CACHE_ENTRIES-1)); +} + +/* + Looks into cache and returns StackwalkCache entry, if current IP is cached. + JIT team guarantees the same ESP offset for the same IPs for different call chains. +*/ +BOOL StackwalkCache::Lookup(UINT_PTR IP) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + +#if defined(TARGET_X86) || defined(TARGET_AMD64) + _ASSERTE(Enabled()); + _ASSERTE(IP); + + unsigned hkey = GetKey(IP); + _ASSERTE(IS_ALIGNED((void*)&g_StackwalkCache[hkey], STACKWALK_CACHE_ENTRY_ALIGN_BOUNDARY)); + // Don't care about m_CacheEntry access atomicity, since it's private to this + // stackwalk/thread + atomicMoveCacheEntry((UINT64*)&g_StackwalkCache[hkey], (UINT64*)&m_CacheEntry); + +#ifdef _DEBUG + if (IP != m_CacheEntry.IP) + { + ClearEntry(); + } +#endif + + return (IP == m_CacheEntry.IP); +#else // TARGET_X86 + return FALSE; +#endif // TARGET_X86 +} + +/* + Caches data provided for current IP. +*/ +void StackwalkCache::Insert(StackwalkCacheEntry *pCacheEntry) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + _ASSERTE(Enabled()); + _ASSERTE(pCacheEntry); + + unsigned hkey = GetKey(pCacheEntry->IP); + _ASSERTE(IS_ALIGNED((void*)&g_StackwalkCache[hkey], STACKWALK_CACHE_ENTRY_ALIGN_BOUNDARY)); + atomicMoveCacheEntry((UINT64*)pCacheEntry, (UINT64*)&g_StackwalkCache[hkey]); +} + +// static +void StackwalkCache::Invalidate(LoaderAllocator * pLoaderAllocator) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + } CONTRACTL_END; + + if (!s_Enabled) + return; + + /* Note that we could just flush the entries corresponding to + pDomain if we wanted to get fancy. To keep things simple for now, + we just invalidate everything + */ + + ZeroMemory(PVOID(&g_StackwalkCache), sizeof(g_StackwalkCache)); +} + //---------------------------------------------------------------------------- // // SetUpRegdisplayForStackWalk - set up Regdisplay for a stack walk diff --git a/src/coreclr/vm/stackwalk.h b/src/coreclr/vm/stackwalk.h index 5e7afd8035fb98..fd0fa7c4c6e63e 100644 --- a/src/coreclr/vm/stackwalk.h +++ b/src/coreclr/vm/stackwalk.h @@ -52,15 +52,22 @@ class AppDomain; // Enumerate all functions. //************************************************************************ +/* This enumerator is meant to be used for the most common cases, i.e. to + enumerate just all the functions of the requested thread. It is just a + cover for the "real" enumerator. + */ + +StackWalkAction StackWalkFunctions(Thread * thread, PSTACKWALKFRAMESCALLBACK pCallback, VOID * pData); + +/*@ISSUE: Maybe use a define instead? +#define StackWalkFunctions(thread, callBack, userdata) thread->StackWalkFrames(METHODSONLY, (callBack),(userData)) +*/ + namespace AsmOffsetsAsserts { class AsmOffsets; }; -#ifdef FEATURE_EH_FUNCLETS -extern "C" void QCALLTYPE AppendExceptionStackFrame(QCall::ObjectHandleOnStack exceptionObj, SIZE_T ip, SIZE_T sp, int flags, ExInfo *pExInfo); -#endif - class CrawlFrame { public: @@ -373,6 +380,20 @@ class CrawlFrame return codeInfo.GetCodeManager(); } + inline StackwalkCacheEntry* GetStackwalkCacheEntry() + { + LIMITED_METHOD_CONTRACT; + _ASSERTE (isCachedMethod != stackWalkCache.IsEmpty()); + if (isCachedMethod && stackWalkCache.m_CacheEntry.IsSafeToUseCache()) + { + return &(stackWalkCache.m_CacheEntry); + } + else + { + return NULL; + } + } + void CheckGSCookies(); inline Thread* GetThread() @@ -438,7 +459,6 @@ class CrawlFrame friend class StackFrameIterator; #ifdef FEATURE_EH_FUNCLETS friend class ExceptionTracker; - friend void QCALLTYPE AppendExceptionStackFrame(QCall::ObjectHandleOnStack exceptionObj, SIZE_T ip, SIZE_T sp, int flags, ExInfo *pExInfo); #endif // FEATURE_EH_FUNCLETS CodeManState codeManState; @@ -473,6 +493,10 @@ class CrawlFrame #endif //FEATURE_EH_FUNCLETS Thread* pThread; + // fields used for stackwalk cache + BOOL isCachedMethod; + StackwalkCache stackWalkCache; + GSCookie *pCurGSCookie; GSCookie *pFirstGSCookie; diff --git a/src/coreclr/vm/stackwalktypes.h b/src/coreclr/vm/stackwalktypes.h index 2263740e67c225..601e001fe34cb3 100644 --- a/src/coreclr/vm/stackwalktypes.h +++ b/src/coreclr/vm/stackwalktypes.h @@ -13,6 +13,7 @@ class CrawlFrame; struct RangeSection; +struct StackwalkCacheEntry; // // This type should be used internally inside the code manager only. EECodeInfo should @@ -71,4 +72,154 @@ typedef StackWalkAction (*PSTACKWALKFRAMESCALLBACK)( ); +/****************************************************************************** + StackwalkCache: new class implements stackwalk perf optimization features. + StackwalkCacheEntry array: very simple per thread hash table, keeping cached data. + StackwalkCacheUnwindInfo: used by EECodeManager::UnwindStackFrame to return + stackwalk cache flags. + Cf. Ilyakoz for any questions. +*/ + +struct StackwalkCacheUnwindInfo +{ +#if defined(TARGET_AMD64) + ULONG RBPOffset; + ULONG RSPOffsetFromUnwindInfo; +#else // !TARGET_AMD64 + BOOL fUseEbp; // Is EBP modified by the method - either for a frame-pointer or for a scratch-register? + BOOL fUseEbpAsFrameReg; // use EBP as the frame pointer? +#endif // !TARGET_AMD64 + + inline StackwalkCacheUnwindInfo() { SUPPORTS_DAC; ZeroMemory(this, sizeof(StackwalkCacheUnwindInfo)); } + StackwalkCacheUnwindInfo(StackwalkCacheEntry * pCacheEntry); +}; + +//************************************************************************ + +#if defined(HOST_64BIT) + #define STACKWALK_CACHE_ENTRY_ALIGN_BOUNDARY 0x10 +#else // !HOST_64BIT + #define STACKWALK_CACHE_ENTRY_ALIGN_BOUNDARY 0x8 +#endif // !HOST_64BIT + +struct +DECLSPEC_ALIGN(STACKWALK_CACHE_ENTRY_ALIGN_BOUNDARY) +StackwalkCacheEntry +{ + // + // don't rearrange the fields, so that invalid value 0x8000000000000000 will never appear + // as StackwalkCacheEntry, it's required for atomicMOVQ using FILD/FISTP instructions + // + UINT_PTR IP; +#if !defined(TARGET_AMD64) + WORD ESPOffset:15; // stack offset (frame size + pending arguments + etc) + WORD fUseEbp:1; // For ESP methods, is EBP touched at all? + WORD fUseEbpAsFrameReg:1; // use EBP as the frame register? + WORD argSize:15; // size of args pushed on stack +#else // TARGET_AMD64 + DWORD RSPOffset; + DWORD RBPOffset; +#endif // TARGET_AMD64 + + inline BOOL Init(UINT_PTR IP, + UINT_PTR SPOffset, + StackwalkCacheUnwindInfo *pUnwindInfo, + UINT_PTR argSize) + { + LIMITED_METHOD_CONTRACT; + + this->IP = IP; + +#if defined(TARGET_X86) + this->ESPOffset = SPOffset; + this->argSize = argSize; + + this->fUseEbp = pUnwindInfo->fUseEbp; + this->fUseEbpAsFrameReg = pUnwindInfo->fUseEbpAsFrameReg; + _ASSERTE(!fUseEbpAsFrameReg || fUseEbp); + + // return success if we fit SPOffset and argSize into + return ((this->ESPOffset == SPOffset) && + (this->argSize == argSize)); +#elif defined(TARGET_AMD64) + // The size of a stack frame is guaranteed to fit in 4 bytes, so we don't need to check RSPOffset and RBPOffset. + + // The actual SP offset may be bigger than the offset we get from the unwind info because of stack allocations. + _ASSERTE(SPOffset >= pUnwindInfo->RSPOffsetFromUnwindInfo); + + _ASSERTE(FitsIn(SPOffset)); + this->RSPOffset = static_cast(SPOffset); + _ASSERTE(FitsIn(pUnwindInfo->RBPOffset + (SPOffset - pUnwindInfo->RSPOffsetFromUnwindInfo))); + this->RBPOffset = static_cast(pUnwindInfo->RBPOffset + (SPOffset - pUnwindInfo->RSPOffsetFromUnwindInfo)); + return TRUE; +#else // !TARGET_X86 && !TARGET_AMD64 + return FALSE; +#endif // !TARGET_X86 && !TARGET_AMD64 + } + + inline BOOL IsSafeToUseCache() + { + LIMITED_METHOD_CONTRACT; + +#if defined(TARGET_X86) + return (!fUseEbp || fUseEbpAsFrameReg); +#elif defined(TARGET_AMD64) + return TRUE; +#else // !TARGET_X86 && !TARGET_AMD64 + return FALSE; +#endif // !TARGET_X86 && !TARGET_AMD64 + } +}; + +#if defined(TARGET_X86) || defined(TARGET_AMD64) +static_assert_no_msg(sizeof(StackwalkCacheEntry) == 2 * sizeof(UINT_PTR)); +#endif // TARGET_X86 || TARGET_AMD64 + +//************************************************************************ + +class StackwalkCache +{ + friend struct _DacGlobals; + + public: + BOOL Lookup(UINT_PTR IP); + void Insert(StackwalkCacheEntry *pCacheEntry); + inline void ClearEntry () { LIMITED_METHOD_DAC_CONTRACT; m_CacheEntry.IP = 0; } + inline BOOL Enabled() { LIMITED_METHOD_DAC_CONTRACT; return s_Enabled; }; + inline BOOL IsEmpty () { LIMITED_METHOD_CONTRACT; return m_CacheEntry.IP == 0; } + +#ifndef DACCESS_COMPILE + StackwalkCache(); +#endif + static void Init(); + + StackwalkCacheEntry m_CacheEntry; // local copy of Global Cache entry for current IP + + static void Invalidate(LoaderAllocator * pLoaderAllocator); + + private: + unsigned GetKey(UINT_PTR IP); + +#ifdef DACCESS_COMPILE + // DAC can't rely on the cache here + const static BOOL s_Enabled; +#else + static BOOL s_Enabled; +#endif +}; + +//************************************************************************ + +inline StackwalkCacheUnwindInfo::StackwalkCacheUnwindInfo(StackwalkCacheEntry * pCacheEntry) +{ + LIMITED_METHOD_CONTRACT; + +#if defined(TARGET_AMD64) + RBPOffset = pCacheEntry->RBPOffset; +#else // !TARGET_AMD64 + fUseEbp = pCacheEntry->fUseEbp; + fUseEbpAsFrameReg = pCacheEntry->fUseEbpAsFrameReg; +#endif // !TARGET_AMD64 +} + #endif // __STACKWALKTYPES_H__ diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index 2a6a87ce1bc7c2..081f5c6cf7a73e 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -7050,12 +7050,12 @@ bool Thread::InitRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, bool validCo } -void Thread::FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, bool fLightUnwind) +void Thread::FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx) { WRAPPER_NO_CONTRACT; SUPPORTS_DAC; - ::FillRegDisplay(pRD, pctx, NULL, fLightUnwind); + ::FillRegDisplay(pRD, pctx); #if defined(DEBUG_REGDISPLAY) && !defined(TARGET_X86) CONSISTENCY_CHECK(!pRD->_pThread || pRD->_pThread == this); @@ -8276,7 +8276,12 @@ void Thread::InitializeSpecialUserModeApc() return; } - s_pfnQueueUserAPC2Proc = pfnQueueUserAPC2Proc; + // In the future, once code paths using the special user-mode APC get some bake time, it should be used regardless of + // whether CET shadow stacks are enabled + if (AreCetShadowStacksEnabled()) + { + s_pfnQueueUserAPC2Proc = pfnQueueUserAPC2Proc; + } } #endif // FEATURE_SPECIAL_USER_MODE_APC diff --git a/src/coreclr/vm/threads.h b/src/coreclr/vm/threads.h index 9d304e2662894d..ef5ba294a1582f 100644 --- a/src/coreclr/vm/threads.h +++ b/src/coreclr/vm/threads.h @@ -2709,16 +2709,14 @@ class Thread #define POPFRAMES 0x0004 + /* use the following flag only if you REALLY know what you are doing !!! */ #define QUICKUNWIND 0x0008 // do not restore all registers during unwind #define HANDLESKIPPEDFRAMES 0x0010 // temporary to handle skipped frames for appdomain unload // stack crawl. Eventually need to always do this but it // breaks the debugger right now. - #define LIGHTUNWIND 0x0020 // Unwind PC+SP+FP only. - // - Implemented on x64 only. - // - Expects the initial context to be outside prolog/epilog. - // - Cannot unwind through methods with stackalloc + #define LIGHTUNWIND 0x0020 // allow using cache schema (see StackwalkCache class) #define NOTIFY_ON_U2M_TRANSITIONS 0x0040 // Provide a callback for native transitions. // This is only useful to a debugger trying to find native code @@ -2796,7 +2794,7 @@ class Thread PTR_Frame pStartFrame = PTR_NULL); bool InitRegDisplay(const PREGDISPLAY, const PT_CONTEXT, bool validContext); - void FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx, bool fLightUnwind = false); + void FillRegDisplay(const PREGDISPLAY pRD, PT_CONTEXT pctx); #ifdef FEATURE_EH_FUNCLETS static PCODE VirtualUnwindCallFrame(T_CONTEXT* pContext, T_KNONVOLATILE_CONTEXT_POINTERS* pContextPointers = NULL, @@ -6222,7 +6220,9 @@ class StackWalkerWalkingThreadHolder #if defined(TARGET_WINDOWS) && defined(TARGET_AMD64) EXTERN_C void STDCALL ClrRestoreNonvolatileContextWorker(PCONTEXT ContextRecord, DWORD64 ssp); #endif +#if !(defined(TARGET_WINDOWS) && defined(TARGET_X86)) void ClrRestoreNonvolatileContext(PCONTEXT ContextRecord); +#endif #endif // DACCESS_COMPILE #endif //__threads_h__ diff --git a/src/coreclr/vm/typedesc.cpp b/src/coreclr/vm/typedesc.cpp index 72bc9e50c29bc7..ccb342ca035892 100644 --- a/src/coreclr/vm/typedesc.cpp +++ b/src/coreclr/vm/typedesc.cpp @@ -1497,7 +1497,7 @@ BOOL TypeVarTypeDesc::SatisfiesConstraints(SigTypeContext *pTypeContextOfConstra return FALSE; } - if (thArg.IsByRefLike() && (specialConstraints & gpAllowByRefLike) == 0) + if (thArg.IsByRefLike() && (specialConstraints & gpAcceptByRefLike) == 0) return FALSE; } diff --git a/src/coreclr/vm/zapsig.cpp b/src/coreclr/vm/zapsig.cpp index 577acbf485a2cb..f0904e6452d0e1 100644 --- a/src/coreclr/vm/zapsig.cpp +++ b/src/coreclr/vm/zapsig.cpp @@ -1341,9 +1341,6 @@ BOOL ZapSig::EncodeMethod( else { Instantiation inst = pMethod->GetMethodInstantiation(); - - pSigBuilder->AppendData(inst.GetNumArgs()); - for (DWORD i = 0; i < inst.GetNumArgs(); i++) { TypeHandle t = inst[i]; diff --git a/src/installer/Directory.Build.targets b/src/installer/Directory.Build.targets index c4e8a8c8fb7011..dccb8277ba7674 100644 --- a/src/installer/Directory.Build.targets +++ b/src/installer/Directory.Build.targets @@ -1,7 +1,7 @@ - $(InstallerName) + $(InstallerName)-pgo $(ArchiveName)-pgo diff --git a/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj b/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj index fec324d46cccbf..4b2f49ec435685 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj +++ b/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj @@ -4,7 +4,7 @@ netstandard2.0 Abstractions for modifying .NET host binaries false - true + true true true true diff --git a/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props b/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props index 7c32807ea03ffb..ffec11de57b317 100644 --- a/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props +++ b/src/installer/pkg/projects/Microsoft.DotNet.ILCompiler/ILCompilerRIDs.props @@ -14,6 +14,5 @@ - diff --git a/src/installer/pkg/projects/netcoreappRIDs.props b/src/installer/pkg/projects/netcoreappRIDs.props index 6c84841697b420..b0f62cee619dec 100644 --- a/src/installer/pkg/projects/netcoreappRIDs.props +++ b/src/installer/pkg/projects/netcoreappRIDs.props @@ -69,11 +69,5 @@ ppc64le - - riscv64 - - - riscv64 - diff --git a/src/installer/pkg/sfx/Directory.Build.props b/src/installer/pkg/sfx/Directory.Build.props index dbf349249cefb3..b0711d7f7ac911 100644 --- a/src/installer/pkg/sfx/Directory.Build.props +++ b/src/installer/pkg/sfx/Directory.Build.props @@ -11,7 +11,7 @@ true true - + true true diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj index 8d492718edd92a..337a0ce1ebee36 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj @@ -7,9 +7,12 @@ true ToolPack $(SharedFrameworkName).Crossgen2 - $(SharedFrameworkName).$(RuntimeIdentifier) + .PGO + $(SharedFrameworkName)$(PgoSuffix).$(RuntimeIdentifier) dotnet-crossgen2 crossgen2 + + linux-x64;linux-musl-x64;linux-arm;linux-musl-arm;linux-arm64;linux-musl-arm64;freebsd-x64;freebsd-arm64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64 false tools/ true @@ -17,6 +20,8 @@ false false + + false diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj index fc7b8b90fe9070..8cd98f995ee0c2 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj @@ -8,7 +8,7 @@ dotnet-apphost-pack dotnet-apphost-pack NetCore.AppHostPack - false + false false + + false true true diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj index 3389feacd0c082..1f6b55038026d4 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Runtime.sfxproj @@ -8,7 +8,7 @@ dotnet-runtime dotnet-runtime-internal true - false + false dotnet-runtime-symbols NetCore.SharedFramework true diff --git a/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj b/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj index 178a37fed055bb..acfcdd089bbef5 100644 --- a/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj +++ b/src/installer/pkg/sfx/bundle/Microsoft.NETCore.App.Bundle.bundleproj @@ -19,7 +19,7 @@ - + diff --git a/src/installer/prepare-artifacts.proj b/src/installer/prepare-artifacts.proj index a9e0931c116010..5413e825c9513b 100644 --- a/src/installer/prepare-artifacts.proj +++ b/src/installer/prepare-artifacts.proj @@ -23,8 +23,6 @@ - - @@ -58,16 +56,6 @@ - - - - DotNetReleaseShipping=true - - - - - - CP0015 - M:System.ComponentModel.TypeDescriptor.CreateDesigner(System.ComponentModel.IComponent,System.Type):[T:System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute] - ref/net9.0/System.ComponentModel.TypeConverter.dll - lib/net9.0/System.ComponentModel.TypeConverter.dll - - \ No newline at end of file diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/Resources/Strings.resx b/src/libraries/System.ComponentModel.TypeConverter/src/Resources/Strings.resx index 6930398b398144..f607a8a2fe91cd 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/Resources/Strings.resx +++ b/src/libraries/System.ComponentModel.TypeConverter/src/Resources/Strings.resx @@ -172,9 +172,6 @@ The {0} culture cannot be converted to a CultureInfo object on this computer. - - Designer support has been disabled in the app configuration and is not supported. - The service instance must derive from or implement {0}. diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/CustomTypeDescriptor.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/CustomTypeDescriptor.cs index 0516dae7767c49..bc5c5ce6ffa67e 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/CustomTypeDescriptor.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/CustomTypeDescriptor.cs @@ -98,7 +98,7 @@ public virtual AttributeCollection GetAttributes() /// The GetEditor method returns an editor of the given type that is /// to be associated with the class this type descriptor is representing. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode)] public virtual object? GetEditor(Type editorBaseType) => _parent?.GetEditor(editorBaseType); ///

diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/Design/IDesignerHost.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/Design/IDesignerHost.cs index 8ab34614a7ebde..b74eeaf316cd86 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/Design/IDesignerHost.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/Design/IDesignerHost.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics.CodeAnalysis; - namespace System.ComponentModel.Design { /// @@ -11,9 +9,6 @@ namespace System.ComponentModel.Design /// public interface IDesignerHost : IServiceContainer { - [FeatureSwitchDefinition("System.ComponentModel.Design.IDesignerHost.IsSupported")] - internal static bool IsSupported => AppContext.TryGetSwitch("System.ComponentModel.Design.IDesignerHost.IsSupported", out bool isSupported) ? isSupported : true; - /// /// Gets or sets a value indicating whether the designer host /// is currently loading the document. diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ICustomTypeDescriptor.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ICustomTypeDescriptor.cs index aaec2986c1fa40..65c7db6623a25e 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ICustomTypeDescriptor.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ICustomTypeDescriptor.cs @@ -47,7 +47,7 @@ public interface ICustomTypeDescriptor /// /// Gets an editor of the specified type for this object. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode)] object? GetEditor(Type editorBaseType); /// diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/PropertyDescriptor.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/PropertyDescriptor.cs index f21ddb67010e32..51fad8dd06d90c 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/PropertyDescriptor.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/PropertyDescriptor.cs @@ -233,7 +233,7 @@ public virtual PropertyDescriptorCollection GetChildProperties(object? instance, /// /// Gets an editor of the specified type. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed + " " + PropertyDescriptorPropertyTypeMessage)] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode + " " + PropertyDescriptorPropertyTypeMessage)] public virtual object? GetEditor(Type editorBaseType) { object? editor = null; diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.ReflectedTypeData.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.ReflectedTypeData.cs index 6cd9c843a6f40a..11449f7489ffc8 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.ReflectedTypeData.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.ReflectedTypeData.cs @@ -292,7 +292,7 @@ internal TypeConverter GetConverter(object? instance) /// /// Retrieves the editor for the given base type. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed + " The Type of instance cannot be statically discovered.")] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode + " The Type of instance cannot be statically discovered.")] internal object? GetEditor(object? instance, Type editorBaseType) { EditorAttribute? typeAttr; @@ -391,7 +391,6 @@ internal TypeConverter GetConverter(object? instance) /// /// Helper method to return an editor attribute of the correct base type. /// - [RequiresUnreferencedCode("The type referenced by the Editor attribute may be trimmed away.")] private static EditorAttribute? GetEditorAttribute(AttributeCollection attributes, Type editorBaseType) { foreach (Attribute attr in attributes) diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.cs index 5dacbaa8a25260..1c9535b83440b1 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ReflectTypeDescriptionProvider.cs @@ -398,7 +398,7 @@ internal TypeConverter GetConverter([DynamicallyAccessedMembers(DynamicallyAcces /// /// Retrieves the editor for the given base type. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed + " The Type of instance cannot be statically discovered.")] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode + " The Type of instance cannot be statically discovered.")] internal object? GetEditor([DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] Type type, object? instance, Type editorBaseType) { ReflectedTypeData td = GetTypeData(type, true)!; @@ -518,7 +518,7 @@ internal TypeConverter GetExtendedConverter(object instance) /// /// Retrieves the editor for the given base type. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed + " The Type of instance cannot be statically discovered.")] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode + " The Type of instance cannot be statically discovered.")] internal object? GetExtendedEditor(object instance, Type editorBaseType) { return GetEditor(instance.GetType(), instance, editorBaseType); @@ -1330,7 +1330,7 @@ internal void Refresh(Type type) /// for types as needed. These instances are stored back into the table /// for the base type, and for the original component type, for fast access. /// - [RequiresUnreferencedCode(TypeDescriptor.DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(TypeDescriptor.EditorRequiresUnreferencedCode)] private static object? GetIntrinsicTypeEditor(Hashtable table, Type callingType) { object? hashEntry = null; diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ToolboxItemAttribute.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ToolboxItemAttribute.cs index 4244338b10f1d5..b6a2244400db68 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ToolboxItemAttribute.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/ToolboxItemAttribute.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.ComponentModel.Design; using System.Diagnostics.CodeAnalysis; namespace System.ComponentModel @@ -17,12 +16,10 @@ public class ToolboxItemAttribute : Attribute [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] private readonly string? _toolboxItemTypeName; - private const string DefaultToolboxItemTypeName = "System.Drawing.Design.ToolboxItem, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"; - /// /// Initializes a new instance of ToolboxItemAttribute and sets the type to /// - public static readonly ToolboxItemAttribute Default = new ToolboxItemAttribute(DefaultToolboxItemTypeName); + public static readonly ToolboxItemAttribute Default = new ToolboxItemAttribute("System.Drawing.Design.ToolboxItem, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"); /// /// Initializes a new instance of ToolboxItemAttribute and sets the type to @@ -33,7 +30,7 @@ public class ToolboxItemAttribute : Attribute /// /// Gets whether the attribute is the default attribute. /// - public override bool IsDefaultAttribute() => _toolboxItemTypeName == DefaultToolboxItemTypeName; + public override bool IsDefaultAttribute() => Equals(Default); /// /// Initializes a new instance of ToolboxItemAttribute and specifies if default values should be used. @@ -42,12 +39,7 @@ public ToolboxItemAttribute(bool defaultType) { if (defaultType) { - if (!IDesignerHost.IsSupported) - { - throw new NotSupportedException(SR.IDesignerHostNotSupported); - } - - _toolboxItemTypeName = DefaultToolboxItemTypeName; + _toolboxItemTypeName = "System.Drawing.Design.ToolboxItem, System.Drawing, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a"; } } diff --git a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs index 0e5bba3b7486e4..948e3f4b386f35 100644 --- a/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs +++ b/src/libraries/System.ComponentModel.TypeConverter/src/System/ComponentModel/TypeDescriptor.cs @@ -20,7 +20,7 @@ namespace System.ComponentModel public sealed class TypeDescriptor { internal const DynamicallyAccessedMemberTypes ReflectTypesDynamicallyAccessedMembers = DynamicallyAccessedMemberTypes.PublicParameterlessConstructor | DynamicallyAccessedMemberTypes.PublicFields; - internal const string DesignTimeAttributeTrimmed = "Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming."; + internal const string EditorRequiresUnreferencedCode = "Editors registered in TypeDescriptor.AddEditorTable may be trimmed."; // Note: this is initialized at class load because we // lock on it for thread safety. It is used from nearly @@ -933,7 +933,7 @@ internal static ICustomTypeDescriptor GetExtendedDescriptor(object component) /// Gets an editor with the specified base type for the /// specified component. /// - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed + " The Type of component cannot be statically discovered.")] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode + " The Type of component cannot be statically discovered.")] public static object? GetEditor(object component, Type editorBaseType) { return GetEditor(component, editorBaseType, false); @@ -944,7 +944,7 @@ internal static ICustomTypeDescriptor GetExtendedDescriptor(object component) /// specified component. /// [EditorBrowsable(EditorBrowsableState.Advanced)] - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed + " The Type of component cannot be statically discovered.")] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode + " The Type of component cannot be statically discovered.")] public static object? GetEditor(object component, Type editorBaseType, bool noCustomTypeDesc) { ArgumentNullException.ThrowIfNull(editorBaseType); @@ -955,7 +955,7 @@ internal static ICustomTypeDescriptor GetExtendedDescriptor(object component) /// /// Gets an editor with the specified base type for the specified type. /// - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode)] public static object? GetEditor( [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.All)] Type type, Type editorBaseType) @@ -2342,7 +2342,7 @@ public static Type ComObjectType get => typeof(TypeDescriptorComObject); } - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode("The Type of component cannot be statically discovered.")] public static IDesigner? CreateDesigner(IComponent component, Type designerBaseType) { Type? type = null; @@ -2635,7 +2635,7 @@ PropertyDescriptor ICustomTypeDescriptor.GetDefaultProperty() return _handler.GetDefaultProperty(_instance); } - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode)] object ICustomTypeDescriptor.GetEditor(Type editorBaseType) { return _handler.GetEditor(_instance, editorBaseType); @@ -2944,7 +2944,7 @@ TypeConverter ICustomTypeDescriptor.GetConverter() /// /// ICustomTypeDescriptor implementation. /// - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode)] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) { ArgumentNullException.ThrowIfNull(editorBaseType); @@ -3307,7 +3307,7 @@ TypeConverter ICustomTypeDescriptor.GetConverter() /// /// ICustomTypeDescriptor implementation. /// - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode)] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) { ArgumentNullException.ThrowIfNull(editorBaseType); @@ -3632,7 +3632,7 @@ public TypeConverter GetConverter() /// /// ICustomTypeDescriptor implementation. /// - [RequiresUnreferencedCode(DesignTimeAttributeTrimmed)] + [RequiresUnreferencedCode(EditorRequiresUnreferencedCode)] public object? GetEditor(Type editorBaseType) { ArgumentNullException.ThrowIfNull(editorBaseType); diff --git a/src/libraries/System.Configuration.ConfigurationManager/src/System/Diagnostics/TraceConfiguration.cs b/src/libraries/System.Configuration.ConfigurationManager/src/System/Diagnostics/TraceConfiguration.cs index de819c301df57c..d02b5a006b3da4 100644 --- a/src/libraries/System.Configuration.ConfigurationManager/src/System/Diagnostics/TraceConfiguration.cs +++ b/src/libraries/System.Configuration.ConfigurationManager/src/System/Diagnostics/TraceConfiguration.cs @@ -55,7 +55,7 @@ private static void InitializingTraceSource(object sender, InitializingTraceSour if (!string.IsNullOrEmpty(sourceElement.SwitchValue)) { - traceSource.Switch.Level = Enum.Parse(sourceElement.SwitchValue); + traceSource.Switch.Level = (SourceLevels)Enum.Parse(typeof(SourceLevels), sourceElement.SwitchValue); } } } @@ -74,7 +74,7 @@ private static void InitializingTraceSource(object sender, InitializingTraceSour // The SwitchValue changed; just update our internalSwitch. if (!string.IsNullOrEmpty(sourceElement.SwitchValue)) { - traceSource.Switch.Level = Enum.Parse(sourceElement.SwitchValue); + traceSource.Switch.Level = (SourceLevels)Enum.Parse(typeof(SourceLevels), sourceElement.SwitchValue); } else { diff --git a/src/libraries/System.Console/src/System/ConsolePal.Unix.cs b/src/libraries/System.Console/src/System/ConsolePal.Unix.cs index 77a679150b297a..23d4b9ba595f8d 100644 --- a/src/libraries/System.Console/src/System/ConsolePal.Unix.cs +++ b/src/libraries/System.Console/src/System/ConsolePal.Unix.cs @@ -355,22 +355,19 @@ private static void GetWindowSize(out int width, out int height) // Invalidate before reading cached values. CheckTerminalSettingsInvalidated(); - if (s_windowWidth == -1) + Interop.Sys.WinSize winsize; + if (s_windowWidth == -1 && + s_terminalHandle != null && + Interop.Sys.GetWindowSize(s_terminalHandle, out winsize) == 0) { - Interop.Sys.WinSize winsize; - if (s_terminalHandle != null && - Interop.Sys.GetWindowSize(s_terminalHandle, out winsize) == 0) - { - s_windowWidth = winsize.Col; - s_windowHeight = winsize.Row; - } - else - { - s_windowWidth = TerminalFormatStringsInstance.Columns; - s_windowHeight = TerminalFormatStringsInstance.Lines; - } + s_windowWidth = winsize.Col; + s_windowHeight = winsize.Row; + } + else + { + s_windowWidth = TerminalFormatStringsInstance.Columns; + s_windowHeight = TerminalFormatStringsInstance.Lines; } - width = s_windowWidth; height = s_windowHeight; } diff --git a/src/libraries/System.Console/src/System/ConsolePal.Windows.cs b/src/libraries/System.Console/src/System/ConsolePal.Windows.cs index ee25d844de866d..820e2fc3991025 100644 --- a/src/libraries/System.Console/src/System/ConsolePal.Windows.cs +++ b/src/libraries/System.Console/src/System/ConsolePal.Windows.cs @@ -108,9 +108,7 @@ public static void SetConsoleInputEncoding(Encoding enc) if (enc.CodePage != UnicodeCodePage) { if (!Interop.Kernel32.SetConsoleCP(enc.CodePage)) - { - HandleSetConsoleEncodingError(Marshal.GetLastPInvokeError()); - } + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastPInvokeError()); } } @@ -124,22 +122,8 @@ public static void SetConsoleOutputEncoding(Encoding enc) if (enc.CodePage != UnicodeCodePage) { if (!Interop.Kernel32.SetConsoleOutputCP(enc.CodePage)) - { - HandleSetConsoleEncodingError(Marshal.GetLastPInvokeError()); - } - } - } - - private static void HandleSetConsoleEncodingError(int lastError) - { - if (lastError == Interop.Errors.ERROR_INVALID_HANDLE - || lastError == Interop.Errors.ERROR_INVALID_ACCESS) - { - // no console, or not a valid handle, so fail silently - return; + throw Win32Marshal.GetExceptionForWin32Error(Marshal.GetLastPInvokeError()); } - - throw Win32Marshal.GetExceptionForWin32Error(lastError); } /// Gets whether Console.In is targeting a terminal display. diff --git a/src/libraries/System.Console/tests/ConsoleEncoding.Windows.cs b/src/libraries/System.Console/tests/ConsoleEncoding.Windows.cs index b32c73bce35fdd..7e19d385d06b4c 100644 --- a/src/libraries/System.Console/tests/ConsoleEncoding.Windows.cs +++ b/src/libraries/System.Console/tests/ConsoleEncoding.Windows.cs @@ -2,8 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Runtime.InteropServices; using System.Text; +using System.Runtime.InteropServices; using Microsoft.DotNet.RemoteExecutor; using Xunit; @@ -38,28 +38,6 @@ public void InputEncoding_SetUnicodeEncoding_SilentlyIgnoredInternally() }).Dispose(); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [PlatformSpecific(TestPlatforms.Windows)] - public void InputEncoding_SetEncodingWhenDetached_ErrorIsSilentlyIgnored() - { - RemoteExecutor.Invoke(() => - { - Encoding encoding = Console.InputEncoding.CodePage != Encoding.ASCII.CodePage - ? Encoding.ASCII - : Encoding.Latin1; - - // use FreeConsole to detach the current console - simulating a process started with the "DETACHED_PROCESS" flag - FreeConsole(); - - // Setting the input encoding should not throw an exception - Console.InputEncoding = encoding; - // The internal state of Console should have updated, despite the failure to change the console's input encoding - Assert.Equal(encoding, Console.InputEncoding); - // Operations on the console are no longer valid - GetConsoleCP fails. - Assert.Equal(0u, GetConsoleCP()); - }).Dispose(); - } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] [PlatformSpecific(TestPlatforms.Windows)] public void OutputEncoding_SetDefaultEncoding_Success() @@ -89,34 +67,9 @@ public void OutputEncoding_SetUnicodeEncoding_SilentlyIgnoredInternally() }).Dispose(); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - [PlatformSpecific(TestPlatforms.Windows)] - public void OutputEncoding_SetEncodingWhenDetached_ErrorIsSilentlyIgnored() - { - RemoteExecutor.Invoke(() => - { - Encoding encoding = Console.OutputEncoding.CodePage != Encoding.ASCII.CodePage - ? Encoding.ASCII - : Encoding.Latin1; - - // use FreeConsole to detach the current console - simulating a process started with the "DETACHED_PROCESS" flag - FreeConsole(); - - // Setting the output encoding should not throw an exception - Console.OutputEncoding = encoding; - // The internal state of Console should have updated, despite the failure to change the console's output encoding - Assert.Equal(encoding, Console.OutputEncoding); - // Operations on the console are no longer valid - GetConsoleOutputCP fails. - Assert.Equal(0u, GetConsoleOutputCP()); - }).Dispose(); - } - [LibraryImport("kernel32.dll")] public static partial uint GetConsoleCP(); [LibraryImport("kernel32.dll")] public static partial uint GetConsoleOutputCP(); - - [LibraryImport("kernel32.dll")] - public static partial int FreeConsole(); } diff --git a/src/libraries/System.Data.Common/ref/System.Data.Common.cs b/src/libraries/System.Data.Common/ref/System.Data.Common.cs index 1c39049c9def4a..cb319e5601ac01 100644 --- a/src/libraries/System.Data.Common/ref/System.Data.Common.cs +++ b/src/libraries/System.Data.Common/ref/System.Data.Common.cs @@ -476,7 +476,7 @@ public void EndEdit() { } System.ComponentModel.EventDescriptor System.ComponentModel.ICustomTypeDescriptor.GetDefaultEvent() { throw null; } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("PropertyDescriptor's PropertyType cannot be statically discovered.")] System.ComponentModel.PropertyDescriptor System.ComponentModel.ICustomTypeDescriptor.GetDefaultProperty() { throw null; } - [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object System.ComponentModel.ICustomTypeDescriptor.GetEditor(System.Type editorBaseType) { throw null; } System.ComponentModel.EventDescriptorCollection System.ComponentModel.ICustomTypeDescriptor.GetEvents() { throw null; } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("The public parameterless constructor or the 'Default' static field may be trimmed from the Attribute's Type.")] @@ -2256,7 +2256,7 @@ void System.Collections.IDictionary.Remove(object keyword) { } System.ComponentModel.EventDescriptor? System.ComponentModel.ICustomTypeDescriptor.GetDefaultEvent() { throw null; } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("PropertyDescriptor's PropertyType cannot be statically discovered.")] System.ComponentModel.PropertyDescriptor? System.ComponentModel.ICustomTypeDescriptor.GetDefaultProperty() { throw null; } - [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object? System.ComponentModel.ICustomTypeDescriptor.GetEditor(System.Type editorBaseType) { throw null; } System.ComponentModel.EventDescriptorCollection System.ComponentModel.ICustomTypeDescriptor.GetEvents() { throw null; } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("The public parameterless constructor or the 'Default' static field may be trimmed from the Attribute's Type.")] @@ -2450,7 +2450,7 @@ protected DbDataRecord() { } System.ComponentModel.EventDescriptor System.ComponentModel.ICustomTypeDescriptor.GetDefaultEvent() { throw null; } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("PropertyDescriptor's PropertyType cannot be statically discovered.")] System.ComponentModel.PropertyDescriptor System.ComponentModel.ICustomTypeDescriptor.GetDefaultProperty() { throw null; } - [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object System.ComponentModel.ICustomTypeDescriptor.GetEditor(System.Type editorBaseType) { throw null; } System.ComponentModel.EventDescriptorCollection System.ComponentModel.ICustomTypeDescriptor.GetEvents() { throw null; } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("The public parameterless constructor or the 'Default' static field may be trimmed from the Attribute's Type.")] diff --git a/src/libraries/System.Data.Common/src/System/Data/Common/DataRecordInternal.cs b/src/libraries/System.Data.Common/src/System/Data/Common/DataRecordInternal.cs index d74645280fceca..825036afdf6b43 100644 --- a/src/libraries/System.Data.Common/src/System/Data/Common/DataRecordInternal.cs +++ b/src/libraries/System.Data.Common/src/System/Data/Common/DataRecordInternal.cs @@ -335,7 +335,7 @@ AttributeCollection ICustomTypeDescriptor.GetAttributes() return null; } - [RequiresUnreferencedCode("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [RequiresUnreferencedCode("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) { return null; diff --git a/src/libraries/System.Data.Common/src/System/Data/Common/DbConnectionStringBuilder.cs b/src/libraries/System.Data.Common/src/System/Data/Common/DbConnectionStringBuilder.cs index 809c9f3132bae6..46b45be82c8435 100644 --- a/src/libraries/System.Data.Common/src/System/Data/Common/DbConnectionStringBuilder.cs +++ b/src/libraries/System.Data.Common/src/System/Data/Common/DbConnectionStringBuilder.cs @@ -608,7 +608,7 @@ AttributeCollection ICustomTypeDescriptor.GetAttributes() { return TypeDescriptor.GetAttributes(this, true); } - [RequiresUnreferencedCode("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [RequiresUnreferencedCode("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) { return TypeDescriptor.GetEditor(this, editorBaseType, true); diff --git a/src/libraries/System.Data.Common/src/System/Data/Common/DbDataRecord.cs b/src/libraries/System.Data.Common/src/System/Data/Common/DbDataRecord.cs index fe8c46bc7ef276..f6211bfe3fe8fa 100644 --- a/src/libraries/System.Data.Common/src/System/Data/Common/DbDataRecord.cs +++ b/src/libraries/System.Data.Common/src/System/Data/Common/DbDataRecord.cs @@ -85,7 +85,7 @@ protected virtual DbDataReader GetDbDataReader(int i) [RequiresUnreferencedCode("PropertyDescriptor's PropertyType cannot be statically discovered.")] PropertyDescriptor? ICustomTypeDescriptor.GetDefaultProperty() => null; - [RequiresUnreferencedCode("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [RequiresUnreferencedCode("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) => null; EventDescriptorCollection ICustomTypeDescriptor.GetEvents() => new EventDescriptorCollection(null); diff --git a/src/libraries/System.Data.Common/src/System/Data/DataRowView.cs b/src/libraries/System.Data.Common/src/System/Data/DataRowView.cs index b47b374994d520..f2d8c97d7edecb 100644 --- a/src/libraries/System.Data.Common/src/System/Data/DataRowView.cs +++ b/src/libraries/System.Data.Common/src/System/Data/DataRowView.cs @@ -242,7 +242,7 @@ public void EndEdit() [RequiresUnreferencedCode("PropertyDescriptor's PropertyType cannot be statically discovered.")] PropertyDescriptor? ICustomTypeDescriptor.GetDefaultProperty() => null; - [RequiresUnreferencedCode("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [RequiresUnreferencedCode("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) => null; EventDescriptorCollection ICustomTypeDescriptor.GetEvents() => new EventDescriptorCollection(null); diff --git a/src/libraries/System.Data.Common/src/System/Data/DataViewManagerListItemTypeDescriptor.cs b/src/libraries/System.Data.Common/src/System/Data/DataViewManagerListItemTypeDescriptor.cs index 724f500755f8fc..afa7277482b18f 100644 --- a/src/libraries/System.Data.Common/src/System/Data/DataViewManagerListItemTypeDescriptor.cs +++ b/src/libraries/System.Data.Common/src/System/Data/DataViewManagerListItemTypeDescriptor.cs @@ -66,7 +66,7 @@ internal DataView GetDataView(DataTable table) /// /// Retrieves the an editor for this object. /// - [RequiresUnreferencedCode("Design-time attributes are not preserved when trimming. Types referenced by attributes like EditorAttribute and DesignerAttribute may not be available after trimming.")] + [RequiresUnreferencedCode("Editors registered in TypeDescriptor.AddEditorTable may be trimmed.")] object? ICustomTypeDescriptor.GetEditor(Type editorBaseType) => null; /// diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/DiagnosticSourceUsersGuide.md b/src/libraries/System.Diagnostics.DiagnosticSource/src/DiagnosticSourceUsersGuide.md index bcd4e96cb8aa98..8f98d156cd1ea6 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/DiagnosticSourceUsersGuide.md +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/DiagnosticSourceUsersGuide.md @@ -180,7 +180,7 @@ Thus the event names only need to be unique within a component. reflection must be used to fetch fields). This is both easier to program and more efficient. Thus in scenarios where there is likely high-volume filtering to be done by the logging listener, having this type available to do the cast is valuable. Note that this type needs to be made public (since - the listener needs to see it). + the listener needs to see it), and should be under the namespace System.Diagnostics.DiagnosticSource.PayloadTypes. Note that if there is doubt about the value DO NOT create an explicit type, as you CAN convert from an anonymous type to a explicit type compatibly in the future, but once you expose the payload type you must keep it forever. The payload type should simply have C# 'TYPE NAME {get; set; }' properties @@ -405,21 +405,6 @@ Thus we could replace the `listener.Subscribe()` call in the previous example wi This very efficiently subscribes to only the 'RequestStart' events. All other events will cause the `DiagnosticSource.IsEnabled()` method to return `false`, and thus be efficiently filtered out. -NOTE: Filtering is only designed as a performance optimization. It is possible for a listener to receive events even when they -do not satisfy the filter. This could occur because some other listener has subscribed to the event or because the source -of the event didn't check IsEnabled() prior to sending it. If you want to be certain that a given event satisfies the filter -you will need to check it inside the callback. For example: - -```C# -Action> callback = (KeyValuePair evnt) => - { - if(predicate(evnt.Key)) // only print out events that satisfy our filter - { - Console.WriteLine("From Listener {0} Received Event {1} with payload {2}", networkListener.Name, evnt.Key, evnt.Value.ToString()); - } - }; -``` - ##### Context-based Filtering Some scenarios require advanced filtering based on extended context. Producers may call `DiagnosticSource.IsEnabled()` overloads and supply additional event properties: diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticListener.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticListener.cs index 1a58f827851cec..3d39054f5694f7 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticListener.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticListener.cs @@ -183,7 +183,7 @@ public virtual void Dispose() // Indicate completion to all subscribers. DiagnosticSubscription? subscriber = null; - subscriber = Interlocked.Exchange(ref _subscriptions, subscriber); + Interlocked.Exchange(ref subscriber, _subscriptions); while (subscriber != null) { subscriber.Observer.OnCompleted(); diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticSourceEventSource.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticSourceEventSource.cs index fbb13c9c9b6db7..510c0d9a21e64e 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticSourceEventSource.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/DiagnosticSourceEventSource.cs @@ -1440,13 +1440,13 @@ public ValueTypedFetchProperty(Type type, PropertyInfo property) : base(type) /// private sealed class ReflectionPropertyFetch : PropertyFetch { - private readonly MethodInvoker _getterInvoker; + private readonly PropertyInfo _property; public ReflectionPropertyFetch(Type type, PropertyInfo property) : base(type) { - _getterInvoker = MethodInvoker.Create(property.GetMethod!); + _property = property; } - public override object? Fetch(object? obj) => _getterInvoker.Invoke(obj); + public override object? Fetch(object? obj) => _property.GetValue(obj); } /// diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs index 3fbe68f5545499..0b446b23ad35a7 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/src/System/Diagnostics/Metrics/MetricsEventSource.cs @@ -737,11 +737,7 @@ private static string FormatQuantiles(QuantileValue[] quantiles) StringBuilder sb = new StringBuilder(); for (int i = 0; i < quantiles.Length; i++) { -#if NETCOREAPP - sb.Append(CultureInfo.InvariantCulture, $"{quantiles[i].Quantile}={quantiles[i].Value}"); -#else - sb.AppendFormat(CultureInfo.InvariantCulture, "{0}={1}", quantiles[i].Quantile, quantiles[i].Value); -#endif + sb.Append(quantiles[i].Quantile).Append('=').Append(quantiles[i].Value); if (i != quantiles.Length - 1) { sb.Append(';'); diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/tests/DiagnosticSourceTests.cs b/src/libraries/System.Diagnostics.DiagnosticSource/tests/DiagnosticSourceTests.cs index 7be9494d77b05e..f8ca97c501ccd5 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/tests/DiagnosticSourceTests.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/tests/DiagnosticSourceTests.cs @@ -116,10 +116,6 @@ public void Completed() listener.Dispose(); Assert.True(observer.Completed); - // Subscriptions are removed when listener is disposed and don't receive further notifications - listener.Write("AnotherNotification", null); - Assert.Equal(1, result.Count); - // confirm that we can unsubscribe without crashing subscription.Dispose(); diff --git a/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs b/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs index 8eec15c60c79ae..0db3af5edb57be 100644 --- a/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs +++ b/src/libraries/System.Diagnostics.DiagnosticSource/tests/MetricEventSourceTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Diagnostics.Tracing; using System.Globalization; @@ -8,7 +9,7 @@ using System.Runtime.CompilerServices; using System.Text; using System.Threading; -using Microsoft.DotNet.RemoteExecutor; +using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -658,59 +659,45 @@ public void MultipleListeners_PublishingInstruments() AssertInitialEnumerationCompleteEventPresent(events2); } - public static bool IsNotBrowserAndRemoteExecuteSupported => PlatformDetection.IsNotBrowser && RemoteExecutor.IsSupported; - - [ConditionalFact(typeof(MetricEventSourceTests), nameof(IsNotBrowserAndRemoteExecuteSupported))] + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] [OuterLoop("Slow and has lots of console spew")] public void EventSourcePublishesTimeSeriesWithEmptyMetadata() { - RemoteExecutor.Invoke(static () => - { - CultureInfo.DefaultThreadCurrentCulture = new CultureInfo("fi-FI"); - - using Meter meter = new Meter("TestMeter1", null, new TagList() { { "Mk1", "Mv1" }, { "Mk2", "Mv2" } }, new object()); - Counter c = meter.CreateCounter("counter1"); - int counterState = 3; - ObservableCounter oc = meter.CreateObservableCounter("observableCounter1", () => { counterState += 7; return counterState; }); - int gaugeState = 0; - ObservableGauge og = meter.CreateObservableGauge("observableGauge1", () => { gaugeState += 9; return gaugeState; }); - Histogram h = meter.CreateHistogram("histogram1"); - UpDownCounter udc = meter.CreateUpDownCounter("upDownCounter1"); - int upDownCounterState = 0; - ObservableUpDownCounter oudc = meter.CreateObservableUpDownCounter("observableUpDownCounter1", () => { upDownCounterState -= 11; return upDownCounterState; }); - - EventWrittenEventArgs[] events; - using (MetricsEventListener listener = new MetricsEventListener(NullTestOutputHelper.Instance, MetricsEventListener.TimeSeriesValues, IntervalSecs, "TestMeter1")) - { - listener.WaitForCollectionStop(s_waitForEventTimeout, 1); - c.Add(5); - h.Record(19); - udc.Add(-33); - listener.WaitForCollectionStop(s_waitForEventTimeout, 2); - c.Add(12); - h.Record(26); - udc.Add(-40); - listener.WaitForCollectionStop(s_waitForEventTimeout, 3); - events = listener.Events.ToArray(); - } + using Meter meter = new Meter("TestMeter1", null, new TagList() { { "Mk1", "Mv1" }, { "Mk2", "Mv2" } }, new object()); + Counter c = meter.CreateCounter("counter1"); + int counterState = 3; + ObservableCounter oc = meter.CreateObservableCounter("observableCounter1", () => { counterState += 7; return counterState; }); + int gaugeState = 0; + ObservableGauge og = meter.CreateObservableGauge("observableGauge1", () => { gaugeState += 9; return gaugeState; }); + Histogram h = meter.CreateHistogram("histogram1"); + UpDownCounter udc = meter.CreateUpDownCounter("upDownCounter1"); + int upDownCounterState = 0; + ObservableUpDownCounter oudc = meter.CreateObservableUpDownCounter("observableUpDownCounter1", () => { upDownCounterState -= 11; return upDownCounterState; }); - AssertBeginInstrumentReportingEventsPresent(events, c, oc, og, h, udc, oudc); - AssertInitialEnumerationCompleteEventPresent(events); - AssertCounterEventsPresent(events, meter.Name, c.Name, "", "", ("5", "5"), ("12", "17")); - AssertCounterEventsPresent(events, meter.Name, oc.Name, "", "", ("", "10"), ("7", "17")); - AssertGaugeEventsPresent(events, meter.Name, og.Name, "", "", "9", "18"); - AssertHistogramEventsPresent(events, meter.Name, h.Name, "", "", ("0.5=19;0.95=19;0.99=19", "1", "19"), ("0.5=26;0.95=26;0.99=26", "1", "26")); - AssertUpDownCounterEventsPresent(events, meter.Name, udc.Name, "", "", ("-33", "-33"), ("-40", "-73")); - AssertUpDownCounterEventsPresent(events, meter.Name, oudc.Name, "", "", ("", "-11"), ("-11", "-22")); - AssertCollectStartStopEventsPresent(events, IntervalSecs, 3); - }).Dispose(); - } + EventWrittenEventArgs[] events; + using (MetricsEventListener listener = new MetricsEventListener(_output, MetricsEventListener.TimeSeriesValues, IntervalSecs, "TestMeter1")) + { + listener.WaitForCollectionStop(s_waitForEventTimeout, 1); + c.Add(5); + h.Record(19); + udc.Add(-33); + listener.WaitForCollectionStop(s_waitForEventTimeout, 2); + c.Add(12); + h.Record(26); + udc.Add(-40); + listener.WaitForCollectionStop(s_waitForEventTimeout, 3); + events = listener.Events.ToArray(); + } - private sealed class NullTestOutputHelper : ITestOutputHelper - { - public static NullTestOutputHelper Instance { get; } = new(); - public void WriteLine(string message) { } - public void WriteLine(string format, params object[] args) { } + AssertBeginInstrumentReportingEventsPresent(events, c, oc, og, h, udc, oudc); + AssertInitialEnumerationCompleteEventPresent(events); + AssertCounterEventsPresent(events, meter.Name, c.Name, "", "", ("5", "5"), ("12", "17")); + AssertCounterEventsPresent(events, meter.Name, oc.Name, "", "", ("", "10"), ("7", "17")); + AssertGaugeEventsPresent(events, meter.Name, og.Name, "", "", "9", "18"); + AssertHistogramEventsPresent(events, meter.Name, h.Name, "", "", ("0.5=19;0.95=19;0.99=19", "1", "19"), ("0.5=26;0.95=26;0.99=26", "1", "26")); + AssertUpDownCounterEventsPresent(events, meter.Name, udc.Name, "", "", ("-33", "-33"), ("-40", "-73")); + AssertUpDownCounterEventsPresent(events, meter.Name, oudc.Name, "", "", ("", "-11"), ("-11", "-22")); + AssertCollectStartStopEventsPresent(events, IntervalSecs, 3); } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] @@ -1484,7 +1471,7 @@ private static string FormatTags(IEnumerable>? tag return sb.ToString(); } - private static void AssertBeginInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) + private void AssertBeginInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) { var beginReportEvents = events.Where(e => e.EventName == "BeginInstrumentReporting").Select(e => new @@ -1516,7 +1503,7 @@ private static void AssertBeginInstrumentReportingEventsPresent(EventWrittenEven Assert.Equal(expectedInstruments.Length, beginReportEvents.Length); } - private static void AssertEndInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) + private void AssertEndInstrumentReportingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) { var beginReportEvents = events.Where(e => e.EventName == "EndInstrumentReporting").Select(e => new @@ -1548,27 +1535,27 @@ private static void AssertEndInstrumentReportingEventsPresent(EventWrittenEventA Assert.Equal(expectedInstruments.Length, beginReportEvents.Length); } - private static void AssertInitialEnumerationCompleteEventPresent(EventWrittenEventArgs[] events, int eventsCount = 1) + private void AssertInitialEnumerationCompleteEventPresent(EventWrittenEventArgs[] events, int eventsCount = 1) { Assert.Equal(eventsCount, events.Where(e => e.EventName == "InitialInstrumentEnumerationComplete").Count()); } - private static void AssertTimeSeriesLimitPresent(EventWrittenEventArgs[] events) + private void AssertTimeSeriesLimitPresent(EventWrittenEventArgs[] events) { Assert.Equal(1, events.Where(e => e.EventName == "TimeSeriesLimitReached").Count()); } - private static void AssertTimeSeriesLimitNotPresent(EventWrittenEventArgs[] events) + private void AssertTimeSeriesLimitNotPresent(EventWrittenEventArgs[] events) { Assert.Equal(0, events.Where(e => e.EventName == "TimeSeriesLimitReached").Count()); } - private static void AssertHistogramLimitPresent(EventWrittenEventArgs[] events) + private void AssertHistogramLimitPresent(EventWrittenEventArgs[] events) { Assert.Equal(1, events.Where(e => e.EventName == "HistogramLimitReached").Count()); } - private static void AssertInstrumentPublishingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) + private void AssertInstrumentPublishingEventsPresent(EventWrittenEventArgs[] events, params Instrument[] expectedInstruments) { var publishEvents = events.Where(e => e.EventName == "InstrumentPublished").Select(e => new @@ -1600,19 +1587,19 @@ private static void AssertInstrumentPublishingEventsPresent(EventWrittenEventArg Assert.Equal(expectedInstruments.Length, publishEvents.Length); } - private static void AssertCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private void AssertCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string)[] expected) { AssertGenericCounterEventsPresent("CounterRateValuePublished", events, meterName, instrumentName, tags, expectedUnit, expected); } - private static void AssertUpDownCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private void AssertUpDownCounterEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string)[] expected) { AssertGenericCounterEventsPresent("UpDownCounterRateValuePublished", events, meterName, instrumentName, tags, expectedUnit, expected); } - private static void AssertGenericCounterEventsPresent(string eventName, EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private void AssertGenericCounterEventsPresent(string eventName, EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string)[] expected) { var counterEvents = events.Where(e => e.EventName == eventName).Select(e => @@ -1636,7 +1623,7 @@ private static void AssertGenericCounterEventsPresent(string eventName, EventWri } } - private static void AssertCounterEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) + private void AssertCounterEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) { var counterEvents = events.Where(e => e.EventName == "CounterRateValuePublished").Select(e => new @@ -1650,7 +1637,7 @@ private static void AssertCounterEventsNotPresent(EventWrittenEventArgs[] events Assert.Equal(0, filteredEvents.Length); } - private static void AssertGaugeEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private void AssertGaugeEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params string[] expectedValues) { var counterEvents = events.Where(e => e.EventName == "GaugeValuePublished").Select(e => @@ -1672,7 +1659,7 @@ private static void AssertGaugeEventsPresent(EventWrittenEventArgs[] events, str } } - private static void AssertHistogramEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, + private void AssertHistogramEventsPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags, string expectedUnit, params (string, string, string)[] expected) { var counterEvents = events.Where(e => e.EventName == "HistogramValuePublished").Select(e => @@ -1698,7 +1685,7 @@ private static void AssertHistogramEventsPresent(EventWrittenEventArgs[] events, } } - private static void AssertHistogramEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) + private void AssertHistogramEventsNotPresent(EventWrittenEventArgs[] events, string meterName, string instrumentName, string tags) { var counterEvents = events.Where(e => e.EventName == "HistogramValuePublished").Select(e => new @@ -1711,7 +1698,7 @@ private static void AssertHistogramEventsNotPresent(EventWrittenEventArgs[] even var filteredEvents = counterEvents.Where(e => e.MeterName == meterName && e.InstrumentName == instrumentName && e.Tags == tags).ToArray(); Assert.Equal(0, filteredEvents.Length); } - private static void AssertCollectStartStopEventsPresent(EventWrittenEventArgs[] events, double expectedIntervalSecs, int expectedPairs) + private void AssertCollectStartStopEventsPresent(EventWrittenEventArgs[] events, double expectedIntervalSecs, int expectedPairs) { int startEventsSeen = 0; int stopEventsSeen = 0; @@ -1740,7 +1727,7 @@ private static void AssertCollectStartStopEventsPresent(EventWrittenEventArgs[] Assert.Equal(expectedPairs, stopEventsSeen); } - private static void AssertObservableCallbackErrorPresent(EventWrittenEventArgs[] events) + private void AssertObservableCallbackErrorPresent(EventWrittenEventArgs[] events) { var errorEvents = events.Where(e => e.EventName == "ObservableInstrumentCallbackError").Select(e => new @@ -1751,7 +1738,7 @@ private static void AssertObservableCallbackErrorPresent(EventWrittenEventArgs[] Assert.Contains("Example user exception", errorEvents[0].ErrorText); } - private static void AssertMultipleSessionsConfiguredIncorrectlyErrorEventsPresent(EventWrittenEventArgs[] events, + private void AssertMultipleSessionsConfiguredIncorrectlyErrorEventsPresent(EventWrittenEventArgs[] events, string expectedMaxHistograms, string actualMaxHistograms, string expectedMaxTimeSeries, string actualMaxTimeSeries, string expectedRefreshInterval, string actualRefreshInterval) { diff --git a/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventData.cs b/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventData.cs index 961a6e4321109f..8497386db5ee17 100644 --- a/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventData.cs +++ b/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventData.cs @@ -39,7 +39,7 @@ public EventLogEntryType EntryType get => _entryType; set { - if (!Enum.IsDefined(value)) + if (!Enum.IsDefined(typeof(EventLogEntryType), value)) throw new InvalidEnumArgumentException(nameof(EntryType), (int)value, typeof(EventLogEntryType)); _entryType = value; diff --git a/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventLogInternal.cs b/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventLogInternal.cs index f34da4ccf018c9..41173ac89234b5 100644 --- a/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventLogInternal.cs +++ b/src/libraries/System.Diagnostics.EventLog/src/System/Diagnostics/EventLogInternal.cs @@ -1292,7 +1292,7 @@ public void WriteEntry(string message, EventLogEntryType type, int eventID, shor if (Source.Length == 0) throw new ArgumentException(SR.NeedSourceToWrite); - if (!Enum.IsDefined(type)) + if (!Enum.IsDefined(typeof(EventLogEntryType), type)) throw new InvalidEnumArgumentException(nameof(type), (int)type, typeof(EventLogEntryType)); string currentMachineName = machineName; diff --git a/src/libraries/System.Diagnostics.PerformanceCounter/src/System/Diagnostics/CounterCreationData.cs b/src/libraries/System.Diagnostics.PerformanceCounter/src/System/Diagnostics/CounterCreationData.cs index 0abb67e55b5f61..33a814625986f1 100644 --- a/src/libraries/System.Diagnostics.PerformanceCounter/src/System/Diagnostics/CounterCreationData.cs +++ b/src/libraries/System.Diagnostics.PerformanceCounter/src/System/Diagnostics/CounterCreationData.cs @@ -33,7 +33,7 @@ public PerformanceCounterType CounterType } set { - if (!Enum.IsDefined(value)) + if (!Enum.IsDefined(typeof(PerformanceCounterType), value)) throw new InvalidEnumArgumentException(nameof(PerformanceCounterType), (int)value, typeof(PerformanceCounterType)); _counterType = value; diff --git a/src/libraries/System.Diagnostics.Tracing/tests/BasicEventSourceTest/Harness/EtwListener.cs b/src/libraries/System.Diagnostics.Tracing/tests/BasicEventSourceTest/Harness/EtwListener.cs index 7e514b20f1a197..cdbb961313ecd7 100644 --- a/src/libraries/System.Diagnostics.Tracing/tests/BasicEventSourceTest/Harness/EtwListener.cs +++ b/src/libraries/System.Diagnostics.Tracing/tests/BasicEventSourceTest/Harness/EtwListener.cs @@ -3,7 +3,6 @@ using Microsoft.Diagnostics.Tracing; using Microsoft.Diagnostics.Tracing.Session; -using Microsoft.DotNet.XUnitExtensions; using System; using System.Collections.Generic; using System.Diagnostics; @@ -40,7 +39,7 @@ public EtwListener(string dataFileName = "EventSourceTestData.etl", string sessi // Today you have to be Admin to turn on ETW events (anyone can write ETW events). if (TraceEventSession.IsElevated() != true) { - throw new SkipTestException("Need to be elevated to run. "); + throw new Exception("Need to be elevated to run. "); } if (dataFileName == null) diff --git a/src/libraries/System.DirectoryServices.Protocols/tests/BerConverterTests.cs b/src/libraries/System.DirectoryServices.Protocols/tests/BerConverterTests.cs index cb6117cbf9361a..ba92d33c2a7b61 100644 --- a/src/libraries/System.DirectoryServices.Protocols/tests/BerConverterTests.cs +++ b/src/libraries/System.DirectoryServices.Protocols/tests/BerConverterTests.cs @@ -124,74 +124,22 @@ public void Encode_InvalidFormat_ThrowsBerConversionException(string format) public static IEnumerable Decode_TestData() { - // Content: zero-length sequence - // Parsed as such yield return new object[] { "{}", new byte[] { 48, 0, 0, 0, 0, 0 }, new object[0] }; - - // Content: sequence containing octet string - // Parsed as such yield return new object[] { "{a}", new byte[] { 48, 132, 0, 0, 0, 5, 4, 3, 97, 98, 99 }, new object[] { "abc" } }; - - // Content: sequence containing integer - // Parsed as such yield return new object[] { "{i}", new byte[] { 48, 132, 0, 0, 0, 3, 2, 1, 10 }, new object[] { 10 } }; - - // Content: sequence containing two booleans - // Parsed as a sequence containing an integer, followed by an enumerated value yield return new object[] { "{ie}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { -1, 0 } }; - - // Content: sequence containing two booleans - // Parsed as such yield return new object[] { "{bb}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { true, false } }; - - // Content: sequence containing two booleans - // Parsed as a sequence containing two octet strings yield return new object[] { "{OO}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { new byte[] { 255 }, new byte[] { 0 } } }; - - // Content: sequence containing two booleans - // Parsed as a sequence containing two bitstrings yield return new object[] { "{BB}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { new byte[] { 255 }, new byte[] { 0 } } }; if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) // vv and VV formats are not supported yet in Linux { - // Content: sequence containing three octet strings - // Parsed as a sequence containing two sequences of octet strings yield return new object[] { "{vv}", new byte[] { 48, 132, 0, 0, 0, 9, 4, 3, 97, 98, 99, 4, 0, 4, 0 }, new object[] { null, null } }; - - // Content: sequence containing three octet strings - // Parsed as two sequences of octet strings - yield return new object[] { "vv", new byte[] { 48, 132, 0, 0, 0, 9, 4, 3, 97, 98, 99, 4, 0, 4, 0 }, new object[] { new string[] { "abc", "" }, null } }; - - // Content: sequence containing two sequences of octet strings - // Parsed as such - yield return new object[] { "{vv}", new byte[] { 48, 14, 48, 5, 4, 3, 97, 98, 99, 48, 5, 4, 3, 100, 101, 102 }, new object[] { new string[] { "abc" }, new string[] { "def" } } }; - - // Content: sequence containing two booleans - // Parsed as a sequence containing two sequences of octet strings yield return new object[] { "{vv}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { new string[] { "\x01" }, null } }; - - // Content: sequence containing two booleans. First boolean has a valid value which is also a valid UTF8 character - // Parsed as two sequences of octet strings - yield return new object[] { "vv", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 48, 1, 1, 0 }, new object[] { new string[] { "\x30", "\x00" }, null } }; - - // Content: sequence of octet strings - // Parsed as a sequence containing two sequences of octet strings (returned as bytes) yield return new object[] { "{VV}", new byte[] { 48, 132, 0, 0, 0, 9, 4, 3, 97, 98, 99, 4, 0, 4, 0 }, new object[] { null, null } }; - - // Content: sequence of octet strings - // Parsed as two sequences of octet strings (returned as bytes) - yield return new object[] { "VV", new byte[] { 48, 132, 0, 0, 0, 9, 4, 3, 97, 98, 99, 4, 0, 4, 0 }, new object[] { new byte[][] { [97, 98, 99], [] }, null } }; - - // Content: sequence containing two booleans - // Parsed as a sequence containing two sequences of octet strings (returned as bytes) - yield return new object[] { "{VV}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { new byte[][] { [1] }, null } }; - - // Content: sequence containing two booleans - // Parsed as two sequences of octet strings (returned as bytes) - yield return new object[] { "VV", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { new byte[][] { [255], [0] }, null } }; + yield return new object[] { "{VV}", new byte[] { 48, 132, 0, 0, 0, 6, 1, 1, 255, 1, 1, 0 }, new object[] { new byte[][] { new byte[] { 1 } }, null } }; } } - [ActiveIssue("https://github.com/dotnet/runtime/issues/99725")] [Theory] [MemberData(nameof(Decode_TestData))] public void Decode_Bytes_ReturnsExpected(string format, byte[] values, object[] expected) @@ -240,24 +188,5 @@ public void Decode_Invalid_ThrowsBerConversionException(string format, byte[] va { Assert.Throws(() => BerConverter.Decode(format, values)); } - - public static IEnumerable Manual_Wrapping_Required_Data() - { - // vv and VV formats are not supported yet in Linux - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - yield return new object[] { "v", new object[] { new string[] { "abc", "def" } } }; - - yield return new object[] { "V", new object[] { new byte[][] { [97, 98, 99], [100, 101, 102] } } }; - } - } - - [Theory] - [MemberData(nameof(Manual_Wrapping_Required_Data))] - public void Must_Manually_Wrap_Several_OctetStrings_In_Sequence(string format, object[] values) - { - Assert.Throws(() => BerConverter.Decode(format, BerConverter.Encode(format, values))); - Assert.Equal(values, BerConverter.Decode(format, BerConverter.Encode("{" + format + "}", values))); - } } } diff --git a/src/libraries/System.DirectoryServices/src/System/DirectoryServices/ActiveDirectory/ActiveDirectorySchemaClass.cs b/src/libraries/System.DirectoryServices/src/System/DirectoryServices/ActiveDirectory/ActiveDirectorySchemaClass.cs index baa3c301fa5c9b..9909ce6616036d 100644 --- a/src/libraries/System.DirectoryServices/src/System/DirectoryServices/ActiveDirectory/ActiveDirectorySchemaClass.cs +++ b/src/libraries/System.DirectoryServices/src/System/DirectoryServices/ActiveDirectory/ActiveDirectorySchemaClass.cs @@ -1429,7 +1429,7 @@ private ArrayList GetPropertyValuesRecursively(string[] propertyNames) // get the properties of the auxiliary classes foreach (string auxSchemaClassName in GetValuesFromCache(PropertyManager.AuxiliaryClass)) { - using ActiveDirectorySchemaClass auxSchemaClass = new ActiveDirectorySchemaClass(_context, auxSchemaClassName, (DirectoryEntry?)null, null); + ActiveDirectorySchemaClass auxSchemaClass = new ActiveDirectorySchemaClass(_context, auxSchemaClassName, (DirectoryEntry?)null, null); foreach (string property in auxSchemaClass.GetPropertyValuesRecursively(propertyNames)) { @@ -1441,7 +1441,8 @@ private ArrayList GetPropertyValuesRecursively(string[] propertyNames) } foreach (string auxSchemaClassName in GetValuesFromCache(PropertyManager.SystemAuxiliaryClass)) { - using ActiveDirectorySchemaClass auxSchemaClass = new ActiveDirectorySchemaClass(_context, auxSchemaClassName, (DirectoryEntry?)null, null); + ActiveDirectorySchemaClass auxSchemaClass = new ActiveDirectorySchemaClass(_context, auxSchemaClassName, (DirectoryEntry?)null, null); + foreach (string property in auxSchemaClass.GetPropertyValuesRecursively(propertyNames)) { if (!values.Contains(property)) diff --git a/src/libraries/System.IO.FileSystem.Watcher/src/System/IO/FileSystemWatcher.Linux.cs b/src/libraries/System.IO.FileSystem.Watcher/src/System/IO/FileSystemWatcher.Linux.cs index b09e0197e04471..65dcefea25dafb 100644 --- a/src/libraries/System.IO.FileSystem.Watcher/src/System/IO/FileSystemWatcher.Linux.cs +++ b/src/libraries/System.IO.FileSystem.Watcher/src/System/IO/FileSystemWatcher.Linux.cs @@ -720,9 +720,9 @@ private bool ProcessEvent(NotifyEvent nextEvent, ref ReadOnlySpan previous break; case Interop.Sys.NotifyEvents.IN_MOVED_TO: - if (!previousEventName.IsEmpty) + if (previousEventName != null) { - // If the previous name from IN_MOVED_FROM is non-empty, then this is a rename. + // If the previous name from IN_MOVED_FROM is non-null, then this is a rename. watcher.NotifyRenameEventArgs(WatcherChangeTypes.Renamed, expandedName, previousEventName); } else diff --git a/src/libraries/System.IO.Packaging/src/System/IO/Packaging/ContentType.cs b/src/libraries/System.IO.Packaging/src/System/IO/Packaging/ContentType.cs index 894fd4826aa80d..c6fe72e7f047df 100644 --- a/src/libraries/System.IO.Packaging/src/System/IO/Packaging/ContentType.cs +++ b/src/libraries/System.IO.Packaging/src/System/IO/Packaging/ContentType.cs @@ -351,6 +351,8 @@ private void ParseParameterAndValue(ReadOnlySpan parameterAndValue) /// private static int GetLengthOfParameterValue(ReadOnlySpan s, int startIndex) { + Debug.Assert(s != null); + int length; //if the parameter value does not start with a '"' then, diff --git a/src/libraries/System.Linq.Expressions/src/ILLink/ILLink.Substitutions.xml b/src/libraries/System.Linq.Expressions/src/ILLink/ILLink.Substitutions.xml index b64cc2d765d67c..600116ec2ba086 100644 --- a/src/libraries/System.Linq.Expressions/src/ILLink/ILLink.Substitutions.xml +++ b/src/libraries/System.Linq.Expressions/src/ILLink/ILLink.Substitutions.xml @@ -1,7 +1,13 @@ + + + + + + diff --git a/src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/TypeUtils.cs b/src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/TypeUtils.cs index 1cb6419a134e36..f15f09139f70a1 100644 --- a/src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/TypeUtils.cs +++ b/src/libraries/System.Linq.Expressions/src/System/Dynamic/Utils/TypeUtils.cs @@ -41,7 +41,10 @@ public static Type LiftPrimitiveOrThrow(this Type type) { if (RuntimeFeature.IsDynamicCodeSupported) { +#pragma warning disable IL3050 + // Analyzer doesn't yet understand feature switches return GetNullableType(type); +#pragma warning restore IL3050 } if (!type.IsValueType || IsNullableType(type)) { diff --git a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Compiler/DelegateHelpers.cs b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Compiler/DelegateHelpers.cs index f4c45f7e85bbf0..5be00ff31ea82a 100644 --- a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Compiler/DelegateHelpers.cs +++ b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Compiler/DelegateHelpers.cs @@ -122,10 +122,13 @@ private static System.Reflection.TypeInfo MakeNewCustomDelegate(Type[] types) const MethodImplAttributes implAttributes = MethodImplAttributes.Runtime | MethodImplAttributes.Managed; const MethodAttributes invokeAttributes = MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.NewSlot | MethodAttributes.Virtual; +#pragma warning disable IL3050 + // Suppress analyzer warnings since they don't currently support feature flags TypeBuilder builder = AssemblyGen.DefineDelegateType("Delegate" + types.Length); builder.DefineConstructor(ctorAttributes, CallingConventions.Standard, delegateCtorSignature).SetImplementationFlags(implAttributes); builder.DefineMethod("Invoke", invokeAttributes, returnType, parameters).SetImplementationFlags(implAttributes); return builder.CreateTypeInfo(); +#pragma warning restore IL3050 } else { diff --git a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/CallInstruction.cs b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/CallInstruction.cs index ce8867e87a43ce..e582660ed3d20d 100644 --- a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/CallInstruction.cs +++ b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/CallInstruction.cs @@ -16,7 +16,6 @@ internal abstract partial class CallInstruction : Instruction /// public abstract int ArgumentCount { get; } - [FeatureGuard(typeof(RequiresDynamicCodeAttribute))] private static bool CanCreateArbitraryDelegates => RuntimeFeature.IsDynamicCodeSupported; #region Construction @@ -51,6 +50,8 @@ public static CallInstruction Create(MethodInfo info, ParameterInfo[] parameters if (!CanCreateArbitraryDelegates) return new MethodInfoCallInstruction(info, argumentCount); + // This code should be unreachable in AOT. The analyzer currently doesn't understand feature switches +#pragma warning disable IL3050 if (!info.IsStatic && info.DeclaringType!.IsValueType) { return new MethodInfoCallInstruction(info, argumentCount); @@ -114,6 +115,7 @@ public static CallInstruction Create(MethodInfo info, ParameterInfo[] parameters s_cache[info] = res; return res; +#pragma warning restore IL3050 } private static CallInstruction GetArrayAccessor(MethodInfo info, int argumentCount) diff --git a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/InstructionList.cs b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/InstructionList.cs index 148e24f812e07e..c9d525a81a0827 100644 --- a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/InstructionList.cs +++ b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/Interpreter/InstructionList.cs @@ -44,6 +44,7 @@ internal sealed class DebugView public DebugView(InstructionArray array) { + ArgumentNullException.ThrowIfNull(array); _array = array; } diff --git a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/LambdaExpression.cs b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/LambdaExpression.cs index a0ccf07bd77ddf..d3f0366a01f386 100644 --- a/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/LambdaExpression.cs +++ b/src/libraries/System.Linq.Expressions/src/System/Linq/Expressions/LambdaExpression.cs @@ -25,7 +25,7 @@ public abstract class LambdaExpression : Expression, IParameterProvider private readonly Expression _body; - [FeatureGuard(typeof(RequiresDynamicCodeAttribute))] + // This can be flipped to false using feature switches at publishing time public static bool CanCompileToIL => RuntimeFeature.IsDynamicCodeSupported; // This could be flipped to false using feature switches at publishing time @@ -138,7 +138,10 @@ public Delegate Compile() { if (CanCompileToIL) { +#pragma warning disable IL3050 + // Analyzer doesn't yet understand feature switches return Compiler.LambdaCompiler.Compile(this); +#pragma warning restore IL3050 } else { @@ -218,7 +221,10 @@ internal Expression(Expression body) { if (CanCompileToIL) { +#pragma warning disable IL3050 + // Analyzer doesn't yet understand feature switches return (TDelegate)(object)Compiler.LambdaCompiler.Compile(this); +#pragma warning restore IL3050 } else { @@ -623,7 +629,10 @@ internal static LambdaExpression CreateLambda(Type delegateType, Expression body MethodInfo create; if (LambdaExpression.CanCompileToIL) { +#pragma warning disable IL3050 + // Analyzer doesn't yet understand feature switches create = typeof(Expression<>).MakeGenericType(delegateType).GetMethod("Create", BindingFlags.Static | BindingFlags.NonPublic)!; +#pragma warning restore IL3050 } else { diff --git a/src/libraries/System.Linq.Expressions/src/System/Runtime/CompilerServices/CallSite.cs b/src/libraries/System.Linq.Expressions/src/System/Runtime/CompilerServices/CallSite.cs index a64bd1e2bde69d..bb3b579e4a1eed 100644 --- a/src/libraries/System.Linq.Expressions/src/System/Runtime/CompilerServices/CallSite.cs +++ b/src/libraries/System.Linq.Expressions/src/System/Runtime/CompilerServices/CallSite.cs @@ -289,7 +289,10 @@ internal T MakeUpdateDelegate() if (System.Linq.Expressions.LambdaExpression.CanCompileToIL && target.IsGenericType && IsSimpleSignature(invoke, out Type[] args)) { +#pragma warning disable IL3050 + // Analyzer doesn't yet understand feature switches return MakeUpdateDelegateWhenCanCompileToIL(); +#pragma warning restore IL3050 } s_cachedNoMatch = CreateCustomNoMatchDelegate(invoke); diff --git a/src/libraries/System.Linq.Expressions/tests/DebugViewTests.cs b/src/libraries/System.Linq.Expressions/tests/DebugViewTests.cs index 4d9167f24940ae..05f74015a26926 100644 --- a/src/libraries/System.Linq.Expressions/tests/DebugViewTests.cs +++ b/src/libraries/System.Linq.Expressions/tests/DebugViewTests.cs @@ -316,7 +316,7 @@ public static void Call() Check(".Call $x.ToString()", Expression.Call(x, typeof(int).GetMethod("ToString", Type.EmptyTypes))); Check(".Call $s.Substring($x)", Expression.Call(s, typeof(string).GetMethod("Substring", new[] { typeof(int) }), x)); Check(".Call $s.Substring(\\r\\n $x,\\r\\n $y)", Expression.Call(s, typeof(string).GetMethod("Substring", new[] { typeof(int), typeof(int) }), x, y)); - Check(".Call System.TimeSpan.FromSeconds($d)", Expression.Call(null, typeof(TimeSpan).GetMethod("FromSeconds", new[] { typeof(double) }), d)); + Check(".Call System.TimeSpan.FromSeconds($d)", Expression.Call(null, typeof(TimeSpan).GetMethod("FromSeconds", new[] { typeof(int) }), d)); } [Fact] diff --git a/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsEnumerableTests.cs b/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsEnumerableTests.cs index 9a1848ea559b1a..962d8745b23892 100644 --- a/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsEnumerableTests.cs +++ b/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsEnumerableTests.cs @@ -50,13 +50,12 @@ public static void AsEnumerable_LinqBinding(Labeled> labeled, { IEnumerable enumerable = labeled.Item.AsEnumerable(); - // The LINQ Cast() retains origin type for ParallelEnumerable and Partitioner when unordered, + // The LINQ Cast() retains origin type for ParallelEnumerable and Partitioner when unordered, // (and all when ordered, due to the extra wrapper) // although aliased as IEnumerable, so further LINQ calls work as expected. // If this test starts failing, update this test, and maybe mention it in release notes. Assert.IsNotType>(enumerable.Cast()); Assert.True(enumerable.Cast() is ParallelQuery); - Assert.True(enumerable.OfType() is ParallelQuery); // for non-nullable value types, OfType is equivalent to Cast Assert.False(enumerable.DefaultIfEmpty() is ParallelQuery); Assert.False(enumerable.Distinct() is ParallelQuery); @@ -65,6 +64,7 @@ public static void AsEnumerable_LinqBinding(Labeled> labeled, Assert.False(enumerable.GroupJoin(Enumerable.Range(0, count), x => x, y => y, (x, g) => x) is ParallelQuery); Assert.False(enumerable.Intersect(Enumerable.Range(0, count)) is ParallelQuery); Assert.False(enumerable.Join(Enumerable.Range(0, count), x => x, y => y, (x, y) => x) is ParallelQuery); + Assert.False(enumerable.OfType() is ParallelQuery); Assert.False(enumerable.OrderBy(x => x) is ParallelQuery); Assert.False(enumerable.OrderByDescending(x => x) is ParallelQuery); Assert.False(enumerable.Reverse() is ParallelQuery); diff --git a/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsSequentialTests.cs b/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsSequentialTests.cs index 0ed93d45be0819..6eada873e8bda9 100644 --- a/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsSequentialTests.cs +++ b/src/libraries/System.Linq.Parallel/tests/QueryOperators/AsSequentialTests.cs @@ -50,13 +50,12 @@ public static void AsSequential_LinqBinding(Labeled> labeled, { IEnumerable seq = labeled.Item.AsSequential(); - // The LINQ Cast() retains origin type for ParallelEnumerable and Partitioner when unordered, + // The LINQ Cast() retains origin type for ParallelEnumerable and Partitioner when unordered, // (and for all sources when ordered, due to the extra wrapper) // although aliased as IEnumerable, so further LINQ calls work as expected. // If this test starts failing, update this test, and maybe mention it in release notes. Assert.IsNotType>(seq.Cast()); Assert.True(seq.Cast() is ParallelQuery); - Assert.True(seq.OfType() is ParallelQuery); // for non-nullable value types, OfType is equivalent to Cast Assert.False(seq.DefaultIfEmpty() is ParallelQuery); Assert.False(seq.Distinct() is ParallelQuery); @@ -65,6 +64,7 @@ public static void AsSequential_LinqBinding(Labeled> labeled, Assert.False(seq.GroupJoin(Enumerable.Range(0, count), x => x, y => y, (x, g) => x) is ParallelQuery); Assert.False(seq.Intersect(Enumerable.Range(0, count)) is ParallelQuery); Assert.False(seq.Join(Enumerable.Range(0, count), x => x, y => y, (x, y) => x) is ParallelQuery); + Assert.False(seq.OfType() is ParallelQuery); Assert.False(seq.OrderBy(x => x) is ParallelQuery); Assert.False(seq.OrderByDescending(x => x) is ParallelQuery); Assert.False(seq.Reverse() is ParallelQuery); diff --git a/src/libraries/System.Linq/System.Linq.sln b/src/libraries/System.Linq/System.Linq.sln index b5283c72e434cd..a81e387e856bd6 100644 --- a/src/libraries/System.Linq/System.Linq.sln +++ b/src/libraries/System.Linq/System.Linq.sln @@ -1,8 +1,4 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.10.34618.27 -MinimumVisualStudioVersion = 10.0.40219.1 +Microsoft Visual Studio Solution File, Format Version 12.00 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestUtilities", "..\Common\tests\TestUtilities\TestUtilities.csproj", "{AF1B1B01-A4EC-45F4-AE51-CC1FA7892181}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Collections", "..\System.Collections\ref\System.Collections.csproj", "{3A8560D8-0E79-4BDE-802A-C96C7FE98258}" @@ -39,11 +35,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{8CA90AB2-58B EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gen", "gen", "{84E98F7C-FA2B-4048-AB7C-9FCDEA9CD37E}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gen", "gen", "{34793393-0347-438D-A832-2476F33C1BE3}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gen", "tools\gen", "{34793393-0347-438D-A832-2476F33C1BE3}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{F8F69023-9ACD-4979-A710-39D16377AEEE}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "tools\src", "{F8F69023-9ACD-4979-A710-39D16377AEEE}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ref", "ref", "{18C4E23D-AB0F-45E5-A6A1-A741F6462E85}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ref", "tools\ref", "{18C4E23D-AB0F-45E5-A6A1-A741F6462E85}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{0ADC596A-5B2E-4E5F-B5B5-DEB65A6C7E9D}" EndProject @@ -115,28 +111,24 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {AF1B1B01-A4EC-45F4-AE51-CC1FA7892181} = {E291F4BF-7B8B-45AD-88F5-FB8B8380C126} + {80A4051B-4A36-4A8B-BA43-A5AB8AA959F3} = {E291F4BF-7B8B-45AD-88F5-FB8B8380C126} {3A8560D8-0E79-4BDE-802A-C96C7FE98258} = {7C5B49B9-F7D9-41FB-A8FA-94328BDDCCD1} {7E4C1F09-B4F2-470E-9E7B-2C386E93D657} = {7C5B49B9-F7D9-41FB-A8FA-94328BDDCCD1} + {D3160C37-FC48-4907-8F4A-F584ED12B275} = {7C5B49B9-F7D9-41FB-A8FA-94328BDDCCD1} {14B966BB-CE23-4432-ADBB-89974389AC1D} = {8CA90AB2-58B9-45E7-A684-EDB60C6924B0} - {80A4051B-4A36-4A8B-BA43-A5AB8AA959F3} = {E291F4BF-7B8B-45AD-88F5-FB8B8380C126} {9A13A12F-C924-43AF-94AF-6F1B33582D27} = {84E98F7C-FA2B-4048-AB7C-9FCDEA9CD37E} {4BEC631E-B5FD-453F-82A0-C95C461798EA} = {84E98F7C-FA2B-4048-AB7C-9FCDEA9CD37E} {C8F0459C-15D5-4624-8CE4-E93ADF96A28C} = {84E98F7C-FA2B-4048-AB7C-9FCDEA9CD37E} - {D3160C37-FC48-4907-8F4A-F584ED12B275} = {7C5B49B9-F7D9-41FB-A8FA-94328BDDCCD1} {E0CA3ED5-EE6C-4F7C-BCE7-EFB1D64A9CD1} = {34793393-0347-438D-A832-2476F33C1BE3} {3EFB74E7-616A-48C1-B43B-3F89AA5013E6} = {34793393-0347-438D-A832-2476F33C1BE3} + {34793393-0347-438D-A832-2476F33C1BE3} = {0ADC596A-5B2E-4E5F-B5B5-DEB65A6C7E9D} {28ABC524-ACEE-4183-A64A-49E3DC830595} = {F8F69023-9ACD-4979-A710-39D16377AEEE} {721DB3D9-8221-424E-BE29-084CDD20D26E} = {F8F69023-9ACD-4979-A710-39D16377AEEE} - {E19B8772-2DBD-4274-8190-F3CC0242A1C0} = {18C4E23D-AB0F-45E5-A6A1-A741F6462E85} - {34793393-0347-438D-A832-2476F33C1BE3} = {0ADC596A-5B2E-4E5F-B5B5-DEB65A6C7E9D} {F8F69023-9ACD-4979-A710-39D16377AEEE} = {0ADC596A-5B2E-4E5F-B5B5-DEB65A6C7E9D} + {E19B8772-2DBD-4274-8190-F3CC0242A1C0} = {18C4E23D-AB0F-45E5-A6A1-A741F6462E85} {18C4E23D-AB0F-45E5-A6A1-A741F6462E85} = {0ADC596A-5B2E-4E5F-B5B5-DEB65A6C7E9D} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A4970D79-BF1C-4343-9070-B409DBB69F93} EndGlobalSection - GlobalSection(SharedMSBuildProjectFiles) = preSolution - ..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.projitems*{3efb74e7-616a-48c1-b43b-3f89aa5013e6}*SharedItemsImports = 5 - ..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.projitems*{721db3d9-8221-424e-be29-084cdd20d26e}*SharedItemsImports = 5 - EndGlobalSection EndGlobal diff --git a/src/libraries/System.Linq/src/CompatibilitySuppressions.xml b/src/libraries/System.Linq/src/CompatibilitySuppressions.xml new file mode 100644 index 00000000000000..0f5e8063636bdd --- /dev/null +++ b/src/libraries/System.Linq/src/CompatibilitySuppressions.xml @@ -0,0 +1,8 @@ + + + + + CP0001 + T:System.Linq.Grouping`2 + + \ No newline at end of file diff --git a/src/libraries/System.Linq/src/System.Linq.csproj b/src/libraries/System.Linq/src/System.Linq.csproj index 68b88631587ace..2fc4153be6890a 100644 --- a/src/libraries/System.Linq/src/System.Linq.csproj +++ b/src/libraries/System.Linq/src/System.Linq.csproj @@ -9,7 +9,6 @@ $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) true - $(DefineConstants);OPTIMIZE_FOR_SIZE @@ -19,22 +18,19 @@ - - - + - @@ -64,16 +60,16 @@ + + - - diff --git a/src/libraries/System.Linq/src/System/Linq/Aggregate.cs b/src/libraries/System.Linq/src/System/Linq/Aggregate.cs index c1a4377c29a8a0..81c3c0aa9a9c80 100644 --- a/src/libraries/System.Linq/src/System/Linq/Aggregate.cs +++ b/src/libraries/System.Linq/src/System/Linq/Aggregate.cs @@ -9,12 +9,12 @@ public static partial class Enumerable { public static TSource Aggregate(this IEnumerable source, Func func) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (func is null) + if (func == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.func); } @@ -38,12 +38,12 @@ public static TSource Aggregate(this IEnumerable source, Func< public static TAccumulate Aggregate(this IEnumerable source, TAccumulate seed, Func func) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (func is null) + if (func == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.func); } @@ -59,17 +59,17 @@ public static TAccumulate Aggregate(this IEnumerable(this IEnumerable source, TAccumulate seed, Func func, Func resultSelector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (func is null) + if (func == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.func); } - if (resultSelector is null) + if (resultSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.resultSelector); } diff --git a/src/libraries/System.Linq/src/System/Linq/AnyAll.cs b/src/libraries/System.Linq/src/System/Linq/AnyAll.cs index 28a3783aa88297..1fd5e5b4ae4479 100644 --- a/src/libraries/System.Linq/src/System/Linq/AnyAll.cs +++ b/src/libraries/System.Linq/src/System/Linq/AnyAll.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections; using System.Collections.Generic; namespace System.Linq @@ -10,47 +9,25 @@ public static partial class Enumerable { public static bool Any(this IEnumerable source) { - if (source is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); - } + return + TryGetNonEnumeratedCount(source, out int count) ? count != 0 : + WithEnumerator(source); - if (source is ICollection gc) + static bool WithEnumerator(IEnumerable source) { - return gc.Count != 0; + using IEnumerator e = source.GetEnumerator(); + return e.MoveNext(); } - -#if !OPTIMIZE_FOR_SIZE - if (source is Iterator iterator) - { - int count = iterator.GetCount(onlyIfCheap: true); - if (count >= 0) - { - return count != 0; - } - - iterator.TryGetFirst(out bool found); - return found; - } -#endif - - if (source is ICollection ngc) - { - return ngc.Count != 0; - } - - using IEnumerator e = source.GetEnumerator(); - return e.MoveNext(); } public static bool Any(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -68,12 +45,12 @@ public static bool Any(this IEnumerable source, Func(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } diff --git a/src/libraries/System.Linq/src/System/Linq/AppendPrepend.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/AppendPrepend.SpeedOpt.cs index 01adb0af01e456..80ee23998603fa 100644 --- a/src/libraries/System.Linq/src/System/Linq/AppendPrepend.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/AppendPrepend.SpeedOpt.cs @@ -8,6 +8,15 @@ namespace System.Linq { public static partial class Enumerable { + private abstract partial class AppendPrependIterator : IIListProvider + { + public abstract TSource[] ToArray(); + + public abstract List ToList(); + + public abstract int GetCount(bool onlyIfCheap); + } + private sealed partial class AppendPrepend1Iterator { private TSource[] LazyToArray() @@ -121,61 +130,14 @@ public override List ToList() public override int GetCount(bool onlyIfCheap) { - if (_source is Iterator iterator) + if (_source is IIListProvider listProv) { - int count = iterator.GetCount(onlyIfCheap); + int count = listProv.GetCount(onlyIfCheap); return count == -1 ? -1 : count + 1; } return !onlyIfCheap || _source is ICollection ? _source.Count() + 1 : -1; } - - public override TSource? TryGetFirst(out bool found) - { - if (_appending) - { - TSource? first = _source.TryGetFirst(out found); - if (found) - { - return first; - } - } - - found = true; - return _item; - } - - public override TSource? TryGetLast(out bool found) - { - if (!_appending) - { - TSource? last = _source.TryGetLast(out found); - if (found) - { - return last; - } - } - - found = true; - return _item; - } - - public override TSource? TryGetElementAt(int index, out bool found) - { - if (!_appending) - { - if (index == 0) - { - found = true; - return _item; - } - - index--; - return _source.TryGetElementAt(index, out found); - } - - return base.TryGetElementAt(index, out found); - } } private sealed partial class AppendPrependN @@ -225,7 +187,7 @@ public override TSource[] ToArray() TSource[] array = new TSource[count]; int index = 0; - for (SingleLinkedNode? node = _prepended; node is not null; node = node.Linked) + for (SingleLinkedNode? node = _prepended; node != null; node = node.Linked) { array[index] = node.Item; ++index; @@ -245,7 +207,7 @@ public override TSource[] ToArray() } index = array.Length; - for (SingleLinkedNode? node = _appended; node is not null; node = node.Linked) + for (SingleLinkedNode? node = _appended; node != null; node = node.Linked) { --index; array[index] = node.Item; @@ -270,9 +232,9 @@ public override List ToList() public override int GetCount(bool onlyIfCheap) { - if (_source is Iterator iterator) + if (_source is IIListProvider listProv) { - int count = iterator.GetCount(onlyIfCheap); + int count = listProv.GetCount(onlyIfCheap); return count == -1 ? -1 : count + _appendCount + _prependCount; } diff --git a/src/libraries/System.Linq/src/System/Linq/AppendPrepend.cs b/src/libraries/System.Linq/src/System/Linq/AppendPrepend.cs index 3e2df29dcae51b..2397fbd39e2828 100644 --- a/src/libraries/System.Linq/src/System/Linq/AppendPrepend.cs +++ b/src/libraries/System.Linq/src/System/Linq/AppendPrepend.cs @@ -10,7 +10,7 @@ public static partial class Enumerable { public static IEnumerable Append(this IEnumerable source, TSource element) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -22,7 +22,7 @@ public static IEnumerable Append(this IEnumerable sou public static IEnumerable Prepend(this IEnumerable source, TSource element) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -43,13 +43,13 @@ private abstract partial class AppendPrependIterator : Iterator source) { - Debug.Assert(source is not null); + Debug.Assert(source != null); _source = source; } protected void GetSourceEnumerator() { - Debug.Assert(_enumerator is null); + Debug.Assert(_enumerator == null); _enumerator = _source.GetEnumerator(); } @@ -59,7 +59,7 @@ protected void GetSourceEnumerator() protected bool LoadFromEnumerator() { - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (_enumerator.MoveNext()) { _current = _enumerator.Current; @@ -72,7 +72,7 @@ protected bool LoadFromEnumerator() public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -98,7 +98,7 @@ public AppendPrepend1Iterator(IEnumerable source, TSource item, bool ap _appending = appending; } - private protected override Iterator Clone() => new AppendPrepend1Iterator(_source, _item, _appending); + public override Iterator Clone() => new AppendPrepend1Iterator(_source, _item, _appending); public override bool MoveNext() { @@ -176,7 +176,7 @@ private sealed partial class AppendPrependN : AppendPrependIterator source, SingleLinkedNode? prepended, SingleLinkedNode? appended, int prependCount, int appendCount) : base(source) { - Debug.Assert(prepended is not null || appended is not null); + Debug.Assert(prepended != null || appended != null); Debug.Assert(prependCount > 0 || appendCount > 0); Debug.Assert(prependCount + appendCount >= 2); Debug.Assert((prepended?.GetCount() ?? 0) == prependCount); @@ -188,7 +188,7 @@ public AppendPrependN(IEnumerable source, SingleLinkedNode? pr _appendCount = appendCount; } - private protected override Iterator Clone() => new AppendPrependN(_source, _prepended, _appended, _prependCount, _appendCount); + public override Iterator Clone() => new AppendPrependN(_source, _prepended, _appended, _prependCount, _appendCount); public override bool MoveNext() { @@ -199,7 +199,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - if (_node is not null) + if (_node != null) { _current = _node.Item; _node = _node.Linked; @@ -215,7 +215,7 @@ public override bool MoveNext() return true; } - if (_appended is null) + if (_appended == null) { return false; } @@ -233,13 +233,13 @@ public override bool MoveNext() public override AppendPrependIterator Append(TSource item) { - var appended = _appended is not null ? _appended.Add(item) : new SingleLinkedNode(item); + var appended = _appended != null ? _appended.Add(item) : new SingleLinkedNode(item); return new AppendPrependN(_source, _prepended, appended, _prependCount, _appendCount + 1); } public override AppendPrependIterator Prepend(TSource item) { - var prepended = _prepended is not null ? _prepended.Add(item) : new SingleLinkedNode(item); + var prepended = _prepended != null ? _prepended.Add(item) : new SingleLinkedNode(item); return new AppendPrependN(_source, prepended, _appended, _prependCount + 1, _appendCount); } } diff --git a/src/libraries/System.Linq/src/System/Linq/Cast.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Cast.SpeedOpt.cs deleted file mode 100644 index 3cded1625e8df0..00000000000000 --- a/src/libraries/System.Linq/src/System/Linq/Cast.SpeedOpt.cs +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections; -using System.Collections.Generic; - -namespace System.Linq -{ - public static partial class Enumerable - { - private sealed partial class CastICollectionIterator - { - public override int GetCount(bool onlyIfCheap) => _source.Count; - - public override TResult[] ToArray() - { - TResult[] array = new TResult[_source.Count]; - - int index = 0; - foreach (TResult item in _source) - { - array[index++] = item; - } - - return array; - } - - public override List ToList() - { - List list = new(_source.Count); - - foreach (TResult item in _source) - { - list.Add(item); - } - - return list; - } - - public override TResult? TryGetElementAt(int index, out bool found) - { - if (index >= 0) - { - IEnumerator e = _source.GetEnumerator(); - try - { - while (e.MoveNext()) - { - if (index == 0) - { - found = true; - return (TResult)e.Current; - } - - index--; - } - } - finally - { - (e as IDisposable)?.Dispose(); - } - } - - found = false; - return default; - } - - public override TResult? TryGetFirst(out bool found) - { - IEnumerator e = _source.GetEnumerator(); - try - { - if (e.MoveNext()) - { - found = true; - return (TResult)e.Current; - } - } - finally - { - (e as IDisposable)?.Dispose(); - } - - found = false; - return default; - } - - public override TResult? TryGetLast(out bool found) - { - IEnumerator e = _source.GetEnumerator(); - try - { - if (e.MoveNext()) - { - TResult last = (TResult)e.Current; - while (e.MoveNext()) - { - last = (TResult)e.Current; - } - - found = true; - return last; - } - - found = false; - return default; - } - finally - { - (e as IDisposable)?.Dispose(); - } - } - } - } -} diff --git a/src/libraries/System.Linq/src/System/Linq/Cast.cs b/src/libraries/System.Linq/src/System/Linq/Cast.cs index 77a001d135742d..0c20609b3eb79d 100644 --- a/src/libraries/System.Linq/src/System/Linq/Cast.cs +++ b/src/libraries/System.Linq/src/System/Linq/Cast.cs @@ -3,12 +3,32 @@ using System.Collections; using System.Collections.Generic; -using System.Diagnostics; namespace System.Linq { public static partial class Enumerable { + public static IEnumerable OfType(this IEnumerable source) + { + if (source == null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); + } + + return OfTypeIterator(source); + } + + private static IEnumerable OfTypeIterator(IEnumerable source) + { + foreach (object? obj in source) + { + if (obj is TResult result) + { + yield return result; + } + } + } + public static IEnumerable< #nullable disable // there's no way to annotate the connection of the nullability of TResult to that of the source TResult @@ -20,16 +40,11 @@ public static IEnumerable< return typedSource; } - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (source is ICollection collection) - { - return new CastICollectionIterator(collection); - } - return CastIterator(source); } @@ -40,46 +55,5 @@ private static IEnumerable CastIterator(IEnumerable source) yield return (TResult)obj; } } - - [DebuggerDisplay("Count = {Count}")] - private sealed partial class CastICollectionIterator(ICollection source) : Iterator - { - private readonly ICollection _source = source; - private IEnumerator? _enumerator; - - private protected override Iterator Clone() => new CastICollectionIterator(_source); - - public override bool MoveNext() - { - switch (_state) - { - case 1: - _enumerator = _source.GetEnumerator(); - _state = 2; - goto case 2; - - case 2: - Debug.Assert(_enumerator is not null); - if (_enumerator.MoveNext()) - { - _current = (TResult)_enumerator.Current; - return true; - } - - Dispose(); - break; - } - - return false; - } - - public override void Dispose() - { - (_enumerator as IDisposable)?.Dispose(); - _enumerator = null; - - base.Dispose(); - } - } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Chunk.cs b/src/libraries/System.Linq/src/System/Linq/Chunk.cs index d1a856bc4e0e42..680c65f72c441c 100644 --- a/src/libraries/System.Linq/src/System/Linq/Chunk.cs +++ b/src/libraries/System.Linq/src/System/Linq/Chunk.cs @@ -35,7 +35,7 @@ public static partial class Enumerable /// public static IEnumerable Chunk(this IEnumerable source, int size) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -45,31 +45,15 @@ public static IEnumerable Chunk(this IEnumerable so ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.size); } - if (source is TSource[] array) + if (IsEmptyArray(source)) { - // Special-case arrays, which have an immutable length. This enables us to not only do an - // empty check and avoid allocating an iterator object when empty, it enables us to have a - // much more efficient (and simpler) implementation for chunking up the array. - return array.Length != 0 ? - ArrayChunkIterator(array, size) : - []; + return []; } - return EnumerableChunkIterator(source, size); + return ChunkIterator(source, size); } - private static IEnumerable ArrayChunkIterator(TSource[] source, int size) - { - int index = 0; - while (index < source.Length) - { - TSource[] chunk = new ReadOnlySpan(source, index, Math.Min(size, source.Length - index)).ToArray(); - index += chunk.Length; - yield return chunk; - } - } - - private static IEnumerable EnumerableChunkIterator(IEnumerable source, int size) + private static IEnumerable ChunkIterator(IEnumerable source, int size) { using IEnumerator e = source.GetEnumerator(); diff --git a/src/libraries/System.Linq/src/System/Linq/Concat.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Concat.SpeedOpt.cs index 452d3fde1ab693..a5ad64f78584f6 100644 --- a/src/libraries/System.Linq/src/System/Linq/Concat.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Concat.SpeedOpt.cs @@ -170,7 +170,7 @@ public override int GetCount(bool onlyIfCheap) // Enumerable.Count() handles ICollections in O(1) time, but check for them here anyway // to avoid a method call because 1) they're common and 2) this code is run in a loop. var collection = source as ICollection; - Debug.Assert(!_hasOnlyCollections || collection is not null); + Debug.Assert(!_hasOnlyCollections || collection != null); int sourceCount = collection?.Count ?? source.Count(); checked @@ -178,7 +178,7 @@ public override int GetCount(bool onlyIfCheap) count += sourceCount; } } - while ((previousN = node.PreviousN) is not null); + while ((previousN = node.PreviousN) != null); Debug.Assert(node._tail is Concat2Iterator); return checked(count + node._tail.GetCount(onlyIfCheap)); @@ -202,7 +202,7 @@ private TSource[] LazyToArray() // On the bright side, the bottleneck will usually be iterating, buffering, and copying // each of the enumerables, so this shouldn't be a noticeable perf hit for most scenarios. IEnumerable? source = GetEnumerable(i); - if (source is null) + if (source == null) { break; } @@ -250,7 +250,7 @@ private TSource[] PreallocatingToArray() source.CopyTo(array, arrayIndex); } } - while ((previousN = node.PreviousN) is not null); + while ((previousN = node.PreviousN) != null); var previous2 = (Concat2Iterator)node._tail; var second = (ICollection)previous2._second; @@ -342,9 +342,13 @@ private TSource[] PreallocatingToArray() } } - private abstract partial class ConcatIterator + private abstract partial class ConcatIterator : IPartition { - public override List ToList() + public abstract int GetCount(bool onlyIfCheap); + + public abstract TSource[] ToArray(); + + public List ToList() { int count = GetCount(onlyIfCheap: true); var list = count != -1 ? new List(count) : new List(); @@ -352,7 +356,7 @@ public override List ToList() for (int i = 0; ; i++) { IEnumerable? source = GetEnumerable(i); - if (source is null) + if (source == null) { break; } @@ -363,6 +367,16 @@ public override List ToList() return list; } + public abstract TSource? TryGetElementAt(int index, out bool found); + + public abstract TSource? TryGetFirst(out bool found); + + public abstract TSource? TryGetLast(out bool found); + + public IPartition? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); + } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Concat.cs b/src/libraries/System.Linq/src/System/Linq/Concat.cs index 7b6dcc009d5efb..e57efd1a047d99 100644 --- a/src/libraries/System.Linq/src/System/Linq/Concat.cs +++ b/src/libraries/System.Linq/src/System/Linq/Concat.cs @@ -10,12 +10,12 @@ public static partial class Enumerable { public static IEnumerable Concat(this IEnumerable first, IEnumerable second) { - if (first is null) + if (first == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.first); } - if (second is null) + if (second == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.second); } @@ -58,14 +58,14 @@ private sealed partial class Concat2Iterator : ConcatIterator /// The second source to concatenate. internal Concat2Iterator(IEnumerable first, IEnumerable second) { - Debug.Assert(first is not null); - Debug.Assert(second is not null); + Debug.Assert(first != null); + Debug.Assert(second != null); _first = first; _second = second; } - private protected override Iterator Clone() => new Concat2Iterator(_first, _second); + public override Iterator Clone() => new Concat2Iterator(_first, _second); internal override ConcatIterator Concat(IEnumerable next) { @@ -139,8 +139,8 @@ private sealed partial class ConcatNIterator : ConcatIterator /// internal ConcatNIterator(ConcatIterator tail, IEnumerable head, int headIndex, bool hasOnlyCollections) { - Debug.Assert(tail is not null); - Debug.Assert(head is not null); + Debug.Assert(tail != null); + Debug.Assert(head != null); Debug.Assert(headIndex >= 2); _tail = tail; @@ -151,7 +151,7 @@ internal ConcatNIterator(ConcatIterator tail, IEnumerable head private ConcatNIterator? PreviousN => _tail as ConcatNIterator; - private protected override Iterator Clone() => new ConcatNIterator(_tail, _head, _headIndex, _hasOnlyCollections); + public override Iterator Clone() => new ConcatNIterator(_tail, _head, _headIndex, _hasOnlyCollections); internal override ConcatIterator Concat(IEnumerable next) { @@ -185,7 +185,7 @@ internal override ConcatIterator Concat(IEnumerable next) return node._head; } } - while ((previousN = node.PreviousN) is not null); + while ((previousN = node.PreviousN) != null); Debug.Assert(index == 0 || index == 1); Debug.Assert(node._tail is Concat2Iterator); @@ -206,7 +206,7 @@ private abstract partial class ConcatIterator : Iterator public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -240,7 +240,7 @@ public override bool MoveNext() { while (true) { - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (_enumerator.MoveNext()) { _current = _enumerator.Current; @@ -248,7 +248,7 @@ public override bool MoveNext() } IEnumerable? next = GetEnumerable(_state++ - 1); - if (next is not null) + if (next != null) { _enumerator.Dispose(); _enumerator = next.GetEnumerator(); diff --git a/src/libraries/System.Linq/src/System/Linq/Contains.cs b/src/libraries/System.Linq/src/System/Linq/Contains.cs index f242c7a35f52aa..a907eea4a99281 100644 --- a/src/libraries/System.Linq/src/System/Linq/Contains.cs +++ b/src/libraries/System.Linq/src/System/Linq/Contains.cs @@ -13,12 +13,12 @@ public static bool Contains(this IEnumerable source, TSource v public static bool Contains(this IEnumerable source, TSource value, IEqualityComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (comparer is null) + if (comparer == null) { foreach (TSource element in source) { diff --git a/src/libraries/System.Linq/src/System/Linq/Count.cs b/src/libraries/System.Linq/src/System/Linq/Count.cs index 6b8819f8c3cbcb..14f3d457f6ea35 100644 --- a/src/libraries/System.Linq/src/System/Linq/Count.cs +++ b/src/libraries/System.Linq/src/System/Linq/Count.cs @@ -10,7 +10,7 @@ public static partial class Enumerable { public static int Count(this IEnumerable source) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -20,12 +20,10 @@ public static int Count(this IEnumerable source) return collectionoft.Count; } -#if !OPTIMIZE_FOR_SIZE - if (source is Iterator iterator) + if (source is IIListProvider listProv) { - return iterator.GetCount(onlyIfCheap: false); + return listProv.GetCount(onlyIfCheap: false); } -#endif if (source is ICollection collection) { @@ -49,12 +47,12 @@ public static int Count(this IEnumerable source) public static int Count(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -96,7 +94,7 @@ public static int Count(this IEnumerable source, Func public static bool TryGetNonEnumeratedCount(this IEnumerable source, out int count) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -107,17 +105,15 @@ public static bool TryGetNonEnumeratedCount(this IEnumerable s return true; } -#if !OPTIMIZE_FOR_SIZE - if (source is Iterator iterator) + if (source is IIListProvider listProv) { - int c = iterator.GetCount(onlyIfCheap: true); + int c = listProv.GetCount(onlyIfCheap: true); if (c >= 0) { count = c; return true; } } -#endif if (source is ICollection collection) { @@ -131,7 +127,7 @@ public static bool TryGetNonEnumeratedCount(this IEnumerable s public static long LongCount(this IEnumerable source) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -153,12 +149,12 @@ public static long LongCount(this IEnumerable source) public static long LongCount(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } diff --git a/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.SpeedOpt.cs index c89d6797581e9b..24619cc438136b 100644 --- a/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.SpeedOpt.cs @@ -8,15 +8,15 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class DefaultIfEmptyIterator + private sealed partial class DefaultIfEmptyIterator : IIListProvider { - public override TSource[] ToArray() + public TSource[] ToArray() { TSource[] array = _source.ToArray(); return array.Length == 0 ? [_default] : array; } - public override List ToList() + public List ToList() { List list = _source.ToList(); if (list.Count == 0) @@ -27,7 +27,7 @@ public override List ToList() return list; } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { int count; if (!onlyIfCheap || _source is ICollection || _source is ICollection) @@ -36,51 +36,11 @@ public override int GetCount(bool onlyIfCheap) } else { - count = _source is Iterator iterator ? iterator.GetCount(onlyIfCheap: true) : -1; + count = _source is IIListProvider listProv ? listProv.GetCount(onlyIfCheap: true) : -1; } return count == 0 ? 1 : count; } - - public override TSource? TryGetFirst(out bool found) - { - TSource? first = _source.TryGetFirst(out found); - if (found) - { - return first; - } - - found = true; - return _default; - } - - public override TSource? TryGetLast(out bool found) - { - TSource? last = _source.TryGetLast(out found); - if (found) - { - return last; - } - - found = true; - return _default; - } - - public override TSource? TryGetElementAt(int index, out bool found) - { - TSource? item = _source.TryGetElementAt(index, out found); - if (found) - { - return item; - } - - if (index == 0) - { - found = true; - } - - return _default; - } } } } diff --git a/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.cs b/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.cs index 1ee5b5fd304db2..593a6b8a67b3d1 100644 --- a/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.cs +++ b/src/libraries/System.Linq/src/System/Linq/DefaultIfEmpty.cs @@ -13,16 +13,11 @@ public static partial class Enumerable public static IEnumerable DefaultIfEmpty(this IEnumerable source, TSource defaultValue) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (source is TSource[] { Length: > 0 }) - { - return source; - } - return new DefaultIfEmptyIterator(source, defaultValue); } @@ -34,12 +29,12 @@ private sealed partial class DefaultIfEmptyIterator : Iterator public DefaultIfEmptyIterator(IEnumerable source, TSource defaultValue) { - Debug.Assert(source is not null); + Debug.Assert(source != null); _source = source; _default = defaultValue; } - private protected override Iterator Clone() => new DefaultIfEmptyIterator(_source, _default); + public override Iterator Clone() => new DefaultIfEmptyIterator(_source, _default); public override bool MoveNext() { @@ -60,7 +55,7 @@ public override bool MoveNext() return true; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (_enumerator.MoveNext()) { _current = _enumerator.Current; @@ -76,7 +71,7 @@ public override bool MoveNext() public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; diff --git a/src/libraries/System.Linq/src/System/Linq/Distinct.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Distinct.SpeedOpt.cs index a3dbb6458969cf..70e96b7ed68fac 100644 --- a/src/libraries/System.Linq/src/System/Linq/Distinct.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Distinct.SpeedOpt.cs @@ -7,15 +7,13 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class DistinctIterator + private sealed partial class DistinctIterator : IIListProvider { - public override TSource[] ToArray() => HashSetToArray(new HashSet(_source, _comparer)); + public TSource[] ToArray() => Enumerable.HashSetToArray(new HashSet(_source, _comparer)); - public override List ToList() => new List(new HashSet(_source, _comparer)); + public List ToList() => new List(new HashSet(_source, _comparer)); - public override int GetCount(bool onlyIfCheap) => onlyIfCheap ? -1 : new HashSet(_source, _comparer).Count; - - public override TSource? TryGetFirst(out bool found) => _source.TryGetFirst(out found); + public int GetCount(bool onlyIfCheap) => onlyIfCheap ? -1 : new HashSet(_source, _comparer).Count; } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Distinct.cs b/src/libraries/System.Linq/src/System/Linq/Distinct.cs index 0408f51da4ba2d..e41973a46ec116 100644 --- a/src/libraries/System.Linq/src/System/Linq/Distinct.cs +++ b/src/libraries/System.Linq/src/System/Linq/Distinct.cs @@ -12,7 +12,7 @@ public static partial class Enumerable public static IEnumerable Distinct(this IEnumerable source, IEqualityComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -101,12 +101,12 @@ private sealed partial class DistinctIterator : Iterator public DistinctIterator(IEnumerable source, IEqualityComparer? comparer) { - Debug.Assert(source is not null); + Debug.Assert(source != null); _source = source; _comparer = comparer; } - private protected override Iterator Clone() => new DistinctIterator(_source, _comparer); + public override Iterator Clone() => new DistinctIterator(_source, _comparer); public override bool MoveNext() { @@ -127,8 +127,8 @@ public override bool MoveNext() _state = 2; return true; case 2: - Debug.Assert(_enumerator is not null); - Debug.Assert(_set is not null); + Debug.Assert(_enumerator != null); + Debug.Assert(_set != null); while (_enumerator.MoveNext()) { element = _enumerator.Current; @@ -148,7 +148,7 @@ public override bool MoveNext() public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; diff --git a/src/libraries/System.Linq/src/System/Linq/ElementAt.cs b/src/libraries/System.Linq/src/System/Linq/ElementAt.cs index 97b87f9eba9999..b33fcaddff924a 100644 --- a/src/libraries/System.Linq/src/System/Linq/ElementAt.cs +++ b/src/libraries/System.Linq/src/System/Linq/ElementAt.cs @@ -11,29 +11,30 @@ public static partial class Enumerable { public static TSource ElementAt(this IEnumerable source, int index) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (source is IList list) + if (source is IPartition partition) + { + TSource? element = partition.TryGetElementAt(index, out bool found); + if (found) + { + return element!; + } + } + else if (source is IList list) { return list[index]; } - - bool found; - TSource? element = -#if !OPTIMIZE_FOR_SIZE - source is Iterator iterator ? iterator.TryGetElementAt(index, out found) : -#endif - TryGetElementAtNonIterator(source, index, out found); - - if (!found) + else if (TryGetElement(source, index, out TSource? element)) { - ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.index); + return element; } - return element!; + ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.index); + return default; } /// Returns the element at a specified index in a sequence. @@ -49,7 +50,7 @@ public static TSource ElementAt(this IEnumerable source, int i /// public static TSource ElementAt(this IEnumerable source, Index index) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -74,12 +75,23 @@ public static TSource ElementAt(this IEnumerable source, Index public static TSource? ElementAtOrDefault(this IEnumerable source, int index) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - return TryGetElementAt(source, index, out _); + if (source is IPartition partition) + { + return partition.TryGetElementAt(index, out bool _); + } + + if (source is IList list) + { + return (uint)index < (uint)list.Count ? list[index] : default; + } + + TryGetElement(source, index, out TSource? element); + return element; } /// Returns the element at a specified index in a sequence or a default value if the index is out of range. @@ -94,7 +106,7 @@ public static TSource ElementAt(this IEnumerable source, Index /// public static TSource? ElementAtOrDefault(this IEnumerable source, Index index) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -113,25 +125,9 @@ public static TSource ElementAt(this IEnumerable source, Index return element; } - private static TSource? TryGetElementAt(this IEnumerable source, int index, out bool found) - { - if (source is IList list) - { - return (found = (uint)index < (uint)list.Count) ? - list[index] : - default; - } - - return -#if !OPTIMIZE_FOR_SIZE - source is Iterator iterator ? iterator.TryGetElementAt(index, out found) : -#endif - TryGetElementAtNonIterator(source, index, out found); - } - - private static TSource? TryGetElementAtNonIterator(IEnumerable source, int index, out bool found) + private static bool TryGetElement(IEnumerable source, int index, [MaybeNullWhen(false)] out TSource element) { - Debug.Assert(source is not null); + Debug.Assert(source != null); if (index >= 0) { @@ -140,21 +136,21 @@ public static TSource ElementAt(this IEnumerable source, Index { if (index == 0) { - found = true; - return e.Current; + element = e.Current; + return true; } index--; } } - found = false; - return default; + element = default; + return false; } private static bool TryGetElementFromEnd(IEnumerable source, int indexFromEnd, [MaybeNullWhen(false)] out TSource element) { - Debug.Assert(source is not null); + Debug.Assert(source != null); if (indexFromEnd > 0) { diff --git a/src/libraries/System.Linq/src/System/Linq/Except.cs b/src/libraries/System.Linq/src/System/Linq/Except.cs index c8517d0b4a787b..b2277ccc92ba50 100644 --- a/src/libraries/System.Linq/src/System/Linq/Except.cs +++ b/src/libraries/System.Linq/src/System/Linq/Except.cs @@ -9,12 +9,12 @@ public static partial class Enumerable { public static IEnumerable Except(this IEnumerable first, IEnumerable second) { - if (first is null) + if (first == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.first); } - if (second is null) + if (second == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.second); } @@ -24,12 +24,12 @@ public static IEnumerable Except(this IEnumerable fir public static IEnumerable Except(this IEnumerable first, IEnumerable second, IEqualityComparer? comparer) { - if (first is null) + if (first == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.first); } - if (second is null) + if (second == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.second); } diff --git a/src/libraries/System.Linq/src/System/Linq/First.cs b/src/libraries/System.Linq/src/System/Linq/First.cs index 74ff81c2fec83b..1c62f547d9a03c 100644 --- a/src/libraries/System.Linq/src/System/Linq/First.cs +++ b/src/libraries/System.Linq/src/System/Linq/First.cs @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; namespace System.Linq { @@ -64,20 +64,16 @@ public static TSource FirstOrDefault(this IEnumerable source, private static TSource? TryGetFirst(this IEnumerable source, out bool found) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - return -#if !OPTIMIZE_FOR_SIZE - source is Iterator iterator ? iterator.TryGetFirst(out found) : -#endif - TryGetFirstNonIterator(source, out found); - } + if (source is IPartition partition) + { + return partition.TryGetFirst(out found); + } - private static TSource? TryGetFirstNonIterator(IEnumerable source, out bool found) - { if (source is IList list) { if (list.Count > 0) @@ -104,12 +100,12 @@ public static TSource FirstOrDefault(this IEnumerable source, private static TSource? TryGetFirst(this IEnumerable source, Func predicate, out bool found) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } diff --git a/src/libraries/System.Linq/src/System/Linq/GroupJoin.cs b/src/libraries/System.Linq/src/System/Linq/GroupJoin.cs index 8bd37bda7c8bd9..d52a2d3740d29d 100644 --- a/src/libraries/System.Linq/src/System/Linq/GroupJoin.cs +++ b/src/libraries/System.Linq/src/System/Linq/GroupJoin.cs @@ -12,27 +12,27 @@ public static IEnumerable GroupJoin(this public static IEnumerable GroupJoin(this IEnumerable outer, IEnumerable inner, Func outerKeySelector, Func innerKeySelector, Func, TResult> resultSelector, IEqualityComparer? comparer) { - if (outer is null) + if (outer == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.outer); } - if (inner is null) + if (inner == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.inner); } - if (outerKeySelector is null) + if (outerKeySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.outerKeySelector); } - if (innerKeySelector is null) + if (innerKeySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.innerKeySelector); } - if (resultSelector is null) + if (resultSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.resultSelector); } diff --git a/src/libraries/System.Linq/src/System/Linq/Grouping.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Grouping.SpeedOpt.cs index d081a09380f956..97e3b115213925 100644 --- a/src/libraries/System.Linq/src/System/Linq/Grouping.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Grouping.SpeedOpt.cs @@ -5,54 +5,63 @@ namespace System.Linq { - public static partial class Enumerable + internal sealed partial class GroupedResultEnumerable : IIListProvider { - private sealed partial class GroupByResultIterator - { - public override TResult[] ToArray() => - Lookup.Create(_source, _keySelector, _elementSelector, _comparer).ToArray(_resultSelector); + public TResult[] ToArray() => + Lookup.Create(_source, _keySelector, _elementSelector, _comparer).ToArray(_resultSelector); - public override List ToList() => - Lookup.Create(_source, _keySelector, _elementSelector, _comparer).ToList(_resultSelector); + public List ToList() => + Lookup.Create(_source, _keySelector, _elementSelector, _comparer).ToList(_resultSelector); - public override int GetCount(bool onlyIfCheap) => - onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _elementSelector, _comparer).Count; - } + public int GetCount(bool onlyIfCheap) => + onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _elementSelector, _comparer).Count; + } - private sealed partial class GroupByResultIterator - { - public override TResult[] ToArray() => - Lookup.Create(_source, _keySelector, _comparer).ToArray(_resultSelector); + internal sealed partial class GroupedResultEnumerable : IIListProvider + { + public TResult[] ToArray() => + Lookup.Create(_source, _keySelector, _comparer).ToArray(_resultSelector); + + public List ToList() => + Lookup.Create(_source, _keySelector, _comparer).ToList(_resultSelector); - public override List ToList() => - Lookup.Create(_source, _keySelector, _comparer).ToList(_resultSelector); + public int GetCount(bool onlyIfCheap) => + onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _comparer).Count; + } - public override int GetCount(bool onlyIfCheap) => - onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _comparer).Count; + internal sealed partial class GroupedEnumerable : IIListProvider> + { + public IGrouping[] ToArray() + { + IIListProvider> lookup = Lookup.Create(_source, _keySelector, _elementSelector, _comparer); + return lookup.ToArray(); } - private sealed partial class GroupByIterator + public List> ToList() { - public override IGrouping[] ToArray() => - Lookup.Create(_source, _keySelector, _elementSelector, _comparer).ToArray(); + IIListProvider> lookup = Lookup.Create(_source, _keySelector, _elementSelector, _comparer); + return lookup.ToList(); + } - public override List> ToList() => - Lookup.Create(_source, _keySelector, _elementSelector, _comparer).ToList(); + public int GetCount(bool onlyIfCheap) => + onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _elementSelector, _comparer).Count; + } - public override int GetCount(bool onlyIfCheap) => - onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _elementSelector, _comparer).Count; + internal sealed partial class GroupedEnumerable : IIListProvider> + { + public IGrouping[] ToArray() + { + IIListProvider> lookup = Lookup.Create(_source, _keySelector, _comparer); + return lookup.ToArray(); } - private sealed partial class GroupByIterator + public List> ToList() { - public override IGrouping[] ToArray() => - Lookup.Create(_source, _keySelector, _comparer).ToArray(); - - public override List> ToList() => - Lookup.Create(_source, _keySelector, _comparer).ToList(); - - public override int GetCount(bool onlyIfCheap) => - onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _comparer).Count; + IIListProvider> lookup = Lookup.Create(_source, _keySelector, _comparer); + return lookup.ToList(); } + + public int GetCount(bool onlyIfCheap) => + onlyIfCheap ? -1 : Lookup.Create(_source, _keySelector, _comparer).Count; } } diff --git a/src/libraries/System.Linq/src/System/Linq/Grouping.cs b/src/libraries/System.Linq/src/System/Linq/Grouping.cs index 6e19e4ab384e44..958642f624d0d3 100644 --- a/src/libraries/System.Linq/src/System/Linq/Grouping.cs +++ b/src/libraries/System.Linq/src/System/Linq/Grouping.cs @@ -29,7 +29,7 @@ public static IEnumerable> GroupBy(this return []; } - return new GroupByIterator(source, keySelector, comparer); + return new GroupedEnumerable(source, keySelector, comparer); } public static IEnumerable> GroupBy(this IEnumerable source, Func keySelector, Func elementSelector) => @@ -57,7 +57,7 @@ public static IEnumerable> GroupBy(source, keySelector, elementSelector, comparer); + return new GroupedEnumerable(source, keySelector, elementSelector, comparer); } public static IEnumerable GroupBy(this IEnumerable source, Func keySelector, Func, TResult> resultSelector) => @@ -85,7 +85,7 @@ public static IEnumerable GroupBy(this IEnumera return []; } - return new GroupByResultIterator(source, keySelector, resultSelector, comparer); + return new GroupedResultEnumerable(source, keySelector, resultSelector, comparer); } public static IEnumerable GroupBy(this IEnumerable source, Func keySelector, Func elementSelector, Func, TResult> resultSelector) => @@ -118,229 +118,7 @@ public static IEnumerable GroupBy(thi return []; } - return new GroupByResultIterator(source, keySelector, elementSelector, resultSelector, comparer); - } - - private sealed partial class GroupByResultIterator : Iterator - { - private readonly IEnumerable _source; - private readonly Func _keySelector; - private readonly Func _elementSelector; - private readonly IEqualityComparer? _comparer; - private readonly Func, TResult> _resultSelector; - - private Lookup? _lookup; - private Grouping? _g; - - public GroupByResultIterator(IEnumerable source, Func keySelector, Func elementSelector, Func, TResult> resultSelector, IEqualityComparer? comparer) - { - _source = source; - _keySelector = keySelector; - _elementSelector = elementSelector; - _comparer = comparer; - _resultSelector = resultSelector; - } - - private protected override Iterator Clone() => new GroupByResultIterator(_source, _keySelector, _elementSelector, _resultSelector, _comparer); - - public override bool MoveNext() - { - switch (_state) - { - case 1: - _lookup = Lookup.Create(_source, _keySelector, _elementSelector, _comparer); - _g = _lookup._lastGrouping; - if (_g is not null) - { - _state = 2; - goto ValidItem; - } - break; - - case 2: - Debug.Assert(_g is not null); - Debug.Assert(_lookup is not null); - if (_g != _lookup._lastGrouping) - { - goto ValidItem; - } - break; - } - - Dispose(); - return false; - - ValidItem: - _g = _g._next; - Debug.Assert(_g is not null); - _g.Trim(); - _current = _resultSelector(_g.Key, _g._elements); - return true; - } - } - - private sealed partial class GroupByResultIterator : Iterator - { - private readonly IEnumerable _source; - private readonly Func _keySelector; - private readonly IEqualityComparer? _comparer; - private readonly Func, TResult> _resultSelector; - - private Lookup? _lookup; - private Grouping? _g; - - public GroupByResultIterator(IEnumerable source, Func keySelector, Func, TResult> resultSelector, IEqualityComparer? comparer) - { - _source = source; - _keySelector = keySelector; - _resultSelector = resultSelector; - _comparer = comparer; - } - - private protected override Iterator Clone() => new GroupByResultIterator(_source, _keySelector, _resultSelector, _comparer); - - public override bool MoveNext() - { - switch (_state) - { - case 1: - _lookup = Lookup.Create(_source, _keySelector, _comparer); - _g = _lookup._lastGrouping; - if (_g is not null) - { - _state = 2; - goto ValidItem; - } - break; - - case 2: - Debug.Assert(_g is not null); - Debug.Assert(_lookup is not null); - if (_g != _lookup._lastGrouping) - { - goto ValidItem; - } - break; - } - - Dispose(); - return false; - - ValidItem: - _g = _g._next; - Debug.Assert(_g is not null); - _g.Trim(); - _current = _resultSelector(_g.Key, _g._elements); - return true; - } - } - - private sealed partial class GroupByIterator : Iterator> - { - private readonly IEnumerable _source; - private readonly Func _keySelector; - private readonly Func _elementSelector; - private readonly IEqualityComparer? _comparer; - - private Lookup? _lookup; - private Grouping? _g; - - public GroupByIterator(IEnumerable source, Func keySelector, Func elementSelector, IEqualityComparer? comparer) - { - _source = source; - _keySelector = keySelector; - _elementSelector = elementSelector; - _comparer = comparer; - } - - private protected override Iterator> Clone() => new GroupByIterator(_source, _keySelector, _elementSelector, _comparer); - - public override bool MoveNext() - { - switch (_state) - { - case 1: - _lookup = Lookup.Create(_source, _keySelector, _elementSelector, _comparer); - _g = _lookup._lastGrouping; - if (_g is not null) - { - _state = 2; - goto ValidItem; - } - break; - - case 2: - Debug.Assert(_g is not null); - Debug.Assert(_lookup is not null); - if (_g != _lookup._lastGrouping) - { - goto ValidItem; - } - break; - } - - Dispose(); - return false; - - ValidItem: - _g = _g._next; - Debug.Assert(_g is not null); - _current = _g; - return true; - } - } - - private sealed partial class GroupByIterator : Iterator> - { - private readonly IEnumerable _source; - private readonly Func _keySelector; - private readonly IEqualityComparer? _comparer; - - private Lookup? _lookup; - private Grouping? _g; - - public GroupByIterator(IEnumerable source, Func keySelector, IEqualityComparer? comparer) - { - _source = source; - _keySelector = keySelector; - _comparer = comparer; - } - - private protected override Iterator> Clone() => new GroupByIterator(_source, _keySelector, _comparer); - - public override bool MoveNext() - { - switch (_state) - { - case 1: - _lookup = Lookup.Create(_source, _keySelector, _comparer); - _g = _lookup._lastGrouping; - if (_g is not null) - { - _state = 2; - goto ValidItem; - } - break; - - case 2: - Debug.Assert(_g is not null); - Debug.Assert(_lookup is not null); - if (_g != _lookup._lastGrouping) - { - goto ValidItem; - } - break; - } - - Dispose(); - return false; - - ValidItem: - _g = _g._next; - Debug.Assert(_g is not null); - _current = _g; - return true; - } + return new GroupedResultEnumerable(source, keySelector, elementSelector, resultSelector, comparer); } } @@ -349,9 +127,15 @@ public interface IGrouping : IEnumerable TKey Key { get; } } + // It is (unfortunately) common to databind directly to Grouping.Key. + // Because of this, we have to declare this internal type public so that we + // can mark the Key property for public reflection. + // + // To limit the damage, the toolchain makes this type appear in a hidden assembly. + // (This is also why it is no longer a nested type of Lookup<,>). [DebuggerDisplay("Key = {Key}")] [DebuggerTypeProxy(typeof(SystemLinq_GroupingDebugView<,>))] - internal sealed class Grouping : IGrouping, IList + public class Grouping : IGrouping, IList { internal readonly TKey _key; internal readonly int _hashCode; @@ -388,12 +172,16 @@ internal void Trim() public IEnumerator GetEnumerator() { - Debug.Assert(_count > 0, "A grouping should only have been created if an element was being added to it."); - return new PartialArrayEnumerator(_elements, _count); + for (int i = 0; i < _count; i++) + { + yield return _elements[i]; + } } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + // DDB195907: implement IGrouping<>.Key implicitly + // so that WPF binding works on this property. public TKey Key => _key; int ICollection.Count => _count; @@ -409,7 +197,11 @@ public IEnumerator GetEnumerator() void ICollection.CopyTo(TElement[] array, int arrayIndex) => Array.Copy(_elements, 0, array, arrayIndex, _count); - bool ICollection.Remove(TElement item) => ThrowHelper.ThrowNotSupportedException_Boolean(); + bool ICollection.Remove(TElement item) + { + ThrowHelper.ThrowNotSupportedException(); + return false; + } int IList.IndexOf(TElement item) => Array.IndexOf(_elements, item, 0, _count); @@ -421,7 +213,7 @@ TElement IList.this[int index] { get { - if ((uint)index >= (uint)_count) + if (index < 0 || index >= _count) { ThrowHelper.ThrowArgumentOutOfRangeException(ExceptionArgument.index); } @@ -429,7 +221,100 @@ TElement IList.this[int index] return _elements[index]; } - set => ThrowHelper.ThrowNotSupportedException(); + set + { + ThrowHelper.ThrowNotSupportedException(); + } } } + + internal sealed partial class GroupedResultEnumerable : IEnumerable + { + private readonly IEnumerable _source; + private readonly Func _keySelector; + private readonly Func _elementSelector; + private readonly IEqualityComparer? _comparer; + private readonly Func, TResult> _resultSelector; + + public GroupedResultEnumerable(IEnumerable source, Func keySelector, Func elementSelector, Func, TResult> resultSelector, IEqualityComparer? comparer) + { + _source = source; + _keySelector = keySelector; + _elementSelector = elementSelector; + _comparer = comparer; + _resultSelector = resultSelector; + } + + public IEnumerator GetEnumerator() + { + Lookup lookup = Lookup.Create(_source, _keySelector, _elementSelector, _comparer); + return lookup.ApplyResultSelector(_resultSelector).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + internal sealed partial class GroupedResultEnumerable : IEnumerable + { + private readonly IEnumerable _source; + private readonly Func _keySelector; + private readonly IEqualityComparer? _comparer; + private readonly Func, TResult> _resultSelector; + + public GroupedResultEnumerable(IEnumerable source, Func keySelector, Func, TResult> resultSelector, IEqualityComparer? comparer) + { + _source = source; + _keySelector = keySelector; + _resultSelector = resultSelector; + _comparer = comparer; + } + + public IEnumerator GetEnumerator() + { + Lookup lookup = Lookup.Create(_source, _keySelector, _comparer); + return lookup.ApplyResultSelector(_resultSelector).GetEnumerator(); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + internal sealed partial class GroupedEnumerable : IEnumerable> + { + private readonly IEnumerable _source; + private readonly Func _keySelector; + private readonly Func _elementSelector; + private readonly IEqualityComparer? _comparer; + + public GroupedEnumerable(IEnumerable source, Func keySelector, Func elementSelector, IEqualityComparer? comparer) + { + _source = source; + _keySelector = keySelector; + _elementSelector = elementSelector; + _comparer = comparer; + } + + public IEnumerator> GetEnumerator() => + Lookup.Create(_source, _keySelector, _elementSelector, _comparer).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + + internal sealed partial class GroupedEnumerable : IEnumerable> + { + private readonly IEnumerable _source; + private readonly Func _keySelector; + private readonly IEqualityComparer? _comparer; + + public GroupedEnumerable(IEnumerable source, Func keySelector, IEqualityComparer? comparer) + { + _source = source; + _keySelector = keySelector; + _comparer = comparer; + } + + public IEnumerator> GetEnumerator() => + Lookup.Create(_source, _keySelector, _comparer).GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } } diff --git a/src/libraries/System.Linq/src/System/Linq/IIListProvider.cs b/src/libraries/System.Linq/src/System/Linq/IIListProvider.cs new file mode 100644 index 00000000000000..9eefc6e61e0ce4 --- /dev/null +++ b/src/libraries/System.Linq/src/System/Linq/IIListProvider.cs @@ -0,0 +1,33 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace System.Linq +{ + /// + /// An iterator that can produce an array or through an optimized path. + /// + internal interface IIListProvider : IEnumerable + { + /// + /// Produce an array of the sequence through an optimized path. + /// + /// The array. + TElement[] ToArray(); + + /// + /// Produce a of the sequence through an optimized path. + /// + /// The . + List ToList(); + + /// + /// Returns the count of elements in the sequence. + /// + /// If true then the count should only be calculated if doing + /// so is quick (sure or likely to be constant time), otherwise -1 should be returned. + /// The number of elements. + int GetCount(bool onlyIfCheap); + } +} diff --git a/src/libraries/System.Linq/src/System/Linq/IPartition.cs b/src/libraries/System.Linq/src/System/Linq/IPartition.cs new file mode 100644 index 00000000000000..86db1921b12f61 --- /dev/null +++ b/src/libraries/System.Linq/src/System/Linq/IPartition.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Linq +{ + /// + /// An iterator that supports random access and can produce a partial sequence of its items through an optimized path. + /// + internal interface IPartition : IIListProvider + { + /// + /// Creates a new partition that skips the specified number of elements from this sequence. + /// + /// The number of elements to skip. + /// An with the first items removed, or null if known empty. + IPartition? Skip(int count); + + /// + /// Creates a new partition that takes the specified number of elements from this sequence. + /// + /// The number of elements to take. + /// An with only the first items, or null if known empty. + IPartition? Take(int count); + + /// + /// Gets the item associated with a 0-based index in this sequence. + /// + /// The 0-based index to access. + /// true if the sequence contains an element at that index, false otherwise. + /// The element if is true, otherwise, the default value of . + TElement? TryGetElementAt(int index, out bool found); + + /// + /// Gets the first item in this sequence. + /// + /// true if the sequence contains an element, false otherwise. + /// The element if is true, otherwise, the default value of . + TElement? TryGetFirst(out bool found); + + /// + /// Gets the last item in this sequence. + /// + /// true if the sequence contains an element, false otherwise. + /// The element if is true, otherwise, the default value of . + TElement? TryGetLast(out bool found); + } +} diff --git a/src/libraries/System.Linq/src/System/Linq/Index.cs b/src/libraries/System.Linq/src/System/Linq/Index.cs index 49339b03d1ad39..1390a764c78d9d 100644 --- a/src/libraries/System.Linq/src/System/Linq/Index.cs +++ b/src/libraries/System.Linq/src/System/Linq/Index.cs @@ -13,7 +13,7 @@ public static partial class Enumerable /// is . public static IEnumerable<(int Index, TSource Item)> Index(this IEnumerable source) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } diff --git a/src/libraries/System.Linq/src/System/Linq/Intersect.cs b/src/libraries/System.Linq/src/System/Linq/Intersect.cs index 15d1179eaeb816..b8db27d7a84729 100644 --- a/src/libraries/System.Linq/src/System/Linq/Intersect.cs +++ b/src/libraries/System.Linq/src/System/Linq/Intersect.cs @@ -11,12 +11,12 @@ public static partial class Enumerable public static IEnumerable Intersect(this IEnumerable first, IEnumerable second, IEqualityComparer? comparer) { - if (first is null) + if (first == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.first); } - if (second is null) + if (second == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.second); } diff --git a/src/libraries/System.Linq/src/System/Linq/Iterator.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Iterator.SpeedOpt.cs deleted file mode 100644 index dfafa98cb27511..00000000000000 --- a/src/libraries/System.Linq/src/System/Linq/Iterator.SpeedOpt.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; - -namespace System.Linq -{ - public static partial class Enumerable - { - private abstract partial class Iterator - { - /// - /// Produce an array of the sequence through an optimized path. - /// - /// The array. - public abstract TSource[] ToArray(); - - /// - /// Produce a of the sequence through an optimized path. - /// - /// The . - public abstract List ToList(); - - /// - /// Returns the count of elements in the sequence. - /// - /// If true then the count should only be calculated if doing - /// so is quick (sure or likely to be constant time), otherwise -1 should be returned. - /// The number of elements. - public abstract int GetCount(bool onlyIfCheap); - - /// - /// Creates a new iterator that skips the specified number of elements from this sequence. - /// - /// The number of elements to skip. - /// An with the first items removed, or null if known empty. - public virtual Iterator? Skip(int count) => new IEnumerableSkipTakeIterator(this, count, -1); - - /// - /// Creates a new iterator that takes the specified number of elements from this sequence. - /// - /// The number of elements to take. - /// An with only the first items, or null if known empty. - public virtual Iterator? Take(int count) => new IEnumerableSkipTakeIterator(this, 0, count - 1); - - /// - /// Gets the item associated with a 0-based index in this sequence. - /// - /// The 0-based index to access. - /// true if the sequence contains an element at that index, false otherwise. - /// The element if is true, otherwise, the default value of . - public virtual TSource? TryGetElementAt(int index, out bool found) => - index == 0 ? TryGetFirst(out found) : - TryGetElementAtNonIterator(this, index, out found); - - /// - /// Gets the first item in this sequence. - /// - /// true if the sequence contains an element, false otherwise. - /// The element if is true, otherwise, the default value of . - public virtual TSource? TryGetFirst(out bool found) => TryGetFirstNonIterator(this, out found); - - /// - /// Gets the last item in this sequence. - /// - /// true if the sequence contains an element, false otherwise. - /// The element if is true, otherwise, the default value of . - public virtual TSource? TryGetLast(out bool found) => TryGetLastNonIterator(this, out found); - } - } -} diff --git a/src/libraries/System.Linq/src/System/Linq/Iterator.cs b/src/libraries/System.Linq/src/System/Linq/Iterator.cs index 8d5982eb0b9b49..b9e8c7b58c0548 100644 --- a/src/libraries/System.Linq/src/System/Linq/Iterator.cs +++ b/src/libraries/System.Linq/src/System/Linq/Iterator.cs @@ -28,12 +28,19 @@ public static partial class Enumerable /// /// /// - private abstract partial class Iterator : IEnumerable, IEnumerator + internal abstract class Iterator : IEnumerable, IEnumerator { - private readonly int _threadId = Environment.CurrentManagedThreadId; + private readonly int _threadId; + internal int _state; + internal TSource _current = default!; - private protected int _state; - private protected TSource _current = default!; + /// + /// Initializes a new instance of the class. + /// + protected Iterator() + { + _threadId = Environment.CurrentManagedThreadId; + } /// /// The item currently yielded by this iterator. @@ -46,7 +53,7 @@ private abstract partial class Iterator : IEnumerable, IEnumer /// /// This method is called if is called more than once. /// - private protected abstract Iterator Clone(); + public abstract Iterator Clone(); /// /// Puts this iterator in a state whereby no further enumeration will take place. @@ -69,7 +76,7 @@ public virtual void Dispose() /// that created this iterator, the result will be this iterator. Otherwise, the result /// will be a shallow copy of this iterator. /// - public Iterator GetEnumerator() + public IEnumerator GetEnumerator() { Iterator enumerator = _state == 0 && _threadId == Environment.CurrentManagedThreadId ? this : Clone(); enumerator._state = 1; @@ -87,24 +94,22 @@ public Iterator GetEnumerator() /// /// The type of the mapped items. /// The selector used to map each item. - public virtual IEnumerable Select(Func selector) => -#if OPTIMIZE_FOR_SIZE - new IEnumerableSelectIterator(this, selector); -#else - new IteratorSelectIterator(this, selector); -#endif - + public virtual IEnumerable Select(Func selector) + { + return new SelectEnumerableIterator(this, selector); + } /// /// Returns an enumerable that filters each item in this iterator based on a predicate. /// /// The predicate used to filter each item. - public virtual IEnumerable Where(Func predicate) => - new IEnumerableWhereIterator(this, predicate); + public virtual IEnumerable Where(Func predicate) + { + return new WhereEnumerableIterator(this, predicate); + } object? IEnumerator.Current => Current; - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); void IEnumerator.Reset() => ThrowHelper.ThrowNotSupportedException(); diff --git a/src/libraries/System.Linq/src/System/Linq/Join.cs b/src/libraries/System.Linq/src/System/Linq/Join.cs index 531a8518e888d2..b1c56e01725a00 100644 --- a/src/libraries/System.Linq/src/System/Linq/Join.cs +++ b/src/libraries/System.Linq/src/System/Linq/Join.cs @@ -12,27 +12,27 @@ public static IEnumerable Join(this IEnu public static IEnumerable Join(this IEnumerable outer, IEnumerable inner, Func outerKeySelector, Func innerKeySelector, Func resultSelector, IEqualityComparer? comparer) { - if (outer is null) + if (outer == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.outer); } - if (inner is null) + if (inner == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.inner); } - if (outerKeySelector is null) + if (outerKeySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.outerKeySelector); } - if (innerKeySelector is null) + if (innerKeySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.innerKeySelector); } - if (resultSelector is null) + if (resultSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.resultSelector); } @@ -58,7 +58,7 @@ private static IEnumerable JoinIterator( { TOuter item = e.Current; Grouping? g = lookup.GetGrouping(outerKeySelector(item), create: false); - if (g is not null) + if (g != null) { int count = g._count; TInner[] elements = g._elements; diff --git a/src/libraries/System.Linq/src/System/Linq/Last.cs b/src/libraries/System.Linq/src/System/Linq/Last.cs index 007ee1659c3f5f..568f0d8670faf8 100644 --- a/src/libraries/System.Linq/src/System/Linq/Last.cs +++ b/src/libraries/System.Linq/src/System/Linq/Last.cs @@ -63,20 +63,16 @@ public static TSource LastOrDefault(this IEnumerable source, F private static TSource? TryGetLast(this IEnumerable source, out bool found) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - return -#if !OPTIMIZE_FOR_SIZE - source is Iterator iterator ? iterator.TryGetLast(out found) : -#endif - TryGetLastNonIterator(source, out found); - } + if (source is IPartition partition) + { + return partition.TryGetLast(out found); + } - private static TSource? TryGetLastNonIterator(IEnumerable source, out bool found) - { if (source is IList list) { int count = list.Count; @@ -111,17 +107,17 @@ public static TSource LastOrDefault(this IEnumerable source, F private static TSource? TryGetLast(this IEnumerable source, Func predicate, out bool found) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } - if (source is OrderedIterator ordered) + if (source is OrderedEnumerable ordered) { return ordered.TryGetLast(predicate, out found); } diff --git a/src/libraries/System.Linq/src/System/Linq/Lookup.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Lookup.SpeedOpt.cs index 42e6420676498a..16a9d7e0a3f4a0 100644 --- a/src/libraries/System.Linq/src/System/Linq/Lookup.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Lookup.SpeedOpt.cs @@ -6,19 +6,34 @@ namespace System.Linq { - public partial class Lookup + public partial class Lookup : IIListProvider> { + IGrouping[] IIListProvider>.ToArray() + { + IGrouping[] array; + if (_count > 0) + { + array = new IGrouping[_count]; + Fill(_lastGrouping, array); + } + else + { + array = []; + } + return array; + } + internal TResult[] ToArray(Func, TResult> resultSelector) { TResult[] array = new TResult[_count]; int index = 0; Grouping? g = _lastGrouping; - if (g is not null) + if (g != null) { do { g = g._next; - Debug.Assert(g is not null); + Debug.Assert(g != null); g.Trim(); array[index] = resultSelector(g._key, g._elements); @@ -29,5 +44,38 @@ internal TResult[] ToArray(Func, TResult> r return array; } + + List> IIListProvider>.ToList() + { + var list = new List>(_count); + if (_count > 0) + { + Fill(_lastGrouping, Enumerable.SetCountAndGetSpan(list, _count)); + } + + return list; + } + + private static void Fill(Grouping? lastGrouping, Span> results) + { + int index = 0; + Grouping? g = lastGrouping; + if (g != null) + { + do + { + g = g._next; + Debug.Assert(g != null); + + results[index] = g; + ++index; + } + while (g != lastGrouping); + } + + Debug.Assert(index == results.Length, "All list elements were not initialized."); + } + + int IIListProvider>.GetCount(bool onlyIfCheap) => _count; } } diff --git a/src/libraries/System.Linq/src/System/Linq/Lookup.cs b/src/libraries/System.Linq/src/System/Linq/Lookup.cs index 7669aaaef7e190..055bf6c61018bd 100644 --- a/src/libraries/System.Linq/src/System/Linq/Lookup.cs +++ b/src/libraries/System.Linq/src/System/Linq/Lookup.cs @@ -14,12 +14,12 @@ public static ILookup ToLookup(this IEnumerable ToLookup(this IEnumerable source, Func keySelector, IEqualityComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (keySelector is null) + if (keySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.keySelector); } @@ -37,17 +37,17 @@ public static ILookup ToLookup(this IEn public static ILookup ToLookup(this IEnumerable source, Func keySelector, Func elementSelector, IEqualityComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (keySelector is null) + if (keySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.keySelector); } - if (elementSelector is null) + if (elementSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.elementSelector); } @@ -76,16 +76,16 @@ public partial class Lookup : ILookup { private readonly IEqualityComparer _comparer; private Grouping[] _groupings; - internal Grouping? _lastGrouping; + private Grouping? _lastGrouping; private int _count; internal static Lookup Create(IEnumerable source, Func keySelector, Func elementSelector, IEqualityComparer? comparer) { - Debug.Assert(source is not null); - Debug.Assert(keySelector is not null); - Debug.Assert(elementSelector is not null); + Debug.Assert(source != null); + Debug.Assert(keySelector != null); + Debug.Assert(elementSelector != null); - var lookup = new CollectionLookup(comparer); + Lookup lookup = new Lookup(comparer); foreach (TSource item in source) { lookup.GetGrouping(keySelector(item), create: true)!.Add(elementSelector(item)); @@ -96,10 +96,10 @@ internal static Lookup Create(IEnumerable sour internal static Lookup Create(IEnumerable source, Func keySelector, IEqualityComparer? comparer) { - Debug.Assert(source is not null); - Debug.Assert(keySelector is not null); + Debug.Assert(source != null); + Debug.Assert(keySelector != null); - var lookup = new CollectionLookup(comparer); + Lookup lookup = new Lookup(comparer); foreach (TElement item in source) { lookup.GetGrouping(keySelector(item), create: true)!.Add(item); @@ -110,11 +110,11 @@ internal static Lookup Create(IEnumerable source, Func internal static Lookup CreateForJoin(IEnumerable source, Func keySelector, IEqualityComparer? comparer) { - var lookup = new CollectionLookup(comparer); + Lookup lookup = new Lookup(comparer); foreach (TElement item in source) { TKey key = keySelector(item); - if (key is not null) + if (key != null) { lookup.GetGrouping(key, create: true)!.Add(item); } @@ -123,7 +123,7 @@ internal static Lookup CreateForJoin(IEnumerable sourc return lookup; } - private protected Lookup(IEqualityComparer? comparer) + private Lookup(IEqualityComparer? comparer) { _comparer = comparer ?? EqualityComparer.Default; _groupings = new Grouping[7]; @@ -133,18 +133,18 @@ private protected Lookup(IEqualityComparer? comparer) public IEnumerable this[TKey key] => GetGrouping(key, create: false) ?? Enumerable.Empty(); - public bool Contains(TKey key) => GetGrouping(key, create: false) is not null; + public bool Contains(TKey key) => GetGrouping(key, create: false) != null; public IEnumerator> GetEnumerator() { Grouping? g = _lastGrouping; - if (g is not null) + if (g != null) { do { g = g._next; - Debug.Assert(g is not null); + Debug.Assert(g != null); yield return g; } while (g != _lastGrouping); @@ -155,7 +155,7 @@ internal List ToList(Func, TResult { List list = new List(_count); Grouping? g = _lastGrouping; - if (g is not null) + if (g != null) { Span span = Enumerable.SetCountAndGetSpan(list, _count); int index = 0; @@ -163,7 +163,7 @@ internal List ToList(Func, TResult { g = g._next; - Debug.Assert(g is not null); + Debug.Assert(g != null); g.Trim(); span[index] = resultSelector(g._key, g._elements); ++index; @@ -179,13 +179,13 @@ internal List ToList(Func, TResult public IEnumerable ApplyResultSelector(Func, TResult> resultSelector) { Grouping? g = _lastGrouping; - if (g is not null) + if (g != null) { do { g = g._next; - Debug.Assert(g is not null); + Debug.Assert(g != null); g.Trim(); yield return resultSelector(g._key, g._elements); } @@ -198,13 +198,13 @@ public IEnumerable ApplyResultSelector(Func? GetGrouping(TKey key, bool create) { int hashCode = InternalGetHashCode(key); - for (Grouping? g = _groupings[(uint)hashCode % _groupings.Length]; g is not null; g = g._hashNext) + for (Grouping? g = _groupings[(uint)hashCode % _groupings.Length]; g != null; g = g._hashNext) { if (g._hashCode == hashCode && _comparer.Equals(g._key, key)) { @@ -223,7 +223,7 @@ private int InternalGetHashCode(TKey key) Grouping g = new Grouping(key, hashCode); g._hashNext = _groupings[index]; _groupings[index] = g; - if (_lastGrouping is null) + if (_lastGrouping == null) { g._next = g; } @@ -259,68 +259,16 @@ private void Resize() } } - internal sealed class CollectionLookup : Lookup, ICollection>, IReadOnlyCollection> - { - internal CollectionLookup(IEqualityComparer? comparer) : base(comparer) { } - - void ICollection>.CopyTo(IGrouping[] array, int arrayIndex) - { - ArgumentNullException.ThrowIfNull(array); - ArgumentOutOfRangeException.ThrowIfNegative(arrayIndex); - ArgumentOutOfRangeException.ThrowIfGreaterThan(arrayIndex, array.Length); - ArgumentOutOfRangeException.ThrowIfLessThan(array.Length - arrayIndex, Count, nameof(arrayIndex)); - - Grouping? g = _lastGrouping; - if (g is not null) - { - do - { - g = g._next; - Debug.Assert(g is not null); - - array[arrayIndex] = g; - ++arrayIndex; - } - while (g != _lastGrouping); - } - } - - bool ICollection>.Contains(IGrouping item) - { - ArgumentNullException.ThrowIfNull(item); - return GetGrouping(item.Key, create: false) is { } grouping && grouping == item; - } - - bool ICollection>.IsReadOnly => true; - void ICollection>.Add(IGrouping item) => throw new NotSupportedException(); - void ICollection>.Clear() => throw new NotSupportedException(); - bool ICollection>.Remove(IGrouping item) => throw new NotSupportedException(); - } - [DebuggerDisplay("Count = 0")] [DebuggerTypeProxy(typeof(SystemLinq_LookupDebugView<,>))] - internal sealed class EmptyLookup : ILookup, ICollection>, IReadOnlyCollection> + internal sealed class EmptyLookup : ILookup { public static readonly EmptyLookup Instance = new(); public IEnumerable this[TKey key] => []; public int Count => 0; - + public bool Contains(TKey key) => false; public IEnumerator> GetEnumerator() => Enumerable.Empty>().GetEnumerator(); IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - - public bool Contains(TKey key) => false; - public bool Contains(IGrouping item) => false; - public void CopyTo(IGrouping[] array, int arrayIndex) - { - ArgumentNullException.ThrowIfNull(array); - ArgumentOutOfRangeException.ThrowIfNegative(arrayIndex); - ArgumentOutOfRangeException.ThrowIfGreaterThan(arrayIndex, array.Length); - } - - public bool IsReadOnly => true; - public void Add(IGrouping item) => throw new NotSupportedException(); - public void Clear() => throw new NotSupportedException(); - public bool Remove(IGrouping item) => throw new NotSupportedException(); } } diff --git a/src/libraries/System.Linq/src/System/Linq/Max.cs b/src/libraries/System.Linq/src/System/Linq/Max.cs index 998e5ff947700d..d0da2f7bfd7415 100644 --- a/src/libraries/System.Linq/src/System/Linq/Max.cs +++ b/src/libraries/System.Linq/src/System/Linq/Max.cs @@ -27,7 +27,7 @@ public static partial class Enumerable private static T? MaxInteger(this IEnumerable source) where T : struct, IBinaryInteger { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -167,7 +167,7 @@ private static T MaxFloat(this IEnumerable source) where T : struct, IFloa private static T? MaxFloat(this IEnumerable source) where T : struct, IFloatingPointIeee754 { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -271,7 +271,7 @@ public static decimal Max(this IEnumerable source) public static decimal? Max(this IEnumerable source) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -322,7 +322,7 @@ public static decimal Max(this IEnumerable source) /// public static TSource? Max(this IEnumerable source, IComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -347,7 +347,7 @@ public static decimal Max(this IEnumerable source) TSource? value = default; using (IEnumerator e = source.GetEnumerator()) { - if (value is null) + if (value == null) { do { @@ -358,12 +358,12 @@ public static decimal Max(this IEnumerable source) value = e.Current; } - while (value is null); + while (value == null); while (e.MoveNext()) { TSource next = e.Current; - if (next is not null && comparer.Compare(next, value) > 0) + if (next != null && comparer.Compare(next, value) > 0) { value = next; } @@ -432,12 +432,12 @@ public static decimal Max(this IEnumerable source) /// public static TSource? MaxBy(this IEnumerable source, Func keySelector, IComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (keySelector is null) + if (keySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.keySelector); } @@ -463,7 +463,7 @@ public static decimal Max(this IEnumerable source) if (default(TKey) is null) { - if (key is null) + if (key == null) { TSource firstValue = value; @@ -478,14 +478,14 @@ public static decimal Max(this IEnumerable source) value = e.Current; key = keySelector(value); } - while (key is null); + while (key == null); } while (e.MoveNext()) { TSource nextValue = e.Current; TKey nextKey = keySelector(nextValue); - if (nextKey is not null && comparer.Compare(nextKey, key) > 0) + if (nextKey != null && comparer.Compare(nextKey, key) > 0) { key = nextKey; value = nextValue; @@ -535,12 +535,12 @@ public static decimal Max(this IEnumerable source) private static TResult MaxInteger(this IEnumerable source, Func selector) where TResult : struct, IBinaryInteger { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -569,12 +569,12 @@ private static TResult MaxInteger(this IEnumerable so private static TResult? MaxInteger(this IEnumerable source, Func selector) where TResult : struct, IBinaryInteger { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -644,12 +644,12 @@ private static TResult MaxInteger(this IEnumerable so private static TResult MaxFloat(this IEnumerable source, Func selector) where TResult : struct, IFloatingPointIeee754 { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -688,12 +688,12 @@ private static TResult MaxFloat(this IEnumerable sour private static TResult? MaxFloat(this IEnumerable source, Func selector) where TResult : struct, IFloatingPointIeee754 { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -747,12 +747,12 @@ private static TResult MaxFloat(this IEnumerable sour public static decimal Max(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -781,12 +781,12 @@ public static decimal Max(this IEnumerable source, Func(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -823,12 +823,12 @@ public static decimal Max(this IEnumerable source, Func(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -836,7 +836,7 @@ public static decimal Max(this IEnumerable source, Func e = source.GetEnumerator()) { - if (value is null) + if (value == null) { do { @@ -847,13 +847,13 @@ public static decimal Max(this IEnumerable source, Func comparer = Comparer.Default; while (e.MoveNext()) { TResult x = selector(e.Current); - if (x is not null && comparer.Compare(x, value) > 0) + if (x != null && comparer.Compare(x, value) > 0) { value = x; } diff --git a/src/libraries/System.Linq/src/System/Linq/Min.cs b/src/libraries/System.Linq/src/System/Linq/Min.cs index 39cf8723a9079f..3a0f2130d966b3 100644 --- a/src/libraries/System.Linq/src/System/Linq/Min.cs +++ b/src/libraries/System.Linq/src/System/Linq/Min.cs @@ -27,7 +27,7 @@ public static partial class Enumerable private static T? MinInteger(this IEnumerable source) where T : struct, IBinaryInteger { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -151,7 +151,7 @@ private static T MinFloat(this IEnumerable source) where T : struct, IFloa private static T? MinFloat(this IEnumerable source) where T : struct, IFloatingPointIeee754 { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -249,7 +249,7 @@ public static decimal Min(this IEnumerable source) public static decimal? Min(this IEnumerable source) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -300,7 +300,7 @@ public static decimal Min(this IEnumerable source) /// public static TSource? Min(this IEnumerable source, IComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -325,7 +325,7 @@ public static decimal Min(this IEnumerable source) TSource? value = default; using (IEnumerator e = source.GetEnumerator()) { - if (value is null) + if (value == null) { do { @@ -336,12 +336,12 @@ public static decimal Min(this IEnumerable source) value = e.Current; } - while (value is null); + while (value == null); while (e.MoveNext()) { TSource next = e.Current; - if (next is not null && comparer.Compare(next, value) < 0) + if (next != null && comparer.Compare(next, value) < 0) { value = next; } @@ -410,12 +410,12 @@ public static decimal Min(this IEnumerable source) /// public static TSource? MinBy(this IEnumerable source, Func keySelector, IComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (keySelector is null) + if (keySelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.keySelector); } @@ -441,7 +441,7 @@ public static decimal Min(this IEnumerable source) if (default(TKey) is null) { - if (key is null) + if (key == null) { TSource firstValue = value; @@ -456,14 +456,14 @@ public static decimal Min(this IEnumerable source) value = e.Current; key = keySelector(value); } - while (key is null); + while (key == null); } while (e.MoveNext()) { TSource nextValue = e.Current; TKey nextKey = keySelector(nextValue); - if (nextKey is not null && comparer.Compare(nextKey, key) < 0) + if (nextKey != null && comparer.Compare(nextKey, key) < 0) { key = nextKey; value = nextValue; @@ -513,12 +513,12 @@ public static decimal Min(this IEnumerable source) private static TResult MinInteger(this IEnumerable source, Func selector) where TResult : struct, IBinaryInteger { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -547,12 +547,12 @@ private static TResult MinInteger(this IEnumerable so private static TResult? MinInteger(this IEnumerable source, Func selector) where TResult : struct, IBinaryInteger { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -604,12 +604,12 @@ private static TResult MinInteger(this IEnumerable so private static TResult MinFloat(this IEnumerable source, Func selector) where TResult : struct, IFloatingPointIeee754 { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -656,12 +656,12 @@ private static TResult MinFloat(this IEnumerable sour private static TResult? MinFloat(this IEnumerable source, Func selector) where TResult : struct, IFloatingPointIeee754 { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -710,12 +710,12 @@ private static TResult MinFloat(this IEnumerable sour public static decimal Min(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -744,12 +744,12 @@ public static decimal Min(this IEnumerable source, Func(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -786,12 +786,12 @@ public static decimal Min(this IEnumerable source, Func(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -799,7 +799,7 @@ public static decimal Min(this IEnumerable source, Func e = source.GetEnumerator()) { - if (value is null) + if (value == null) { do { @@ -810,13 +810,13 @@ public static decimal Min(this IEnumerable source, Func comparer = Comparer.Default; while (e.MoveNext()) { TResult x = selector(e.Current); - if (x is not null && comparer.Compare(x, value) < 0) + if (x != null && comparer.Compare(x, value) < 0) { value = x; } diff --git a/src/libraries/System.Linq/src/System/Linq/OfType.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/OfType.SpeedOpt.cs deleted file mode 100644 index 834fcdad006d7d..00000000000000 --- a/src/libraries/System.Linq/src/System/Linq/OfType.SpeedOpt.cs +++ /dev/null @@ -1,175 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Linq -{ - public static partial class Enumerable - { - private sealed partial class OfTypeIterator - { - public override int GetCount(bool onlyIfCheap) - { - if (onlyIfCheap) - { - return -1; - } - - int count = 0; - foreach (object? item in _source) - { - if (item is TResult) - { - checked { count++; } - } - } - - return count; - } - - public override TResult[] ToArray() - { - SegmentedArrayBuilder.ScratchBuffer scratch = default; - SegmentedArrayBuilder builder = new(scratch); - - foreach (object? item in _source) - { - if (item is TResult castItem) - { - builder.Add(castItem); - } - } - - TResult[] result = builder.ToArray(); - builder.Dispose(); - - return result; - } - - public override List ToList() - { - var list = new List(); - - foreach (object? item in _source) - { - if (item is TResult castItem) - { - list.Add(castItem); - } - } - - return list; - } - - public override TResult? TryGetFirst(out bool found) - { - foreach (object? item in _source) - { - if (item is TResult castItem) - { - found = true; - return castItem; - } - } - - found = false; - return default; - } - - public override TResult? TryGetLast(out bool found) - { - IEnumerator e = _source.GetEnumerator(); - try - { - if (e.MoveNext()) - { - do - { - if (e.Current is TResult last) - { - found = true; - - while (e.MoveNext()) - { - if (e.Current is TResult castCurrent) - { - last = castCurrent; - } - } - - return last; - } - } - while (e.MoveNext()); - } - } - finally - { - (e as IDisposable)?.Dispose(); - } - - found = false; - return default; - } - - public override TResult? TryGetElementAt(int index, out bool found) - { - if (index >= 0) - { - foreach (object? item in _source) - { - if (item is TResult castItem) - { - if (index == 0) - { - found = true; - return castItem; - } - - index--; - } - } - } - - found = false; - return default; - } - - public override IEnumerable Select(Func selector) - { - // If the source is any generic enumerable of a reference type, which should be the 90% case, it'll covariantly - // implement IEnumerable, and we can optimize the OfType().Select case by treating the OfType instead like - // a Where, using the same WhereSelectIterators that are used for Where.Select. - if (!typeof(TResult).IsValueType && _source is IEnumerable objectSource) - { - // Unsafe.As here is safe because we're only dealing with reference types, and we know by construction that only - // TResult instances will be passed in. Using Unsafe.As allows us to avoid an extra closure and delegate allocation. - Func localSelector = -#if DEBUG - o => - { - Debug.Assert(o is TResult); - return selector((TResult)o); - }; -#else - Unsafe.As>(selector); -#endif - - // We can special-case arrays and IEnumerable to use the corresponding WhereSelectIterators because - // they're covariant. It's not worthwhile checking for List to use the ListWhereSelectIterator - // because List<> is not covariant. - Func isTResult = static o => o is TResult; - return objectSource is object[] array ? - new ArrayWhereSelectIterator(array, isTResult, localSelector) : - new IEnumerableWhereSelectIterator(objectSource, isTResult, localSelector); - } - - return base.Select(selector); - } - } - } -} diff --git a/src/libraries/System.Linq/src/System/Linq/OfType.cs b/src/libraries/System.Linq/src/System/Linq/OfType.cs deleted file mode 100644 index ec4db225b2ee72..00000000000000 --- a/src/libraries/System.Linq/src/System/Linq/OfType.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; - -namespace System.Linq -{ - public static partial class Enumerable - { - public static IEnumerable OfType(this IEnumerable source) - { - if (source is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); - } - - if (default(TResult) is not null && source is IEnumerable typedSource) - { - // The source was already an IEnumerable and TResult can't be null. As - // such, all values the original input can yield are valid, and we can just return - // the strongly-typed input directly as if this were Cast rather than OfType. - return typedSource; - } - - return new OfTypeIterator(source); - } - - private sealed partial class OfTypeIterator(IEnumerable source) : Iterator - { - private readonly IEnumerable _source = source; - private IEnumerator? _enumerator; - - private protected override Iterator Clone() => new OfTypeIterator(_source); - - public override bool MoveNext() - { - switch (_state) - { - case 1: - _enumerator = _source.GetEnumerator(); - _state = 2; - goto case 2; - - case 2: - Debug.Assert(_enumerator is not null); - while (_enumerator.MoveNext()) - { - if (_enumerator.Current is TResult result) - { - _current = result; - return true; - } - } - - Dispose(); - break; - } - - return false; - } - - public override void Dispose() - { - (_enumerator as IDisposable)?.Dispose(); - _enumerator = null; - - base.Dispose(); - } - } - } -} diff --git a/src/libraries/System.Linq/src/System/Linq/OrderBy.cs b/src/libraries/System.Linq/src/System/Linq/OrderBy.cs index b276539109973d..aa7a08ee81c9f6 100644 --- a/src/libraries/System.Linq/src/System/Linq/OrderBy.cs +++ b/src/libraries/System.Linq/src/System/Linq/OrderBy.cs @@ -44,14 +44,14 @@ public static IOrderedEnumerable Order(this IEnumerable source) => /// public static IOrderedEnumerable Order(this IEnumerable source, IComparer? comparer) => TypeIsImplicitlyStable() && (comparer is null || comparer == Comparer.Default) ? - new ImplicitlyStableOrderedIterator(source, descending: false) : + new OrderedImplicitlyStableEnumerable(source, descending: false) : OrderBy(source, EnumerableSorter.IdentityFunc, comparer); public static IOrderedEnumerable OrderBy(this IEnumerable source, Func keySelector) - => new OrderedIterator(source, keySelector, null, false, null); + => new OrderedEnumerable(source, keySelector, null, false, null); public static IOrderedEnumerable OrderBy(this IEnumerable source, Func keySelector, IComparer? comparer) - => new OrderedIterator(source, keySelector, comparer, false, null); + => new OrderedEnumerable(source, keySelector, comparer, false, null); /// /// Sorts the elements of a sequence in descending order. @@ -89,18 +89,18 @@ public static IOrderedEnumerable OrderDescending(this IEnumerable sourc /// public static IOrderedEnumerable OrderDescending(this IEnumerable source, IComparer? comparer) => TypeIsImplicitlyStable() && (comparer is null || comparer == Comparer.Default) ? - new ImplicitlyStableOrderedIterator(source, descending: true) : + new OrderedImplicitlyStableEnumerable(source, descending: true) : OrderByDescending(source, EnumerableSorter.IdentityFunc, comparer); public static IOrderedEnumerable OrderByDescending(this IEnumerable source, Func keySelector) => - new OrderedIterator(source, keySelector, null, true, null); + new OrderedEnumerable(source, keySelector, null, true, null); public static IOrderedEnumerable OrderByDescending(this IEnumerable source, Func keySelector, IComparer? comparer) => - new OrderedIterator(source, keySelector, comparer, true, null); + new OrderedEnumerable(source, keySelector, comparer, true, null); public static IOrderedEnumerable ThenBy(this IOrderedEnumerable source, Func keySelector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -110,7 +110,7 @@ public static IOrderedEnumerable ThenBy(this IOrderedEnu public static IOrderedEnumerable ThenBy(this IOrderedEnumerable source, Func keySelector, IComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -120,7 +120,7 @@ public static IOrderedEnumerable ThenBy(this IOrderedEnu public static IOrderedEnumerable ThenByDescending(this IOrderedEnumerable source, Func keySelector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -130,7 +130,7 @@ public static IOrderedEnumerable ThenByDescending(this I public static IOrderedEnumerable ThenByDescending(this IOrderedEnumerable source, Func keySelector, IComparer? comparer) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -140,27 +140,14 @@ public static IOrderedEnumerable ThenByDescending(this I /// Gets whether the results of an unstable sort will be observably the same as a stable sort. [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool TypeIsImplicitlyStable() - { - Type t = typeof(T); - if (typeof(T).IsEnum) - { - t = typeof(T).GetEnumUnderlyingType(); - } - - // Check for integral primitive types that compare equally iff they have the same bit pattern. - // bool is included because, even though technically it can have 256 different values, anything - // other than 0/1 is only producible using unsafe code. It's tempting to include a type like string - // here, as it's so commonly used with ordering, but two different string objects can compare equally, - // and their reference identity can be observable in a stable vs unstable sort. - return - t == typeof(sbyte) || t == typeof(byte) || t == typeof(bool) || - t == typeof(short) || t == typeof(ushort) || t == typeof(char) || - t == typeof(int) || t == typeof(uint) || - t == typeof(long) || t == typeof(ulong) || - t == typeof(Int128) || t == typeof(UInt128) || - t == typeof(nint) || t == typeof(nuint); - } + internal static bool TypeIsImplicitlyStable() => + typeof(T) == typeof(sbyte) || typeof(T) == typeof(byte) || + typeof(T) == typeof(int) || typeof(T) == typeof(uint) || + typeof(T) == typeof(short) || typeof(T) == typeof(ushort) || + typeof(T) == typeof(long) || typeof(T) == typeof(ulong) || + typeof(T) == typeof(Int128) || typeof(T) == typeof(UInt128) || + typeof(T) == typeof(nint) || typeof(T) == typeof(nuint) || + typeof(T) == typeof(bool) || typeof(T) == typeof(char); } public interface IOrderedEnumerable : IEnumerable diff --git a/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.SpeedOpt.cs index 57f70b75f3ac89..615c196cced3d6 100644 --- a/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.SpeedOpt.cs @@ -8,539 +8,428 @@ namespace System.Linq { - public static partial class Enumerable + internal abstract partial class OrderedEnumerable : IPartition { - private abstract partial class OrderedIterator + public virtual TElement[] ToArray() { - public override TElement[] ToArray() + TElement[] buffer = _source.ToArray(); + if (buffer.Length == 0) { - TElement[] buffer = _source.ToArray(); - if (buffer.Length == 0) - { - return buffer; - } - - TElement[] array = new TElement[buffer.Length]; - Fill(buffer, array); - return array; + return buffer; } - public override List ToList() - { - TElement[] buffer = _source.ToArray(); + TElement[] array = new TElement[buffer.Length]; + Fill(buffer, array); + return array; + } - List list = new(); - if (buffer.Length > 0) - { - Fill(buffer, SetCountAndGetSpan(list, buffer.Length)); - } + public virtual List ToList() + { + TElement[] buffer = _source.ToArray(); - return list; + List list = new(); + if (buffer.Length > 0) + { + Fill(buffer, Enumerable.SetCountAndGetSpan(list, buffer.Length)); } - private void Fill(TElement[] buffer, Span destination) + return list; + } + + private void Fill(TElement[] buffer, Span destination) + { + int[] map = SortedMap(buffer); + for (int i = 0; i < destination.Length; i++) { - int[] map = SortedMap(buffer); - for (int i = 0; i < destination.Length; i++) - { - destination[i] = buffer[map[i]]; - } + destination[i] = buffer[map[i]]; } + } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) + { + if (_source is IIListProvider listProv) { - if (_source is Iterator iterator) - { - return iterator.GetCount(onlyIfCheap); - } + return listProv.GetCount(onlyIfCheap); + } + + return !onlyIfCheap || _source is ICollection || _source is ICollection ? _source.Count() : -1; + } - return !onlyIfCheap || _source is ICollection || _source is ICollection ? _source.Count() : -1; + internal TElement[] ToArray(int minIdx, int maxIdx) + { + TElement[] buffer = _source.ToArray(); + if (buffer.Length <= minIdx) + { + return []; } - internal TElement[] ToArray(int minIdx, int maxIdx) + if (buffer.Length <= maxIdx) { - TElement[] buffer = _source.ToArray(); - if (buffer.Length <= minIdx) - { - return []; - } + maxIdx = buffer.Length - 1; + } - if (buffer.Length <= maxIdx) - { - maxIdx = buffer.Length - 1; - } + if (minIdx == maxIdx) + { + return [GetEnumerableSorter().ElementAt(buffer, buffer.Length, minIdx)]; + } - if (minIdx == maxIdx) - { - return [GetEnumerableSorter().ElementAt(buffer, buffer.Length, minIdx)]; - } + TElement[] array = new TElement[maxIdx - minIdx + 1]; - TElement[] array = new TElement[maxIdx - minIdx + 1]; + Fill(minIdx, maxIdx, buffer, array); - Fill(minIdx, maxIdx, buffer, array); + return array; + } - return array; + internal List ToList(int minIdx, int maxIdx) + { + TElement[] buffer = _source.ToArray(); + if (buffer.Length <= minIdx) + { + return new List(); } - internal List ToList(int minIdx, int maxIdx) + if (buffer.Length <= maxIdx) { - TElement[] buffer = _source.ToArray(); - if (buffer.Length <= minIdx) - { - return new List(); - } + maxIdx = buffer.Length - 1; + } - if (buffer.Length <= maxIdx) - { - maxIdx = buffer.Length - 1; - } + if (minIdx == maxIdx) + { + return new List(1) { GetEnumerableSorter().ElementAt(buffer, buffer.Length, minIdx) }; + } - if (minIdx == maxIdx) - { - return new List(1) { GetEnumerableSorter().ElementAt(buffer, buffer.Length, minIdx) }; - } + List list = new(); + Fill(minIdx, maxIdx, buffer, Enumerable.SetCountAndGetSpan(list, maxIdx - minIdx + 1)); + return list; + } - List list = new(); - Fill(minIdx, maxIdx, buffer, SetCountAndGetSpan(list, maxIdx - minIdx + 1)); - return list; + private void Fill(int minIdx, int maxIdx, TElement[] buffer, Span destination) + { + int[] map = SortedMap(buffer, minIdx, maxIdx); + int idx = 0; + while (minIdx <= maxIdx) + { + destination[idx] = buffer[map[minIdx]]; + ++idx; + ++minIdx; } + } - private void Fill(int minIdx, int maxIdx, TElement[] buffer, Span destination) + internal int GetCount(int minIdx, int maxIdx, bool onlyIfCheap) + { + int count = GetCount(onlyIfCheap); + if (count <= 0) { - int[] map = SortedMap(buffer, minIdx, maxIdx); - int idx = 0; - while (minIdx <= maxIdx) - { - destination[idx] = buffer[map[minIdx]]; - ++idx; - ++minIdx; - } + return count; } - internal int GetCount(int minIdx, int maxIdx, bool onlyIfCheap) + if (count <= minIdx) { - int count = GetCount(onlyIfCheap); - if (count <= 0) - { - return count; - } - - if (count <= minIdx) - { - return 0; - } - - return (count <= maxIdx ? count : maxIdx + 1) - minIdx; + return 0; } - public override Iterator Skip(int count) => new SkipTakeOrderedIterator(this, count, int.MaxValue); + return (count <= maxIdx ? count : maxIdx + 1) - minIdx; + } - public override Iterator Take(int count) => new SkipTakeOrderedIterator(this, 0, count - 1); + public IPartition Skip(int count) => new OrderedPartition(this, count, int.MaxValue); - public override TElement? TryGetElementAt(int index, out bool found) - { - if (index == 0) - { - return TryGetFirst(out found); - } - - if (index > 0) - { - TElement[] buffer = _source.ToArray(); - if (index < buffer.Length) - { - found = true; - return GetEnumerableSorter().ElementAt(buffer, buffer.Length, index); - } - } + public IPartition Take(int count) => new OrderedPartition(this, 0, count - 1); - found = false; - return default; + public TElement? TryGetElementAt(int index, out bool found) + { + if (index == 0) + { + return TryGetFirst(out found); } - public override TElement? TryGetFirst(out bool found) + if (index > 0) { - CachingComparer comparer = GetComparer(); - using (IEnumerator e = _source.GetEnumerator()) + TElement[] buffer = _source.ToArray(); + if (index < buffer.Length) { - if (!e.MoveNext()) - { - found = false; - return default; - } - - TElement value = e.Current; - comparer.SetElement(value); - while (e.MoveNext()) - { - TElement x = e.Current; - if (comparer.Compare(x, true) < 0) - { - value = x; - } - } - found = true; - return value; + return GetEnumerableSorter().ElementAt(buffer, buffer.Length, index); } } - public override TElement? TryGetLast(out bool found) + found = false; + return default; + } + + public virtual TElement? TryGetFirst(out bool found) + { + CachingComparer comparer = GetComparer(); + using (IEnumerator e = _source.GetEnumerator()) { - using (IEnumerator e = _source.GetEnumerator()) + if (!e.MoveNext()) { - if (!e.MoveNext()) - { - found = false; - return default; - } + found = false; + return default; + } - CachingComparer comparer = GetComparer(); - TElement value = e.Current; - comparer.SetElement(value); - while (e.MoveNext()) + TElement value = e.Current; + comparer.SetElement(value); + while (e.MoveNext()) + { + TElement x = e.Current; + if (comparer.Compare(x, true) < 0) { - TElement current = e.Current; - if (comparer.Compare(current, false) >= 0) - { - value = current; - } + value = x; } - - found = true; - return value; } + + found = true; + return value; } + } - public TElement? TryGetLast(int minIdx, int maxIdx, out bool found) + public virtual TElement? TryGetLast(out bool found) + { + using (IEnumerator e = _source.GetEnumerator()) { - TElement[] buffer = _source.ToArray(); - if (minIdx < buffer.Length) + if (!e.MoveNext()) { - found = true; - return (maxIdx < buffer.Length - 1) ? - GetEnumerableSorter().ElementAt(buffer, buffer.Length, maxIdx) : - Last(buffer); + found = false; + return default; } - found = false; - return default; - } - - private TElement Last(TElement[] items) - { CachingComparer comparer = GetComparer(); - - TElement value = items[0]; + TElement value = e.Current; comparer.SetElement(value); - - for (int i = 1; i < items.Length; ++i) + while (e.MoveNext()) { - TElement x = items[i]; - if (comparer.Compare(x, cacheLower: false) >= 0) + TElement current = e.Current; + if (comparer.Compare(current, false) >= 0) { - value = x; + value = current; } } + found = true; return value; } } - private sealed partial class OrderedIterator : OrderedIterator + public TElement? TryGetLast(int minIdx, int maxIdx, out bool found) { - // For complicated cases, rely on the base implementation that's more comprehensive. - // For the simple case of OrderBy(...).First() or OrderByDescending(...).First() (i.e. where - // there's just a single comparer we need to factor in), we can just do the iteration directly. - - public override TElement? TryGetFirst(out bool found) + TElement[] buffer = _source.ToArray(); + if (minIdx < buffer.Length) { - if (_parent is not null) - { - return base.TryGetFirst(out found); - } + found = true; + return (maxIdx < buffer.Length - 1) ? + GetEnumerableSorter().ElementAt(buffer, buffer.Length, maxIdx) : + Last(buffer); + } - using IEnumerator e = _source.GetEnumerator(); + found = false; + return default; + } - if (e.MoveNext()) - { - IComparer comparer = _comparer; - Func keySelector = _keySelector; + private TElement Last(TElement[] items) + { + CachingComparer comparer = GetComparer(); - TElement resultValue = e.Current; - TKey resultKey = keySelector(resultValue); + TElement value = items[0]; + comparer.SetElement(value); - if (_descending) - { - while (e.MoveNext()) - { - TElement nextValue = e.Current; - TKey nextKey = keySelector(nextValue); - if (comparer.Compare(nextKey, resultKey) > 0) - { - resultKey = nextKey; - resultValue = nextValue; - } - } - } - else - { - while (e.MoveNext()) - { - TElement nextValue = e.Current; - TKey nextKey = keySelector(nextValue); - if (comparer.Compare(nextKey, resultKey) < 0) - { - resultKey = nextKey; - resultValue = nextValue; - } - } - } - - found = true; - return resultValue; + for (int i = 1; i < items.Length; ++i) + { + TElement x = items[i]; + if (comparer.Compare(x, cacheLower: false) >= 0) + { + value = x; } - - found = false; - return default; } - public override TElement? TryGetLast(out bool found) + return value; + } + } + + internal sealed partial class OrderedEnumerable : OrderedEnumerable + { + // For complicated cases, rely on the base implementation that's more comprehensive. + // For the simple case of OrderBy(...).First() or OrderByDescending(...).First() (i.e. where + // there's just a single comparer we need to factor in), we can just do the iteration directly. + + public override TElement? TryGetFirst(out bool found) + { + if (_parent is not null) { - if (_parent is not null) - { - return base.TryGetLast(out found); - } + return base.TryGetFirst(out found); + } - using IEnumerator e = _source.GetEnumerator(); + using IEnumerator e = _source.GetEnumerator(); - if (e.MoveNext()) - { - IComparer comparer = _comparer; - Func keySelector = _keySelector; + if (e.MoveNext()) + { + IComparer comparer = _comparer; + Func keySelector = _keySelector; - TElement resultValue = e.Current; - TKey resultKey = keySelector(resultValue); + TElement resultValue = e.Current; + TKey resultKey = keySelector(resultValue); - if (_descending) + if (_descending) + { + while (e.MoveNext()) { - while (e.MoveNext()) + TElement nextValue = e.Current; + TKey nextKey = keySelector(nextValue); + if (comparer.Compare(nextKey, resultKey) > 0) { - TElement nextValue = e.Current; - TKey nextKey = keySelector(nextValue); - if (comparer.Compare(nextKey, resultKey) <= 0) - { - resultKey = nextKey; - resultValue = nextValue; - } + resultKey = nextKey; + resultValue = nextValue; } } - else + } + else + { + while (e.MoveNext()) { - while (e.MoveNext()) + TElement nextValue = e.Current; + TKey nextKey = keySelector(nextValue); + if (comparer.Compare(nextKey, resultKey) < 0) { - TElement nextValue = e.Current; - TKey nextKey = keySelector(nextValue); - if (comparer.Compare(nextKey, resultKey) >= 0) - { - resultKey = nextKey; - resultValue = nextValue; - } + resultKey = nextKey; + resultValue = nextValue; } } - - found = true; - return resultValue; } - found = false; - return default; + found = true; + return resultValue; } + + found = false; + return default; } - private sealed partial class ImplicitlyStableOrderedIterator : OrderedIterator + public override TElement? TryGetLast(out bool found) { - public override TElement[] ToArray() + if (_parent is not null) { - TElement[] array = _source.ToArray(); - Sort(array, _descending); - return array; + return base.TryGetLast(out found); } - public override List ToList() - { - List list = _source.ToList(); - Sort(CollectionsMarshal.AsSpan(list), _descending); - return list; - } + using IEnumerator e = _source.GetEnumerator(); - public override TElement? TryGetFirst(out bool found) => - TryGetFirstOrLast(out found, first: !_descending); + if (e.MoveNext()) + { + IComparer comparer = _comparer; + Func keySelector = _keySelector; - public override TElement? TryGetLast(out bool found) => - TryGetFirstOrLast(out found, first: _descending); + TElement resultValue = e.Current; + TKey resultKey = keySelector(resultValue); - private TElement? TryGetFirstOrLast(out bool found, bool first) - { - if (TryGetSpan(_source, out ReadOnlySpan span)) + if (_descending) { - if (span.Length != 0) + while (e.MoveNext()) { - Debug.Assert(TypeIsImplicitlyStable(), "Using Min/Max has different semantics for floating-point values."); - - found = true; - return first ? - Min(_source) : - Max(_source); + TElement nextValue = e.Current; + TKey nextKey = keySelector(nextValue); + if (comparer.Compare(nextKey, resultKey) <= 0) + { + resultKey = nextKey; + resultValue = nextValue; + } } } else { - using IEnumerator e = _source.GetEnumerator(); - - if (e.MoveNext()) + while (e.MoveNext()) { - TElement resultValue = e.Current; - - if (first) + TElement nextValue = e.Current; + TKey nextKey = keySelector(nextValue); + if (comparer.Compare(nextKey, resultKey) >= 0) { - while (e.MoveNext()) - { - TElement nextValue = e.Current; - if (Comparer.Default.Compare(nextValue, resultValue) < 0) - { - resultValue = nextValue; - } - } - } - else - { - while (e.MoveNext()) - { - TElement nextValue = e.Current; - if (Comparer.Default.Compare(nextValue, resultValue) >= 0) - { - resultValue = nextValue; - } - } + resultKey = nextKey; + resultValue = nextValue; } - - found = true; - return resultValue; } } - found = false; - return default; + found = true; + return resultValue; } + + found = false; + return default; } + } - private sealed class SkipTakeOrderedIterator : Iterator + internal sealed partial class OrderedImplicitlyStableEnumerable : OrderedEnumerable + { + public override TElement[] ToArray() { - private readonly OrderedIterator _source; - private readonly int _minIndexInclusive; - private readonly int _maxIndexInclusive; + TElement[] array = _source.ToArray(); + Sort(array, _descending); + return array; + } - private TElement[]? _buffer; - private int[]? _map; - private int _maxIdx; + public override List ToList() + { + List list = _source.ToList(); + Sort(CollectionsMarshal.AsSpan(list), _descending); + return list; + } - public SkipTakeOrderedIterator(OrderedIterator source, int minIdxInclusive, int maxIdxInclusive) - { - _source = source; - _minIndexInclusive = minIdxInclusive; - _maxIndexInclusive = maxIdxInclusive; - } + public override TElement? TryGetFirst(out bool found) => + TryGetFirstOrLast(out found, first: !_descending); - private protected override Iterator Clone() => new SkipTakeOrderedIterator(_source, _minIndexInclusive, _maxIndexInclusive); + public override TElement? TryGetLast(out bool found) => + TryGetFirstOrLast(out found, first: _descending); - public override bool MoveNext() + private TElement? TryGetFirstOrLast(out bool found, bool first) + { + if (Enumerable.TryGetSpan(_source, out ReadOnlySpan span)) { - int state = _state; - - Initialized: - if (state > 1) + if (span.Length != 0) { - Debug.Assert(_buffer is not null); - Debug.Assert(_map is not null); + Debug.Assert(Enumerable.TypeIsImplicitlyStable(), "Using Min/Max has different semantics for floating-point values."); - int[] map = _map; - int i = state - 2 + _minIndexInclusive; - if (i <= _maxIdx) - { - _current = _buffer[map[i]]; - _state++; - return true; - } + found = true; + return first ? + Enumerable.Min(_source) : + Enumerable.Max(_source); } - else if (state == 1) + } + else + { + using IEnumerator e = _source.GetEnumerator(); + + if (e.MoveNext()) { - TElement[] buffer = _source.ToArray(); - int count = buffer.Length; - if (count > _minIndexInclusive) + TElement resultValue = e.Current; + + if (first) { - _maxIdx = _maxIndexInclusive; - if (count <= _maxIdx) + while (e.MoveNext()) { - _maxIdx = count - 1; + TElement nextValue = e.Current; + if (Comparer.Default.Compare(nextValue, resultValue) < 0) + { + resultValue = nextValue; + } } - - if (_minIndexInclusive == _maxIdx) + } + else + { + while (e.MoveNext()) { - _current = _source.GetEnumerableSorter().ElementAt(buffer, count, _minIndexInclusive); - _state = -1; - return true; + TElement nextValue = e.Current; + if (Comparer.Default.Compare(nextValue, resultValue) >= 0) + { + resultValue = nextValue; + } } - - _map = _source.SortedMap(buffer, _minIndexInclusive, _maxIdx); - _buffer = buffer; - _state = state = 2; - goto Initialized; } - } - - Dispose(); - return false; - } - - public override Iterator? Skip(int count) - { - int minIndex = _minIndexInclusive + count; - return (uint)minIndex > (uint)_maxIndexInclusive ? null : new SkipTakeOrderedIterator(_source, minIndex, _maxIndexInclusive); - } - public override Iterator Take(int count) - { - int maxIndex = _minIndexInclusive + count - 1; - if ((uint)maxIndex >= (uint)_maxIndexInclusive) - { - return this; - } - - return new SkipTakeOrderedIterator(_source, _minIndexInclusive, maxIndex); - } - - public override TElement? TryGetElementAt(int index, out bool found) - { - if ((uint)index <= (uint)(_maxIndexInclusive - _minIndexInclusive)) - { - return _source.TryGetElementAt(index + _minIndexInclusive, out found); + found = true; + return resultValue; } - - found = false; - return default; } - public override TElement? TryGetFirst(out bool found) => _source.TryGetElementAt(_minIndexInclusive, out found); - - public override TElement? TryGetLast(out bool found) => - _source.TryGetLast(_minIndexInclusive, _maxIndexInclusive, out found); - - public override TElement[] ToArray() => _source.ToArray(_minIndexInclusive, _maxIndexInclusive); - - public override List ToList() => _source.ToList(_minIndexInclusive, _maxIndexInclusive); - - public override int GetCount(bool onlyIfCheap) => _source.GetCount(_minIndexInclusive, _maxIndexInclusive, onlyIfCheap); + found = false; + return default; } } } diff --git a/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.cs b/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.cs index 91d51da2e58582..6b6b83bac93f70 100644 --- a/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.cs +++ b/src/libraries/System.Linq/src/System/Linq/OrderedEnumerable.cs @@ -7,654 +7,618 @@ namespace System.Linq { - public static partial class Enumerable + internal abstract partial class OrderedEnumerable : IOrderedEnumerable { - private abstract partial class OrderedIterator : Iterator, IOrderedEnumerable - { - internal readonly IEnumerable _source; - - protected OrderedIterator(IEnumerable source) => _source = source; - - private protected int[] SortedMap(TElement[] buffer) => GetEnumerableSorter().Sort(buffer, buffer.Length); - - internal int[] SortedMap(TElement[] buffer, int minIdx, int maxIdx) => - GetEnumerableSorter().Sort(buffer, buffer.Length, minIdx, maxIdx); + internal IEnumerable _source; - internal abstract EnumerableSorter GetEnumerableSorter(EnumerableSorter? next = null); + protected OrderedEnumerable(IEnumerable source) => _source = source; - internal abstract CachingComparer GetComparer(CachingComparer? childComparer = null); + private int[] SortedMap(TElement[] buffer) => GetEnumerableSorter().Sort(buffer, buffer.Length); - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + private int[] SortedMap(TElement[] buffer, int minIdx, int maxIdx) => + GetEnumerableSorter().Sort(buffer, buffer.Length, minIdx, maxIdx); - IOrderedEnumerable IOrderedEnumerable.CreateOrderedEnumerable(Func keySelector, IComparer? comparer, bool descending) => - new OrderedIterator(_source, keySelector, comparer, @descending, this); - - public TElement? TryGetLast(Func predicate, out bool found) + public virtual IEnumerator GetEnumerator() + { + TElement[] buffer = _source.ToArray(); + if (buffer.Length > 0) { - CachingComparer comparer = GetComparer(); - using (IEnumerator e = _source.GetEnumerator()) + int[] map = SortedMap(buffer); + for (int i = 0; i < buffer.Length; i++) { - TElement value; - do - { - if (!e.MoveNext()) - { - found = false; - return default; - } - - value = e.Current; - } - while (!predicate(value)); - - comparer.SetElement(value); - while (e.MoveNext()) - { - TElement x = e.Current; - if (predicate(x) && comparer.Compare(x, false) >= 0) - { - value = x; - } - } - - found = true; - return value; + yield return buffer[map[i]]; } } } - private sealed partial class OrderedIterator : OrderedIterator + internal IEnumerator GetEnumerator(int minIdx, int maxIdx) { - private readonly OrderedIterator? _parent; - private readonly Func _keySelector; - private readonly IComparer _comparer; - private readonly bool _descending; - private TElement[]? _buffer; - private int[]? _map; - - internal OrderedIterator(IEnumerable source, Func keySelector, IComparer? comparer, bool descending, OrderedIterator? parent) : - base(source) + TElement[] buffer = _source.ToArray(); + int count = buffer.Length; + if (count > minIdx) { - if (source is null) + if (count <= maxIdx) { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); + maxIdx = count - 1; } - if (keySelector is null) + + if (minIdx == maxIdx) { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.keySelector); + yield return GetEnumerableSorter().ElementAt(buffer, count, minIdx); + } + else + { + int[] map = SortedMap(buffer, minIdx, maxIdx); + while (minIdx <= maxIdx) + { + yield return buffer[map[minIdx]]; + ++minIdx; + } } - - _parent = parent; - _keySelector = keySelector; - _comparer = comparer ?? Comparer.Default; - _descending = descending; } + } - private protected override Iterator Clone() => new OrderedIterator(_source, _keySelector, _comparer, _descending, _parent); + private EnumerableSorter GetEnumerableSorter() => GetEnumerableSorter(null); - internal override EnumerableSorter GetEnumerableSorter(EnumerableSorter? next) - { - // Special case the common use of string with default comparer. Comparer.Default checks the - // thread's Culture on each call which is an overhead which is not required, because we are about to - // do a sort which remains on the current thread (and EnumerableSorter is not used afterwards). - IComparer comparer = _comparer; - if (typeof(TKey) == typeof(string) && comparer == Comparer.Default) - { - comparer = (IComparer)StringComparer.CurrentCulture; - } + internal abstract EnumerableSorter GetEnumerableSorter(EnumerableSorter? next); - EnumerableSorter sorter = new EnumerableSorter(_keySelector, comparer, _descending, next); - if (_parent is not null) - { - sorter = _parent.GetEnumerableSorter(sorter); - } + internal abstract CachingComparer GetComparer(CachingComparer? childComparer = null); - return sorter; - } + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - internal override CachingComparer GetComparer(CachingComparer? childComparer) - { - CachingComparer cmp = childComparer is null - ? new CachingComparer(_keySelector, _comparer, _descending) - : new CachingComparerWithChild(_keySelector, _comparer, _descending, childComparer); - return _parent is not null ? _parent.GetComparer(cmp) : cmp; - } + IOrderedEnumerable IOrderedEnumerable.CreateOrderedEnumerable(Func keySelector, IComparer? comparer, bool descending) => + new OrderedEnumerable(_source, keySelector, comparer, @descending, this); - public override bool MoveNext() + public TElement? TryGetLast(Func predicate, out bool found) + { + CachingComparer comparer = GetComparer(); + using (IEnumerator e = _source.GetEnumerator()) { - int state = _state; - - Initialized: - if (state > 1) + TElement value; + do { - Debug.Assert(_buffer is not null); - Debug.Assert(_map is not null); - Debug.Assert(_map.Length == _buffer.Length); - - int[] map = _map; - int i = state - 2; - if ((uint)i < (uint)map.Length) + if (!e.MoveNext()) { - _current = _buffer[map[i]]; - _state++; - return true; + found = false; + return default; } + + value = e.Current; } - else if (state == 1) + while (!predicate(value)); + + comparer.SetElement(value); + while (e.MoveNext()) { - TElement[] buffer = _source.ToArray(); - if (buffer.Length != 0) + TElement x = e.Current; + if (predicate(x) && comparer.Compare(x, false) >= 0) { - _map = SortedMap(buffer); - _buffer = buffer; - _state = state = 2; - goto Initialized; + value = x; } } - Dispose(); - return false; - } - - public override void Dispose() - { - _buffer = null; - _map = null; - base.Dispose(); + found = true; + return value; } } + } - /// An ordered enumerable used by Order/OrderDescending for Ts that are bitwise indistinguishable for any considered equal. - private sealed partial class ImplicitlyStableOrderedIterator : OrderedIterator - { - private readonly bool _descending; - private TElement[]? _buffer; + internal sealed partial class OrderedEnumerable : OrderedEnumerable + { + private readonly OrderedEnumerable? _parent; + private readonly Func _keySelector; + private readonly IComparer _comparer; + private readonly bool _descending; - public ImplicitlyStableOrderedIterator(IEnumerable source, bool descending) : base(source) + internal OrderedEnumerable(IEnumerable source, Func keySelector, IComparer? comparer, bool descending, OrderedEnumerable? parent) : + base(source) + { + if (source is null) { - Debug.Assert(TypeIsImplicitlyStable()); - - if (source is null) - { - ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); - } - - _descending = descending; + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - - private protected override Iterator Clone() => new ImplicitlyStableOrderedIterator(_source, _descending); - - internal override CachingComparer GetComparer(CachingComparer? childComparer) => - childComparer is null ? - new CachingComparer(EnumerableSorter.IdentityFunc, Comparer.Default, _descending) : - new CachingComparerWithChild(EnumerableSorter.IdentityFunc, Comparer.Default, _descending, childComparer); - - internal override EnumerableSorter GetEnumerableSorter(EnumerableSorter? next) => - new EnumerableSorter(EnumerableSorter.IdentityFunc, Comparer.Default, _descending, next); - - public override bool MoveNext() + if (keySelector is null) { - int state = _state; - TElement[]? buffer; - - Initialized: - if (state > 1) - { - buffer = _buffer; - Debug.Assert(buffer is not null); - - int i = state - 2; - if ((uint)i < (uint)buffer.Length) - { - _current = buffer[i]; - _state++; - return true; - } - } - else if (state == 1) - { - buffer = _source.ToArray(); - if (buffer.Length != 0) - { - Sort(buffer, _descending); - _buffer = buffer; - _state = state = 2; - goto Initialized; - } - } - - Dispose(); - return false; + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.keySelector); } - public override void Dispose() + _parent = parent; + _keySelector = keySelector; + _comparer = comparer ?? Comparer.Default; + _descending = descending; + } + + internal override EnumerableSorter GetEnumerableSorter(EnumerableSorter? next) + { + // Special case the common use of string with default comparer. Comparer.Default checks the + // thread's Culture on each call which is an overhead which is not required, because we are about to + // do a sort which remains on the current thread (and EnumerableSorter is not used afterwards). + IComparer comparer = _comparer; + if (typeof(TKey) == typeof(string) && comparer == Comparer.Default) { - _buffer = null; - base.Dispose(); + comparer = (IComparer)StringComparer.CurrentCulture; } - private static void Sort(Span span, bool descending) + EnumerableSorter sorter = new EnumerableSorter(_keySelector, comparer, _descending, next); + if (_parent != null) { - if (descending) - { - span.Sort(static (a, b) => Comparer.Default.Compare(b, a)); - } - else - { - span.Sort(); - } + sorter = _parent.GetEnumerableSorter(sorter); } + + return sorter; } - // A comparer that chains comparisons, and pushes through the last element found to be - // lower or higher (depending on use), so as to represent the sort of comparisons - // done by OrderBy().ThenBy() combinations. - private abstract class CachingComparer + internal override CachingComparer GetComparer(CachingComparer? childComparer) { - internal abstract int Compare(TElement element, bool cacheLower); - - internal abstract void SetElement(TElement element); + CachingComparer cmp = childComparer == null + ? new CachingComparer(_keySelector, _comparer, _descending) + : new CachingComparerWithChild(_keySelector, _comparer, _descending, childComparer); + return _parent != null ? _parent.GetComparer(cmp) : cmp; } + } - private class CachingComparer : CachingComparer + /// An ordered enumerable used by Order/OrderDescending for Ts that are bitwise indistinguishable for any considered equal. + internal sealed partial class OrderedImplicitlyStableEnumerable : OrderedEnumerable + { + private readonly bool _descending; + + public OrderedImplicitlyStableEnumerable(IEnumerable source, bool descending) : base(source) { - protected readonly Func _keySelector; - protected readonly IComparer _comparer; - protected readonly bool _descending; - protected TKey? _lastKey; + Debug.Assert(Enumerable.TypeIsImplicitlyStable()); - public CachingComparer(Func keySelector, IComparer comparer, bool descending) + if (source is null) { - _keySelector = keySelector; - _comparer = comparer; - _descending = descending; + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - internal override int Compare(TElement element, bool cacheLower) + _descending = descending; + } + + internal override CachingComparer GetComparer(CachingComparer? childComparer) => + childComparer == null ? + new CachingComparer(EnumerableSorter.IdentityFunc, Comparer.Default, _descending) : + new CachingComparerWithChild(EnumerableSorter.IdentityFunc, Comparer.Default, _descending, childComparer); + + internal override EnumerableSorter GetEnumerableSorter(EnumerableSorter? next) => + new EnumerableSorter(EnumerableSorter.IdentityFunc, Comparer.Default, _descending, next); + + public override IEnumerator GetEnumerator() + { + TElement[] buffer = _source.ToArray(); + if (buffer.Length > 0) { - TKey newKey = _keySelector(element); - int cmp = _descending ? _comparer.Compare(_lastKey, newKey) : _comparer.Compare(newKey, _lastKey); - if (cacheLower == cmp < 0) + Sort(buffer, _descending); + for (int i = 0; i < buffer.Length; i++) { - _lastKey = newKey; + yield return buffer[i]; } - - return cmp; } + } - internal override void SetElement(TElement element) + private static void Sort(Span span, bool descending) + { + if (descending) + { + span.Sort(static (a, b) => Comparer.Default.Compare(b, a)); + } + else { - _lastKey = _keySelector(element); + span.Sort(); } } + } + + // A comparer that chains comparisons, and pushes through the last element found to be + // lower or higher (depending on use), so as to represent the sort of comparisons + // done by OrderBy().ThenBy() combinations. + internal abstract class CachingComparer + { + internal abstract int Compare(TElement element, bool cacheLower); - private sealed class CachingComparerWithChild : CachingComparer + internal abstract void SetElement(TElement element); + } + + internal class CachingComparer : CachingComparer + { + protected readonly Func _keySelector; + protected readonly IComparer _comparer; + protected readonly bool _descending; + protected TKey? _lastKey; + + public CachingComparer(Func keySelector, IComparer comparer, bool descending) { - private readonly CachingComparer _child; + _keySelector = keySelector; + _comparer = comparer; + _descending = descending; + } - public CachingComparerWithChild(Func keySelector, IComparer comparer, bool descending, CachingComparer child) - : base(keySelector, comparer, descending) + internal override int Compare(TElement element, bool cacheLower) + { + TKey newKey = _keySelector(element); + int cmp = _descending ? _comparer.Compare(_lastKey, newKey) : _comparer.Compare(newKey, _lastKey); + if (cacheLower == cmp < 0) { - _child = child; + _lastKey = newKey; } - internal override int Compare(TElement element, bool cacheLower) - { - TKey newKey = _keySelector(element); - int cmp = _descending ? _comparer.Compare(_lastKey, newKey) : _comparer.Compare(newKey, _lastKey); - if (cmp == 0) - { - return _child.Compare(element, cacheLower); - } + return cmp; + } - if (cacheLower == cmp < 0) - { - _lastKey = newKey; - _child.SetElement(element); - } + internal override void SetElement(TElement element) + { + _lastKey = _keySelector(element); + } + } - return cmp; + internal sealed class CachingComparerWithChild : CachingComparer + { + private readonly CachingComparer _child; + + public CachingComparerWithChild(Func keySelector, IComparer comparer, bool descending, CachingComparer child) + : base(keySelector, comparer, descending) + { + _child = child; + } + + internal override int Compare(TElement element, bool cacheLower) + { + TKey newKey = _keySelector(element); + int cmp = _descending ? _comparer.Compare(_lastKey, newKey) : _comparer.Compare(newKey, _lastKey); + if (cmp == 0) + { + return _child.Compare(element, cacheLower); } - internal override void SetElement(TElement element) + if (cacheLower == cmp < 0) { - base.SetElement(element); + _lastKey = newKey; _child.SetElement(element); } + + return cmp; } - private abstract class EnumerableSorter + internal override void SetElement(TElement element) { - /// Function that returns its input unmodified. - /// - /// Used for reference equality in order to avoid unnecessary computation when a caller - /// can benefit from knowing that the produced value is identical to the input. - /// - internal static readonly Func IdentityFunc = e => e; - - internal abstract void ComputeKeys(TElement[] elements, int count); + base.SetElement(element); + _child.SetElement(element); + } + } - internal abstract int CompareAnyKeys(int index1, int index2); + internal abstract class EnumerableSorter + { + /// Function that returns its input unmodified. + /// + /// Used for reference equality in order to avoid unnecessary computation when a caller + /// can benefit from knowing that the produced value is identical to the input. + /// + internal static readonly Func IdentityFunc = e => e; - private int[] ComputeMap(TElement[] elements, int count) - { - ComputeKeys(elements, count); + internal abstract void ComputeKeys(TElement[] elements, int count); - int[] map = new int[count]; - FillIncrementing(map, 0); - return map; - } + internal abstract int CompareAnyKeys(int index1, int index2); - internal int[] Sort(TElement[] elements, int count) + private int[] ComputeMap(TElement[] elements, int count) + { + ComputeKeys(elements, count); + int[] map = new int[count]; + for (int i = 0; i < map.Length; i++) { - int[] map = ComputeMap(elements, count); - QuickSort(map, 0, count - 1); - return map; + map[i] = i; } - internal int[] Sort(TElement[] elements, int count, int minIdx, int maxIdx) - { - int[] map = ComputeMap(elements, count); - PartialQuickSort(map, 0, count - 1, minIdx, maxIdx); - return map; - } + return map; + } - internal TElement ElementAt(TElement[] elements, int count, int idx) - { - int[] map = ComputeMap(elements, count); - return idx == 0 ? - elements[Min(map, count)] : - elements[QuickSelect(map, count - 1, idx)]; - } + internal int[] Sort(TElement[] elements, int count) + { + int[] map = ComputeMap(elements, count); + QuickSort(map, 0, count - 1); + return map; + } - protected abstract void QuickSort(int[] map, int left, int right); + internal int[] Sort(TElement[] elements, int count, int minIdx, int maxIdx) + { + int[] map = ComputeMap(elements, count); + PartialQuickSort(map, 0, count - 1, minIdx, maxIdx); + return map; + } - // Sorts the k elements between minIdx and maxIdx without sorting all elements - // Time complexity: O(n + k log k) best and average case. O(n^2) worse case. - protected abstract void PartialQuickSort(int[] map, int left, int right, int minIdx, int maxIdx); + internal TElement ElementAt(TElement[] elements, int count, int idx) + { + int[] map = ComputeMap(elements, count); + return idx == 0 ? + elements[Min(map, count)] : + elements[QuickSelect(map, count - 1, idx)]; + } - // Finds the element that would be at idx if the collection was sorted. - // Time complexity: O(n) best and average case. O(n^2) worse case. - protected abstract int QuickSelect(int[] map, int right, int idx); + protected abstract void QuickSort(int[] map, int left, int right); - protected abstract int Min(int[] map, int count); - } + // Sorts the k elements between minIdx and maxIdx without sorting all elements + // Time complexity: O(n + k log k) best and average case. O(n^2) worse case. + protected abstract void PartialQuickSort(int[] map, int left, int right, int minIdx, int maxIdx); - private sealed class EnumerableSorter : EnumerableSorter - { - private readonly Func _keySelector; - private readonly IComparer _comparer; - private readonly bool _descending; - private readonly EnumerableSorter? _next; - private TKey[]? _keys; + // Finds the element that would be at idx if the collection was sorted. + // Time complexity: O(n) best and average case. O(n^2) worse case. + protected abstract int QuickSelect(int[] map, int right, int idx); - internal EnumerableSorter(Func keySelector, IComparer comparer, bool descending, EnumerableSorter? next) - { - _keySelector = keySelector; - _comparer = comparer; - _descending = descending; - _next = next; - } + protected abstract int Min(int[] map, int count); + } + + internal sealed class EnumerableSorter : EnumerableSorter + { + private readonly Func _keySelector; + private readonly IComparer _comparer; + private readonly bool _descending; + private readonly EnumerableSorter? _next; + private TKey[]? _keys; + + internal EnumerableSorter(Func keySelector, IComparer comparer, bool descending, EnumerableSorter? next) + { + _keySelector = keySelector; + _comparer = comparer; + _descending = descending; + _next = next; + } - internal override void ComputeKeys(TElement[] elements, int count) + internal override void ComputeKeys(TElement[] elements, int count) + { + Func keySelector = _keySelector; + if (!ReferenceEquals(keySelector, IdentityFunc)) { - Func keySelector = _keySelector; - if (!ReferenceEquals(keySelector, IdentityFunc)) + var keys = new TKey[count]; + for (int i = 0; i < keys.Length; i++) { - var keys = new TKey[count]; - for (int i = 0; i < keys.Length; i++) - { - keys[i] = keySelector(elements[i]); - } - _keys = keys; + keys[i] = keySelector(elements[i]); } - else - { - // The key selector is our known identity function, which means we don't - // need to invoke the key selector for every element. Further, we can just - // use the original array as the keys (even if count is smaller, as the additional - // values will just be ignored). - Debug.Assert(typeof(TKey) == typeof(TElement)); - _keys = (TKey[])(object)elements; - } - - _next?.ComputeKeys(elements, count); + _keys = keys; } - - internal override int CompareAnyKeys(int index1, int index2) + else { - TKey[]? keys = _keys; - Debug.Assert(keys is not null); + // The key selector is our known identity function, which means we don't + // need to invoke the key selector for every element. Further, we can just + // use the original array as the keys (even if count is smaller, as the additional + // values will just be ignored). + Debug.Assert(typeof(TKey) == typeof(TElement)); + _keys = (TKey[])(object)elements; + } - int c = _comparer.Compare(keys[index1], keys[index2]); - if (c == 0) - { - if (_next is null) - { - return index1 - index2; // ensure stability of sort - } + _next?.ComputeKeys(elements, count); + } + + internal override int CompareAnyKeys(int index1, int index2) + { + TKey[]? keys = _keys; + Debug.Assert(keys != null); - return _next.CompareAnyKeys(index1, index2); + int c = _comparer.Compare(keys[index1], keys[index2]); + if (c == 0) + { + if (_next == null) + { + return index1 - index2; // ensure stability of sort } - // -c will result in a negative value for int.MinValue (-int.MinValue == int.MinValue). - // Flipping keys earlier is more likely to trigger something strange in a comparer, - // particularly as it comes to the sort being stable. - return (_descending != (c > 0)) ? 1 : -1; + return _next.CompareAnyKeys(index1, index2); } - private int CompareAnyKeys_DefaultComparer_NoNext_Ascending(int index1, int index2) - { - Debug.Assert(typeof(TKey).IsValueType); - Debug.Assert(_comparer == Comparer.Default); - Debug.Assert(_next is null); - Debug.Assert(!_descending); - - TKey[]? keys = _keys; - Debug.Assert(keys is not null); - - int c = Comparer.Default.Compare(keys[index1], keys[index2]); - return - c == 0 ? index1 - index2 : // ensure stability of sort - c; - } + // -c will result in a negative value for int.MinValue (-int.MinValue == int.MinValue). + // Flipping keys earlier is more likely to trigger something strange in a comparer, + // particularly as it comes to the sort being stable. + return (_descending != (c > 0)) ? 1 : -1; + } - private int CompareAnyKeys_DefaultComparer_NoNext_Descending(int index1, int index2) - { - Debug.Assert(typeof(TKey).IsValueType); - Debug.Assert(_comparer == Comparer.Default); - Debug.Assert(_next is null); - Debug.Assert(_descending); - - TKey[]? keys = _keys; - Debug.Assert(keys is not null); - - int c = Comparer.Default.Compare(keys[index2], keys[index1]); - return - c == 0 ? index1 - index2 : // ensure stability of sort - c; - } + private int CompareAnyKeys_DefaultComparer_NoNext_Ascending(int index1, int index2) + { + Debug.Assert(typeof(TKey).IsValueType); + Debug.Assert(_comparer == Comparer.Default); + Debug.Assert(_next is null); + Debug.Assert(!_descending); + + TKey[]? keys = _keys; + Debug.Assert(keys != null); + + int c = Comparer.Default.Compare(keys[index1], keys[index2]); + return + c == 0 ? index1 - index2 : // ensure stability of sort + c; + } - private int CompareKeys(int index1, int index2) => index1 == index2 ? 0 : CompareAnyKeys(index1, index2); + private int CompareAnyKeys_DefaultComparer_NoNext_Descending(int index1, int index2) + { + Debug.Assert(typeof(TKey).IsValueType); + Debug.Assert(_comparer == Comparer.Default); + Debug.Assert(_next is null); + Debug.Assert(_descending); + + TKey[]? keys = _keys; + Debug.Assert(keys != null); + + int c = Comparer.Default.Compare(keys[index2], keys[index1]); + return + c == 0 ? index1 - index2 : // ensure stability of sort + c; + } - protected override void QuickSort(int[] keys, int lo, int hi) - { - Comparison comparison; + private int CompareKeys(int index1, int index2) => index1 == index2 ? 0 : CompareAnyKeys(index1, index2); + + protected override void QuickSort(int[] keys, int lo, int hi) + { + Comparison comparison; - if (typeof(TKey).IsValueType && _next is null && _comparer == Comparer.Default) + if (typeof(TKey).IsValueType && _next is null && _comparer == Comparer.Default) + { + // We can use Comparer.Default.Compare and benefit from devirtualization and inlining. + // We can also avoid extra steps to check whether we need to deal with a subsequent tie breaker (_next). + if (!_descending) { - // We can use Comparer.Default.Compare and benefit from devirtualization and inlining. - // We can also avoid extra steps to check whether we need to deal with a subsequent tie breaker (_next). - if (!_descending) - { - comparison = CompareAnyKeys_DefaultComparer_NoNext_Ascending; - } - else - { - comparison = CompareAnyKeys_DefaultComparer_NoNext_Descending; - } + comparison = CompareAnyKeys_DefaultComparer_NoNext_Ascending; } else { - comparison = CompareAnyKeys; + comparison = CompareAnyKeys_DefaultComparer_NoNext_Descending; } - - new Span(keys, lo, hi - lo + 1).Sort(comparison); } + else + { + comparison = CompareAnyKeys; + } + + new Span(keys, lo, hi - lo + 1).Sort(comparison); + } - // Sorts the k elements between minIdx and maxIdx without sorting all elements - // Time complexity: O(n + k log k) best and average case. O(n^2) worse case. - protected override void PartialQuickSort(int[] map, int left, int right, int minIdx, int maxIdx) + // Sorts the k elements between minIdx and maxIdx without sorting all elements + // Time complexity: O(n + k log k) best and average case. O(n^2) worse case. + protected override void PartialQuickSort(int[] map, int left, int right, int minIdx, int maxIdx) + { + do { + int i = left; + int j = right; + int x = map[i + ((j - i) >> 1)]; do { - int i = left; - int j = right; - int x = map[i + ((j - i) >> 1)]; - do + while (i < map.Length && CompareKeys(x, map[i]) > 0) { - while (i < map.Length && CompareKeys(x, map[i]) > 0) - { - i++; - } - - while (j >= 0 && CompareKeys(x, map[j]) < 0) - { - j--; - } - - if (i > j) - { - break; - } - - if (i < j) - { - int temp = map[i]; - map[i] = map[j]; - map[j] = temp; - } - i++; - j--; } - while (i <= j); - if (minIdx >= i) + while (j >= 0 && CompareKeys(x, map[j]) < 0) { - left = i + 1; + j--; } - else if (maxIdx <= j) + + if (i > j) { - right = j - 1; + break; } - if (j - left <= right - i) + if (i < j) { - if (left < j) - { - PartialQuickSort(map, left, j, minIdx, maxIdx); - } - - left = i; + int temp = map[i]; + map[i] = map[j]; + map[j] = temp; } - else + + i++; + j--; + } + while (i <= j); + + if (minIdx >= i) + { + left = i + 1; + } + else if (maxIdx <= j) + { + right = j - 1; + } + + if (j - left <= right - i) + { + if (left < j) { - if (i < right) - { - PartialQuickSort(map, i, right, minIdx, maxIdx); - } + PartialQuickSort(map, left, j, minIdx, maxIdx); + } - right = j; + left = i; + } + else + { + if (i < right) + { + PartialQuickSort(map, i, right, minIdx, maxIdx); } + + right = j; } - while (left < right); } + while (left < right); + } - // Finds the element that would be at idx if the collection was sorted. - // Time complexity: O(n) best and average case. O(n^2) worse case. - protected override int QuickSelect(int[] map, int right, int idx) + // Finds the element that would be at idx if the collection was sorted. + // Time complexity: O(n) best and average case. O(n^2) worse case. + protected override int QuickSelect(int[] map, int right, int idx) + { + int left = 0; + do { - int left = 0; + int i = left; + int j = right; + int x = map[i + ((j - i) >> 1)]; do { - int i = left; - int j = right; - int x = map[i + ((j - i) >> 1)]; - do + while (i < map.Length && CompareKeys(x, map[i]) > 0) { - while (i < map.Length && CompareKeys(x, map[i]) > 0) - { - i++; - } - - while (j >= 0 && CompareKeys(x, map[j]) < 0) - { - j--; - } - - if (i > j) - { - break; - } - - if (i < j) - { - int temp = map[i]; - map[i] = map[j]; - map[j] = temp; - } - i++; - j--; } - while (i <= j); - if (i <= idx) + while (j >= 0 && CompareKeys(x, map[j]) < 0) { - left = i + 1; + j--; } - else + + if (i > j) { - right = j - 1; + break; } - if (j - left <= right - i) + if (i < j) { - if (left < j) - { - right = j; - } - - left = i; + int temp = map[i]; + map[i] = map[j]; + map[j] = temp; } - else - { - if (i < right) - { - left = i; - } + i++; + j--; + } + while (i <= j); + + if (i <= idx) + { + left = i + 1; + } + else + { + right = j - 1; + } + + if (j - left <= right - i) + { + if (left < j) + { right = j; } + + left = i; } - while (left < right); + else + { + if (i < right) + { + left = i; + } - return map[idx]; + right = j; + } } + while (left < right); + + return map[idx]; + } - protected override int Min(int[] map, int count) + protected override int Min(int[] map, int count) + { + int index = 0; + for (int i = 1; i < count; i++) { - int index = 0; - for (int i = 1; i < count; i++) + if (CompareKeys(map[i], map[index]) < 0) { - if (CompareKeys(map[i], map[index]) < 0) - { - index = i; - } + index = i; } - return map[index]; } + return map[index]; } } } diff --git a/src/libraries/System.Linq/src/System/Linq/PartialArrayEnumerator.cs b/src/libraries/System.Linq/src/System/Linq/PartialArrayEnumerator.cs deleted file mode 100644 index 7a002d115c69d9..00000000000000 --- a/src/libraries/System.Linq/src/System/Linq/PartialArrayEnumerator.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; - -namespace System.Linq -{ - /// Enumerator for iterating through part of an array. - internal sealed class PartialArrayEnumerator : IEnumerator - { - private readonly T[] _array; - private readonly int _count; - private int _index = -1; - - public PartialArrayEnumerator(T[] array, int count) - { - Debug.Assert(array is not null); - _array = array; - _count = count; - } - - public bool MoveNext() - { - if (_index + 1 < _count) - { - _index++; - return true; - } - - return false; - } - - public T Current => _array[_index]; - object? IEnumerator.Current => Current; - - public void Dispose() { } - - public void Reset() => _index = -1; - } -} diff --git a/src/libraries/System.Linq/src/System/Linq/SkipTake.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Partition.SpeedOpt.cs similarity index 77% rename from src/libraries/System.Linq/src/System/Linq/SkipTake.SpeedOpt.cs rename to src/libraries/System.Linq/src/System/Linq/Partition.SpeedOpt.cs index 1a49488b09fbb7..202fb803881edb 100644 --- a/src/libraries/System.Linq/src/System/Linq/SkipTake.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Partition.SpeedOpt.cs @@ -1,11 +1,69 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections; using System.Collections.Generic; using System.Diagnostics; namespace System.Linq { + internal sealed class OrderedPartition : IPartition + { + private readonly OrderedEnumerable _source; + private readonly int _minIndexInclusive; + private readonly int _maxIndexInclusive; + + public OrderedPartition(OrderedEnumerable source, int minIdxInclusive, int maxIdxInclusive) + { + _source = source; + _minIndexInclusive = minIdxInclusive; + _maxIndexInclusive = maxIdxInclusive; + } + + public IEnumerator GetEnumerator() => _source.GetEnumerator(_minIndexInclusive, _maxIndexInclusive); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + public IPartition? Skip(int count) + { + int minIndex = _minIndexInclusive + count; + return (uint)minIndex > (uint)_maxIndexInclusive ? null : new OrderedPartition(_source, minIndex, _maxIndexInclusive); + } + + public IPartition Take(int count) + { + int maxIndex = _minIndexInclusive + count - 1; + if ((uint)maxIndex >= (uint)_maxIndexInclusive) + { + return this; + } + + return new OrderedPartition(_source, _minIndexInclusive, maxIndex); + } + + public TElement? TryGetElementAt(int index, out bool found) + { + if ((uint)index <= (uint)(_maxIndexInclusive - _minIndexInclusive)) + { + return _source.TryGetElementAt(index + _minIndexInclusive, out found); + } + + found = false; + return default; + } + + public TElement? TryGetFirst(out bool found) => _source.TryGetElementAt(_minIndexInclusive, out found); + + public TElement? TryGetLast(out bool found) => + _source.TryGetLast(_minIndexInclusive, _maxIndexInclusive, out found); + + public TElement[] ToArray() => _source.ToArray(_minIndexInclusive, _maxIndexInclusive); + + public List ToList() => _source.ToList(_minIndexInclusive, _maxIndexInclusive); + + public int GetCount(bool onlyIfCheap) => _source.GetCount(_minIndexInclusive, _maxIndexInclusive, onlyIfCheap); + } + public static partial class Enumerable { /// @@ -13,15 +71,15 @@ public static partial class Enumerable /// /// The type of the source list. [DebuggerDisplay("Count = {Count}")] - private sealed class IListSkipTakeIterator : Iterator, IList, IReadOnlyList + private sealed class ListPartition : Iterator, IPartition, IList, IReadOnlyList { private readonly IList _source; private readonly int _minIndexInclusive; private readonly int _maxIndexInclusive; - public IListSkipTakeIterator(IList source, int minIndexInclusive, int maxIndexInclusive) + public ListPartition(IList source, int minIndexInclusive, int maxIndexInclusive) { - Debug.Assert(source is not null); + Debug.Assert(source != null); Debug.Assert(minIndexInclusive >= 0); Debug.Assert(minIndexInclusive <= maxIndexInclusive); _source = source; @@ -29,8 +87,8 @@ public IListSkipTakeIterator(IList source, int minIndexInclusive, int m _maxIndexInclusive = maxIndexInclusive; } - private protected override Iterator Clone() => - new IListSkipTakeIterator(_source, _minIndexInclusive, _maxIndexInclusive); + public override Iterator Clone() => + new ListPartition(_source, _minIndexInclusive, _maxIndexInclusive); public override bool MoveNext() { @@ -50,21 +108,21 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new IListSkipTakeSelectIterator(_source, selector, _minIndexInclusive, _maxIndexInclusive); + new SelectListPartitionIterator(_source, selector, _minIndexInclusive, _maxIndexInclusive); - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { int minIndex = _minIndexInclusive + count; - return (uint)minIndex > (uint)_maxIndexInclusive ? null : new IListSkipTakeIterator(_source, minIndex, _maxIndexInclusive); + return (uint)minIndex > (uint)_maxIndexInclusive ? null : new ListPartition(_source, minIndex, _maxIndexInclusive); } - public override Iterator Take(int count) + public IPartition Take(int count) { int maxIndex = _minIndexInclusive + count - 1; - return (uint)maxIndex >= (uint)_maxIndexInclusive ? this : new IListSkipTakeIterator(_source, _minIndexInclusive, maxIndex); + return (uint)maxIndex >= (uint)_maxIndexInclusive ? this : new ListPartition(_source, _minIndexInclusive, maxIndex); } - public override TSource? TryGetElementAt(int index, out bool found) + public TSource? TryGetElementAt(int index, out bool found) { if ((uint)index <= (uint)(_maxIndexInclusive - _minIndexInclusive) && index < _source.Count - _minIndexInclusive) { @@ -76,7 +134,7 @@ public override Iterator Take(int count) return default; } - public override TSource? TryGetFirst(out bool found) + public TSource? TryGetFirst(out bool found) { if (_source.Count > _minIndexInclusive) { @@ -88,7 +146,7 @@ public override Iterator Take(int count) return default; } - public override TSource? TryGetLast(out bool found) + public TSource? TryGetLast(out bool found) { int lastIndex = _source.Count - 1; if (lastIndex >= _minIndexInclusive) @@ -115,9 +173,9 @@ public int Count } } - public override int GetCount(bool onlyIfCheap) => Count; + public int GetCount(bool onlyIfCheap) => Count; - public override TSource[] ToArray() + public TSource[] ToArray() { int count = Count; if (count == 0) @@ -130,16 +188,16 @@ public override TSource[] ToArray() return array; } - public override List ToList() + public List ToList() { int count = Count; - - List list = []; - if (count != 0) + if (count == 0) { - Fill(_source, SetCountAndGetSpan(list, count), _minIndexInclusive); + return new List(); } + List list = new List(count); + Fill(_source, SetCountAndGetSpan(list, count), _minIndexInclusive); return list; } @@ -199,7 +257,7 @@ public TSource this[int index] /// An iterator that yields the items of part of an . /// /// The type of the source enumerable. - private sealed class IEnumerableSkipTakeIterator : Iterator + private sealed class EnumerablePartition : Iterator, IPartition { private readonly IEnumerable _source; private readonly int _minIndexInclusive; @@ -207,9 +265,9 @@ private sealed class IEnumerableSkipTakeIterator : Iterator // If this is -1, it's impossible to set a limit on the count. private IEnumerator? _enumerator; - internal IEnumerableSkipTakeIterator(IEnumerable source, int minIndexInclusive, int maxIndexInclusive) + internal EnumerablePartition(IEnumerable source, int minIndexInclusive, int maxIndexInclusive) { - Debug.Assert(source is not null); + Debug.Assert(source != null); Debug.Assert(!(source is IList), $"The caller needs to check for {nameof(IList)}."); Debug.Assert(minIndexInclusive >= 0); Debug.Assert(maxIndexInclusive >= -1); @@ -230,12 +288,12 @@ internal IEnumerableSkipTakeIterator(IEnumerable source, int minIndexIn private int Limit => _maxIndexInclusive + 1 - _minIndexInclusive; // This is that upper bound. - private protected override Iterator Clone() => - new IEnumerableSkipTakeIterator(_source, _minIndexInclusive, _maxIndexInclusive); + public override Iterator Clone() => + new EnumerablePartition(_source, _minIndexInclusive, _maxIndexInclusive); public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -244,7 +302,7 @@ public override void Dispose() base.Dispose(); } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { if (onlyIfCheap) { @@ -261,7 +319,7 @@ public override int GetCount(bool onlyIfCheap) using (IEnumerator en = _source.GetEnumerator()) { // We only want to iterate up to _maxIndexInclusive + 1. - // Past that, we know the enumerable will be able to fit this subset, + // Past that, we know the enumerable will be able to fit this partition, // so the count will just be _maxIndexInclusive + 1 - _minIndexInclusive. // Note that it is possible for _maxIndexInclusive to be int.MaxValue here, @@ -273,6 +331,7 @@ public override int GetCount(bool onlyIfCheap) Debug.Assert(count != (uint)int.MaxValue + 1 || _minIndexInclusive > 0, "Our return value will be incorrect."); return Math.Max((int)count - _minIndexInclusive, 0); } + } public override bool MoveNext() @@ -293,7 +352,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (!SkipBeforeFirst(_enumerator)) { // Reached the end before we finished skipping. @@ -303,7 +362,7 @@ public override bool MoveNext() _state = 3; goto default; default: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if ((!HasLimit || taken < Limit) && _enumerator.MoveNext()) { if (HasLimit) @@ -324,7 +383,10 @@ public override bool MoveNext() return false; } - public override Iterator? Skip(int count) + public override IEnumerable Select(Func selector) => + new SelectIPartitionIterator(this, selector); + + public IPartition? Skip(int count) { int minIndex = _minIndexInclusive + count; @@ -335,7 +397,7 @@ public override bool MoveNext() // If we don't know our max count and minIndex can no longer fit in a positive int, // then we will need to wrap ourselves in another iterator. // This can happen, for example, during e.Skip(int.MaxValue).Skip(int.MaxValue). - return new IEnumerableSkipTakeIterator(this, count, -1); + return new EnumerablePartition(this, count, -1); } } else if ((uint)minIndex > (uint)_maxIndexInclusive) @@ -347,10 +409,10 @@ public override bool MoveNext() } Debug.Assert(minIndex >= 0, $"We should have taken care of all cases when {nameof(minIndex)} overflows."); - return new IEnumerableSkipTakeIterator(_source, minIndex, _maxIndexInclusive); + return new EnumerablePartition(_source, minIndex, _maxIndexInclusive); } - public override Iterator Take(int count) + public IPartition Take(int count) { int maxIndex = _minIndexInclusive + count - 1; if (!HasLimit) @@ -363,7 +425,7 @@ public override Iterator Take(int count) // _minIndexInclusive (which is count - 1) must fit in an int. // Example: e.Skip(50).Take(int.MaxValue). - return new IEnumerableSkipTakeIterator(this, 0, count - 1); + return new EnumerablePartition(this, 0, count - 1); } } else if ((uint)maxIndex >= (uint)_maxIndexInclusive) @@ -375,23 +437,18 @@ public override Iterator Take(int count) } Debug.Assert(maxIndex >= 0, $"We should have taken care of all cases when {nameof(maxIndex)} overflows."); - return new IEnumerableSkipTakeIterator(_source, _minIndexInclusive, maxIndex); + return new EnumerablePartition(_source, _minIndexInclusive, maxIndex); } - public override TSource? TryGetElementAt(int index, out bool found) + public TSource? TryGetElementAt(int index, out bool found) { // If the index is negative or >= our max count, return early. if (index >= 0 && (!HasLimit || index < Limit)) { - Debug.Assert(_minIndexInclusive + index >= 0, $"Adding {nameof(index)} caused {nameof(_minIndexInclusive)} to overflow."); - - if (_source is Iterator iterator) - { - return iterator.TryGetElementAt(_minIndexInclusive + index, out found); - } - using (IEnumerator en = _source.GetEnumerator()) { + Debug.Assert(_minIndexInclusive + index >= 0, $"Adding {nameof(index)} caused {nameof(_minIndexInclusive)} to overflow."); + if (SkipBefore(_minIndexInclusive + index, en) && en.MoveNext()) { found = true; @@ -404,15 +461,8 @@ public override Iterator Take(int count) return default; } - public override TSource? TryGetFirst(out bool found) + public TSource? TryGetFirst(out bool found) { - Debug.Assert(!HasLimit || Limit > 0); - - if (_source is Iterator iterator) - { - return iterator.TryGetElementAt(_minIndexInclusive, out found); - } - using (IEnumerator en = _source.GetEnumerator()) { if (SkipBeforeFirst(en) && en.MoveNext()) @@ -426,17 +476,8 @@ public override Iterator Take(int count) return default; } - public override TSource? TryGetLast(out bool found) + public TSource? TryGetLast(out bool found) { - if (_source is Iterator iterator && - iterator.GetCount(onlyIfCheap: true) is int count && - count >= _minIndexInclusive) - { - return !HasLimit ? - iterator.TryGetLast(out found) : - iterator.TryGetElementAt(_maxIndexInclusive, out found); - } - using (IEnumerator en = _source.GetEnumerator()) { if (SkipBeforeFirst(en) && en.MoveNext()) @@ -461,7 +502,7 @@ public override Iterator Take(int count) return default; } - public override TSource[] ToArray() + public TSource[] ToArray() { using (IEnumerator en = _source.GetEnumerator()) { @@ -489,7 +530,7 @@ public override TSource[] ToArray() return []; } - public override List ToList() + public List ToList() { var list = new List(); @@ -524,7 +565,7 @@ private static int SkipAndCount(int index, IEnumerator en) private static uint SkipAndCount(uint index, IEnumerator en) { - Debug.Assert(en is not null); + Debug.Assert(en != null); for (uint i = 0; i < index; i++) { diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index 41d1336018bd33..c125673e16d36e 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -2,42 +2,76 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace System.Linq { public static partial class Enumerable { - private sealed partial class RangeIterator : IList, IReadOnlyList + private sealed partial class RangeIterator : IPartition, IList, IReadOnlyList { public override IEnumerable Select(Func selector) { - return new RangeSelectIterator(_start, _end, selector); + return new SelectRangeIterator(_start, _end, selector); } - public override int[] ToArray() + public int[] ToArray() { int start = _start; int[] array = new int[_end - start]; - FillIncrementing(array, start); + Fill(array, start); return array; } - public override List ToList() + public List ToList() { (int start, int end) = (_start, _end); List list = new List(end - start); - FillIncrementing(SetCountAndGetSpan(list, end - start), start); + Fill(SetCountAndGetSpan(list, end - start), start); return list; } public void CopyTo(int[] array, int arrayIndex) => - FillIncrementing(array.AsSpan(arrayIndex, _end - _start), _start); + Fill(array.AsSpan(arrayIndex, _end - _start), _start); - public override int GetCount(bool onlyIfCheap) => _end - _start; + private static void Fill(Span destination, int value) + { + ref int pos = ref MemoryMarshal.GetReference(destination); + ref int end = ref Unsafe.Add(ref pos, destination.Length); + + if (Vector.IsHardwareAccelerated && + destination.Length >= Vector.Count) + { + Vector init = Vector.Indices; + Vector current = new Vector(value) + init; + Vector increment = new Vector(Vector.Count); + + ref int oneVectorFromEnd = ref Unsafe.Subtract(ref end, Vector.Count); + do + { + current.StoreUnsafe(ref pos); + current += increment; + pos = ref Unsafe.Add(ref pos, Vector.Count); + } + while (!Unsafe.IsAddressGreaterThan(ref pos, ref oneVectorFromEnd)); + + value = current[0]; + } + + while (Unsafe.IsAddressLessThan(ref pos, ref end)) + { + pos = value++; + pos = ref Unsafe.Add(ref pos, 1); + } + } + + public int GetCount(bool onlyIfCheap) => _end - _start; public int Count => _end - _start; - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { if (count >= _end - _start) { @@ -47,7 +81,7 @@ public void CopyTo(int[] array, int arrayIndex) => return new RangeIterator(_start + count, _end - _start - count); } - public override Iterator Take(int count) + public IPartition Take(int count) { int curCount = _end - _start; if (count >= curCount) @@ -58,7 +92,7 @@ public override Iterator Take(int count) return new RangeIterator(_start, count); } - public override int TryGetElementAt(int index, out bool found) + public int TryGetElementAt(int index, out bool found) { if ((uint)index < (uint)(_end - _start)) { @@ -70,13 +104,13 @@ public override int TryGetElementAt(int index, out bool found) return 0; } - public override int TryGetFirst(out bool found) + public int TryGetFirst(out bool found) { found = true; return _start; } - public override int TryGetLast(out bool found) + public int TryGetLast(out bool found) { found = true; return _end - 1; diff --git a/src/libraries/System.Linq/src/System/Linq/Range.cs b/src/libraries/System.Linq/src/System/Linq/Range.cs index 1bf57b0e520d01..206f90415d52a1 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.cs @@ -3,9 +3,6 @@ using System.Collections.Generic; using System.Diagnostics; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace System.Linq { @@ -45,7 +42,7 @@ public RangeIterator(int start, int count) private int CountForDebugger => _end - _start; - private protected override Iterator Clone() => new RangeIterator(_start, _end - _start); + public override Iterator Clone() => new RangeIterator(_start, _end - _start); public override bool MoveNext() { @@ -74,37 +71,5 @@ public override void Dispose() _state = -1; // Don't reset current } } - - /// Fills the with incrementing numbers, starting from . - private static void FillIncrementing(Span destination, int value) - { - ref int pos = ref MemoryMarshal.GetReference(destination); - ref int end = ref Unsafe.Add(ref pos, destination.Length); - - if (Vector.IsHardwareAccelerated && - destination.Length >= Vector.Count) - { - Vector init = Vector.Indices; - Vector current = new Vector(value) + init; - Vector increment = new Vector(Vector.Count); - - ref int oneVectorFromEnd = ref Unsafe.Subtract(ref end, Vector.Count); - do - { - current.StoreUnsafe(ref pos); - current += increment; - pos = ref Unsafe.Add(ref pos, Vector.Count); - } - while (!Unsafe.IsAddressGreaterThan(ref pos, ref oneVectorFromEnd)); - - value = current[0]; - } - - while (Unsafe.IsAddressLessThan(ref pos, ref end)) - { - pos = value++; - pos = ref Unsafe.Add(ref pos, 1); - } - } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Repeat.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Repeat.SpeedOpt.cs index 2e98ae6b8fecd5..3c25ee20ba5f2c 100644 --- a/src/libraries/System.Linq/src/System/Linq/Repeat.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Repeat.SpeedOpt.cs @@ -8,12 +8,15 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class RepeatIterator : IList, IReadOnlyList + private sealed partial class RepeatIterator : IPartition, IList, IReadOnlyList { - public override TResult[] ToArray() + public override IEnumerable Select(Func selector) => + new SelectIPartitionIterator(this, selector); + + public TResult[] ToArray() { TResult[] array = new TResult[_count]; - if (_current is not null) + if (_current != null) { Array.Fill(array, _current); } @@ -21,7 +24,7 @@ public override TResult[] ToArray() return array; } - public override List ToList() + public List ToList() { List list = new List(_count); SetCountAndGetSpan(list, _count).Fill(_current); @@ -29,11 +32,11 @@ public override List ToList() return list; } - public override int GetCount(bool onlyIfCheap) => _count; + public int GetCount(bool onlyIfCheap) => _count; public int Count => _count; - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { Debug.Assert(count > 0); @@ -45,7 +48,7 @@ public override List ToList() return new RepeatIterator(_current, _count - count); } - public override Iterator Take(int count) + public IPartition Take(int count) { Debug.Assert(count > 0); @@ -57,7 +60,7 @@ public override Iterator Take(int count) return new RepeatIterator(_current, count); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { if ((uint)index < (uint)_count) { @@ -69,13 +72,13 @@ public override Iterator Take(int count) return default; } - public override TResult TryGetFirst(out bool found) + public TResult TryGetFirst(out bool found) { found = true; return _current; } - public override TResult TryGetLast(out bool found) + public TResult TryGetLast(out bool found) { found = true; return _current; diff --git a/src/libraries/System.Linq/src/System/Linq/Repeat.cs b/src/libraries/System.Linq/src/System/Linq/Repeat.cs index ed6a9a696a30a2..d0ef9941e6ef09 100644 --- a/src/libraries/System.Linq/src/System/Linq/Repeat.cs +++ b/src/libraries/System.Linq/src/System/Linq/Repeat.cs @@ -39,7 +39,7 @@ public RepeatIterator(TResult element, int count) _count = count; } - private protected override Iterator Clone() + public override Iterator Clone() { return new RepeatIterator(_current, _count); } diff --git a/src/libraries/System.Linq/src/System/Linq/Reverse.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Reverse.SpeedOpt.cs index d1ec26de879afd..bb301cc3084899 100644 --- a/src/libraries/System.Linq/src/System/Linq/Reverse.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Reverse.SpeedOpt.cs @@ -7,28 +7,28 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class ReverseIterator + private sealed partial class ReverseIterator : IPartition { - public override TSource[] ToArray() + public TSource[] ToArray() { TSource[] array = _source.ToArray(); Array.Reverse(array); return array; } - public override List ToList() + public List ToList() { List list = _source.ToList(); list.Reverse(); return list; } - public override int GetCount(bool onlyIfCheap) => + public int GetCount(bool onlyIfCheap) => !onlyIfCheap ? _source.Count() : TryGetNonEnumeratedCount(_source, out int count) ? count : -1; - public override TSource? TryGetElementAt(int index, out bool found) + public TSource? TryGetElementAt(int index, out bool found) { if (_source is IList list) { @@ -53,11 +53,11 @@ public override int GetCount(bool onlyIfCheap) => return default; } - public override TSource? TryGetFirst(out bool found) + public TSource? TryGetFirst(out bool found) { - if (_source is Iterator iterator) + if (_source is IPartition partition) { - return iterator.TryGetLast(out found); + return partition.TryGetLast(out found); } else if (_source is IList list) { @@ -89,11 +89,11 @@ public override int GetCount(bool onlyIfCheap) => return default; } - public override TSource? TryGetLast(out bool found) + public TSource? TryGetLast(out bool found) { - if (_source is Iterator iterator) + if (_source is IPartition partition) { - return iterator.TryGetFirst(out found); + return partition.TryGetFirst(out found); } else if (_source is IList list) { @@ -116,6 +116,10 @@ public override int GetCount(bool onlyIfCheap) => found = false; return default; } + + public IPartition? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Reverse.cs b/src/libraries/System.Linq/src/System/Linq/Reverse.cs index 0eba290b7c7f49..2a02115daca12a 100644 --- a/src/libraries/System.Linq/src/System/Linq/Reverse.cs +++ b/src/libraries/System.Linq/src/System/Linq/Reverse.cs @@ -10,7 +10,7 @@ public static partial class Enumerable { public static IEnumerable Reverse(this IEnumerable source) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -34,11 +34,11 @@ private sealed partial class ReverseIterator : Iterator public ReverseIterator(IEnumerable source) { - Debug.Assert(source is not null); + Debug.Assert(source != null); _source = source; } - private protected override Iterator Clone() => new ReverseIterator(_source); + public override Iterator Clone() => new ReverseIterator(_source); public override bool MoveNext() { @@ -70,7 +70,7 @@ public override bool MoveNext() int index = _state - 3; if (index != -1) { - Debug.Assert(_buffer is not null); + Debug.Assert(_buffer != null); _current = _buffer[index]; --_state; return true; diff --git a/src/libraries/System.Linq/src/System/Linq/Select.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Select.SpeedOpt.cs index f491f1f0de015a..06f87db9c0f786 100644 --- a/src/libraries/System.Linq/src/System/Linq/Select.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Select.SpeedOpt.cs @@ -10,9 +10,15 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class IEnumerableSelectIterator + static partial void CreateSelectIPartitionIterator( + Func selector, IPartition partition, ref IEnumerable? result) { - public override TResult[] ToArray() + result = new SelectIPartitionIterator(partition, selector); + } + + private sealed partial class SelectEnumerableIterator : IIListProvider + { + public TResult[] ToArray() { SegmentedArrayBuilder.ScratchBuffer scratch = default; SegmentedArrayBuilder builder = new(scratch); @@ -29,7 +35,7 @@ public override TResult[] ToArray() return result; } - public override List ToList() + public List ToList() { var list = new List(); @@ -42,7 +48,7 @@ public override List ToList() return list; } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { // In case someone uses Count() to force evaluation of // the selector, run it provided `onlyIfCheap` is false. @@ -65,73 +71,11 @@ public override int GetCount(bool onlyIfCheap) return count; } - - public override TResult? TryGetElementAt(int index, out bool found) - { - if (index >= 0) - { - IEnumerator e = _source.GetEnumerator(); - try - { - while (e.MoveNext()) - { - if (index == 0) - { - found = true; - return _selector(e.Current); - } - - index--; - } - } - finally - { - (e as IDisposable)?.Dispose(); - } - } - - found = false; - return default; - } - - public override TResult? TryGetFirst(out bool found) - { - using IEnumerator e = _source.GetEnumerator(); - if (e.MoveNext()) - { - found = true; - return _selector(e.Current); - } - - found = false; - return default; - } - - public override TResult? TryGetLast(out bool found) - { - using IEnumerator e = _source.GetEnumerator(); - - if (e.MoveNext()) - { - found = true; - TSource last = e.Current; - - while (e.MoveNext()) - { - last = e.Current; - } - - return _selector(last); - } - - found = false; - return default; - } } - private sealed partial class ArraySelectIterator + private sealed partial class SelectArrayIterator : IPartition { - public override TResult[] ToArray() + public TResult[] ToArray() { // See assert in constructor. // Since _source should never be empty, we don't check for 0/return Array.Empty. @@ -144,7 +88,7 @@ public override TResult[] ToArray() return results; } - public override List ToList() + public List ToList() { TSource[] source = _source; Debug.Assert(source.Length > 0); @@ -163,7 +107,7 @@ private static void Fill(ReadOnlySpan source, Span destination } } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { // In case someone uses Count() to force evaluation of // the selector, run it provided `onlyIfCheap` is false. @@ -179,7 +123,7 @@ public override int GetCount(bool onlyIfCheap) return _source.Length; } - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { Debug.Assert(count > 0); if (count >= _source.Length) @@ -187,31 +131,30 @@ public override int GetCount(bool onlyIfCheap) return null; } - return new IListSkipTakeSelectIterator(_source, _selector, count, int.MaxValue); + return new SelectListPartitionIterator(_source, _selector, count, int.MaxValue); } - public override Iterator Take(int count) + public IPartition Take(int count) { Debug.Assert(count > 0); return count >= _source.Length ? this : - new IListSkipTakeSelectIterator(_source, _selector, 0, count - 1); + new SelectListPartitionIterator(_source, _selector, 0, count - 1); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { - TSource[] source = _source; - if ((uint)index < (uint)source.Length) + if ((uint)index < (uint)_source.Length) { found = true; - return _selector(source[index]); + return _selector(_source[index]); } found = false; return default; } - public override TResult TryGetFirst(out bool found) + public TResult TryGetFirst(out bool found) { Debug.Assert(_source.Length > 0); // See assert in constructor @@ -219,34 +162,34 @@ public override TResult TryGetFirst(out bool found) return _selector(_source[0]); } - public override TResult TryGetLast(out bool found) + public TResult TryGetLast(out bool found) { Debug.Assert(_source.Length > 0); // See assert in constructor found = true; - return _selector(_source[^1]); + return _selector(_source[_source.Length - 1]); } } - private sealed partial class RangeSelectIterator : Iterator + private sealed partial class SelectRangeIterator : Iterator, IPartition { private readonly int _start; private readonly int _end; private readonly Func _selector; - public RangeSelectIterator(int start, int end, Func selector) + public SelectRangeIterator(int start, int end, Func selector) { Debug.Assert(start < end); Debug.Assert((uint)(end - start) <= (uint)int.MaxValue); - Debug.Assert(selector is not null); + Debug.Assert(selector != null); _start = start; _end = end; _selector = selector; } - private protected override Iterator Clone() => - new RangeSelectIterator(_start, _end, _selector); + public override Iterator Clone() => + new SelectRangeIterator(_start, _end, _selector); public override bool MoveNext() { @@ -263,9 +206,9 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new RangeSelectIterator(_start, _end, CombineSelectors(_selector, selector)); + new SelectRangeIterator(_start, _end, CombineSelectors(_selector, selector)); - public override TResult[] ToArray() + public TResult[] ToArray() { var results = new TResult[_end - _start]; Fill(results, _start, _selector); @@ -273,7 +216,7 @@ public override TResult[] ToArray() return results; } - public override List ToList() + public List ToList() { var results = new List(_end - _start); Fill(SetCountAndGetSpan(results, _end - _start), _start, _selector); @@ -289,7 +232,7 @@ private static void Fill(Span results, int start, Func fu } } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { // In case someone uses Count() to force evaluation of the selector, // run it provided `onlyIfCheap` is false. @@ -304,7 +247,7 @@ public override int GetCount(bool onlyIfCheap) return _end - _start; } - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { Debug.Assert(count > 0); @@ -313,10 +256,10 @@ public override int GetCount(bool onlyIfCheap) return null; } - return new RangeSelectIterator(_start + count, _end, _selector); + return new SelectRangeIterator(_start + count, _end, _selector); } - public override Iterator Take(int count) + public IPartition Take(int count) { Debug.Assert(count > 0); @@ -325,10 +268,10 @@ public override Iterator Take(int count) return this; } - return new RangeSelectIterator(_start, _start + count, _selector); + return new SelectRangeIterator(_start, _start + count, _selector); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { if ((uint)index < (uint)(_end - _start)) { @@ -340,14 +283,14 @@ public override Iterator Take(int count) return default; } - public override TResult TryGetFirst(out bool found) + public TResult TryGetFirst(out bool found) { Debug.Assert(_end > _start); found = true; return _selector(_start); } - public override TResult TryGetLast(out bool found) + public TResult TryGetLast(out bool found) { Debug.Assert(_end > _start); found = true; @@ -355,9 +298,9 @@ public override TResult TryGetLast(out bool found) } } - private sealed partial class ListSelectIterator + private sealed partial class SelectListIterator : IPartition { - public override TResult[] ToArray() + public TResult[] ToArray() { ReadOnlySpan source = CollectionsMarshal.AsSpan(_source); if (source.Length == 0) @@ -371,7 +314,7 @@ public override TResult[] ToArray() return results; } - public override List ToList() + public List ToList() { ReadOnlySpan source = CollectionsMarshal.AsSpan(_source); @@ -389,7 +332,7 @@ private static void Fill(ReadOnlySpan source, Span destination } } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { // In case someone uses Count() to force evaluation of // the selector, run it provided `onlyIfCheap` is false. @@ -407,19 +350,19 @@ public override int GetCount(bool onlyIfCheap) return count; } - public override Iterator Skip(int count) + public IPartition Skip(int count) { Debug.Assert(count > 0); - return new IListSkipTakeSelectIterator(_source, _selector, count, int.MaxValue); + return new SelectListPartitionIterator(_source, _selector, count, int.MaxValue); } - public override Iterator Take(int count) + public IPartition Take(int count) { Debug.Assert(count > 0); - return new IListSkipTakeSelectIterator(_source, _selector, 0, count - 1); + return new SelectListPartitionIterator(_source, _selector, 0, count - 1); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { if ((uint)index < (uint)_source.Count) { @@ -431,7 +374,7 @@ public override Iterator Take(int count) return default; } - public override TResult? TryGetFirst(out bool found) + public TResult? TryGetFirst(out bool found) { if (_source.Count != 0) { @@ -443,7 +386,7 @@ public override Iterator Take(int count) return default; } - public override TResult? TryGetLast(out bool found) + public TResult? TryGetLast(out bool found) { int len = _source.Count; if (len != 0) @@ -457,9 +400,9 @@ public override Iterator Take(int count) } } - private sealed partial class IListSelectIterator + private sealed partial class SelectIListIterator : IPartition { - public override TResult[] ToArray() + public TResult[] ToArray() { int count = _source.Count; if (count == 0) @@ -473,7 +416,7 @@ public override TResult[] ToArray() return results; } - public override List ToList() + public List ToList() { IList source = _source; int count = _source.Count; @@ -492,7 +435,7 @@ private static void Fill(IList source, Span results, Func Skip(int count) + public IPartition Skip(int count) { Debug.Assert(count > 0); - return new IListSkipTakeSelectIterator(_source, _selector, count, int.MaxValue); + return new SelectListPartitionIterator(_source, _selector, count, int.MaxValue); } - public override Iterator Take(int count) + public IPartition Take(int count) { Debug.Assert(count > 0); - return new IListSkipTakeSelectIterator(_source, _selector, 0, count - 1); + return new SelectListPartitionIterator(_source, _selector, 0, count - 1); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { if ((uint)index < (uint)_source.Count) { @@ -534,7 +477,7 @@ public override Iterator Take(int count) return default; } - public override TResult? TryGetFirst(out bool found) + public TResult? TryGetFirst(out bool found) { if (_source.Count != 0) { @@ -546,7 +489,7 @@ public override Iterator Take(int count) return default; } - public override TResult? TryGetLast(out bool found) + public TResult? TryGetLast(out bool found) { int len = _source.Count; if (len != 0) @@ -561,26 +504,26 @@ public override Iterator Take(int count) } /// - /// An iterator that maps each item of an . + /// An iterator that maps each item of an . /// - /// The type of the source elements. + /// The type of the source partition. /// The type of the mapped items. - private sealed class IteratorSelectIterator : Iterator + private sealed class SelectIPartitionIterator : Iterator, IPartition { - private readonly Iterator _source; + private readonly IPartition _source; private readonly Func _selector; - private Iterator? _enumerator; + private IEnumerator? _enumerator; - public IteratorSelectIterator(Iterator source, Func selector) + public SelectIPartitionIterator(IPartition source, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); _source = source; _selector = selector; } - private protected override Iterator Clone() => - new IteratorSelectIterator(_source, _selector); + public override Iterator Clone() => + new SelectIPartitionIterator(_source, _selector); public override bool MoveNext() { @@ -591,7 +534,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (_enumerator.MoveNext()) { _current = _selector(_enumerator.Current); @@ -607,7 +550,7 @@ public override bool MoveNext() public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -617,23 +560,23 @@ public override void Dispose() } public override IEnumerable Select(Func selector) => - new IteratorSelectIterator(_source, CombineSelectors(_selector, selector)); + new SelectIPartitionIterator(_source, CombineSelectors(_selector, selector)); - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { Debug.Assert(count > 0); - Iterator? source = _source.Skip(count); - return source is null ? null : new IteratorSelectIterator(source, _selector); + IPartition? source = _source.Skip(count); + return source is null ? null : new SelectIPartitionIterator(source, _selector); } - public override Iterator? Take(int count) + public IPartition? Take(int count) { Debug.Assert(count > 0); - Iterator? source = _source.Take(count); - return source is null ? null : new IteratorSelectIterator(source, _selector); + IPartition? source = _source.Take(count); + return source is null ? null : new SelectIPartitionIterator(source, _selector); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { bool sourceFound; TSource? input = _source.TryGetElementAt(index, out sourceFound); @@ -641,7 +584,7 @@ public override IEnumerable Select(Func s return sourceFound ? _selector(input!) : default!; } - public override TResult? TryGetFirst(out bool found) + public TResult? TryGetFirst(out bool found) { bool sourceFound; TSource? input = _source.TryGetFirst(out sourceFound); @@ -649,7 +592,7 @@ public override IEnumerable Select(Func s return sourceFound ? _selector(input!) : default!; } - public override TResult? TryGetLast(out bool found) + public TResult? TryGetLast(out bool found) { bool sourceFound; TSource? input = _source.TryGetLast(out sourceFound); @@ -686,7 +629,7 @@ private TResult[] PreallocatingToArray(int count) return array; } - public override TResult[] ToArray() + public TResult[] ToArray() { int count = _source.GetCount(onlyIfCheap: true); return count switch @@ -697,7 +640,7 @@ public override TResult[] ToArray() }; } - public override List ToList() + public List ToList() { int count = _source.GetCount(onlyIfCheap: true); List list; @@ -722,7 +665,7 @@ public override List ToList() return list; } - private static void Fill(Iterator source, Span results, Func func) + private static void Fill(IPartition source, Span results, Func func) { int index = 0; foreach (TSource item in source) @@ -734,7 +677,7 @@ private static void Fill(Iterator source, Span results, FuncThe type of the source list. /// The type of the mapped items. [DebuggerDisplay("Count = {Count}")] - private sealed class IListSkipTakeSelectIterator : Iterator + private sealed class SelectListPartitionIterator : Iterator, IPartition { private readonly IList _source; private readonly Func _selector; private readonly int _minIndexInclusive; private readonly int _maxIndexInclusive; - public IListSkipTakeSelectIterator(IList source, Func selector, int minIndexInclusive, int maxIndexInclusive) + public SelectListPartitionIterator(IList source, Func selector, int minIndexInclusive, int maxIndexInclusive) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); Debug.Assert(minIndexInclusive >= 0); Debug.Assert(minIndexInclusive <= maxIndexInclusive); _source = source; @@ -781,8 +724,8 @@ public IListSkipTakeSelectIterator(IList source, Func _maxIndexInclusive = maxIndexInclusive; } - private protected override Iterator Clone() => - new IListSkipTakeSelectIterator(_source, _selector, _minIndexInclusive, _maxIndexInclusive); + public override Iterator Clone() => + new SelectListPartitionIterator(_source, _selector, _minIndexInclusive, _maxIndexInclusive); public override bool MoveNext() { @@ -802,23 +745,23 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new IListSkipTakeSelectIterator(_source, CombineSelectors(_selector, selector), _minIndexInclusive, _maxIndexInclusive); + new SelectListPartitionIterator(_source, CombineSelectors(_selector, selector), _minIndexInclusive, _maxIndexInclusive); - public override Iterator? Skip(int count) + public IPartition? Skip(int count) { Debug.Assert(count > 0); int minIndex = _minIndexInclusive + count; - return (uint)minIndex > (uint)_maxIndexInclusive ? null : new IListSkipTakeSelectIterator(_source, _selector, minIndex, _maxIndexInclusive); + return (uint)minIndex > (uint)_maxIndexInclusive ? null : new SelectListPartitionIterator(_source, _selector, minIndex, _maxIndexInclusive); } - public override Iterator Take(int count) + public IPartition Take(int count) { Debug.Assert(count > 0); int maxIndex = _minIndexInclusive + count - 1; - return (uint)maxIndex >= (uint)_maxIndexInclusive ? this : new IListSkipTakeSelectIterator(_source, _selector, _minIndexInclusive, maxIndex); + return (uint)maxIndex >= (uint)_maxIndexInclusive ? this : new SelectListPartitionIterator(_source, _selector, _minIndexInclusive, maxIndex); } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { if ((uint)index <= (uint)(_maxIndexInclusive - _minIndexInclusive) && index < _source.Count - _minIndexInclusive) { @@ -830,7 +773,7 @@ public override Iterator Take(int count) return default; } - public override TResult? TryGetFirst(out bool found) + public TResult? TryGetFirst(out bool found) { if (_source.Count > _minIndexInclusive) { @@ -842,7 +785,7 @@ public override Iterator Take(int count) return default; } - public override TResult? TryGetLast(out bool found) + public TResult? TryGetLast(out bool found) { int lastIndex = _source.Count - 1; if (lastIndex >= _minIndexInclusive) @@ -869,7 +812,7 @@ private int Count } } - public override TResult[] ToArray() + public TResult[] ToArray() { int count = Count; if (count == 0) @@ -883,7 +826,7 @@ public override TResult[] ToArray() return array; } - public override List ToList() + public List ToList() { int count = Count; if (count == 0) @@ -905,7 +848,7 @@ private static void Fill(IList source, Span destination, Func< } } - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { // In case someone uses Count() to force evaluation of // the selector, run it provided `onlyIfCheap` is false. diff --git a/src/libraries/System.Linq/src/System/Linq/Select.cs b/src/libraries/System.Linq/src/System/Linq/Select.cs index 37a41c450f873c..3059fa7f0a8e02 100644 --- a/src/libraries/System.Linq/src/System/Linq/Select.cs +++ b/src/libraries/System.Linq/src/System/Linq/Select.cs @@ -13,12 +13,12 @@ public static partial class Enumerable public static IEnumerable Select( this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -37,28 +37,43 @@ public static IEnumerable Select( return []; } - return new ArraySelectIterator(array, selector); + return new SelectArrayIterator(array, selector); } if (source is List list) { - return new ListSelectIterator(list, selector); + return new SelectListIterator(list, selector); } - return new IListSelectIterator(ilist, selector); + return new SelectIListIterator(ilist, selector); } - return new IEnumerableSelectIterator(source, selector); + if (source is IPartition partition) + { + IEnumerable? result = null; + CreateSelectIPartitionIterator(selector, partition, ref result); + if (result != null) + { + return result; + } + } + + return new SelectEnumerableIterator(source, selector); } +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6177 + static partial void CreateSelectIPartitionIterator( + Func selector, IPartition partition, [NotNull] ref IEnumerable? result); +#pragma warning restore IDE0060 + public static IEnumerable Select(this IEnumerable source, Func selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -90,26 +105,26 @@ private static IEnumerable SelectIterator(IEnumerable /// /// The type of the source enumerable. /// The type of the mapped items. - private sealed partial class IEnumerableSelectIterator : Iterator + private sealed partial class SelectEnumerableIterator : Iterator { private readonly IEnumerable _source; private readonly Func _selector; private IEnumerator? _enumerator; - public IEnumerableSelectIterator(IEnumerable source, Func selector) + public SelectEnumerableIterator(IEnumerable source, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); _source = source; _selector = selector; } - private protected override Iterator Clone() => - new IEnumerableSelectIterator(_source, _selector); + public override Iterator Clone() => + new SelectEnumerableIterator(_source, _selector); public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -127,7 +142,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (_enumerator.MoveNext()) { _current = _selector(_enumerator.Current); @@ -142,7 +157,7 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new IEnumerableSelectIterator(_source, CombineSelectors(_selector, selector)); + new SelectEnumerableIterator(_source, CombineSelectors(_selector, selector)); } /// @@ -151,15 +166,15 @@ public override IEnumerable Select(Func s /// The type of the source array. /// The type of the mapped items. [DebuggerDisplay("Count = {CountForDebugger}")] - private sealed partial class ArraySelectIterator : Iterator + private sealed partial class SelectArrayIterator : Iterator { private readonly TSource[] _source; private readonly Func _selector; - public ArraySelectIterator(TSource[] source, Func selector) + public SelectArrayIterator(TSource[] source, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); Debug.Assert(source.Length > 0); // Caller should check this beforehand and return a cached result _source = source; _selector = selector; @@ -167,7 +182,7 @@ public ArraySelectIterator(TSource[] source, Func selector) private int CountForDebugger => _source.Length; - private protected override Iterator Clone() => new ArraySelectIterator(_source, _selector); + public override Iterator Clone() => new SelectArrayIterator(_source, _selector); public override bool MoveNext() { @@ -185,7 +200,7 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new ArraySelectIterator(_source, CombineSelectors(_selector, selector)); + new SelectArrayIterator(_source, CombineSelectors(_selector, selector)); } /// @@ -194,23 +209,23 @@ public override IEnumerable Select(Func s /// The type of the source list. /// The type of the mapped items. [DebuggerDisplay("Count = {CountForDebugger}")] - private sealed partial class ListSelectIterator : Iterator + private sealed partial class SelectListIterator : Iterator { private readonly List _source; private readonly Func _selector; private List.Enumerator _enumerator; - public ListSelectIterator(List source, Func selector) + public SelectListIterator(List source, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); _source = source; _selector = selector; } private int CountForDebugger => _source.Count; - private protected override Iterator Clone() => new ListSelectIterator(_source, _selector); + public override Iterator Clone() => new SelectListIterator(_source, _selector); public override bool MoveNext() { @@ -235,7 +250,7 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new ListSelectIterator(_source, CombineSelectors(_selector, selector)); + new SelectListIterator(_source, CombineSelectors(_selector, selector)); } /// @@ -244,23 +259,23 @@ public override IEnumerable Select(Func s /// The type of the source list. /// The type of the mapped items. [DebuggerDisplay("Count = {CountForDebugger}")] - private sealed partial class IListSelectIterator : Iterator + private sealed partial class SelectIListIterator : Iterator { private readonly IList _source; private readonly Func _selector; private IEnumerator? _enumerator; - public IListSelectIterator(IList source, Func selector) + public SelectIListIterator(IList source, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); _source = source; _selector = selector; } private int CountForDebugger => _source.Count; - private protected override Iterator Clone() => new IListSelectIterator(_source, _selector); + public override Iterator Clone() => new SelectIListIterator(_source, _selector); public override bool MoveNext() { @@ -271,7 +286,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); if (_enumerator.MoveNext()) { _current = _selector(_enumerator.Current); @@ -287,7 +302,7 @@ public override bool MoveNext() public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -297,7 +312,7 @@ public override void Dispose() } public override IEnumerable Select(Func selector) => - new IListSelectIterator(_source, CombineSelectors(_selector, selector)); + new SelectIListIterator(_source, CombineSelectors(_selector, selector)); } } } diff --git a/src/libraries/System.Linq/src/System/Linq/SelectMany.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/SelectMany.SpeedOpt.cs index ae0bf35ef8f1af..050ae6a4e06b61 100644 --- a/src/libraries/System.Linq/src/System/Linq/SelectMany.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/SelectMany.SpeedOpt.cs @@ -7,9 +7,9 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class SelectManySingleSelectorIterator + private sealed partial class SelectManySingleSelectorIterator : IIListProvider { - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { if (onlyIfCheap) { @@ -29,7 +29,7 @@ public override int GetCount(bool onlyIfCheap) return count; } - public override TResult[] ToArray() + public TResult[] ToArray() { SegmentedArrayBuilder.ScratchBuffer scratch = default; SegmentedArrayBuilder builder = new(scratch); @@ -46,7 +46,7 @@ public override TResult[] ToArray() return result; } - public override List ToList() + public List ToList() { var list = new List(); diff --git a/src/libraries/System.Linq/src/System/Linq/SelectMany.cs b/src/libraries/System.Linq/src/System/Linq/SelectMany.cs index d8845e8334db3e..748a9030f060a5 100644 --- a/src/libraries/System.Linq/src/System/Linq/SelectMany.cs +++ b/src/libraries/System.Linq/src/System/Linq/SelectMany.cs @@ -10,12 +10,12 @@ public static partial class Enumerable { public static IEnumerable SelectMany(this IEnumerable source, Func> selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -30,12 +30,12 @@ public static IEnumerable SelectMany(this IEnumerable public static IEnumerable SelectMany(this IEnumerable source, Func> selector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (selector is null) + if (selector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.selector); } @@ -67,17 +67,17 @@ private static IEnumerable SelectManyIterator(IEnumer public static IEnumerable SelectMany(this IEnumerable source, Func> collectionSelector, Func resultSelector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (collectionSelector is null) + if (collectionSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.collectionSelector); } - if (resultSelector is null) + if (resultSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.resultSelector); } @@ -109,17 +109,17 @@ private static IEnumerable SelectManyIterator SelectMany(this IEnumerable source, Func> collectionSelector, Func resultSelector) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (collectionSelector is null) + if (collectionSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.collectionSelector); } - if (resultSelector is null) + if (resultSelector == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.resultSelector); } @@ -152,27 +152,27 @@ private sealed partial class SelectManySingleSelectorIterator internal SelectManySingleSelectorIterator(IEnumerable source, Func> selector) { - Debug.Assert(source is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(selector != null); _source = source; _selector = selector; } - private protected override Iterator Clone() + public override Iterator Clone() { return new SelectManySingleSelectorIterator(_source, _selector); } public override void Dispose() { - if (_subEnumerator is not null) + if (_subEnumerator != null) { _subEnumerator.Dispose(); _subEnumerator = null; } - if (_sourceEnumerator is not null) + if (_sourceEnumerator != null) { _sourceEnumerator.Dispose(); _sourceEnumerator = null; @@ -192,7 +192,7 @@ public override bool MoveNext() goto case 2; case 2: // Take the next element from the source enumerator. - Debug.Assert(_sourceEnumerator is not null); + Debug.Assert(_sourceEnumerator != null); if (!_sourceEnumerator.MoveNext()) { break; @@ -206,7 +206,7 @@ public override bool MoveNext() goto case 3; case 3: // Take the next element from the sub-collection and yield. - Debug.Assert(_subEnumerator is not null); + Debug.Assert(_subEnumerator != null); if (!_subEnumerator.MoveNext()) { _subEnumerator.Dispose(); diff --git a/src/libraries/System.Linq/src/System/Linq/SequenceEqual.cs b/src/libraries/System.Linq/src/System/Linq/SequenceEqual.cs index 16bf60e686145e..ff4c32200a5c34 100644 --- a/src/libraries/System.Linq/src/System/Linq/SequenceEqual.cs +++ b/src/libraries/System.Linq/src/System/Linq/SequenceEqual.cs @@ -12,12 +12,12 @@ public static bool SequenceEqual(this IEnumerable first, IEnum public static bool SequenceEqual(this IEnumerable first, IEnumerable second, IEqualityComparer? comparer) { - if (first is null) + if (first == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.first); } - if (second is null) + if (second == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.second); } diff --git a/src/libraries/System.Linq/src/System/Linq/Single.cs b/src/libraries/System.Linq/src/System/Linq/Single.cs index 9c7328cc3ff16e..9e35009e6175f4 100644 --- a/src/libraries/System.Linq/src/System/Linq/Single.cs +++ b/src/libraries/System.Linq/src/System/Linq/Single.cs @@ -107,12 +107,12 @@ public static TSource SingleOrDefault(this IEnumerable source, private static TSource? TryGetSingle(this IEnumerable source, Func predicate, out bool found) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } diff --git a/src/libraries/System.Linq/src/System/Linq/SingleLinkedNode.cs b/src/libraries/System.Linq/src/System/Linq/SingleLinkedNode.cs index 144d1a81889bda..e0bf5e54c51ee0 100644 --- a/src/libraries/System.Linq/src/System/Linq/SingleLinkedNode.cs +++ b/src/libraries/System.Linq/src/System/Linq/SingleLinkedNode.cs @@ -27,7 +27,7 @@ public SingleLinkedNode(TSource item) /// The item to place in this node. private SingleLinkedNode(SingleLinkedNode linked, TSource item) { - Debug.Assert(linked is not null); + Debug.Assert(linked != null); Linked = linked; Item = item; } @@ -54,7 +54,7 @@ private SingleLinkedNode(SingleLinkedNode linked, TSource item) public int GetCount() { int count = 0; - for (SingleLinkedNode? node = this; node is not null; node = node.Linked) + for (SingleLinkedNode? node = this; node != null; node = node.Linked) { count++; } @@ -77,7 +77,7 @@ public SingleLinkedNode GetNode(int index) for (; index > 0; index--) { node = node.Linked!; - Debug.Assert(node is not null); + Debug.Assert(node != null); } return node; @@ -103,7 +103,7 @@ public TSource[] ToArray(int count) public void Fill(Span span) { int index = 0; - for (SingleLinkedNode? node = this; node is not null; node = node.Linked) + for (SingleLinkedNode? node = this; node != null; node = node.Linked) { span[index] = node.Item; index++; @@ -117,7 +117,7 @@ public void Fill(Span span) public void FillReversed(Span span) { int index = span.Length; - for (SingleLinkedNode? node = this; node is not null; node = node.Linked) + for (SingleLinkedNode? node = this; node != null; node = node.Linked) { --index; span[index] = node.Item; diff --git a/src/libraries/System.Linq/src/System/Linq/Skip.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Skip.SpeedOpt.cs index 74ff73a068242c..1596dc0cc7cfe0 100644 --- a/src/libraries/System.Linq/src/System/Linq/Skip.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Skip.SpeedOpt.cs @@ -9,7 +9,7 @@ public static partial class Enumerable { private static IEnumerable SkipIterator(IEnumerable source, int count) => source is IList sourceList ? - (IEnumerable)new IListSkipTakeIterator(sourceList, count, int.MaxValue) : - new IEnumerableSkipTakeIterator(source, count, -1); + (IEnumerable)new ListPartition(sourceList, count, int.MaxValue) : + new EnumerablePartition(source, count, -1); } } diff --git a/src/libraries/System.Linq/src/System/Linq/Skip.cs b/src/libraries/System.Linq/src/System/Linq/Skip.cs index c87bea9e25e68a..3652d1da3e79e8 100644 --- a/src/libraries/System.Linq/src/System/Linq/Skip.cs +++ b/src/libraries/System.Linq/src/System/Linq/Skip.cs @@ -9,7 +9,7 @@ public static partial class Enumerable { public static IEnumerable Skip(this IEnumerable source, int count) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -23,31 +23,29 @@ public static IEnumerable Skip(this IEnumerable sourc { // Return source if not actually skipping, but only if it's a type from here, to avoid // issues if collections are used as keys or otherwise must not be aliased. - if (source is Iterator) + if (source is Iterator || source is IPartition) { return source; } count = 0; } -#if !OPTIMIZE_FOR_SIZE - else if (source is Iterator iterator) + else if (source is IPartition partition) { - return iterator.Skip(count) ?? Empty(); + return partition.Skip(count) ?? Empty(); } -#endif return SkipIterator(source, count); } public static IEnumerable SkipWhile(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -83,12 +81,12 @@ private static IEnumerable SkipWhileIterator(IEnumerable SkipWhile(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -130,7 +128,7 @@ private static IEnumerable SkipWhileIterator(IEnumerable SkipLast(this IEnumerable source, int count) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } diff --git a/src/libraries/System.Linq/src/System/Linq/Take.SizeOpt.cs b/src/libraries/System.Linq/src/System/Linq/Take.SizeOpt.cs index f61b01ee57773a..65ad659f1a2ce6 100644 --- a/src/libraries/System.Linq/src/System/Linq/Take.SizeOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Take.SizeOpt.cs @@ -21,7 +21,7 @@ private static IEnumerable TakeIterator(IEnumerable s private static IEnumerable TakeRangeIterator(IEnumerable source, int startIndex, int endIndex) { - Debug.Assert(source is not null); + Debug.Assert(source != null); Debug.Assert(startIndex >= 0 && startIndex < endIndex); using IEnumerator e = source.GetEnumerator(); diff --git a/src/libraries/System.Linq/src/System/Linq/Take.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Take.SpeedOpt.cs index 81025ee21d0b4b..f97c5295f75a78 100644 --- a/src/libraries/System.Linq/src/System/Linq/Take.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Take.SpeedOpt.cs @@ -10,30 +10,30 @@ public static partial class Enumerable { private static IEnumerable TakeIterator(IEnumerable source, int count) { - Debug.Assert(source is not null && !IsEmptyArray(source)); + Debug.Assert(source != null && !IsEmptyArray(source)); Debug.Assert(count > 0); return - source is Iterator iterator ? (iterator.Take(count) ?? Empty()) : - source is IList sourceList ? new IListSkipTakeIterator(sourceList, 0, count - 1) : - new IEnumerableSkipTakeIterator(source, 0, count - 1); + source is IPartition partition ? (partition.Take(count) ?? Empty()) : + source is IList sourceList ? new ListPartition(sourceList, 0, count - 1) : + new EnumerablePartition(source, 0, count - 1); } private static IEnumerable TakeRangeIterator(IEnumerable source, int startIndex, int endIndex) { - Debug.Assert(source is not null && !IsEmptyArray(source)); + Debug.Assert(source != null && !IsEmptyArray(source)); Debug.Assert(startIndex >= 0 && startIndex < endIndex); return - source is Iterator iterator ? TakeIteratorRange(iterator, startIndex, endIndex) : - source is IList sourceList ? new IListSkipTakeIterator(sourceList, startIndex, endIndex - 1) : - new IEnumerableSkipTakeIterator(source, startIndex, endIndex - 1); + source is IPartition partition ? TakePartitionRange(partition, startIndex, endIndex) : + source is IList sourceList ? new ListPartition(sourceList, startIndex, endIndex - 1) : + new EnumerablePartition(source, startIndex, endIndex - 1); - static IEnumerable TakeIteratorRange(Iterator iterator, int startIndex, int endIndex) + static IEnumerable TakePartitionRange(IPartition partition, int startIndex, int endIndex) { - Iterator? source; + IPartition? source; if (endIndex != 0 && - (source = iterator.Take(endIndex)) is not null && + (source = partition.Take(endIndex)) is not null && (startIndex == 0 || (source = source!.Skip(startIndex)) is not null)) { return source; diff --git a/src/libraries/System.Linq/src/System/Linq/Take.cs b/src/libraries/System.Linq/src/System/Linq/Take.cs index 65feb8a3eb08b8..627d5bf03ed78f 100644 --- a/src/libraries/System.Linq/src/System/Linq/Take.cs +++ b/src/libraries/System.Linq/src/System/Linq/Take.cs @@ -10,7 +10,7 @@ public static partial class Enumerable { public static IEnumerable Take(this IEnumerable source, int count) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -32,7 +32,7 @@ public static IEnumerable Take(this IEnumerable sourc /// public static IEnumerable Take(this IEnumerable source, Range range) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } @@ -70,7 +70,7 @@ public static IEnumerable Take(this IEnumerable sourc private static IEnumerable TakeRangeFromEndIterator(IEnumerable source, bool isStartIndexFromEnd, int startIndex, bool isEndIndexFromEnd, int endIndex) { - Debug.Assert(source is not null); + Debug.Assert(source != null); Debug.Assert(isStartIndexFromEnd || isEndIndexFromEnd); Debug.Assert(isStartIndexFromEnd ? startIndex > 0 && (!isEndIndexFromEnd || startIndex > endIndex) @@ -189,12 +189,12 @@ static int CalculateEndIndex(bool isEndIndexFromEnd, int endIndex, int count) => public static IEnumerable TakeWhile(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -222,12 +222,12 @@ private static IEnumerable TakeWhileIterator(IEnumerable TakeWhile(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -261,7 +261,7 @@ private static IEnumerable TakeWhileIterator(IEnumerable TakeLast(this IEnumerable source, int count) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } diff --git a/src/libraries/System.Linq/src/System/Linq/ToCollection.cs b/src/libraries/System.Linq/src/System/Linq/ToCollection.cs index afbf1d4e158a9b..043cac8f0038b0 100644 --- a/src/libraries/System.Linq/src/System/Linq/ToCollection.cs +++ b/src/libraries/System.Linq/src/System/Linq/ToCollection.cs @@ -16,12 +16,10 @@ public static TSource[] ToArray(this IEnumerable source) ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } -#if !OPTIMIZE_FOR_SIZE - if (source is Iterator iterator) + if (source is IIListProvider arrayProvider) { - return iterator.ToArray(); + return arrayProvider.ToArray(); } -#endif if (source is ICollection collection) { @@ -59,12 +57,10 @@ public static List ToList(this IEnumerable source) ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } -#if !OPTIMIZE_FOR_SIZE - if (source is Iterator iterator) + if (source is IIListProvider listProvider) { - return iterator.ToList(); + return listProvider.ToList(); } -#endif return new List(source); } diff --git a/src/libraries/System.Linq/src/System/Linq/Union.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Union.SpeedOpt.cs index e907c63562ddb3..6acf199e665b2a 100644 --- a/src/libraries/System.Linq/src/System/Linq/Union.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Union.SpeedOpt.cs @@ -7,7 +7,7 @@ namespace System.Linq { public static partial class Enumerable { - private abstract partial class UnionIterator + private abstract partial class UnionIterator : IIListProvider { private HashSet FillSet() { @@ -15,7 +15,7 @@ private HashSet FillSet() for (int index = 0; ; ++index) { IEnumerable? enumerable = GetEnumerable(index); - if (enumerable is null) + if (enumerable == null) { return set; } @@ -24,27 +24,11 @@ private HashSet FillSet() } } - public override TSource[] ToArray() => HashSetToArray(FillSet()); + public TSource[] ToArray() => Enumerable.HashSetToArray(FillSet()); - public override List ToList() => new List(FillSet()); + public List ToList() => new List(FillSet()); - public override int GetCount(bool onlyIfCheap) => onlyIfCheap ? -1 : FillSet().Count; - - public override TSource? TryGetFirst(out bool found) - { - IEnumerable? source; - for (int i = 0; (source = GetEnumerable(i)) is not null; i++) - { - TSource? result = source.TryGetFirst(out found); - if (found) - { - return result; - } - } - - found = false; - return default; - } + public int GetCount(bool onlyIfCheap) => onlyIfCheap ? -1 : FillSet().Count; } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Union.cs b/src/libraries/System.Linq/src/System/Linq/Union.cs index a8cb34ac5173a5..6031498bbce54f 100644 --- a/src/libraries/System.Linq/src/System/Linq/Union.cs +++ b/src/libraries/System.Linq/src/System/Linq/Union.cs @@ -13,12 +13,12 @@ public static partial class Enumerable public static IEnumerable Union(this IEnumerable first, IEnumerable second, IEqualityComparer? comparer) { - if (first is null) + if (first == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.first); } - if (second is null) + if (second == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.second); } @@ -111,7 +111,7 @@ protected UnionIterator(IEqualityComparer? comparer) public sealed override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -134,7 +134,7 @@ private void SetEnumerator(IEnumerator enumerator) private void StoreFirst() { - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); var set = new HashSet(DefaultInternalSetCapacity, _comparer); TSource element = _enumerator.Current; @@ -145,8 +145,8 @@ private void StoreFirst() private bool GetNext() { - Debug.Assert(_enumerator is not null); - Debug.Assert(_set is not null); + Debug.Assert(_enumerator != null); + Debug.Assert(_set != null); HashSet set = _set; @@ -167,7 +167,7 @@ public sealed override bool MoveNext() { if (_state == 1) { - for (IEnumerable? enumerable = GetEnumerable(0); enumerable is not null; enumerable = GetEnumerable(_state - 1)) + for (IEnumerable? enumerable = GetEnumerable(0); enumerable != null; enumerable = GetEnumerable(_state - 1)) { IEnumerator enumerator = enumerable.GetEnumerator(); SetEnumerator(enumerator); @@ -190,7 +190,7 @@ public sealed override bool MoveNext() } IEnumerable? enumerable = GetEnumerable(_state - 1); - if (enumerable is null) + if (enumerable == null) { break; } @@ -217,13 +217,13 @@ private sealed class UnionIterator2 : UnionIterator public UnionIterator2(IEnumerable first, IEnumerable second, IEqualityComparer? comparer) : base(comparer) { - Debug.Assert(first is not null); - Debug.Assert(second is not null); + Debug.Assert(first != null); + Debug.Assert(second != null); _first = first; _second = second; } - private protected override Iterator Clone() => new UnionIterator2(_first, _second, _comparer); + public override Iterator Clone() => new UnionIterator2(_first, _second, _comparer); internal override IEnumerable? GetEnumerable(int index) { @@ -262,7 +262,7 @@ public UnionIteratorN(SingleLinkedNode> sources, int headIn _headIndex = headIndex; } - private protected override Iterator Clone() => new UnionIteratorN(_sources, _headIndex, _comparer); + public override Iterator Clone() => new UnionIteratorN(_sources, _headIndex, _comparer); internal override IEnumerable? GetEnumerable(int index) => index > _headIndex ? null : _sources.GetNode(_headIndex - index).Item; diff --git a/src/libraries/System.Linq/src/System/Linq/Utilities.cs b/src/libraries/System.Linq/src/System/Linq/Utilities.cs index 987d0004d4ec3a..208d6878040a48 100644 --- a/src/libraries/System.Linq/src/System/Linq/Utilities.cs +++ b/src/libraries/System.Linq/src/System/Linq/Utilities.cs @@ -26,7 +26,7 @@ public static bool AreEqualityComparersEqual(IEqualityComparer var defaultComparer = EqualityComparer.Default; - if (left is null) + if (left == null) { // Micro-opt: Typically it's impossible to get a different instance // of the default comparer without reflection/serialization. @@ -34,7 +34,7 @@ public static bool AreEqualityComparersEqual(IEqualityComparer return right == defaultComparer || right!.Equals(defaultComparer); } - if (right is null) + if (right == null) { return left == defaultComparer || left.Equals(defaultComparer); } diff --git a/src/libraries/System.Linq/src/System/Linq/Where.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Where.SpeedOpt.cs index 40a05db9abf8ff..5bf0a718088386 100644 --- a/src/libraries/System.Linq/src/System/Linq/Where.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Where.SpeedOpt.cs @@ -8,9 +8,9 @@ namespace System.Linq { public static partial class Enumerable { - private sealed partial class IEnumerableWhereIterator + private sealed partial class WhereEnumerableIterator : IPartition { - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { if (onlyIfCheap) { @@ -33,7 +33,7 @@ public override int GetCount(bool onlyIfCheap) return count; } - public override TSource[] ToArray() + public TSource[] ToArray() { SegmentedArrayBuilder.ScratchBuffer scratch = default; SegmentedArrayBuilder builder = new(scratch); @@ -53,7 +53,7 @@ public override TSource[] ToArray() return result; } - public override List ToList() + public List ToList() { var list = new List(); @@ -69,7 +69,7 @@ public override List ToList() return list; } - public override TSource? TryGetFirst(out bool found) + public TSource? TryGetFirst(out bool found) { Func predicate = _predicate; @@ -86,7 +86,7 @@ public override List ToList() return default; } - public override TSource? TryGetLast(out bool found) + public TSource? TryGetLast(out bool found) { using IEnumerator e = _source.GetEnumerator(); @@ -121,7 +121,7 @@ public override List ToList() return default; } - public override TSource? TryGetElementAt(int index, out bool found) + public TSource? TryGetElementAt(int index, out bool found) { if (index >= 0) { @@ -145,11 +145,15 @@ public override List ToList() found = false; return default; } + + public IPartition? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } - private sealed partial class ArrayWhereIterator + internal sealed partial class WhereArrayIterator : IPartition { - public override int GetCount(bool onlyIfCheap) => GetCount(onlyIfCheap, _source, _predicate); + public int GetCount(bool onlyIfCheap) => GetCount(onlyIfCheap, _source, _predicate); public static int GetCount(bool onlyIfCheap, ReadOnlySpan source, Func predicate) { @@ -174,7 +178,7 @@ public static int GetCount(bool onlyIfCheap, ReadOnlySpan source, Func< return count; } - public override TSource[] ToArray() => ToArray(_source, _predicate); + public TSource[] ToArray() => ToArray(_source, _predicate); public static TSource[] ToArray(ReadOnlySpan source, Func predicate) { @@ -195,7 +199,7 @@ public static TSource[] ToArray(ReadOnlySpan source, Func ToList() => ToList(_source, _predicate); + public List ToList() => ToList(_source, _predicate); public static List ToList(ReadOnlySpan source, Func predicate) { @@ -212,7 +216,7 @@ public static List ToList(ReadOnlySpan source, Func predicate = _predicate; @@ -229,7 +233,7 @@ public static List ToList(ReadOnlySpan source, Func predicate = _predicate; @@ -247,7 +251,7 @@ public static List ToList(ReadOnlySpan source, Func= 0) { @@ -271,17 +275,21 @@ public static List ToList(ReadOnlySpan source, Func? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } - private sealed partial class ListWhereIterator : Iterator + private sealed partial class WhereListIterator : Iterator, IPartition { - public override int GetCount(bool onlyIfCheap) => ArrayWhereIterator.GetCount(onlyIfCheap, CollectionsMarshal.AsSpan(_source), _predicate); + public int GetCount(bool onlyIfCheap) => WhereArrayIterator.GetCount(onlyIfCheap, CollectionsMarshal.AsSpan(_source), _predicate); - public override TSource[] ToArray() => ArrayWhereIterator.ToArray(CollectionsMarshal.AsSpan(_source), _predicate); + public TSource[] ToArray() => WhereArrayIterator.ToArray(CollectionsMarshal.AsSpan(_source), _predicate); - public override List ToList() => ArrayWhereIterator.ToList(CollectionsMarshal.AsSpan(_source), _predicate); + public List ToList() => WhereArrayIterator.ToList(CollectionsMarshal.AsSpan(_source), _predicate); - public override TSource? TryGetFirst(out bool found) + public TSource? TryGetFirst(out bool found) { Func predicate = _predicate; @@ -298,7 +306,7 @@ private sealed partial class ListWhereIterator : Iterator return default; } - public override TSource? TryGetLast(out bool found) + public TSource? TryGetLast(out bool found) { ReadOnlySpan source = CollectionsMarshal.AsSpan(_source); Func predicate = _predicate; @@ -316,7 +324,7 @@ private sealed partial class ListWhereIterator : Iterator return default; } - public override TSource? TryGetElementAt(int index, out bool found) + public TSource? TryGetElementAt(int index, out bool found) { if (index >= 0) { @@ -340,11 +348,15 @@ private sealed partial class ListWhereIterator : Iterator found = false; return default; } + + public IPartition? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } - private sealed partial class ArrayWhereSelectIterator + private sealed partial class WhereSelectArrayIterator : IPartition { - public override int GetCount(bool onlyIfCheap) => GetCount(onlyIfCheap, _source, _predicate, _selector); + public int GetCount(bool onlyIfCheap) => GetCount(onlyIfCheap, _source, _predicate, _selector); public static int GetCount(bool onlyIfCheap, ReadOnlySpan source, Func predicate, Func selector) { @@ -373,7 +385,7 @@ public static int GetCount(bool onlyIfCheap, ReadOnlySpan source, Func< return count; } - public override TResult[] ToArray() => ToArray(_source, _predicate, _selector); + public TResult[] ToArray() => ToArray(_source, _predicate, _selector); public static TResult[] ToArray(ReadOnlySpan source, Func predicate, Func selector) { @@ -394,7 +406,7 @@ public static TResult[] ToArray(ReadOnlySpan source, Func ToList() => ToList(_source, _predicate, _selector); + public List ToList() => ToList(_source, _predicate, _selector); public static List ToList(ReadOnlySpan source, Func predicate, Func selector) { @@ -411,7 +423,7 @@ public static List ToList(ReadOnlySpan source, Func TryGetFirst(_source, _predicate, _selector, out found); + public TResult? TryGetFirst(out bool found) => TryGetFirst(_source, _predicate, _selector, out found); public static TResult? TryGetFirst(ReadOnlySpan source, Func predicate, Func selector, out bool found) { @@ -428,7 +440,7 @@ public static List ToList(ReadOnlySpan source, Func TryGetLast(_source, _predicate, _selector, out found); + public TResult? TryGetLast(out bool found) => TryGetLast(_source, _predicate, _selector, out found); public static TResult? TryGetLast(ReadOnlySpan source, Func predicate, Func selector, out bool found) { @@ -445,7 +457,7 @@ public static List ToList(ReadOnlySpan source, Func TryGetElementAt(_source, _predicate, _selector, index, out found); + public TResult? TryGetElementAt(int index, out bool found) => TryGetElementAt(_source, _predicate, _selector, index, out found); public static TResult? TryGetElementAt(ReadOnlySpan source, Func predicate, Func selector, int index, out bool found) { @@ -469,26 +481,34 @@ public static List ToList(ReadOnlySpan source, Func? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } - private sealed partial class ListWhereSelectIterator + private sealed partial class WhereSelectListIterator : IPartition { - public override int GetCount(bool onlyIfCheap) => ArrayWhereSelectIterator.GetCount(onlyIfCheap, CollectionsMarshal.AsSpan(_source), _predicate, _selector); + public int GetCount(bool onlyIfCheap) => WhereSelectArrayIterator.GetCount(onlyIfCheap, CollectionsMarshal.AsSpan(_source), _predicate, _selector); + + public TResult[] ToArray() => WhereSelectArrayIterator.ToArray(CollectionsMarshal.AsSpan(_source), _predicate, _selector); - public override TResult[] ToArray() => ArrayWhereSelectIterator.ToArray(CollectionsMarshal.AsSpan(_source), _predicate, _selector); + public List ToList() => WhereSelectArrayIterator.ToList(CollectionsMarshal.AsSpan(_source), _predicate, _selector); - public override List ToList() => ArrayWhereSelectIterator.ToList(CollectionsMarshal.AsSpan(_source), _predicate, _selector); + public TResult? TryGetElementAt(int index, out bool found) => WhereSelectArrayIterator.TryGetElementAt(CollectionsMarshal.AsSpan(_source), _predicate, _selector, index, out found); - public override TResult? TryGetElementAt(int index, out bool found) => ArrayWhereSelectIterator.TryGetElementAt(CollectionsMarshal.AsSpan(_source), _predicate, _selector, index, out found); + public TResult? TryGetFirst(out bool found) => WhereSelectArrayIterator.TryGetFirst(CollectionsMarshal.AsSpan(_source), _predicate, _selector, out found); - public override TResult? TryGetFirst(out bool found) => ArrayWhereSelectIterator.TryGetFirst(CollectionsMarshal.AsSpan(_source), _predicate, _selector, out found); + public TResult? TryGetLast(out bool found) => WhereSelectArrayIterator.TryGetLast(CollectionsMarshal.AsSpan(_source), _predicate, _selector, out found); - public override TResult? TryGetLast(out bool found) => ArrayWhereSelectIterator.TryGetLast(CollectionsMarshal.AsSpan(_source), _predicate, _selector, out found); + public IPartition? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } - private sealed partial class IEnumerableWhereSelectIterator + private sealed partial class WhereSelectEnumerableIterator : IPartition { - public override int GetCount(bool onlyIfCheap) + public int GetCount(bool onlyIfCheap) { // In case someone uses Count() to force evaluation of // the selector, run it provided `onlyIfCheap` is false. @@ -515,7 +535,7 @@ public override int GetCount(bool onlyIfCheap) return count; } - public override TResult[] ToArray() + public TResult[] ToArray() { SegmentedArrayBuilder.ScratchBuffer scratch = default; SegmentedArrayBuilder builder = new(scratch); @@ -536,7 +556,7 @@ public override TResult[] ToArray() return result; } - public override List ToList() + public List ToList() { var list = new List(); @@ -553,7 +573,7 @@ public override List ToList() return list; } - public override TResult? TryGetFirst(out bool found) + public TResult? TryGetFirst(out bool found) { Func predicate = _predicate; @@ -570,7 +590,7 @@ public override List ToList() return default; } - public override TResult? TryGetLast(out bool found) + public TResult? TryGetLast(out bool found) { using IEnumerator e = _source.GetEnumerator(); @@ -605,7 +625,7 @@ public override List ToList() return default; } - public override TResult? TryGetElementAt(int index, out bool found) + public TResult? TryGetElementAt(int index, out bool found) { if (index >= 0) { @@ -629,6 +649,10 @@ public override List ToList() found = false; return default; } + + public IPartition? Skip(int count) => new EnumerablePartition(this, count, -1); + + public IPartition? Take(int count) => new EnumerablePartition(this, 0, count - 1); } } } diff --git a/src/libraries/System.Linq/src/System/Linq/Where.cs b/src/libraries/System.Linq/src/System/Linq/Where.cs index 4371af8299fb2e..aec6370a330f8e 100644 --- a/src/libraries/System.Linq/src/System/Linq/Where.cs +++ b/src/libraries/System.Linq/src/System/Linq/Where.cs @@ -11,12 +11,12 @@ public static partial class Enumerable { public static IEnumerable Where(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -33,25 +33,25 @@ public static IEnumerable Where(this IEnumerable sour return []; } - return new ArrayWhereIterator(array, predicate); + return new WhereArrayIterator(array, predicate); } if (source is List list) { - return new ListWhereIterator(list, predicate); + return new WhereListIterator(list, predicate); } - return new IEnumerableWhereIterator(source, predicate); + return new WhereEnumerableIterator(source, predicate); } public static IEnumerable Where(this IEnumerable source, Func predicate) { - if (source is null) + if (source == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.source); } - if (predicate is null) + if (predicate == null) { ThrowHelper.ThrowArgumentNullException(ExceptionArgument.predicate); } @@ -85,25 +85,25 @@ private static IEnumerable WhereIterator(IEnumerable /// An iterator that filters each item of an . /// /// The type of the source enumerable. - private sealed partial class IEnumerableWhereIterator : Iterator + private sealed partial class WhereEnumerableIterator : Iterator { private readonly IEnumerable _source; private readonly Func _predicate; private IEnumerator? _enumerator; - public IEnumerableWhereIterator(IEnumerable source, Func predicate) + public WhereEnumerableIterator(IEnumerable source, Func predicate) { - Debug.Assert(source is not null); - Debug.Assert(predicate is not null); + Debug.Assert(source != null); + Debug.Assert(predicate != null); _source = source; _predicate = predicate; } - private protected override Iterator Clone() => new IEnumerableWhereIterator(_source, _predicate); + public override Iterator Clone() => new WhereEnumerableIterator(_source, _predicate); public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -121,7 +121,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); while (_enumerator.MoveNext()) { TSource item = _enumerator.Current; @@ -140,31 +140,31 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new IEnumerableWhereSelectIterator(_source, _predicate, selector); + new WhereSelectEnumerableIterator(_source, _predicate, selector); public override IEnumerable Where(Func predicate) => - new IEnumerableWhereIterator(_source, CombinePredicates(_predicate, predicate)); + new WhereEnumerableIterator(_source, CombinePredicates(_predicate, predicate)); } /// /// An iterator that filters each item of an array. /// /// The type of the source array. - private sealed partial class ArrayWhereIterator : Iterator + internal sealed partial class WhereArrayIterator : Iterator { private readonly TSource[] _source; private readonly Func _predicate; - public ArrayWhereIterator(TSource[] source, Func predicate) + public WhereArrayIterator(TSource[] source, Func predicate) { - Debug.Assert(source is not null && source.Length > 0); - Debug.Assert(predicate is not null); + Debug.Assert(source != null && source.Length > 0); + Debug.Assert(predicate != null); _source = source; _predicate = predicate; } - private protected override Iterator Clone() => - new ArrayWhereIterator(_source, _predicate); + public override Iterator Clone() => + new WhereArrayIterator(_source, _predicate); public override bool MoveNext() { @@ -187,32 +187,32 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new ArrayWhereSelectIterator(_source, _predicate, selector); + new WhereSelectArrayIterator(_source, _predicate, selector); public override IEnumerable Where(Func predicate) => - new ArrayWhereIterator(_source, CombinePredicates(_predicate, predicate)); + new WhereArrayIterator(_source, CombinePredicates(_predicate, predicate)); } /// /// An iterator that filters each item of a . /// /// The type of the source list. - private sealed partial class ListWhereIterator : Iterator + private sealed partial class WhereListIterator : Iterator { private readonly List _source; private readonly Func _predicate; private List.Enumerator _enumerator; - public ListWhereIterator(List source, Func predicate) + public WhereListIterator(List source, Func predicate) { - Debug.Assert(source is not null); - Debug.Assert(predicate is not null); + Debug.Assert(source != null); + Debug.Assert(predicate != null); _source = source; _predicate = predicate; } - private protected override Iterator Clone() => - new ListWhereIterator(_source, _predicate); + public override Iterator Clone() => + new WhereListIterator(_source, _predicate); public override bool MoveNext() { @@ -241,10 +241,10 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new ListWhereSelectIterator(_source, _predicate, selector); + new WhereSelectListIterator(_source, _predicate, selector); public override IEnumerable Where(Func predicate) => - new ListWhereIterator(_source, CombinePredicates(_predicate, predicate)); + new WhereListIterator(_source, CombinePredicates(_predicate, predicate)); } /// @@ -252,24 +252,24 @@ public override IEnumerable Where(Func predicate) => /// /// The type of the source array. /// The type of the mapped items. - private sealed partial class ArrayWhereSelectIterator : Iterator + private sealed partial class WhereSelectArrayIterator : Iterator { private readonly TSource[] _source; private readonly Func _predicate; private readonly Func _selector; - public ArrayWhereSelectIterator(TSource[] source, Func predicate, Func selector) + public WhereSelectArrayIterator(TSource[] source, Func predicate, Func selector) { - Debug.Assert(source is not null && source.Length > 0); - Debug.Assert(predicate is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null && source.Length > 0); + Debug.Assert(predicate != null); + Debug.Assert(selector != null); _source = source; _predicate = predicate; _selector = selector; } - private protected override Iterator Clone() => - new ArrayWhereSelectIterator(_source, _predicate, _selector); + public override Iterator Clone() => + new WhereSelectArrayIterator(_source, _predicate, _selector); public override bool MoveNext() { @@ -292,7 +292,7 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new ArrayWhereSelectIterator(_source, _predicate, CombineSelectors(_selector, selector)); + new WhereSelectArrayIterator(_source, _predicate, CombineSelectors(_selector, selector)); } /// @@ -300,25 +300,25 @@ public override IEnumerable Select(Func s /// /// The type of the source list. /// The type of the mapped items. - private sealed partial class ListWhereSelectIterator : Iterator + private sealed partial class WhereSelectListIterator : Iterator { private readonly List _source; private readonly Func _predicate; private readonly Func _selector; private List.Enumerator _enumerator; - public ListWhereSelectIterator(List source, Func predicate, Func selector) + public WhereSelectListIterator(List source, Func predicate, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(predicate is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(predicate != null); + Debug.Assert(selector != null); _source = source; _predicate = predicate; _selector = selector; } - private protected override Iterator Clone() => - new ListWhereSelectIterator(_source, _predicate, _selector); + public override Iterator Clone() => + new WhereSelectListIterator(_source, _predicate, _selector); public override bool MoveNext() { @@ -347,7 +347,7 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new ListWhereSelectIterator(_source, _predicate, CombineSelectors(_selector, selector)); + new WhereSelectListIterator(_source, _predicate, CombineSelectors(_selector, selector)); } /// @@ -355,29 +355,29 @@ public override IEnumerable Select(Func s /// /// The type of the source enumerable. /// The type of the mapped items. - private sealed partial class IEnumerableWhereSelectIterator : Iterator + private sealed partial class WhereSelectEnumerableIterator : Iterator { private readonly IEnumerable _source; private readonly Func _predicate; private readonly Func _selector; private IEnumerator? _enumerator; - public IEnumerableWhereSelectIterator(IEnumerable source, Func predicate, Func selector) + public WhereSelectEnumerableIterator(IEnumerable source, Func predicate, Func selector) { - Debug.Assert(source is not null); - Debug.Assert(predicate is not null); - Debug.Assert(selector is not null); + Debug.Assert(source != null); + Debug.Assert(predicate != null); + Debug.Assert(selector != null); _source = source; _predicate = predicate; _selector = selector; } - private protected override Iterator Clone() => - new IEnumerableWhereSelectIterator(_source, _predicate, _selector); + public override Iterator Clone() => + new WhereSelectEnumerableIterator(_source, _predicate, _selector); public override void Dispose() { - if (_enumerator is not null) + if (_enumerator != null) { _enumerator.Dispose(); _enumerator = null; @@ -395,7 +395,7 @@ public override bool MoveNext() _state = 2; goto case 2; case 2: - Debug.Assert(_enumerator is not null); + Debug.Assert(_enumerator != null); while (_enumerator.MoveNext()) { TSource item = _enumerator.Current; @@ -414,7 +414,7 @@ public override bool MoveNext() } public override IEnumerable Select(Func selector) => - new IEnumerableWhereSelectIterator(_source, _predicate, CombineSelectors(_selector, selector)); + new WhereSelectEnumerableIterator(_source, _predicate, CombineSelectors(_selector, selector)); } } } diff --git a/src/libraries/System.Linq/tests/AggregateByTests.cs b/src/libraries/System.Linq/tests/AggregateByTests.cs index 6232ce24a6df49..daae145f775508 100644 --- a/src/libraries/System.Linq/tests/AggregateByTests.cs +++ b/src/libraries/System.Linq/tests/AggregateByTests.cs @@ -8,16 +8,6 @@ namespace System.Linq.Tests { public class AggregateByTests : EnumerableTests { - [Fact] - public void Empty() - { - Assert.All(IdentityTransforms(), transform => - { - Assert.Equal(Enumerable.Empty>(), transform(Enumerable.Empty()).AggregateBy(i => i, i => i, (a, i) => a + i)); - Assert.Equal(Enumerable.Empty>(), transform(Enumerable.Empty()).AggregateBy(i => i, 0, (a, i) => a + i)); - }); - } - [Fact] public void AggregateBy_SourceNull_ThrowsArgumentNullException() { @@ -25,26 +15,22 @@ public void AggregateBy_SourceNull_ThrowsArgumentNullException() AssertExtensions.Throws("source", () => first.AggregateBy(x => x, string.Empty, (x, y) => x + y)); AssertExtensions.Throws("source", () => first.AggregateBy(x => x, string.Empty, (x, y) => x + y, new AnagramEqualityComparer())); - AssertExtensions.Throws("source", () => first.AggregateBy(x => x, x => x, (x, y) => x + y)); - AssertExtensions.Throws("source", () => first.AggregateBy(x => x, x => x, (x, y) => x + y, new AnagramEqualityComparer())); } [Fact] public void AggregateBy_KeySelectorNull_ThrowsArgumentNullException() { - string[] source = ["test"]; + string[] source = { }; Func keySelector = null; AssertExtensions.Throws("keySelector", () => source.AggregateBy(keySelector, string.Empty, (x, y) => x + y)); AssertExtensions.Throws("keySelector", () => source.AggregateBy(keySelector, string.Empty, (x, y) => x + y, new AnagramEqualityComparer())); - AssertExtensions.Throws("keySelector", () => source.AggregateBy(keySelector, x => x, (x, y) => x + y)); - AssertExtensions.Throws("keySelector", () => source.AggregateBy(keySelector, x => x, (x, y) => x + y, new AnagramEqualityComparer())); } [Fact] public void AggregateBy_SeedSelectorNull_ThrowsArgumentNullException() { - string[] source = ["test"]; + string[] source = { }; Func seedSelector = null; AssertExtensions.Throws("seedSelector", () => source.AggregateBy(x => x, seedSelector, (x, y) => x + y)); @@ -54,13 +40,11 @@ public void AggregateBy_SeedSelectorNull_ThrowsArgumentNullException() [Fact] public void AggregateBy_FuncNull_ThrowsArgumentNullException() { - string[] source = ["test"]; + string[] source = { }; Func func = null; AssertExtensions.Throws("func", () => source.AggregateBy(x => x, string.Empty, func)); AssertExtensions.Throws("func", () => source.AggregateBy(x => x, string.Empty, func, new AnagramEqualityComparer())); - AssertExtensions.Throws("func", () => source.AggregateBy(x => x, x => x, func)); - AssertExtensions.Throws("func", () => source.AggregateBy(x => x, x => x, func, new AnagramEqualityComparer())); } [Fact] diff --git a/src/libraries/System.Linq/tests/AnyTests.cs b/src/libraries/System.Linq/tests/AnyTests.cs index 555b5064c81570..4d1732d3e33971 100644 --- a/src/libraries/System.Linq/tests/AnyTests.cs +++ b/src/libraries/System.Linq/tests/AnyTests.cs @@ -102,7 +102,7 @@ public static IEnumerable TestDataWithPredicate() [MemberData(nameof(TestDataWithPredicate))] public void Any_Predicate(IEnumerable source, Func predicate, bool expected) { - if (predicate is null) + if (predicate == null) { Assert.Equal(expected, source.Any()); } @@ -115,7 +115,7 @@ public void Any_Predicate(IEnumerable source, Func predicate, bo [Theory, MemberData(nameof(TestDataWithPredicate))] public void AnyRunOnce(IEnumerable source, Func predicate, bool expected) { - if (predicate is null) + if (predicate == null) { Assert.Equal(expected, source.RunOnce().Any()); } diff --git a/src/libraries/System.Linq/tests/AppendPrependTests.cs b/src/libraries/System.Linq/tests/AppendPrependTests.cs index e1ac1ae11c861f..9df154f46ff47a 100644 --- a/src/libraries/System.Linq/tests/AppendPrependTests.cs +++ b/src/libraries/System.Linq/tests/AppendPrependTests.cs @@ -89,7 +89,7 @@ public void ForcedToEnumeratorDoesntEnumeratePrepend() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Prepend(4); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -98,7 +98,7 @@ public void ForcedToEnumeratorDoesntEnumerateAppend() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Append(4); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -107,7 +107,7 @@ public void ForcedToEnumeratorDoesntEnumerateMultipleAppendsAndPrepends() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Append(4).Append(5).Prepend(-1).Prepend(-2); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -263,27 +263,5 @@ public void AppendPrependRunOnce() source = NumberRangeGuaranteedNotCollectionType(2, 2).Prepend(1).Prepend(0).Append(4).Append(5).RunOnce(); Assert.Equal(Enumerable.Range(0, 6), source.ToList()); } - - [Fact] - public void AppendPrepend_First_Last_ElementAt() - { - Assert.Equal(42, new int[] { 42 }.Append(84).First()); - Assert.Equal(42, new int[] { 84 }.Prepend(42).First()); - Assert.Equal(84, new int[] { 42 }.Append(84).Last()); - Assert.Equal(84, new int[] { 84 }.Prepend(42).Last()); - Assert.Equal(42, new int[] { 42 }.Append(84).ElementAt(0)); - Assert.Equal(42, new int[] { 84 }.Prepend(42).ElementAt(0)); - Assert.Equal(84, new int[] { 42 }.Append(84).ElementAt(1)); - Assert.Equal(84, new int[] { 84 }.Prepend(42).ElementAt(1)); - - Assert.Equal(42, NumberRangeGuaranteedNotCollectionType(42, 1).Append(84).First()); - Assert.Equal(42, NumberRangeGuaranteedNotCollectionType(84, 1).Prepend(42).First()); - Assert.Equal(84, NumberRangeGuaranteedNotCollectionType(42, 1).Append(84).Last()); - Assert.Equal(84, NumberRangeGuaranteedNotCollectionType(84, 1).Prepend(42).Last()); - Assert.Equal(42, NumberRangeGuaranteedNotCollectionType(42, 1).Append(84).ElementAt(0)); - Assert.Equal(42, NumberRangeGuaranteedNotCollectionType(84, 1).Prepend(42).ElementAt(0)); - Assert.Equal(84, NumberRangeGuaranteedNotCollectionType(42, 1).Append(84).ElementAt(1)); - Assert.Equal(84, NumberRangeGuaranteedNotCollectionType(84, 1).Prepend(42).ElementAt(1)); - } } } diff --git a/src/libraries/System.Linq/tests/CastTests.cs b/src/libraries/System.Linq/tests/CastTests.cs index 125c2b57f43f32..6577d2e72e35d2 100644 --- a/src/libraries/System.Linq/tests/CastTests.cs +++ b/src/libraries/System.Linq/tests/CastTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using Xunit; @@ -229,106 +230,10 @@ public void NullSource() [Fact] public void ForcedToEnumeratorDoesntEnumerate() { - var iterator = new object[0].Where(i => i is not null).Cast(); + var iterator = new object[0].Where(i => i != null).Cast(); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); - } - - [Fact] - public void TargetTypeIsSourceType_Nop() - { - object[] values = new string[] { "hello", "world" }; - Assert.Same(values, values.Cast()); - } - - [Fact] - public void CastOnMultidimensionalArraySucceeds() - { - Array array = Array.CreateInstance(typeof(int), 2, 3); - for (int i = 0; i < 2; i++) - { - for (int j = 0; j < 3; j++) - { - array.SetValue(i * 3 + j, i, j); - } - } - - int[] result = array.Cast().ToArray(); - for (int i = 0; i < 6; i++) - { - Assert.Equal(i, result[i]); - } - } - - [Fact] - public void CastCountReturnsExpectedLength() - { - object[] objects = new object[] { "hello", "world" }; - Assert.Equal(2, objects.Cast().Count()); - } - - [Fact] - public void CastFirstReturnsFirstElement() - { - object[] objects = new object[] { "hello", "world" }; - Assert.Equal("hello", objects.Cast().First()); - } - - [Fact] - public void CastFirstOnEmptySequenceThrows() - { - object[] objects = Array.Empty(); - Assert.Throws(() => objects.Cast().First()); - } - - [Fact] - public void CastLastReturnsLastElement() - { - object[] objects = new object[] { "hello", "world" }; - Assert.Equal("world", objects.Cast().Last()); - } - - [Fact] - public void CastElementAtReturnsExpectedElement() - { - object[] objects = new object[] { "hello", "world" }; - Assert.Equal("world", objects.Cast().ElementAt(1)); - } - - [Fact] - public void CastElementAtOutOfRangeThrows() - { - object[] objects = new object[] { "hello", "world" }; - Assert.Throws(() => objects.Cast().ElementAt(2)); - } - - [Fact] - public void CastLastOnEmptySequenceThrows() - { - object[] objects = Array.Empty(); - Assert.Throws(() => objects.Cast().Last()); - } - - [Fact] - public void CastSelectProcessesEachElement() - { - object[] objects = new object[] { "hello", "world!" }; - Assert.Equal(new[] { 5, 6 }, objects.Cast().Select(s => s.Length)); - } - - [Fact] - public void CastSkipSkipsElements() - { - object[] objects = new object[] { "hello", "there", "world" }; - Assert.Equal(new[] { "world" }, objects.Cast().Skip(2)); - } - - [Fact] - public void CastTakeTakesElements() - { - object[] objects = new object[] { "hello", "there", "world" }; - Assert.Equal(new[] { "hello", "there" }, objects.Cast().Take(2)); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/ChunkTests.cs b/src/libraries/System.Linq/tests/ChunkTests.cs index 31433ddabff30f..ee348604192792 100644 --- a/src/libraries/System.Linq/tests/ChunkTests.cs +++ b/src/libraries/System.Linq/tests/ChunkTests.cs @@ -7,12 +7,6 @@ namespace System.Linq.Tests { public class ChunkTests : EnumerableTests { - [Fact] - public void Empty() - { - Assert.Equal(Enumerable.Empty(), Enumerable.Empty().Chunk(4)); - } - [Fact] public void ThrowsOnNullSource() { @@ -38,82 +32,88 @@ public void ChunkSourceLazily() Assert.True(chunks.MoveNext()); } - [Theory] - [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345})] - public void ChunkSourceRepeatCalls(int[] array) + private static IEnumerable ConvertToType(T[] array, Type type) { - Assert.All(IdentityTransforms(), t => + return type switch { - IEnumerable source = t(array); + {} x when x == typeof(TestReadOnlyCollection) => new TestReadOnlyCollection(array), + {} x when x == typeof(TestCollection) => new TestCollection(array), + {} x when x == typeof(TestEnumerable) => new TestEnumerable(array), + _ => throw new Exception() + }; + } + + [Theory] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345}, typeof(TestReadOnlyCollection))] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345}, typeof(TestCollection))] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345}, typeof(TestEnumerable))] + public void ChunkSourceRepeatCalls(int[] array, Type type) + { + IEnumerable source = ConvertToType(array, type); - Assert.Equal(source.Chunk(3), source.Chunk(3)); - }); + Assert.Equal(source.Chunk(3), source.Chunk(3)); } [Theory] - [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345})] - public void ChunkSourceEvenly(int[] array) + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345}, typeof(TestReadOnlyCollection))] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345}, typeof(TestCollection))] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2, -12345}, typeof(TestEnumerable))] + public void ChunkSourceEvenly(int[] array, Type type) { - Assert.All(IdentityTransforms(), t => - { - IEnumerable source = t(array); - - using IEnumerator chunks = source.Chunk(3).GetEnumerator(); - chunks.MoveNext(); - Assert.Equal(new[] { 9999, 0, 888 }, chunks.Current); - chunks.MoveNext(); - Assert.Equal(new[] { -1, 66, -777 }, chunks.Current); - chunks.MoveNext(); - Assert.Equal(new[] { 1, 2, -12345 }, chunks.Current); - Assert.False(chunks.MoveNext()); - }); + IEnumerable source = ConvertToType(array, type); + + using IEnumerator chunks = source.Chunk(3).GetEnumerator(); + chunks.MoveNext(); + Assert.Equal(new[] {9999, 0, 888}, chunks.Current); + chunks.MoveNext(); + Assert.Equal(new[] {-1, 66, -777}, chunks.Current); + chunks.MoveNext(); + Assert.Equal(new[] {1, 2, -12345}, chunks.Current); + Assert.False(chunks.MoveNext()); } [Theory] - [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2})] - public void ChunkSourceUnevenly(int[] array) + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2}, typeof(TestReadOnlyCollection))] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2}, typeof(TestCollection))] + [InlineData(new[] {9999, 0, 888, -1, 66, -777, 1, 2}, typeof(TestEnumerable))] + public void ChunkSourceUnevenly(int[] array, Type type) { - Assert.All(IdentityTransforms(), t => - { - IEnumerable source = t(array); - - using IEnumerator chunks = source.Chunk(3).GetEnumerator(); - chunks.MoveNext(); - Assert.Equal(new[] { 9999, 0, 888 }, chunks.Current); - chunks.MoveNext(); - Assert.Equal(new[] { -1, 66, -777 }, chunks.Current); - chunks.MoveNext(); - Assert.Equal(new[] { 1, 2 }, chunks.Current); - Assert.False(chunks.MoveNext()); - }); + IEnumerable source = ConvertToType(array, type); + + using IEnumerator chunks = source.Chunk(3).GetEnumerator(); + chunks.MoveNext(); + Assert.Equal(new[] {9999, 0, 888}, chunks.Current); + chunks.MoveNext(); + Assert.Equal(new[] {-1, 66, -777}, chunks.Current); + chunks.MoveNext(); + Assert.Equal(new[] {1, 2}, chunks.Current); + Assert.False(chunks.MoveNext()); } [Theory] - [InlineData(new[] {9999, 0})] - public void ChunkSourceSmallerThanMaxSize(int[] array) + [InlineData(new[] {9999, 0}, typeof(TestReadOnlyCollection))] + [InlineData(new[] {9999, 0}, typeof(TestCollection))] + [InlineData(new[] {9999, 0}, typeof(TestEnumerable))] + public void ChunkSourceSmallerThanMaxSize(int[] array, Type type) { - Assert.All(IdentityTransforms(), t => - { - IEnumerable source = t(array); + IEnumerable source = ConvertToType(array, type); - using IEnumerator chunks = source.Chunk(3).GetEnumerator(); - chunks.MoveNext(); - Assert.Equal(new[] { 9999, 0 }, chunks.Current); - Assert.False(chunks.MoveNext()); - }); + using IEnumerator chunks = source.Chunk(3).GetEnumerator(); + chunks.MoveNext(); + Assert.Equal(new[] {9999, 0}, chunks.Current); + Assert.False(chunks.MoveNext()); } [Theory] - [InlineData(new int[0])] - public void EmptySourceYieldsNoChunks(int[] array) + [InlineData(new int[] {}, typeof(TestReadOnlyCollection))] + [InlineData(new int[] {}, typeof(TestCollection))] + [InlineData(new int[] {}, typeof(TestEnumerable))] + public void EmptySourceYieldsNoChunks(int[] array, Type type) { - Assert.All(IdentityTransforms(), t => - { - IEnumerable source = t(array); + IEnumerable source = ConvertToType(array, type); - using IEnumerator chunks = source.Chunk(3).GetEnumerator(); - Assert.False(chunks.MoveNext()); - }); + using IEnumerator chunks = source.Chunk(3).GetEnumerator(); + Assert.False(chunks.MoveNext()); } [Fact] diff --git a/src/libraries/System.Linq/tests/ConcatTests.cs b/src/libraries/System.Linq/tests/ConcatTests.cs index 357da40f2ea659..6209d846ff2438 100644 --- a/src/libraries/System.Linq/tests/ConcatTests.cs +++ b/src/libraries/System.Linq/tests/ConcatTests.cs @@ -51,7 +51,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Concat(Enumerable.Range(0, 3)); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -83,55 +83,6 @@ public void VerifyEquals(IEnumerable expected, IEnumerable actual) VerifyEqualsWorker(expected, actual); } - [Theory] - [MemberData(nameof(ArraySourcesData))] - [MemberData(nameof(SelectArraySourcesData))] - [MemberData(nameof(EnumerableSourcesData))] - [MemberData(nameof(NonCollectionSourcesData))] - [MemberData(nameof(ListSourcesData))] - [MemberData(nameof(ConcatOfConcatsData))] - [MemberData(nameof(ConcatWithSelfData))] - [MemberData(nameof(ChainedCollectionConcatData))] - [MemberData(nameof(AppendedPrependedConcatAlternationsData))] - public void First_Last_ElementAt(IEnumerable _, IEnumerable actual) - { - int count = actual.Count(); - if (count == 0) - { - Assert.Throws(() => actual.First()); - Assert.Throws(() => actual.Last()); - Assert.Throws(() => actual.ElementAt(0)); - } - else - { - int first = actual.First(); - int last = actual.Last(); - int elementAt = actual.ElementAt(count / 2); - - int enumeratedFirst = 0, enumeratedLast = 0, enumeratedElementAt = 0; - int i = 0; - foreach (int item in actual) - { - if (i == 0) - { - enumeratedFirst = item; - } - - if (i == count / 2) - { - enumeratedElementAt = item; - } - - enumeratedLast = item; - i++; - } - - Assert.Equal(enumeratedFirst, first); - Assert.Equal(enumeratedLast, last); - Assert.Equal(enumeratedElementAt, elementAt); - } - } - private static void VerifyEqualsWorker(IEnumerable expected, IEnumerable actual) { // Returns a list of functions that, when applied to enumerable, should return diff --git a/src/libraries/System.Linq/tests/ConsistencyTests.cs b/src/libraries/System.Linq/tests/ConsistencyTests.cs index 746efa61439438..b3c75affa6a9a5 100644 --- a/src/libraries/System.Linq/tests/ConsistencyTests.cs +++ b/src/libraries/System.Linq/tests/ConsistencyTests.cs @@ -18,7 +18,7 @@ public static void MatchSequencePattern() { MethodInfo enumerableNotInQueryable = GetMissingExtensionMethod(typeof(Enumerable), typeof(Queryable), GetExcludedMethods()); - Assert.True(enumerableNotInQueryable is null, string.Format("Enumerable method {0} not defined by Queryable", enumerableNotInQueryable)); + Assert.True(enumerableNotInQueryable == null, string.Format("Enumerable method {0} not defined by Queryable", enumerableNotInQueryable)); MethodInfo queryableNotInEnumerable = GetMissingExtensionMethod( typeof(Queryable), @@ -28,7 +28,7 @@ public static void MatchSequencePattern() } ); - Assert.True(queryableNotInEnumerable is null, string.Format("Queryable method {0} not defined by Enumerable", queryableNotInEnumerable)); + Assert.True(queryableNotInEnumerable == null, string.Format("Queryable method {0} not defined by Enumerable", queryableNotInEnumerable)); } // If a change to Enumerable has required a change to the exception list in this test diff --git a/src/libraries/System.Linq/tests/ContainsTests.cs b/src/libraries/System.Linq/tests/ContainsTests.cs index eea9c04ac50867..9151284c0a13d7 100644 --- a/src/libraries/System.Linq/tests/ContainsTests.cs +++ b/src/libraries/System.Linq/tests/ContainsTests.cs @@ -71,7 +71,7 @@ public static IEnumerable String_TestData() [MemberData(nameof(String_TestData))] public void String(IEnumerable source, IEqualityComparer comparer, string value, bool expected) { - if (comparer is null) + if (comparer == null) { Assert.Equal(expected, source.Contains(value)); } @@ -81,7 +81,7 @@ public void String(IEnumerable source, IEqualityComparer compare [Theory, MemberData(nameof(String_TestData))] public void StringRunOnce(IEnumerable source, IEqualityComparer comparer, string value, bool expected) { - if (comparer is null) + if (comparer == null) { Assert.Equal(expected, source.RunOnce().Contains(value)); } diff --git a/src/libraries/System.Linq/tests/CountTests.cs b/src/libraries/System.Linq/tests/CountTests.cs index 88404efb782f6a..2a93644c87e34e 100644 --- a/src/libraries/System.Linq/tests/CountTests.cs +++ b/src/libraries/System.Linq/tests/CountTests.cs @@ -48,7 +48,7 @@ public static IEnumerable Int_TestData() [MemberData(nameof(Int_TestData))] public void Int(IEnumerable source, Func predicate, int expected) { - if (predicate is null) + if (predicate == null) { Assert.Equal(expected, source.Count()); } @@ -61,7 +61,7 @@ public void Int(IEnumerable source, Func predicate, int expected [Theory, MemberData(nameof(Int_TestData))] public void IntRunOnce(IEnumerable source, Func predicate, int expected) { - if (predicate is null) + if (predicate == null) { Assert.Equal(expected, source.RunOnce().Count()); } diff --git a/src/libraries/System.Linq/tests/DefaultIfEmptyTests.cs b/src/libraries/System.Linq/tests/DefaultIfEmptyTests.cs index 3dadd6d679378e..b4e9e13f804240 100644 --- a/src/libraries/System.Linq/tests/DefaultIfEmptyTests.cs +++ b/src/libraries/System.Linq/tests/DefaultIfEmptyTests.cs @@ -103,26 +103,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).DefaultIfEmpty(); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); - } - - [Fact] - public void First_Last_ElementAt() - { - IEnumerable nonEmpty = Enumerable.Range(1, 3); - Assert.Equal(1, nonEmpty.First()); - Assert.Equal(3, nonEmpty.Last()); - Assert.Equal(1, nonEmpty.ElementAt(0)); - Assert.Equal(2, nonEmpty.ElementAt(1)); - Assert.Equal(3, nonEmpty.ElementAt(2)); - Assert.Throws(() => nonEmpty.ElementAt(-1)); - Assert.Throws(() => nonEmpty.ElementAt(4)); - - IEnumerable empty = Enumerable.Empty(); - Assert.Equal(42, empty.DefaultIfEmpty(42).First()); - Assert.Equal(42, empty.DefaultIfEmpty(42).Last()); - Assert.Equal(42, empty.DefaultIfEmpty(42).ElementAt(0)); - Assert.Throws(() => empty.DefaultIfEmpty(42).ElementAt(1)); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/DistinctTests.cs b/src/libraries/System.Linq/tests/DistinctTests.cs index 24ba295f3d47cc..7408e96ddb38ce 100644 --- a/src/libraries/System.Linq/tests/DistinctTests.cs +++ b/src/libraries/System.Linq/tests/DistinctTests.cs @@ -230,7 +230,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Distinct(); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -303,12 +303,6 @@ public static void DistinctBy_RunOnce_HasExpectedOutput(IEnumerab public static IEnumerable DistinctBy_TestData() { - yield return WrapArgs( - source: Array.Empty(), - keySelector: x => x, - comparer: null, - expected: Enumerable.Empty()); - yield return WrapArgs( source: Enumerable.Range(0, 10), keySelector: x => x, diff --git a/src/libraries/System.Linq/tests/EnumerableTests.cs b/src/libraries/System.Linq/tests/EnumerableTests.cs index f647d037eb6da6..16869e2654a056 100644 --- a/src/libraries/System.Linq/tests/EnumerableTests.cs +++ b/src/libraries/System.Linq/tests/EnumerableTests.cs @@ -87,7 +87,7 @@ protected class AnagramEqualityComparer : IEqualityComparer public bool Equals(string x, string y) { if (ReferenceEquals(x, y)) return true; - if (x is null | y is null) return false; + if (x == null | y == null) return false; int length = x.Length; if (length != y.Length) return false; using (var en = x.OrderBy(i => i).GetEnumerator()) @@ -103,7 +103,7 @@ public bool Equals(string x, string y) public int GetHashCode(string obj) { - if (obj is null) return 0; + if (obj == null) return 0; int hash = obj.Length; foreach (char c in obj) hash ^= c; diff --git a/src/libraries/System.Linq/tests/ExceptTests.cs b/src/libraries/System.Linq/tests/ExceptTests.cs index eeba02e81da433..36976b7cc57a54 100644 --- a/src/libraries/System.Linq/tests/ExceptTests.cs +++ b/src/libraries/System.Linq/tests/ExceptTests.cs @@ -42,7 +42,7 @@ public static IEnumerable Int_TestData() [MemberData(nameof(Int_TestData))] public void Int(IEnumerable first, IEnumerable second, IEqualityComparer comparer, IEnumerable expected) { - if (comparer is null) + if (comparer == null) { Assert.Equal(expected, first.Except(second)); } @@ -64,7 +64,7 @@ public static IEnumerable String_TestData() [MemberData(nameof(String_TestData))] public void String(IEnumerable first, IEnumerable second, IEqualityComparer comparer, IEnumerable expected) { - if (comparer is null) + if (comparer == null) { Assert.Equal(expected, first.Except(second)); } @@ -119,7 +119,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Except(Enumerable.Range(0, 3)); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] diff --git a/src/libraries/System.Linq/tests/GroupByTests.cs b/src/libraries/System.Linq/tests/GroupByTests.cs index a1567cc4ecedd9..4b8967a28a82e7 100644 --- a/src/libraries/System.Linq/tests/GroupByTests.cs +++ b/src/libraries/System.Linq/tests/GroupByTests.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; using System.Reflection; using Xunit; @@ -16,7 +17,7 @@ private static void AssertGroupingCorrect(IEnumerable keys private static void AssertGroupingCorrect(IEnumerable keys, IEnumerable elements, IEnumerable> grouping, IEqualityComparer keyComparer) { - if (grouping is null) + if (grouping == null) { Assert.Null(elements); Assert.Null(keys); @@ -37,7 +38,7 @@ private static void AssertGroupingCorrect(IEnumerable keys TKey key = keyEn.Current; - if (key is null) + if (key == null) { groupingForNullKeys.Add(elEn.Current); } @@ -57,7 +58,7 @@ private static void AssertGroupingCorrect(IEnumerable keys TKey key = group.Key; List list; - if (key is null) + if (key == null) { Assert.Equal(groupingForNullKeys, group); groupingForNullKeys.Clear(); @@ -863,56 +864,5 @@ public static void GroupingKeyIsPublic() PropertyInfo key = grouptype.GetProperty("Key", BindingFlags.Instance | BindingFlags.Public); Assert.NotNull(key); } - - [Fact] - public void MultipleIterationsOfSameEnumerable() - { - foreach (IEnumerable> e1 in new[] { Enumerable.Range(0, 10).GroupBy(i => i), Enumerable.Range(0, 10).GroupBy(i => i, i => i) }) - { - for (int trial = 0; trial < 3; trial++) - { - int count = 0; - foreach (IGrouping g in e1) count++; - Assert.Equal(10, count); - } - } - - foreach (IEnumerable e2 in new[] { Enumerable.Range(0, 10).GroupBy(i => i, (i, e) => i), Enumerable.Range(0, 10).GroupBy(i => i, i => i, (i, e) => i) }) - { - for (int trial = 0; trial < 3; trial++) - { - int count = 0; - foreach (int i in e2) count++; - Assert.Equal(10, count); - } - } - } - - [Fact] - public void EnumerateGrouping() - { - IGrouping g = Enumerable.Range(0, 42).GroupBy(i => "onegroup").First(); - Assert.Equal("onegroup", g.Key); - Assert.Equal(42, g.Count()); - - using IEnumerator e = g.GetEnumerator(); - - var values = new HashSet(); - - for (int trial = 0; trial < 3; trial++) - { - values.Clear(); - - while (e.MoveNext()) - { - Assert.True(values.Add(e.Current)); - } - - Assert.Equal(42, values.Count); - Assert.Equal(Enumerable.Range(0, 42), values.Order()); - - e.Reset(); - } - } } } diff --git a/src/libraries/System.Linq/tests/GroupJoinTests.cs b/src/libraries/System.Linq/tests/GroupJoinTests.cs index a6fc58ed8d38f8..b800eb928ddd63 100644 --- a/src/libraries/System.Linq/tests/GroupJoinTests.cs +++ b/src/libraries/System.Linq/tests/GroupJoinTests.cs @@ -44,24 +44,24 @@ public override int GetHashCode() public bool Equals(JoinRec other) { if (!string.Equals(name, other.name)) return false; - if (orderID is null) + if (orderID == null) { - if (other.orderID is not null) return false; + if (other.orderID != null) return false; } else { - if (other.orderID is null) return false; + if (other.orderID == null) return false; if (orderID.Length != other.orderID.Length) return false; for (int i = 0; i != other.orderID.Length; ++i) if (orderID[i] != other.orderID[i]) return false; } - if (total is null) + if (total == null) { - if (other.total is not null) return false; + if (other.total != null) return false; } else { - if (other.total is null) return false; + if (other.total == null) return false; if (total.Length != other.total.Length) return false; for (int i = 0; i != other.total.Length; ++i) if (total[i] != other.total[i]) return false; @@ -511,7 +511,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).GroupJoin(Enumerable.Empty(), i => i, i => i, (o, i) => i); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator>; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/IndexTests.cs b/src/libraries/System.Linq/tests/IndexTests.cs index 0742569f787d40..4b08820fe0e30c 100644 --- a/src/libraries/System.Linq/tests/IndexTests.cs +++ b/src/libraries/System.Linq/tests/IndexTests.cs @@ -8,12 +8,6 @@ namespace System.Linq.Tests { public class IndexTests : EnumerableTests { - [Fact] - public void Empty() - { - Assert.Empty(Enumerable.Empty().Index()); - } - [Fact] public void Index_SourceIsNull_ArgumentNullExceptionThrown() { diff --git a/src/libraries/System.Linq/tests/IntersectTests.cs b/src/libraries/System.Linq/tests/IntersectTests.cs index 9d1011a93b591c..7b1d5fdfbd873d 100644 --- a/src/libraries/System.Linq/tests/IntersectTests.cs +++ b/src/libraries/System.Linq/tests/IntersectTests.cs @@ -61,7 +61,7 @@ public static IEnumerable String_TestData() [MemberData(nameof(String_TestData))] public void String(IEnumerable first, IEnumerable second, IEqualityComparer comparer, string[] expected) { - if (comparer is null) + if (comparer == null) { Assert.Equal(expected, first.Intersect(second)); } @@ -116,7 +116,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Intersect(Enumerable.Range(0, 3)); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] diff --git a/src/libraries/System.Linq/tests/JoinTests.cs b/src/libraries/System.Linq/tests/JoinTests.cs index 496f49a6fcb5a7..05ef7059eb1a56 100644 --- a/src/libraries/System.Linq/tests/JoinTests.cs +++ b/src/libraries/System.Linq/tests/JoinTests.cs @@ -415,7 +415,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Join(Enumerable.Empty(), i => i, i => i, (o, i) => i); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/LongCountTests.cs b/src/libraries/System.Linq/tests/LongCountTests.cs index 308ad764674b20..4c7eb8e323c27e 100644 --- a/src/libraries/System.Linq/tests/LongCountTests.cs +++ b/src/libraries/System.Linq/tests/LongCountTests.cs @@ -45,7 +45,7 @@ public static IEnumerable LongCount_TestData() [MemberData(nameof(LongCount_TestData))] public static void LongCount(IEnumerable source, Func predicate, long expected) { - if (predicate is null) + if (predicate == null) { Assert.Equal(expected, source.LongCount()); } @@ -59,7 +59,7 @@ public static void LongCount(IEnumerable source, Func predicate, [MemberData(nameof(LongCount_TestData))] public static void LongCountRunOnce(IEnumerable source, Func predicate, long expected) { - if (predicate is null) + if (predicate == null) { Assert.Equal(expected, source.RunOnce().LongCount()); } diff --git a/src/libraries/System.Linq/tests/MaxTests.cs b/src/libraries/System.Linq/tests/MaxTests.cs index bb70a14d684c72..a1509855091d11 100644 --- a/src/libraries/System.Linq/tests/MaxTests.cs +++ b/src/libraries/System.Linq/tests/MaxTests.cs @@ -251,8 +251,6 @@ public void Max_Float_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Max()); Assert.Throws(() => Enumerable.Empty().Max(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max(x => x)); Assert.Throws(() => Array.Empty().Max()); Assert.Throws(() => new List().Max()); } @@ -333,8 +331,6 @@ public void Max_Double_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Max()); Assert.Throws(() => Enumerable.Empty().Max(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max(x => x)); Assert.Throws(() => Array.Empty().Max()); Assert.Throws(() => new List().Max()); } @@ -401,8 +397,6 @@ public void Max_Decimal_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Max()); Assert.Throws(() => Enumerable.Empty().Max(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max(x => x)); Assert.Throws(() => Array.Empty().Max()); Assert.Throws(() => new List().Max(x => x)); } @@ -628,8 +622,6 @@ public void Max_DateTime_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Max()); Assert.Throws(() => Enumerable.Empty().Max(i => i)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max(i => i)); } public static IEnumerable Max_String_TestData() @@ -896,7 +888,6 @@ public void Max_String_WithSelectorAccessingProperty() public void Max_Boolean_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Max()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Max()); } [Fact] diff --git a/src/libraries/System.Linq/tests/MinTests.cs b/src/libraries/System.Linq/tests/MinTests.cs index 0cc72fa43a1001..feca6994d066d6 100644 --- a/src/libraries/System.Linq/tests/MinTests.cs +++ b/src/libraries/System.Linq/tests/MinTests.cs @@ -136,8 +136,6 @@ public void Min_Int_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); Assert.Throws(() => Enumerable.Empty().Min(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min(x => x)); Assert.Throws(() => Array.Empty().Min()); Assert.Throws(() => new List().Min()); } @@ -184,8 +182,6 @@ public void Min_Long_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); Assert.Throws(() => Enumerable.Empty().Min(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min(x => x)); Assert.Throws(() => Array.Empty().Min()); Assert.Throws(() => new List().Min()); } @@ -254,8 +250,6 @@ public void Min_Float_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); Assert.Throws(() => Enumerable.Empty().Min(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min(x => x)); Assert.Throws(() => Array.Empty().Min()); Assert.Throws(() => new List().Min()); } @@ -322,8 +316,6 @@ public void Min_Double_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); Assert.Throws(() => Enumerable.Empty().Min(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min(x => x)); Assert.Throws(() => Array.Empty().Min()); Assert.Throws(() => new List().Min()); } @@ -363,8 +355,6 @@ public void Min_Decimal_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); Assert.Throws(() => Enumerable.Empty().Min(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min(x => x)); Assert.Throws(() => Array.Empty().Min()); Assert.Throws(() => new List().Min()); } @@ -605,8 +595,6 @@ public void Min_DateTime_EmptySource_ThrowsInvalidOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); Assert.Throws(() => Enumerable.Empty().Min(x => x)); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min(x => x)); Assert.Throws(() => Array.Empty().Min()); Assert.Throws(() => new List().Min()); } @@ -870,7 +858,6 @@ public void Min_String_NullSelector_ThrowsArgumentNullException() public void Min_Bool_EmptySource_ThrowsInvalodOperationException() { Assert.Throws(() => Enumerable.Empty().Min()); - Assert.Throws(() => ForceNotCollection(Enumerable.Empty()).Min()); } [Fact] diff --git a/src/libraries/System.Linq/tests/OfTypeTests.cs b/src/libraries/System.Linq/tests/OfTypeTests.cs index 3f1c7eb51da2a7..3111d6a79c50c8 100644 --- a/src/libraries/System.Linq/tests/OfTypeTests.cs +++ b/src/libraries/System.Linq/tests/OfTypeTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using Xunit; @@ -130,103 +131,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).OfType(); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); - } - - [Fact] - public void ValueType_ReturnsOriginal() - { - IEnumerable e = Enumerable.Range(0, 10); - Assert.Same(e, e.OfType()); - } - - [Fact] - public void NullableValueType_ReturnsNewEnumerable() - { - IEnumerable e = Enumerable.Range(0, 10).Select(i => (int?)i); - Assert.NotSame(e, e.OfType()); - Assert.NotSame(e, e.OfType()); - } - - [Fact] - public void ReferenceType_ReturnsNewEnumerable() - { - IEnumerable e = Enumerable.Range(0, 10).Select(i => (object)i); - Assert.NotSame(e, e.OfType()); - Assert.NotSame(e, e.OfType()); - Assert.NotSame(e, e.OfType()); - Assert.NotSame(e, e.OfType()); - } - - [Fact] - public void ToArray() - { - IEnumerable source = new object[] { 1, 2, 3, 4, 5 }; - Assert.Equal(new int[] { 1, 2, 3, 4, 5 }, source.OfType().ToArray()); - Assert.Empty(source.OfType().ToArray()); - } - - [Fact] - public void ToList() - { - IEnumerable source = new object[] { 1, 2, 3, 4, 5 }; - Assert.Equal(new int[] { 1, 2, 3, 4, 5 }, source.OfType().ToList()); - Assert.Empty(source.OfType().ToList()); - } - - [Fact] - public void Count() - { - Assert.Equal(0, new object[] { }.OfType().Count()); - Assert.Equal(1, new object[] { "abc" }.OfType().Count()); - Assert.Equal(2, new object[] { "abc", "def" }.OfType().Count()); - Assert.Equal(2, new object[] { "abc", 42, "def" }.OfType().Count()); - Assert.Equal(2, new object[] { "abc", 42, null, "def" }.OfType().Count()); - Assert.Equal(3, new object[] { null, new object(), null, new object(), new object(), null }.OfType().Count()); - - Assert.False(new object[] { "abc" }.OfType().TryGetNonEnumeratedCount(out _)); - Assert.False(new object[] { "abc" }.OfType().TryGetNonEnumeratedCount(out _)); - Assert.False(new int[] { 42 }.OfType().TryGetNonEnumeratedCount(out _)); - } - - [Fact] - public void First_Last_ElementAt() - { - IEnumerable source = new object[] { 1, 2, 3, 4, 5 }; - - Assert.Equal(1, source.OfType().First()); - Assert.Equal(0, source.OfType().FirstOrDefault()); - - Assert.Equal(5, source.OfType().Last()); - Assert.Equal(0, source.OfType().LastOrDefault()); - - Assert.Equal(4, source.OfType().ElementAt(3)); - Assert.Equal(0, source.OfType().ElementAtOrDefault(6)); - } - - [Fact] - public void OfTypeSelect() - { - IEnumerable objects = new object[] { "1", null, "22", null, 3, 4, "55555" }; - Assert.Equal(new int[] { 1, 2, 5 }, objects.OfType().Select(s => s.Length)); - - Assert.Equal(new int[] { 1, 2, 3, 4, 5 }, new int[] { 1, 2, 3, 4, 5 }.OfType().Select(o => (int)o)); - } - - [Fact] - public void MultipleIterations() - { - var orig = new object[] { null, null, null, null, null }; - IEnumerable objects = orig.OfType(); - - for (int i = 0; i < orig.Length; i++) - { - orig[i] = i.ToString(); - - int count = 0; - foreach (object o in objects) count++; - Assert.Equal(i + 1, count); - } + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/OrderTests.cs b/src/libraries/System.Linq/tests/OrderTests.cs index dee76efe73823d..ed2dd9bfc8765e 100644 --- a/src/libraries/System.Linq/tests/OrderTests.cs +++ b/src/libraries/System.Linq/tests/OrderTests.cs @@ -196,9 +196,6 @@ public void FirstOnOrdered() { Assert.Equal(0, Enumerable.Range(0, 10).Shuffle().Order().First()); Assert.Equal(9, Enumerable.Range(0, 10).Shuffle().OrderDescending().First()); - - Assert.Equal(0, ForceNotCollection(Enumerable.Range(0, 10).Shuffle()).Order().First()); - Assert.Equal(9, ForceNotCollection(Enumerable.Range(0, 10).Shuffle()).OrderDescending().First()); } [Fact] @@ -284,9 +281,6 @@ public void LastOnOrdered() { Assert.Equal(9, Enumerable.Range(0, 10).Shuffle().Order().Last()); Assert.Equal(0, Enumerable.Range(0, 10).Shuffle().OrderDescending().Last()); - - Assert.Equal(9, ForceNotCollection(Enumerable.Range(0, 10).Shuffle()).Order().Last()); - Assert.Equal(0, ForceNotCollection(Enumerable.Range(0, 10).Shuffle()).OrderDescending().Last()); } [Fact] @@ -313,16 +307,6 @@ public void LastOrDefaultOnOrdered() Assert.Equal(0, Enumerable.Empty().Order().LastOrDefault()); } - [Fact] - public void ElementAtOnOrdered() - { - Assert.Equal(4, Enumerable.Range(0, 10).Shuffle().Order().ElementAt(4)); - Assert.Equal(5, Enumerable.Range(0, 10).Shuffle().OrderDescending().ElementAt(4)); - - Assert.Equal(4, ForceNotCollection(Enumerable.Range(0, 10).Shuffle()).Order().ElementAt(4)); - Assert.Equal(5, ForceNotCollection(Enumerable.Range(0, 10).Shuffle()).OrderDescending().ElementAt(4)); - } - [Fact] public void EnumeratorDoesntContinue() { diff --git a/src/libraries/System.Linq/tests/OrderedSubsetting.cs b/src/libraries/System.Linq/tests/OrderedSubsetting.cs index c39a61a86301a9..6c3d5f57cd69dd 100644 --- a/src/libraries/System.Linq/tests/OrderedSubsetting.cs +++ b/src/libraries/System.Linq/tests/OrderedSubsetting.cs @@ -416,7 +416,7 @@ public void SelectForcedToEnumeratorDoesntEnumerate() var iterator = Enumerable.Range(-1, 8).Shuffle().OrderBy(i => i).Skip(1).Take(5).Select(i => i * 2); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] diff --git a/src/libraries/System.Linq/tests/RangeTests.cs b/src/libraries/System.Linq/tests/RangeTests.cs index 2e331cfee7ecd7..8421a66ba890c9 100644 --- a/src/libraries/System.Linq/tests/RangeTests.cs +++ b/src/libraries/System.Linq/tests/RangeTests.cs @@ -243,7 +243,6 @@ static void Validate(IEnumerable e, int[] expected) Assert.Throws(() => list.Insert(0, 42)); Assert.Throws(() => list.Clear()); Assert.Throws(() => list.Remove(42)); - Assert.Throws(() => list.RemoveAt(0)); Assert.Throws(() => list[0] = 42); AssertExtensions.Throws("index", () => list[-1]); AssertExtensions.Throws("index", () => list[expected.Length]); @@ -256,8 +255,6 @@ static void Validate(IEnumerable e, int[] expected) Assert.False(list.Contains(expected[0] - 1)); Assert.False(list.Contains(expected[^1] + 1)); - Assert.Equal(-1, list.IndexOf(expected[0] - 1)); - Assert.Equal(-1, list.IndexOf(expected[^1] + 1)); Assert.All(expected, i => Assert.True(list.Contains(i))); Assert.All(expected, i => Assert.Equal(Array.IndexOf(expected, i), list.IndexOf(i))); for (int i = 0; i < expected.Length; i++) diff --git a/src/libraries/System.Linq/tests/RepeatTests.cs b/src/libraries/System.Linq/tests/RepeatTests.cs index df8eebda35691e..625dff376de311 100644 --- a/src/libraries/System.Linq/tests/RepeatTests.cs +++ b/src/libraries/System.Linq/tests/RepeatTests.cs @@ -255,7 +255,6 @@ static void Validate(IEnumerable e, int[] expected) Assert.Throws(() => list.Insert(0, 42)); Assert.Throws(() => list.Clear()); Assert.Throws(() => list.Remove(42)); - Assert.Throws(() => list.RemoveAt(0)); Assert.Throws(() => list[0] = 42); AssertExtensions.Throws("index", () => list[-1]); AssertExtensions.Throws("index", () => list[expected.Length]); @@ -268,8 +267,6 @@ static void Validate(IEnumerable e, int[] expected) Assert.False(list.Contains(expected[0] - 1)); Assert.False(list.Contains(expected[^1] + 1)); - Assert.Equal(-1, list.IndexOf(expected[0] - 1)); - Assert.Equal(-1, list.IndexOf(expected[^1] + 1)); Assert.All(expected, i => Assert.True(list.Contains(i))); Assert.All(expected, i => Assert.Equal(Array.IndexOf(expected, i), list.IndexOf(i))); for (int i = 0; i < expected.Length; i++) diff --git a/src/libraries/System.Linq/tests/ReverseTests.cs b/src/libraries/System.Linq/tests/ReverseTests.cs index 1cd337a9750361..6ecb3874dad542 100644 --- a/src/libraries/System.Linq/tests/ReverseTests.cs +++ b/src/libraries/System.Linq/tests/ReverseTests.cs @@ -82,7 +82,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Reverse(); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/SelectManyTests.cs b/src/libraries/System.Linq/tests/SelectManyTests.cs index 9a1446c1eb4e07..246826ac9ed5de 100644 --- a/src/libraries/System.Linq/tests/SelectManyTests.cs +++ b/src/libraries/System.Linq/tests/SelectManyTests.cs @@ -345,7 +345,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).SelectMany(i => new int[0]); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -353,7 +353,7 @@ public void ForcedToEnumeratorDoesntEnumerateIndexed() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).SelectMany((e, i) => new int[0]); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -361,7 +361,7 @@ public void ForcedToEnumeratorDoesntEnumerateResultSel() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).SelectMany(i => new int[0], (e, i) => e); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -369,7 +369,7 @@ public void ForcedToEnumeratorDoesntEnumerateIndexedResultSel() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).SelectMany((e, i) => new int[0], (e, i) => e); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Theory] diff --git a/src/libraries/System.Linq/tests/SelectTests.cs b/src/libraries/System.Linq/tests/SelectTests.cs index e1c642ae31620c..6cf77214a5b33b 100644 --- a/src/libraries/System.Linq/tests/SelectTests.cs +++ b/src/libraries/System.Linq/tests/SelectTests.cs @@ -747,7 +747,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Select(i => i); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -756,7 +756,7 @@ public void ForcedToEnumeratorDoesntEnumerateIndexed() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Select((e, i) => i); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -764,7 +764,7 @@ public void ForcedToEnumeratorDoesntEnumerateArray() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToArray().Select(i => i); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -772,7 +772,7 @@ public void ForcedToEnumeratorDoesntEnumerateList() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Select(i => i); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -780,7 +780,7 @@ public void ForcedToEnumeratorDoesntEnumerateIList() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().AsReadOnly().Select(i => i); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -788,7 +788,7 @@ public void ForcedToEnumeratorDoesntEnumerateIPartition() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().AsReadOnly().Select(i => i).Skip(1); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] diff --git a/src/libraries/System.Linq/tests/SequenceEqualTests.cs b/src/libraries/System.Linq/tests/SequenceEqualTests.cs index 7393d18947aa8f..380916550efd60 100644 --- a/src/libraries/System.Linq/tests/SequenceEqualTests.cs +++ b/src/libraries/System.Linq/tests/SequenceEqualTests.cs @@ -1,7 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.ObjectModel; +using System; +using System.Collections.Generic; using Xunit; namespace System.Linq.Tests @@ -245,31 +246,5 @@ public void ByteArrays_SpecialCasedButExpectedBehavior() } } } - - [Fact] - public void ICollectionsCompareCorrectly() - { - Assert.True(new TestCollection([]).SequenceEqual(new TestCollection([]))); - Assert.True(new TestCollection([1]).SequenceEqual(new TestCollection([1]))); - Assert.True(new TestCollection([1, 2, 3]).SequenceEqual(new TestCollection([1, 2, 3]))); - - Assert.False(new TestCollection([1, 2, 3, 4]).SequenceEqual(new TestCollection([1, 2, 3]))); - Assert.False(new TestCollection([1, 2, 3]).SequenceEqual(new TestCollection([1, 2, 3, 4]))); - Assert.False(new TestCollection([1, 2, 3]).SequenceEqual(new TestCollection([1, 2, 4]))); - Assert.False(new TestCollection([-1, 2, 3]).SequenceEqual(new TestCollection([-2, 2, 3]))); - } - - [Fact] - public void IListsCompareCorrectly() - { - Assert.True(new ReadOnlyCollection([]).SequenceEqual(new ReadOnlyCollection([]))); - Assert.True(new ReadOnlyCollection([1]).SequenceEqual(new ReadOnlyCollection([1]))); - Assert.True(new ReadOnlyCollection([1, 2, 3]).SequenceEqual(new ReadOnlyCollection([1, 2, 3]))); - - Assert.False(new ReadOnlyCollection([1, 2, 3, 4]).SequenceEqual(new ReadOnlyCollection([1, 2, 3]))); - Assert.False(new ReadOnlyCollection([1, 2, 3]).SequenceEqual(new ReadOnlyCollection([1, 2, 3, 4]))); - Assert.False(new ReadOnlyCollection([1, 2, 3]).SequenceEqual(new ReadOnlyCollection([1, 2, 4]))); - Assert.False(new ReadOnlyCollection([-1, 2, 3]).SequenceEqual(new ReadOnlyCollection([-2, 2, 3]))); - } } } diff --git a/src/libraries/System.Linq/tests/SkipLastTests.cs b/src/libraries/System.Linq/tests/SkipLastTests.cs index c4770410870d47..fe9652a875e9d9 100644 --- a/src/libraries/System.Linq/tests/SkipLastTests.cs +++ b/src/libraries/System.Linq/tests/SkipLastTests.cs @@ -9,12 +9,6 @@ namespace System.Linq.Tests { public class SkipLastTests : EnumerableTests { - [Fact] - public void SkipLastThrowsOnNull() - { - AssertExtensions.Throws("source", () => ((IEnumerable)null).SkipLast(10)); - } - [Theory] [MemberData(nameof(EnumerableData), MemberType = typeof(SkipTakeData))] public void SkipLast(IEnumerable source, int count) diff --git a/src/libraries/System.Linq/tests/SkipTests.cs b/src/libraries/System.Linq/tests/SkipTests.cs index cf4d16b1309907..b2dd1cf62f4478 100644 --- a/src/libraries/System.Linq/tests/SkipTests.cs +++ b/src/libraries/System.Linq/tests/SkipTests.cs @@ -210,7 +210,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Skip(2); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -219,7 +219,7 @@ public void ForcedToEnumeratorDoesntEnumerateIList() var iterator = (new[] { 0, 1, 2 }).Skip(2); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -497,7 +497,7 @@ public void IteratorStateShouldNotChangeIfNumberOfElementsIsUnbounded() // On platforms that do not have this change, the optimization may not be present // and the iterator may not have a field named _state. In that case, nop. - if (state is not null) + if (state != null) { state.SetValue(iterator, int.MaxValue); diff --git a/src/libraries/System.Linq/tests/SkipWhileTests.cs b/src/libraries/System.Linq/tests/SkipWhileTests.cs index a13a686cef3f34..26281efc5f862a 100644 --- a/src/libraries/System.Linq/tests/SkipWhileTests.cs +++ b/src/libraries/System.Linq/tests/SkipWhileTests.cs @@ -1,20 +1,15 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections; using System.Collections.Generic; using Xunit; +using Xunit.Abstractions; namespace System.Linq.Tests { public class SkipWhileTests : EnumerableTests { - [Fact] - public void Empty() - { - Assert.Equal(Enumerable.Empty(), Enumerable.Empty().SkipWhile(i => i < 40)); - Assert.Equal(Enumerable.Empty(), Enumerable.Empty().SkipWhile((i, index) => i < 40)); - } - [Fact] public void SkipWhileAllTrue() { @@ -166,7 +161,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).SkipWhile(e => true); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -175,7 +170,7 @@ public void ForcedToEnumeratorDoesntEnumerateIndexed() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).SkipWhile((e, i) => true); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/TakeLastTests.cs b/src/libraries/System.Linq/tests/TakeLastTests.cs index b39d59e94263a6..31b58d5bf017ae 100644 --- a/src/libraries/System.Linq/tests/TakeLastTests.cs +++ b/src/libraries/System.Linq/tests/TakeLastTests.cs @@ -9,12 +9,6 @@ namespace System.Linq.Tests { public class TakeLastTests : EnumerableTests { - [Fact] - public void SkipLastThrowsOnNull() - { - AssertExtensions.Throws("source", () => ((IEnumerable)null).TakeLast(10)); - } - [Theory] [MemberData(nameof(EnumerableData), MemberType = typeof(SkipTakeData))] public void TakeLast(IEnumerable source, int count) diff --git a/src/libraries/System.Linq/tests/TakeTests.cs b/src/libraries/System.Linq/tests/TakeTests.cs index d9408a157124bf..93a0405bfaf312 100644 --- a/src/libraries/System.Linq/tests/TakeTests.cs +++ b/src/libraries/System.Linq/tests/TakeTests.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.Collections.ObjectModel; using Xunit; namespace System.Linq.Tests @@ -271,23 +270,23 @@ public void ForcedToEnumeratorDoesNotEnumerate() var iterator1 = NumberRangeGuaranteedNotCollectionType(0, 3).Take(2); // Don't insist on this behaviour, but check it's correct if it happens var en1 = iterator1 as IEnumerator; - Assert.False(en1 is not null && en1.MoveNext()); + Assert.False(en1 != null && en1.MoveNext()); var iterator2 = NumberRangeGuaranteedNotCollectionType(0, 3).Take(0..2); var en2 = iterator2 as IEnumerator; - Assert.False(en2 is not null && en2.MoveNext()); + Assert.False(en2 != null && en2.MoveNext()); var iterator3 = NumberRangeGuaranteedNotCollectionType(0, 3).Take(^3..2); var en3 = iterator3 as IEnumerator; - Assert.False(en3 is not null && en3.MoveNext()); + Assert.False(en3 != null && en3.MoveNext()); var iterator4 = NumberRangeGuaranteedNotCollectionType(0, 3).Take(0..^1); var en4 = iterator4 as IEnumerator; - Assert.False(en4 is not null && en4.MoveNext()); + Assert.False(en4 != null && en4.MoveNext()); var iterator5 = NumberRangeGuaranteedNotCollectionType(0, 3).Take(^3..^1); var en5 = iterator5 as IEnumerator; - Assert.False(en5 is not null && en5.MoveNext()); + Assert.False(en5 != null && en5.MoveNext()); } [Fact] @@ -320,23 +319,23 @@ public void ForcedToEnumeratorDoesntEnumerateIList() var iterator1 = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Take(2); // Don't insist on this behaviour, but check it's correct if it happens var en1 = iterator1 as IEnumerator; - Assert.False(en1 is not null && en1.MoveNext()); + Assert.False(en1 != null && en1.MoveNext()); var iterator2 = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Take(0..2); var en2 = iterator2 as IEnumerator; - Assert.False(en2 is not null && en2.MoveNext()); + Assert.False(en2 != null && en2.MoveNext()); var iterator3 = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Take(^3..2); var en3 = iterator3 as IEnumerator; - Assert.False(en3 is not null && en3.MoveNext()); + Assert.False(en3 != null && en3.MoveNext()); var iterator4 = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Take(0..^1); var en4 = iterator4 as IEnumerator; - Assert.False(en4 is not null && en4.MoveNext()); + Assert.False(en4 != null && en4.MoveNext()); var iterator5 = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Take(^3..^1); var en5 = iterator5 as IEnumerator; - Assert.False(en5 is not null && en5.MoveNext()); + Assert.False(en5 != null && en5.MoveNext()); } [Fact] @@ -2032,37 +2031,5 @@ public void EmptySource_DoNotThrowException_EnumerablePartition() Assert.Empty(EnumerablePartitionOrEmpty(source).Take(3..^8)); Assert.Empty(EnumerablePartitionOrEmpty(source).Take(^6..^7)); } - - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsSpeedOptimized))] - public void SkipTakeOnIListIsIList() - { - IList list = new ReadOnlyCollection(Enumerable.Range(0, 100).ToList()); - IList skipTake = Assert.IsAssignableFrom>(list.Skip(10).Take(20)); - - Assert.True(skipTake.IsReadOnly); - Assert.Equal(20, skipTake.Count); - int[] results = new int[20]; - skipTake.CopyTo(results, 0); - for (int i = 0; i < 20; i++) - { - Assert.Equal(i + 10, skipTake[i]); - Assert.Equal(i + 10, results[i]); - Assert.True(skipTake.Contains(i + 10)); - Assert.True(skipTake.IndexOf(i + 10) == i); - } - - Assert.False(skipTake.Contains(9)); - Assert.False(skipTake.Contains(30)); - - Assert.Throws(() => skipTake[-1]); - Assert.Throws(() => skipTake[20]); - - Assert.Throws(() => skipTake.Add(42)); - Assert.Throws(() => skipTake.Clear()); - Assert.Throws(() => skipTake.Insert(0, 42)); - Assert.Throws(() => skipTake.Remove(42)); - Assert.Throws(() => skipTake.RemoveAt(0)); - Assert.Throws(() => skipTake[0] = 42); - } } } diff --git a/src/libraries/System.Linq/tests/TakeWhileTests.cs b/src/libraries/System.Linq/tests/TakeWhileTests.cs index 357b31a34e2546..55f02459978a00 100644 --- a/src/libraries/System.Linq/tests/TakeWhileTests.cs +++ b/src/libraries/System.Linq/tests/TakeWhileTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections; using System.Collections.Generic; using Xunit; @@ -8,13 +9,6 @@ namespace System.Linq.Tests { public class TakeWhileTests : EnumerableTests { - [Fact] - public void Empty() - { - Assert.Equal(Enumerable.Empty(), Enumerable.Empty().TakeWhile(i => i < 40)); - Assert.Equal(Enumerable.Empty(), Enumerable.Empty().TakeWhile((i, index) => i < 40)); - } - [Fact] public void SameResultsRepeatCallsIntQuery() { @@ -174,7 +168,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).TakeWhile(e => true); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -183,7 +177,7 @@ public void ForcedToEnumeratorDoesntEnumerateIndexed() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).TakeWhile((e, i) => true); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } } } diff --git a/src/libraries/System.Linq/tests/TestExtensions.cs b/src/libraries/System.Linq/tests/TestExtensions.cs index b27dd3b0e49e31..4c0cfe6626a7ba 100644 --- a/src/libraries/System.Linq/tests/TestExtensions.cs +++ b/src/libraries/System.Linq/tests/TestExtensions.cs @@ -10,10 +10,10 @@ namespace System.Linq.Tests public static class TestExtensions { public static IEnumerable RunOnce(this IEnumerable source) => - source is null ? null : (source as IList)?.RunOnce() ?? new RunOnceEnumerable(source); + source == null ? null : (source as IList)?.RunOnce() ?? new RunOnceEnumerable(source); public static IEnumerable RunOnce(this IList source) - => source is null ? null : new RunOnceList(source); + => source == null ? null : new RunOnceList(source); private class RunOnceEnumerable : IEnumerable { diff --git a/src/libraries/System.Linq/tests/ToArrayTests.cs b/src/libraries/System.Linq/tests/ToArrayTests.cs index 23f51f503af2aa..bc1c47ffcf91a9 100644 --- a/src/libraries/System.Linq/tests/ToArrayTests.cs +++ b/src/libraries/System.Linq/tests/ToArrayTests.cs @@ -152,8 +152,8 @@ public void ToArray_ArrayWhereSelect(int[] sourceIntegers, string[] convertedStr Assert.Equal(convertedStrings, sourceIntegers.Where(i => true).Select(i => i.ToString()).ToArray()); Assert.Equal(Array.Empty(), sourceIntegers.Where(i => false).Select(i => i.ToString()).ToArray()); - Assert.Equal(convertedStrings, sourceIntegers.Select(i => i.ToString()).Where(s => s is not null).ToArray()); - Assert.Equal(Array.Empty(), sourceIntegers.Select(i => i.ToString()).Where(s => s is null).ToArray()); + Assert.Equal(convertedStrings, sourceIntegers.Select(i => i.ToString()).Where(s => s != null).ToArray()); + Assert.Equal(Array.Empty(), sourceIntegers.Select(i => i.ToString()).Where(s => s == null).ToArray()); } [Theory] @@ -172,8 +172,8 @@ public void ToArray_ListWhereSelect(int[] sourceIntegers, string[] convertedStri Assert.Equal(convertedStrings, sourceList.Where(i => true).Select(i => i.ToString()).ToArray()); Assert.Equal(Array.Empty(), sourceList.Where(i => false).Select(i => i.ToString()).ToArray()); - Assert.Equal(convertedStrings, sourceList.Select(i => i.ToString()).Where(s => s is not null).ToArray()); - Assert.Equal(Array.Empty(), sourceList.Select(i => i.ToString()).Where(s => s is null).ToArray()); + Assert.Equal(convertedStrings, sourceList.Select(i => i.ToString()).Where(s => s != null).ToArray()); + Assert.Equal(Array.Empty(), sourceList.Select(i => i.ToString()).Where(s => s == null).ToArray()); } [Fact] diff --git a/src/libraries/System.Linq/tests/ToListTests.cs b/src/libraries/System.Linq/tests/ToListTests.cs index ec8d9b16650833..9d6b49ad128fe7 100644 --- a/src/libraries/System.Linq/tests/ToListTests.cs +++ b/src/libraries/System.Linq/tests/ToListTests.cs @@ -121,8 +121,8 @@ public void ToList_ArrayWhereSelect(int[] sourceIntegers, string[] convertedStri Assert.Equal(convertedList, sourceIntegers.Where(i => true).Select(i => i.ToString()).ToList()); Assert.Equal(emptyStringsList, sourceIntegers.Where(i => false).Select(i => i.ToString()).ToList()); - Assert.Equal(convertedList, sourceIntegers.Select(i => i.ToString()).Where(s => s is not null).ToList()); - Assert.Equal(emptyStringsList, sourceIntegers.Select(i => i.ToString()).Where(s => s is null).ToList()); + Assert.Equal(convertedList, sourceIntegers.Select(i => i.ToString()).Where(s => s != null).ToList()); + Assert.Equal(emptyStringsList, sourceIntegers.Select(i => i.ToString()).Where(s => s == null).ToList()); } [Theory] @@ -145,8 +145,8 @@ public void ToList_ListWhereSelect(int[] sourceIntegers, string[] convertedStrin Assert.Equal(convertedList, sourceList.Where(i => true).Select(i => i.ToString()).ToList()); Assert.Equal(emptyStringsList, sourceList.Where(i => false).Select(i => i.ToString()).ToList()); - Assert.Equal(convertedList, sourceList.Select(i => i.ToString()).Where(s => s is not null).ToList()); - Assert.Equal(emptyStringsList, sourceList.Select(i => i.ToString()).Where(s => s is null).ToList()); + Assert.Equal(convertedList, sourceList.Select(i => i.ToString()).Where(s => s != null).ToList()); + Assert.Equal(emptyStringsList, sourceList.Select(i => i.ToString()).Where(s => s == null).ToList()); } [Theory] @@ -166,8 +166,8 @@ public void ToList_IListWhereSelect(int[] sourceIntegers, string[] convertedStri Assert.Equal(convertedList, sourceList.Where(i => true).Select(i => i.ToString()).ToList()); Assert.Equal(ReadOnlyCollection.Empty, sourceList.Where(i => false).Select(i => i.ToString()).ToList()); - Assert.Equal(convertedList, sourceList.Select(i => i.ToString()).Where(s => s is not null).ToList()); - Assert.Equal(ReadOnlyCollection.Empty, sourceList.Select(i => i.ToString()).Where(s => s is null).ToList()); + Assert.Equal(convertedList, sourceList.Select(i => i.ToString()).Where(s => s != null).ToList()); + Assert.Equal(ReadOnlyCollection.Empty, sourceList.Select(i => i.ToString()).Where(s => s == null).ToList()); } [Fact] diff --git a/src/libraries/System.Linq/tests/ToLookupTests.cs b/src/libraries/System.Linq/tests/ToLookupTests.cs index 5361d6dc6ae5e9..458aaa9e4dded3 100644 --- a/src/libraries/System.Linq/tests/ToLookupTests.cs +++ b/src/libraries/System.Linq/tests/ToLookupTests.cs @@ -1,8 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections; +using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Reflection; using Xunit; namespace System.Linq.Tests @@ -51,14 +53,6 @@ from x4 in q2 Assert.Equal(q.ToLookup(e => e.a1), q.ToLookup(e => e.a1)); } - [Fact] - public void Empty() - { - AssertMatches(Enumerable.Empty(), Enumerable.Empty(), Enumerable.Empty().ToLookup(i => i)); - Assert.False(Enumerable.Empty().ToLookup(i => i).Contains(0)); - Assert.Empty(Enumerable.Empty().ToLookup(i => i)[0]); - } - [Fact] public void NullKeyIncluded() { @@ -295,59 +289,6 @@ public void ApplyResultSelectorForGroup(int enumType) Assert.Equal(expected, result); } - [Fact] - public void ApplyResultSelector() - { - Lookup lookup = (Lookup)new int[] { 1, 2, 2, 3, 3, 3 }.ToLookup(i => i); - IEnumerable sums = lookup.ApplyResultSelector((key, elements) => - { - Assert.Equal(key, elements.Count()); - return elements.Sum(); - }); - Assert.Equal([1, 4, 9], sums); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - [InlineData(10)] - public void LookupImplementsICollection(int count) - { - Assert.IsAssignableFrom>>(Enumerable.Range(0, count).ToLookup(i => i.ToString())); - Assert.IsAssignableFrom>>(Enumerable.Range(0, count).ToLookup(i => i.ToString(), StringComparer.OrdinalIgnoreCase)); - Assert.IsAssignableFrom>>(Enumerable.Range(0, count).ToLookup(i => i.ToString(), i => i)); - Assert.IsAssignableFrom>>(Enumerable.Range(0, count).ToLookup(i => i.ToString(), i => i, StringComparer.OrdinalIgnoreCase)); - - var collection = (ICollection>)Enumerable.Range(0, count).ToLookup(i => i.ToString()); - Assert.Equal(count, collection.Count); - Assert.True(collection.IsReadOnly); - Assert.Throws(() => collection.Add(null)); - Assert.Throws(() => collection.Remove(null)); - Assert.Throws(() => collection.Clear()); - - if (count > 0) - { - IGrouping first = collection.First(); - IGrouping last = collection.Last(); - Assert.True(collection.Contains(first)); - Assert.True(collection.Contains(last)); - } - Assert.False(collection.Contains(new NopGrouping())); - - IGrouping[] items = new IGrouping[count]; - collection.CopyTo(items, 0); - Assert.Equal(collection.Select(i => i), items); - Assert.Equal(items, Enumerable.Range(0, count).ToLookup(i => i.ToString()).ToArray()); - Assert.Equal(items, Enumerable.Range(0, count).ToLookup(i => i.ToString()).ToList()); - } - - private sealed class NopGrouping : IGrouping - { - public string Key => ""; - public IEnumerator GetEnumerator() => ((IList)Array.Empty()).GetEnumerator(); - IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - } - public class Membership { public int Id { get; set; } @@ -360,7 +301,7 @@ public class Role : IEquatable { public int Id { get; set; } - public bool Equals(Role other) => other is not null && Id == other.Id; + public bool Equals(Role other) => other != null && Id == other.Id; public override bool Equals(object obj) => Equals(obj as Role); @@ -374,7 +315,7 @@ public class RoleMetadata : IEquatable public int CountrB { get; set; } public bool Equals(RoleMetadata other) - => other is not null && Role.Equals(other.Role) && CountA == other.CountA && CountrB == other.CountrB; + => other != null && Role.Equals(other.Role) && CountA == other.CountA && CountrB == other.CountrB; public override bool Equals(object obj) => Equals(obj as RoleMetadata); diff --git a/src/libraries/System.Linq/tests/UnionTests.cs b/src/libraries/System.Linq/tests/UnionTests.cs index f1a45effa45ac6..9dbfd822b16395 100644 --- a/src/libraries/System.Linq/tests/UnionTests.cs +++ b/src/libraries/System.Linq/tests/UnionTests.cs @@ -302,7 +302,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Union(Enumerable.Range(0, 3)); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -311,7 +311,7 @@ public void ForcedToEnumeratorDoesntEnumerateMultipleUnions() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Union(Enumerable.Range(0, 3)).Union(Enumerable.Range(2, 4)).Union(new[] { 9, 2, 4 }); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] diff --git a/src/libraries/System.Linq/tests/WhereTests.cs b/src/libraries/System.Linq/tests/WhereTests.cs index 0cf0ff50cd38a2..a6bc625e9e4936 100644 --- a/src/libraries/System.Linq/tests/WhereTests.cs +++ b/src/libraries/System.Linq/tests/WhereTests.cs @@ -1016,7 +1016,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Where(i => true); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -1024,7 +1024,7 @@ public void ForcedToEnumeratorDoesntEnumerateArray() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToArray().Where(i => true); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -1032,7 +1032,7 @@ public void ForcedToEnumeratorDoesntEnumerateList() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Where(i => true); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -1040,7 +1040,7 @@ public void ForcedToEnumeratorDoesntEnumerateIndexed() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Where((e, i) => true); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -1048,7 +1048,7 @@ public void ForcedToEnumeratorDoesntEnumerateWhereSelect() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Where(i => true).Select(i => i); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -1056,7 +1056,7 @@ public void ForcedToEnumeratorDoesntEnumerateWhereSelectArray() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToArray().Where(i => true).Select(i => i); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] @@ -1064,7 +1064,7 @@ public void ForcedToEnumeratorDoesntEnumerateWhereSelectList() { var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).ToList().Where(i => true).Select(i => i); var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Theory] @@ -1094,50 +1094,6 @@ public void ToCollection(IEnumerable source) } } - [Fact] - public void WhereFirstLast() - { - Assert.All(IdentityTransforms(), transform => - { - IEnumerable data = transform(Enumerable.Range(0, 10)); - - Assert.Equal(3, data.Where(i => i == 3).First()); - Assert.Equal(0, data.Where(i => i % 2 == 0).First()); - - Assert.Equal(3, data.Where(i => i == 3).Last()); - Assert.Equal(8, data.Where(i => i % 2 == 0).Last()); - - Assert.Equal(3, data.Where(i => i == 3).ElementAt(0)); - Assert.Equal(8, data.Where(i => i % 2 == 0).ElementAt(4)); - - Assert.Throws(() => data.Where(i => i == 10).First()); - Assert.Throws(() => data.Where(i => i == 10).Last()); - Assert.Throws(() => data.Where(i => i == 10).ElementAt(0)); - }); - } - - [Fact] - public void WhereSelectFirstLast() - { - Assert.All(IdentityTransforms(), transform => - { - IEnumerable data = transform(Enumerable.Range(0, 10)); - - Assert.Equal(6, data.Where(i => i == 3).Select(i => i * 2).First()); - Assert.Equal(0, data.Where(i => i % 2 == 0).Select(i => i * 2).First()); - - Assert.Equal(6, data.Where(i => i == 3).Select(i => i * 2).Last()); - Assert.Equal(16, data.Where(i => i % 2 == 0).Select(i => i * 2).Last()); - - Assert.Equal(6, data.Where(i => i == 3).Select(i => i * 2).ElementAt(0)); - Assert.Equal(16, data.Where(i => i % 2 == 0).Select(i => i * 2).ElementAt(4)); - - Assert.Throws(() => data.Where(i => i == 10).Select(i => i * 2).First()); - Assert.Throws(() => data.Where(i => i == 10).Select(i => i * 2).Last()); - Assert.Throws(() => data.Where(i => i == 10).Select(i => i * 2).ElementAt(0)); - }); - } - public static IEnumerable ToCollectionData() { IEnumerable seq = GenerateRandomSequnce(seed: 0xdeadbeef, count: 10); diff --git a/src/libraries/System.Linq/tests/ZipTests.cs b/src/libraries/System.Linq/tests/ZipTests.cs index f15461931ad974..cfaa8bade6577b 100644 --- a/src/libraries/System.Linq/tests/ZipTests.cs +++ b/src/libraries/System.Linq/tests/ZipTests.cs @@ -374,7 +374,7 @@ public void ForcedToEnumeratorDoesntEnumerate() var iterator = NumberRangeGuaranteedNotCollectionType(0, 3).Zip(Enumerable.Range(0, 3), (x, y) => x + y); // Don't insist on this behaviour, but check it's correct if it happens var en = iterator as IEnumerator; - Assert.False(en is not null && en.MoveNext()); + Assert.False(en != null && en.MoveNext()); } [Fact] diff --git a/src/libraries/System.Management/src/System/Management/ManagementDateTime.cs b/src/libraries/System.Management/src/System/Management/ManagementDateTime.cs index 2ba36e619f186b..34860c2ff7e2bb 100644 --- a/src/libraries/System.Management/src/System/Management/ManagementDateTime.cs +++ b/src/libraries/System.Management/src/System/Management/ManagementDateTime.cs @@ -193,7 +193,7 @@ public static DateTime ToDateTime(string dmtfDate) throw new ArgumentOutOfRangeException(nameof(dmtfDate)); } - // codeql[cs/leap-year/unsafe-date-construction-from-two-elements] - DateTime not constructed from multiple elements - it's parsed from a string with defaults that are stable DateTime.MinValue. It would be intentional to throw if an invalid combination occurred. + var datetime = new DateTime(year, month, day, hour, minute, second, 0, DateTimeKind.Local); // Then add the ticks calculated from the microseconds datetime = datetime.AddTicks(ticks); diff --git a/src/libraries/System.Memory/ref/System.Memory.cs b/src/libraries/System.Memory/ref/System.Memory.cs index 8aac4764a1498a..991678fe583485 100644 --- a/src/libraries/System.Memory/ref/System.Memory.cs +++ b/src/libraries/System.Memory/ref/System.Memory.cs @@ -326,7 +326,7 @@ public static void CopyTo(this T[]? source, System.Span destination) { } public static int LastIndexOfAnyExceptInRange(this System.Span span, T lowInclusive, T highInclusive) where T : System.IComparable { throw null; } public static int LastIndexOf(this System.ReadOnlySpan span, System.ReadOnlySpan value) where T : System.IEquatable? { throw null; } public static int LastIndexOf(this System.ReadOnlySpan span, T value) where T : System.IEquatable? { throw null; } - public static int LastIndexOf(this System.Span span, System.ReadOnlySpan value) where T : System.IEquatable? { throw null; } + public static int LastIndexOf(this System.Span span, System.ReadOnlySpan value) where T : System.IEquatable { throw null; } public static int LastIndexOf(this System.Span span, T value) where T : System.IEquatable? { throw null; } public static int LastIndexOfAnyInRange(this System.ReadOnlySpan span, T lowInclusive, T highInclusive) where T : System.IComparable { throw null; } public static int LastIndexOfAnyInRange(this System.Span span, T lowInclusive, T highInclusive) where T : System.IComparable { throw null; } diff --git a/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs b/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs index f609e38dae3440..0cf99cbab5e4d5 100644 --- a/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs +++ b/src/libraries/System.Memory/tests/ArrayBufferWriter/ArrayBufferWriterTests.T.cs @@ -424,7 +424,14 @@ public void MultipleCallsToGetSpan() Assert.True(span.Length >= 256); Span newSpan = output.GetSpan(); Assert.Equal(span.Length, newSpan.Length); - Assert.Equal(0, Unsafe.ByteOffset(ref MemoryMarshal.GetReference(span), ref MemoryMarshal.GetReference(newSpan))); + + unsafe + { + void* pSpan = Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)); + void* pNewSpan = Unsafe.AsPointer(ref MemoryMarshal.GetReference(newSpan)); + Assert.Equal((IntPtr)pSpan, (IntPtr)pNewSpan); + } + Assert.Equal(span.Length, output.GetSpan().Length); } finally diff --git a/src/libraries/System.Memory/tests/MemoryMarshal/CreateFromPinnedArray.cs b/src/libraries/System.Memory/tests/MemoryMarshal/CreateFromPinnedArray.cs index e56a24e5eea28d..3610ea0dbfbf61 100644 --- a/src/libraries/System.Memory/tests/MemoryMarshal/CreateFromPinnedArray.cs +++ b/src/libraries/System.Memory/tests/MemoryMarshal/CreateFromPinnedArray.cs @@ -190,7 +190,6 @@ public static unsafe void CreateFromPinnedArrayVerifyPinning() int[] pinnedArray = { 90, 91, 92, 93, 94, 95, 96, 97, 98 }; GCHandle pinnedGCHandle = GCHandle.Alloc(pinnedArray, GCHandleType.Pinned); - // Unsafe.AsPointer is used to ensure we catch if the GC moves the memory Memory pinnedMemory = MemoryMarshal.CreateFromPinnedArray(pinnedArray, 0, 2); void* pinnedPtr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(pinnedMemory.Span)); void* memoryHandlePinnedPtr = pinnedMemory.Pin().Pointer; @@ -198,8 +197,8 @@ public static unsafe void CreateFromPinnedArrayVerifyPinning() GC.Collect(); GC.Collect(2); - Assert.Equal((IntPtr)pinnedPtr, (IntPtr)Unsafe.AsPointer(ref MemoryMarshal.GetReference(pinnedMemory.Span))); - Assert.Equal((IntPtr)memoryHandlePinnedPtr, pinnedGCHandle.AddrOfPinnedObject()); + Assert.Equal((int)pinnedPtr, (int)Unsafe.AsPointer(ref MemoryMarshal.GetReference(pinnedMemory.Span))); + Assert.Equal((int)memoryHandlePinnedPtr, (int)pinnedGCHandle.AddrOfPinnedObject().ToPointer()); pinnedGCHandle.Free(); } diff --git a/src/libraries/System.Memory/tests/MemoryMarshal/GetArrayDataReference.cs b/src/libraries/System.Memory/tests/MemoryMarshal/GetArrayDataReference.cs index c3478421c358c1..0fc55c64c44513 100644 --- a/src/libraries/System.Memory/tests/MemoryMarshal/GetArrayDataReference.cs +++ b/src/libraries/System.Memory/tests/MemoryMarshal/GetArrayDataReference.cs @@ -43,7 +43,7 @@ public static unsafe void GetArrayDataReference_EmptyInput_ReturnsRefToWhereFirs ref int theRef = ref MemoryMarshal.GetArrayDataReference(theArray); - Assert.False(Unsafe.IsNullRef(ref theRef)); + Assert.True(Unsafe.AsPointer(ref theRef) != null); Assert.True(Unsafe.AreSame(ref theRef, ref MemoryMarshal.GetReference(theArray.AsSpan()))); ref int theMdArrayRef = ref Unsafe.As(ref MemoryMarshal.GetArrayDataReference((Array)theArray)); // szarray passed to generalized Array helper diff --git a/src/libraries/System.Memory/tests/MemoryPool/MemoryPool.cs b/src/libraries/System.Memory/tests/MemoryPool/MemoryPool.cs index 4ee663dc1dd885..e040a363493bf1 100644 --- a/src/libraries/System.Memory/tests/MemoryPool/MemoryPool.cs +++ b/src/libraries/System.Memory/tests/MemoryPool/MemoryPool.cs @@ -52,7 +52,6 @@ public static void MemoryPoolSpan() { unsafe { - // Unsafe.AsPointer is safe here since it's pinned void* pSpan = Unsafe.AsPointer(ref MemoryMarshal.GetReference(sp)); Assert.Equal((IntPtr)newMemoryHandle.Pointer, (IntPtr)pSpan); } @@ -78,7 +77,6 @@ public static void MemoryPoolPin(int elementIndex) { unsafe { - // Unsafe.AsPointer is safe here since it's pinned void* pSpan = Unsafe.AsPointer(ref MemoryMarshal.GetReference(sp.Slice(elementIndex))); Assert.Equal((IntPtr)pSpan, ((IntPtr)newMemoryHandle.Pointer)); } @@ -114,7 +112,6 @@ public static void MemoryPoolPinOffsetAtEnd() { unsafe { - // Unsafe.AsPointer is safe here since it's pinned void* pSpan = Unsafe.AsPointer(ref MemoryMarshal.GetReference(sp.Slice(elementIndex))); Assert.Equal((IntPtr)pSpan, ((IntPtr)newMemoryHandle.Pointer)); } @@ -222,7 +219,11 @@ public static void MemoryPoolTryGetArray() unsafe { Assert.True(MemoryMarshal.TryGetArray(memory, out arraySegment)); - Assert.Equal(0, Unsafe.ByteOffset(ref MemoryMarshal.GetArrayDataReference(arraySegment.Array), ref MemoryMarshal.GetReference(memory.Span))); + fixed (int* pArray = arraySegment.Array) + { + void* pSpan = Unsafe.AsPointer(ref MemoryMarshal.GetReference(memory.Span)); + Assert.Equal((IntPtr)pSpan, (IntPtr)pArray); + } } } } diff --git a/src/libraries/System.Memory/tests/ReadOnlySpan/AsSpan.cs b/src/libraries/System.Memory/tests/ReadOnlySpan/AsSpan.cs index 83387f22a2d202..d6070d47c3ba99 100644 --- a/src/libraries/System.Memory/tests/ReadOnlySpan/AsSpan.cs +++ b/src/libraries/System.Memory/tests/ReadOnlySpan/AsSpan.cs @@ -91,7 +91,6 @@ static unsafe void Validate(string text, int start, int length, ReadOnlySpan strings = new string[] { "Hello", "World" }; - ReadOnlySpan span = ReadOnlySpan.CastUp(strings); - span.ValidateReferenceType("Hello", "World"); - } - } -} diff --git a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs index c0b80b5ab06776..5d1a51bde210aa 100644 --- a/src/libraries/System.Memory/tests/Span/StringSearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/StringSearchValues.cs @@ -43,18 +43,15 @@ public static void Values_ImplementsSearchValuesBase(StringComparison comparison foreach (string value in values) { - Assert.True(stringValues.Contains(value)); - string differentCase = value.ToLowerInvariant(); if (value == differentCase) { differentCase = value.ToUpperInvariant(); + Assert.NotEqual(value, differentCase); } - if (value != differentCase) - { - Assert.Equal(comparisonType == StringComparison.OrdinalIgnoreCase, stringValues.Contains(differentCase)); - } + Assert.True(stringValues.Contains(value)); + Assert.Equal(comparisonType == StringComparison.OrdinalIgnoreCase, stringValues.Contains(differentCase)); AssertIndexOfAnyAndFriends(new[] { value }, 0, -1, 0, -1); AssertIndexOfAnyAndFriends(new[] { value, value }, 0, -1, 1, -1); @@ -76,7 +73,7 @@ public static void Values_ImplementsSearchValuesBase(StringComparison comparison AssertIndexOfAnyAndFriends(new[] { ValueNotInSet, differentCase, ValueNotInSet }, 1, 0, 1, 2); AssertIndexOfAnyAndFriends(new[] { differentCase, ValueNotInSet, differentCase }, 0, 1, 2, 1); } - else if (value != differentCase) + else { AssertIndexOfAnyAndFriends(new[] { differentCase }, -1, 0, -1, 0); AssertIndexOfAnyAndFriends(new[] { differentCase, differentCase }, -1, 0, -1, 1); @@ -176,122 +173,6 @@ public static void IndexOfAny(StringComparison comparisonType, int expected, str Assert.Equal(expected >= 0, text.AsSpan().ContainsAny(stringValues)); Assert.Equal(expected >= 0, textSpan.ContainsAny(stringValues)); - - if (values is null || stringValues.Contains(string.Empty)) - { - // The tests below don't work if an empty string is in the set. - return; - } - - // The tests below assume none of the values contain these characters. - Assert.Equal(-1, IndexOfAnyReferenceImpl(new string('\0', 100), valuesArray, comparisonType)); - Assert.Equal(-1, IndexOfAnyReferenceImpl(new string('\u00FC', 100), valuesArray, comparisonType)); - - string[] valuesWithDifferentCases = valuesArray; - - if (comparisonType == StringComparison.OrdinalIgnoreCase) - { - valuesWithDifferentCases = valuesArray - .SelectMany(v => new[] { v, v.ToUpperInvariant(), v.ToLowerInvariant() }) - .Distinct() - // Invariant conversions may produce values that don't match under ordinal rules. Filter them out. - .Where(v => valuesArray.Any(original => v.Equals(original, StringComparison.OrdinalIgnoreCase))) - .ToArray(); - } - - // Test cases where the implementation changes based on the haystack length (e.g. swapping from Teddy to Rabin-Karp). - for (int haystackLength = 0; haystackLength < 50; haystackLength++) - { - TestWithPoisonPages(PoisonPagePlacement.Before, haystackLength); - TestWithPoisonPages(PoisonPagePlacement.After, haystackLength); - } - - void TestWithPoisonPages(PoisonPagePlacement poisonPlacement, int haystackLength) - { - using BoundedMemory memory = BoundedMemory.Allocate(haystackLength, poisonPlacement); - Span haystack = memory.Span; - - char asciiNumberNotInSet = Enumerable.Range('0', 10).Select(c => (char)c) - .First(c => !values.Contains(c)); - - char asciiLetterLowerNotInSet; - char asciiLetterUpperNotInSet; - - if (comparisonType == StringComparison.Ordinal) - { - asciiLetterLowerNotInSet = Enumerable.Range('a', 26).Select(c => (char)c).First(c => !values.Contains(c)); - asciiLetterUpperNotInSet = Enumerable.Range('A', 26).Select(c => (char)c).First(c => !values.Contains(c)); - } - else - { - asciiLetterLowerNotInSet = Enumerable.Range('a', 26).Select(c => (char)c) - .First(c => !values.AsSpan().ContainsAny(c, char.ToUpperInvariant(c))); - - asciiLetterUpperNotInSet = Enumerable.Range(0, 26).Select(c => (char)('Z' - c)) - .First(c => !values.AsSpan().ContainsAny(c, char.ToLowerInvariant(c))); - } - - TestWithDifferentMarkerChars(haystack, '\0'); - TestWithDifferentMarkerChars(haystack, '\u00FC'); - TestWithDifferentMarkerChars(haystack, asciiNumberNotInSet); - TestWithDifferentMarkerChars(haystack, asciiLetterLowerNotInSet); - TestWithDifferentMarkerChars(haystack, asciiLetterUpperNotInSet); - } - - void TestWithDifferentMarkerChars(Span haystack, char marker) - { - haystack.Fill(marker); - Assert.True(haystack.IndexOfAny(stringValues) == -1, marker.ToString()); - - string shortestValue = valuesArray.MinBy(value => value.Length); - - // Test every value individually at every offset in the haystack. - foreach (string value in valuesWithDifferentCases) - { - for (int startOffset = 0; startOffset <= haystack.Length - value.Length; startOffset++) - { - haystack.Fill(marker); - - // Place an unrelated matching value at the end of the haystack. It shouldn't affect the result. - shortestValue.CopyTo(haystack.Slice(haystack.Length - shortestValue.Length)); - - // Place a matching value at the offset position. - value.CopyTo(haystack.Slice(startOffset)); - - int actual = haystack.IndexOfAny(stringValues); - if (startOffset != actual) - { - StringSearchValuesTestHelper.AssertionFailed(haystack, valuesArray, stringValues, comparisonType, startOffset, actual); - } - } - } - - if (text == valuesArray[0]) - { - // Already tested above. - return; - } - - // Test the provided test case at various offsets in the haystack. - for (int startOffset = 0; startOffset <= haystack.Length - text.Length; startOffset++) - { - haystack.Fill(marker); - - // Place the test case text at the end of the haystack. It shouldn't affect the result. - text.CopyTo(haystack.Slice(haystack.Length - text.Length)); - - // Place the test text at the offset position. - text.CopyTo(haystack.Slice(startOffset)); - - int expectedAtOffset = expected == -1 ? -1 : startOffset + expected; - - int actual = haystack.IndexOfAny(stringValues); - if (expectedAtOffset != actual) - { - StringSearchValuesTestHelper.AssertionFailed(haystack, valuesArray, stringValues, comparisonType, expectedAtOffset, actual); - } - } - } } [Fact] @@ -316,102 +197,6 @@ public static void IndexOfAny_InvalidUtf16() IndexOfAny(StringComparison.OrdinalIgnoreCase, 1, "\uD801\uDCD8\uD8FB\uDCD8", "foo, \uDCD8"); } - [Theory] - // Single value of various lengths - [InlineData("a")] - [InlineData("!")] - [InlineData("\u00F6")] - [InlineData("ab")] - [InlineData("a!")] - [InlineData("!a")] - [InlineData("!%")] - [InlineData("a\u00F6")] - [InlineData("\u00F6\u00F6")] - [InlineData("abc")] - [InlineData("ab!")] - [InlineData("a\u00F6b")] - [InlineData("\u00F6a\u00F6")] - [InlineData("abcd")] - [InlineData("ab!cd")] - [InlineData("abcde")] - [InlineData("abcd!")] - [InlineData("abcdefgh")] - [InlineData("abcdefghi")] - // Multiple values, but they all share the same prefix - [InlineData("abc", "ab", "abcd")] - // These should hit the Aho-Corasick implementation - [InlineData("a", "b")] - [InlineData("ab", "c")] - // Simple Teddy cases - [InlineData("abc", "cde")] - [InlineData("abc", "cd")] - // Teddy where all starting chars are letters, but not all other characters are - [InlineData("ab", "de%", "ghi", "jkl!")] - [InlineData("abc", "def%", "ghi", "jkl!")] - // Teddy where starting chars aren't only letters - [InlineData("ab", "d%e", "ghi", "jkl!")] - [InlineData("abc", "def%", "ghi", "!jkl")] - // Teddy where the starting chars aren't affected by case conversion - [InlineData("12", "45b", "789")] - [InlineData("123", "456", "789")] - [InlineData("123", "456a", "789b")] - // We'll expand these values to all case permutations - [InlineData("ab", "bc")] - [InlineData("ab", "c!")] - [InlineData("ab", "c!", "!%")] - // These won't be expanded as they would produce more than 8 permutations - [InlineData("ab", "bc", "c!")] - [InlineData("abc", "bc")] - // Rabin-Karp where one of the values is longer than what the implementation can match (17) - [InlineData("abc", "a012345678012345678")] - // Rabin-Karp where all of the values are longer than what the implementation can match (17) - [InlineData("a012345678012345678", "bc012345678012345678")] - // Teddy with exactly 8 values (filling all 8 buckets) - [InlineData("ab", "bc", "def", "ghi", "jkl", "mno", "pqr", "stu")] - [InlineData("abc", "def", "ghi", "jkl", "mno", "pqr", "stu", "vwx")] - // Teddy with more than 8 values - [InlineData("ab", "bc", "def", "ghi", "jkl", "mno", "pqr", "stu", "vwx")] - [InlineData("abc", "def", "ghi", "jkl", "mno", "pqr", "stu", "vwx", "yab")] - public static void SimpleIndexOfAnyValues(params string[] valuesArray) - { - TestCore(valuesArray); - - // Test cases where the implementation differs for ASCII letters, different cases, non-letters. - if (valuesArray.Any(v => v.Contains('a'))) - { - TestCore(valuesArray.Select(v => v.Replace('a', 'A')).ToArray()); - TestCore(valuesArray.Select(v => v.Replace('a', '7')).ToArray()); - } - - int offset = valuesArray.Length / 2; - string original = valuesArray[offset]; - - // Test non-ASCII values - valuesArray[offset] = $"{original}\u00F6"; - TestCore(valuesArray); - - valuesArray[offset] = $"\u00F6{original}"; - TestCore(valuesArray); - - valuesArray[offset] = $"{original[0]}\u00F6{original.AsSpan(1)}"; - TestCore(valuesArray); - - // Test null chars in values - valuesArray[offset] = $"{original[0]}\0{original.AsSpan(1)}"; - TestCore(valuesArray); - - static void TestCore(string[] valuesArray) - { - Values_ImplementsSearchValuesBase(StringComparison.Ordinal, valuesArray); - Values_ImplementsSearchValuesBase(StringComparison.OrdinalIgnoreCase, valuesArray); - - string values = string.Join(", ", valuesArray); - - IndexOfAny(StringComparison.Ordinal, 0, valuesArray[0], values); - IndexOfAny(StringComparison.OrdinalIgnoreCase, 0, valuesArray[0], values); - } - } - [Fact] [SkipOnPlatform(TestPlatforms.LinuxBionic, "Remote executor has problems with exit codes")] public static void IndexOfAny_CanProduceDifferentResultsUnderNls() @@ -710,7 +495,7 @@ private static ReadOnlySpan GetRandomSlice(Random rng, ReadOnlySpan spa return slice.Slice(0, Math.Min(slice.Length, rng.Next(maxLength + 1))); } - public static void AssertionFailed(ReadOnlySpan haystack, string[] needle, SearchValues searchValues, StringComparison comparisonType, int expected, int actual) + private static void AssertionFailed(ReadOnlySpan haystack, string[] needle, SearchValues searchValues, StringComparison comparisonType, int expected, int actual) { Type implType = searchValues.GetType(); string impl = $"{implType.Name} [{string.Join(", ", implType.GenericTypeArguments.Select(t => t.Name))}]"; diff --git a/src/libraries/System.Memory/tests/System.Memory.Tests.csproj b/src/libraries/System.Memory/tests/System.Memory.Tests.csproj index 843d5e1b479ceb..ba793f07c8f5c5 100644 --- a/src/libraries/System.Memory/tests/System.Memory.Tests.csproj +++ b/src/libraries/System.Memory/tests/System.Memory.Tests.csproj @@ -24,7 +24,6 @@ - diff --git a/src/libraries/System.Memory/tests/TestHelpers.cs b/src/libraries/System.Memory/tests/TestHelpers.cs index 5fac5143a3b8a8..54d314355f0531 100644 --- a/src/libraries/System.Memory/tests/TestHelpers.cs +++ b/src/libraries/System.Memory/tests/TestHelpers.cs @@ -37,7 +37,7 @@ public static unsafe void ValidateNonNullEmpty(this Span span) Assert.True(span.IsEmpty); // Validate that empty Span is not normalized to null - Assert.False(Unsafe.IsNullRef(ref MemoryMarshal.GetReference(span))); + Assert.True(Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)) != null); } public delegate void AssertThrowsAction(Span span); @@ -98,7 +98,7 @@ public static unsafe void ValidateNonNullEmpty(this ReadOnlySpan span) Assert.True(span.IsEmpty); // Validate that empty Span is not normalized to null - Assert.False(Unsafe.IsNullRef(ref MemoryMarshal.GetReference(span))); + Assert.True(Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)) != null); } public delegate void AssertThrowsActionReadOnly(ReadOnlySpan span); diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpHandler.cs b/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpHandler.cs index 050913e6f86843..bbcd625d036d6a 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpHandler.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpHandler.cs @@ -511,6 +511,7 @@ public BrowserHttpReadStream(BrowserHttpController controller) public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) { + ArgumentNullException.ThrowIfNull(buffer, nameof(buffer)); _controller.ThrowIfDisposed(); MemoryHandle pinBuffer = buffer.Pin(); diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpInterop.cs b/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpInterop.cs index c003cb7615e28f..eee79765c246f3 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpInterop.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/BrowserHttpHandler/BrowserHttpInterop.cs @@ -147,7 +147,7 @@ public static async Task CancellationHelper(Task promise, CancellationToken canc } }, (promise, jsController))) { - await promise.ConfigureAwait(false); + await promise.ConfigureAwait(true); } } catch (OperationCanceledException oce) when (cancellationToken.IsCancellationRequested) diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs b/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs index 3dfc789919b33c..9989bc4568e617 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/HttpTelemetry.cs @@ -339,13 +339,13 @@ private unsafe void WriteEvent(int eventId, byte arg1, byte arg2, long arg3, str arg5 ??= ""; arg7 ??= ""; + const int NumEventDatas = 7; + EventData* descrs = stackalloc EventData[NumEventDatas]; + fixed (char* arg4Ptr = arg4) fixed (char* arg5Ptr = arg5) fixed (char* arg7Ptr = arg7) { - const int NumEventDatas = 7; - EventData* descrs = stackalloc EventData[NumEventDatas]; - descrs[0] = new EventData { DataPointer = (IntPtr)(&arg1), @@ -381,9 +381,9 @@ private unsafe void WriteEvent(int eventId, byte arg1, byte arg2, long arg3, str DataPointer = (IntPtr)arg7Ptr, Size = (arg7.Length + 1) * sizeof(char) }; - - WriteEventCore(eventId, NumEventDatas, descrs); } + + WriteEventCore(eventId, NumEventDatas, descrs); } } } diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs index af64584f02c68e..4d84e54748e3ab 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs @@ -765,12 +765,9 @@ private void ProcessDataFrame(FrameHeader frameHeader) // Just ignore the frame in this case. ReadOnlySpan frameData = GetFrameData(_incomingBuffer.ActiveSpan.Slice(0, frameHeader.PayloadLength), hasPad: frameHeader.PaddedFlag, hasPriority: false); - bool endStream = frameHeader.EndStreamFlag; - if (frameData.Length > 0 || endStream) - { - http2Stream?.OnResponseData(frameData, endStream); - } + bool endStream = frameHeader.EndStreamFlag; + http2Stream?.OnResponseData(frameData, endStream); if (frameData.Length > 0) { @@ -1444,14 +1441,6 @@ private int WriteHeaderCollection(HttpRequestMessage request, HttpHeaders header continue; } - // Extended connect requests will use the response content stream for bidirectional communication. - // We will ignore any content set for such requests in Http2Stream.SendRequestBodyAsync, as it has no defined semantics. - // Drop the Content-Length header as well in the unlikely case it was set. - if (knownHeader == KnownHeaders.ContentLength && request.IsExtendedConnectRequest) - { - continue; - } - // For all other known headers, send them via their pre-encoded name and the associated value. WriteBytes(knownHeader.Http2EncodedName, ref headerBuffer); string? separator = null; diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs index 4b1f4ccb3ee695..8f8d81ec6b24f0 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Stream.cs @@ -105,9 +105,7 @@ public Http2Stream(HttpRequestMessage request, Http2Connection connection) _headerBudgetRemaining = connection._pool.Settings.MaxResponseHeadersByteLength; - // Extended connect requests will use the response content stream for bidirectional communication. - // We will ignore any content set for such requests in SendRequestBodyAsync, as it has no defined semantics. - if (_request.Content == null || _request.IsExtendedConnectRequest) + if (_request.Content == null) { _requestCompletionState = StreamCompletionState.Completed; if (_request.IsExtendedConnectRequest) @@ -175,9 +173,7 @@ public HttpResponseMessage GetAndClearResponse() public async Task SendRequestBodyAsync(CancellationToken cancellationToken) { - // Extended connect requests will use the response content stream for bidirectional communication. - // Ignore any content set for such requests, as it has no defined semantics. - if (_request.Content == null || _request.IsExtendedConnectRequest) + if (_request.Content == null) { Debug.Assert(_requestCompletionState == StreamCompletionState.Completed); return; @@ -254,7 +250,6 @@ public async Task SendRequestBodyAsync(CancellationToken cancellationToken) // and we also don't want to propagate any error to the caller, in particular for non-duplex scenarios. Debug.Assert(_responseCompletionState == StreamCompletionState.Completed); _requestCompletionState = StreamCompletionState.Completed; - Debug.Assert(!ConnectProtocolEstablished); Complete(); return; } @@ -266,7 +261,6 @@ public async Task SendRequestBodyAsync(CancellationToken cancellationToken) _requestCompletionState = StreamCompletionState.Failed; SendReset(); - Debug.Assert(!ConnectProtocolEstablished); Complete(); } @@ -319,7 +313,6 @@ public async Task SendRequestBodyAsync(CancellationToken cancellationToken) if (complete) { - Debug.Assert(!ConnectProtocolEstablished); Complete(); } } @@ -427,17 +420,7 @@ private void Cancel() if (sendReset) { SendReset(); - - // Extended CONNECT notes: - // - // To prevent from calling it *twice*, Extended CONNECT stream's Complete() is only - // called from CloseResponseBody(), as CloseResponseBody() is *always* called - // from Extended CONNECT stream's Dispose(). - - if (!ConnectProtocolEstablished) - { - Complete(); - } + Complete(); } } @@ -827,20 +810,7 @@ public void OnHeadersComplete(bool endStream) Debug.Assert(_responseCompletionState == StreamCompletionState.InProgress, $"Response already completed with state={_responseCompletionState}"); _responseCompletionState = StreamCompletionState.Completed; - - // Extended CONNECT notes: - // - // To prevent from calling it *prematurely*, Extended CONNECT stream's Complete() is only - // called from CloseResponseBody(), as CloseResponseBody() is *only* called - // from Extended CONNECT stream's Dispose(). - // - // Due to bidirectional streaming nature of the Extended CONNECT request, - // the *write side* of the stream can only be completed by calling Dispose(). - // - // The streaming in both ways happens over the single "response" stream instance, which makes - // _requestCompletionState *not indicative* of the actual state of the write side of the stream. - - if (_requestCompletionState == StreamCompletionState.Completed && !ConnectProtocolEstablished) + if (_requestCompletionState == StreamCompletionState.Completed) { Complete(); } @@ -901,20 +871,7 @@ public void OnResponseData(ReadOnlySpan buffer, bool endStream) Debug.Assert(_responseCompletionState == StreamCompletionState.InProgress, $"Response already completed with state={_responseCompletionState}"); _responseCompletionState = StreamCompletionState.Completed; - - // Extended CONNECT notes: - // - // To prevent from calling it *prematurely*, Extended CONNECT stream's Complete() is only - // called from CloseResponseBody(), as CloseResponseBody() is *only* called - // from Extended CONNECT stream's Dispose(). - // - // Due to bidirectional streaming nature of the Extended CONNECT request, - // the *write side* of the stream can only be completed by calling Dispose(). - // - // The streaming in both ways happens over the single "response" stream instance, which makes - // _requestCompletionState *not indicative* of the actual state of the write side of the stream. - - if (_requestCompletionState == StreamCompletionState.Completed && !ConnectProtocolEstablished) + if (_requestCompletionState == StreamCompletionState.Completed) { Complete(); } @@ -1079,17 +1036,17 @@ public async Task ReadResponseHeadersAsync(CancellationToken cancellationToken) Debug.Assert(_response != null && _response.Content != null); // Start to process the response body. var responseContent = (HttpConnectionResponseContent)_response.Content; - if (ConnectProtocolEstablished) - { - responseContent.SetStream(new Http2ReadWriteStream(this, closeResponseBodyOnDispose: true)); - } - else if (emptyResponse) + if (emptyResponse) { // If there are any trailers, copy them over to the response. Normally this would be handled by // the response stream hitting EOF, but if there is no response body, we do it here. MoveTrailersToResponseMessage(_response); responseContent.SetStream(EmptyReadStream.Instance); } + else if (ConnectProtocolEstablished) + { + responseContent.SetStream(new Http2ReadWriteStream(this)); + } else { responseContent.SetStream(new Http2ReadStream(this)); @@ -1352,25 +1309,8 @@ private async ValueTask SendDataAsync(ReadOnlyMemory buffer, CancellationT } } - // This method should only be called from Http2ReadWriteStream.Dispose() private void CloseResponseBody() { - // Extended CONNECT notes: - // - // Due to bidirectional streaming nature of the Extended CONNECT request, - // the *write side* of the stream can only be completed by calling Dispose() - // (which, for Extended CONNECT case, will in turn call CloseResponseBody()) - // - // Similarly to QuicStream, disposal *gracefully* closes the write side of the stream - // (unless we've received RST_STREAM before) and *abortively* closes the read side - // of the stream (unless we've received EOS before). - - if (ConnectProtocolEstablished && _resetException is null) - { - // Gracefully close the write side of the Extended CONNECT stream - _connection.LogExceptions(_connection.SendEndStreamAsync(StreamId)); - } - // Check if the response body has been fully consumed. bool fullyConsumed = false; Debug.Assert(!Monitor.IsEntered(SyncObject)); @@ -1383,7 +1323,6 @@ private void CloseResponseBody() } // If the response body isn't completed, cancel it now. - // This includes aborting the read side of the Extended CONNECT stream. if (!fullyConsumed) { Cancel(); @@ -1398,12 +1337,6 @@ private void CloseResponseBody() lock (SyncObject) { - if (ConnectProtocolEstablished) - { - // This should be the only place where Extended Connect stream is completed - Complete(); - } - _responseBuffer.Dispose(); } } @@ -1497,7 +1430,10 @@ private enum StreamCompletionState : byte private sealed class Http2ReadStream : Http2ReadWriteStream { - public Http2ReadStream(Http2Stream http2Stream) : base(http2Stream, closeResponseBodyOnDispose: true) { } + public Http2ReadStream(Http2Stream http2Stream) : base(http2Stream) + { + base.CloseResponseBodyOnDispose = true; + } public override bool CanWrite => false; @@ -1546,13 +1482,12 @@ public class Http2ReadWriteStream : HttpBaseStream private Http2Stream? _http2Stream; private readonly HttpResponseMessage _responseMessage; - public Http2ReadWriteStream(Http2Stream http2Stream, bool closeResponseBodyOnDispose = false) + public Http2ReadWriteStream(Http2Stream http2Stream) { Debug.Assert(http2Stream != null); Debug.Assert(http2Stream._response != null); _http2Stream = http2Stream; _responseMessage = _http2Stream._response; - CloseResponseBodyOnDispose = closeResponseBodyOnDispose; } ~Http2ReadWriteStream() @@ -1568,7 +1503,7 @@ public Http2ReadWriteStream(Http2Stream http2Stream, bool closeResponseBodyOnDis } } - protected bool CloseResponseBodyOnDispose { get; private init; } + protected bool CloseResponseBodyOnDispose { get; set; } protected override void Dispose(bool disposing) { diff --git a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs index c9a6deab2e77fb..0c70813838b47d 100644 --- a/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs +++ b/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/HttpConnectionPool.cs @@ -655,7 +655,6 @@ private async Task HandleHttp11Downgrade(HttpRequestMessage request, Stream stre { if (NetEventSource.Log.IsEnabled()) Trace("Discarding downgraded HTTP/1.1 connection because HTTP/1.1 connection limit is exceeded"); stream.Dispose(); - return; } HttpConnection http11Connection; diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs index 3036935ef5533d..33c94856d270cb 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/HttpClientHandlerTest.Http2.cs @@ -205,50 +205,6 @@ public async Task Http2_ZeroLengthResponseBody_Success() } } - [Fact] - public async Task Http2_DataFrameOnlyPadding_Success() - { - using (Http2LoopbackServer server = Http2LoopbackServer.CreateServer()) - using (HttpClient client = CreateHttpClient()) - { - Task sendTask = client.GetAsync(server.Address, HttpCompletionOption.ResponseHeadersRead); - - Http2LoopbackConnection connection = await server.EstablishConnectionAsync(); - - int streamId = await connection.ReadRequestHeaderAsync(); - - await connection.SendDefaultResponseHeadersAsync(streamId); - - // Send zero-length DATA frame with padding - byte paddingLength = byte.MaxValue; - int dataLength = 1024; - DataFrame frame = new DataFrame(new byte[0], FrameFlags.Padded, paddingLength, streamId); - await connection.WriteFrameAsync(frame); - - HttpResponseMessage response = await sendTask; - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - - using var responseStream = response.Content.ReadAsStream(); - - // The read must pend because we havent received any data yet. - var buffer = new byte[dataLength]; - var readTask = responseStream.ReadAtLeastAsync(buffer, dataLength); - Assert.False(readTask.IsCompleted); - - // Send DATA frame with padding - frame = new DataFrame(new byte[dataLength], FrameFlags.Padded, paddingLength, streamId); - await connection.WriteFrameAsync(frame); - - Assert.Equal(dataLength, await readTask); - - // Send zero-length, end-stream DATA frame with padding - frame = new DataFrame(new byte[0], FrameFlags.Padded | FrameFlags.EndStream, paddingLength, streamId); - await connection.WriteFrameAsync(frame); - - Assert.Equal(0, await responseStream.ReadAsync(buffer)); - } - } - [Theory] [InlineData("Client content", null)] [InlineData("Client content", "Server content")] diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs index f3a46d4d08507e..0ea6ae9e13f60b 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.Http2ExtendedConnect.cs @@ -2,10 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Net.Test.Common; -using System.Threading; using System.Threading.Tasks; using Xunit; using Xunit.Abstractions; @@ -33,7 +31,6 @@ public static IEnumerable UseSsl_MemberData() [MemberData(nameof(UseSsl_MemberData))] public async Task Connect_ReadWriteResponseStream(bool useSsl) { - const int MessageCount = 3; byte[] clientMessage = new byte[] { 1, 2, 3 }; byte[] serverMessage = new byte[] { 4, 5, 6, 7 }; @@ -46,62 +43,34 @@ await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync(async uri HttpRequestMessage request = CreateRequest(HttpMethod.Connect, uri, UseVersion, exactVersion: true); request.Headers.Protocol = "foo"; - bool readFromContentStream = false; - - // We won't send the content bytes, but we will send content headers. - // Since we're dropping the content, we'll also drop the Content-Length header. - request.Content = new StreamContent(new DelegateStream( - readAsyncFunc: (_, _, _, _) => - { - readFromContentStream = true; - throw new UnreachableException(); - })); - - request.Headers.Add("User-Agent", "foo"); - request.Content.Headers.Add("Content-Language", "bar"); - request.Content.Headers.ContentLength = 42; - using HttpResponseMessage response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); using Stream responseStream = await response.Content.ReadAsStreamAsync(); - for (int i = 0; i < MessageCount; i++) - { - await responseStream.WriteAsync(clientMessage); - await responseStream.FlushAsync(); + await responseStream.WriteAsync(clientMessage); + await responseStream.FlushAsync(); - byte[] readBuffer = new byte[serverMessage.Length]; - await responseStream.ReadExactlyAsync(readBuffer); - Assert.Equal(serverMessage, readBuffer); - } + byte[] readBuffer = new byte[serverMessage.Length]; + await responseStream.ReadExactlyAsync(readBuffer); + Assert.Equal(serverMessage, readBuffer); // Receive server's EOS - Assert.Equal(0, await responseStream.ReadAsync(new byte[1])); - - Assert.False(readFromContentStream); + Assert.Equal(0, await responseStream.ReadAsync(readBuffer)); clientCompleted.SetResult(); }, async server => { await using Http2LoopbackConnection connection = await ((Http2LoopbackServer)server).EstablishConnectionAsync(new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }); - connection.IgnoreWindowUpdates(); - - (int streamId, HttpRequestData request) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); - Assert.Equal("foo", request.GetSingleHeaderValue("User-Agent")); - Assert.Equal("bar", request.GetSingleHeaderValue("Content-Language")); - Assert.Equal(0, request.GetHeaderValueCount("Content-Length")); + (int streamId, _) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); await connection.SendResponseHeadersAsync(streamId, endStream: false).ConfigureAwait(false); - for (int i = 0; i < MessageCount; i++) - { - DataFrame dataFrame = await connection.ReadDataFrameAsync(); - Assert.Equal(clientMessage, dataFrame.Data.ToArray()); + DataFrame dataFrame = await connection.ReadDataFrameAsync(); + Assert.Equal(clientMessage, dataFrame.Data.ToArray()); - await connection.SendResponseDataAsync(streamId, serverMessage, endStream: i == MessageCount - 1); - } + await connection.SendResponseDataAsync(streamId, serverMessage, endStream: true); await clientCompleted.Task.WaitAsync(TestHelper.PassingTestTimeout); }, options: new GenericLoopbackOptions { UseSsl = useSsl }); @@ -194,114 +163,5 @@ await server.AcceptConnectionAsync(async connection => await new[] { serverTask, clientTask }.WhenAllOrAnyFailed().WaitAsync(TestHelper.PassingTestTimeout); } - - [Theory] - [MemberData(nameof(UseSsl_MemberData))] - public async Task Connect_ServerSideEOS_ReceivedByClient(bool useSsl) - { - var timeoutTcs = new CancellationTokenSource(TestHelper.PassingTestTimeout); - var serverReceivedEOS = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync( - clientFunc: async uri => - { - var client = CreateHttpClient(); - var request = CreateRequest(HttpMethod.Connect, uri, UseVersion, exactVersion: true); - request.Headers.Protocol = "foo"; - - var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutTcs.Token); - var responseStream = await response.Content.ReadAsStreamAsync(timeoutTcs.Token); - - // receive server's EOS - Assert.Equal(0, await responseStream.ReadAsync(new byte[1], timeoutTcs.Token)); - - // send client's EOS - responseStream.Dispose(); - - // wait for "ack" from server - await serverReceivedEOS.Task.WaitAsync(timeoutTcs.Token); - - // can dispose handler now - client.Dispose(); - }, - serverFunc: async server => - { - await using var connection = await ((Http2LoopbackServer)server).EstablishConnectionAsync( - new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }); - connection.IgnoreWindowUpdates(); - - (int streamId, _) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); - await connection.SendResponseHeadersAsync(streamId, endStream: false); - - // send server's EOS - await connection.SendResponseDataAsync(streamId, Array.Empty(), endStream: true); - - // receive client's EOS "in response" to server's EOS - var eosFrame = Assert.IsType(await connection.ReadFrameAsync(timeoutTcs.Token)); - Assert.Equal(streamId, eosFrame.StreamId); - Assert.Equal(0, eosFrame.Data.Length); - Assert.True(eosFrame.EndStreamFlag); - - serverReceivedEOS.SetResult(); - - // on handler dispose, client should shutdown the connection without sending additional frames - await connection.WaitForClientDisconnectAsync().WaitAsync(timeoutTcs.Token); - }, - options: new GenericLoopbackOptions { UseSsl = useSsl }); - } - - [Theory] - [MemberData(nameof(UseSsl_MemberData))] - public async Task Connect_ClientSideEOS_ReceivedByServer(bool useSsl) - { - var timeoutTcs = new CancellationTokenSource(TestHelper.PassingTestTimeout); - var serverReceivedRst = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - await Http2LoopbackServerFactory.Singleton.CreateClientAndServerAsync( - clientFunc: async uri => - { - var client = CreateHttpClient(); - var request = CreateRequest(HttpMethod.Connect, uri, UseVersion, exactVersion: true); - request.Headers.Protocol = "foo"; - - var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, timeoutTcs.Token); - var responseStream = await response.Content.ReadAsStreamAsync(timeoutTcs.Token); - - // send client's EOS - // this will also send RST_STREAM as we didn't receive server's EOS before - responseStream.Dispose(); - - // wait for "ack" from server - await serverReceivedRst.Task.WaitAsync(timeoutTcs.Token); - - // can dispose handler now - client.Dispose(); - }, - serverFunc: async server => - { - await using var connection = await ((Http2LoopbackServer)server).EstablishConnectionAsync( - new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }); - connection.IgnoreWindowUpdates(); - - (int streamId, _) = await connection.ReadAndParseRequestHeaderAsync(readBody: false); - await connection.SendResponseHeadersAsync(streamId, endStream: false); - - // receive client's EOS - var eosFrame = Assert.IsType(await connection.ReadFrameAsync(timeoutTcs.Token)); - Assert.Equal(streamId, eosFrame.StreamId); - Assert.Equal(0, eosFrame.Data.Length); - Assert.True(eosFrame.EndStreamFlag); - - // receive client's RST_STREAM as we didn't send server's EOS before - var rstFrame = Assert.IsType(await connection.ReadFrameAsync(timeoutTcs.Token)); - Assert.Equal(streamId, rstFrame.StreamId); - - serverReceivedRst.SetResult(); - - // on handler dispose, client should shutdown the connection without sending additional frames - await connection.WaitForClientDisconnectAsync().WaitAsync(timeoutTcs.Token); - }, - options: new GenericLoopbackOptions { UseSsl = useSsl }); - } } } diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs index c44df51433c9e3..0fd9636a7cabc7 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs @@ -3531,76 +3531,6 @@ public sealed class SocketsHttpHandlerTest_ConnectCallback_Http2 : SocketsHttpHa { public SocketsHttpHandlerTest_ConnectCallback_Http2(ITestOutputHelper output) : base(output) { } protected override Version UseVersion => HttpVersion.Version20; - - [Fact] - public async Task Http2Connection_DroppedWhenDowngradingToHttp11WhenAtMaxConnections() - { - // Regression test for https://github.com/dotnet/runtime/issues/99401 - await LoopbackServer.CreateClientAndServerAsync( - async uri => - { - using HttpClientHandler handler = CreateHttpClientHandler(); - using HttpClient client = CreateHttpClient(handler); - - int connectionCount = 0; - - GetUnderlyingSocketsHttpHandler(handler).MaxConnectionsPerServer = 1; - - GetUnderlyingSocketsHttpHandler(handler).ConnectCallback = async (context, token) => - { - connectionCount++; - - var socket = new Socket(SocketType.Stream, ProtocolType.Tcp) { NoDelay = true }; - await socket.ConnectAsync(context.DnsEndPoint, token); - var stream = new NetworkStream(socket, ownsSocket: true); - - // Not using ALPN, so the client will attempt to downgrade to HTTP/1.1. - var options = new SslClientAuthenticationOptions(); - options.RemoteCertificateValidationCallback = (_, _, _, _) => true; - options.TargetHost = context.DnsEndPoint.Host; - - var sslStream = new SslStream(stream); - await sslStream.AuthenticateAsClientAsync(options); - return sslStream; - }; - - // Send a request to establish the first HTTP/1.1 connection. - using HttpResponseMessage response1 = await client.SendAsync(CreateRequest(HttpMethod.Get, uri, HttpVersion.Version11)); - Assert.Equal(1, connectionCount); - Assert.Equal("1", await response1.Content.ReadAsStringAsync()); - - // Send an HTTP/2 request that will be downgraded to HTTP/1.1. - // The new connection should be thrown away as we're at the connection limit, - // and the request should be handled using the existing HTTP/1.1 connection. - using HttpResponseMessage response2 = await client.SendAsync(CreateRequest(HttpMethod.Get, uri, HttpVersion.Version20)); - Assert.Equal(2, connectionCount); - Assert.Equal("2", await response2.Content.ReadAsStringAsync()); - - // If we now block the first connection, the second request should wait without attempting to open a new connection. - Task firstRequestTask = client.GetStringAsync(uri); - Task secondRequestTask = client.GetStringAsync(uri); - - Assert.Equal("3", await firstRequestTask); - Assert.Equal("4", await secondRequestTask); - - Assert.Equal(2, connectionCount); - }, - async server => - { - await using LoopbackServer.Connection connection = await server.EstablishConnectionAsync(); - - await connection.ReadRequestHeaderAndSendResponseAsync(content: "1"); - - // The client should throw away this connection as soon as it notices there's no ALPN for HTTP/2. - await using var secondConnection = await server.EstablishConnectionAsync(); - await Assert.ThrowsAnyAsync(() => secondConnection.ReadRequestDataAsync()); - - await connection.ReadRequestHeaderAndSendResponseAsync(content: "2"); - await connection.ReadRequestHeaderAndSendResponseAsync(content: "3"); - await connection.ReadRequestHeaderAndSendResponseAsync(content: "4"); - }, - new LoopbackServer.Options { UseSsl = true }); - } } public abstract class SocketsHttpHandlerTest_PlaintextStreamFilter : HttpClientHandlerTestBase diff --git a/src/libraries/System.Net.HttpListener/src/System/Net/Windows/CookieExtensions.cs b/src/libraries/System.Net.HttpListener/src/System/Net/Windows/CookieExtensions.cs index 0e5366f9bc149d..e5b3783c19e72e 100644 --- a/src/libraries/System.Net.HttpListener/src/System/Net/Windows/CookieExtensions.cs +++ b/src/libraries/System.Net.HttpListener/src/System/Net/Windows/CookieExtensions.cs @@ -14,7 +14,7 @@ internal static class CookieExtensions public static string ToServerString(this Cookie cookie) { - s_toServerStringFunc ??= (Func)typeof(Cookie).GetMethod("ToServerString", BindingFlags.Instance | BindingFlags.NonPublic)?.CreateDelegate(typeof(Func))!; + s_toServerStringFunc ??= (Func)typeof(Cookie).GetMethod("ToServerString", BindingFlags.Instance | BindingFlags.NonPublic)!.CreateDelegate(typeof(Func)); Debug.Assert(s_toServerStringFunc != null, "Reflection failed for Cookie.ToServerString()."); return s_toServerStringFunc(cookie); } @@ -23,7 +23,7 @@ public static string ToServerString(this Cookie cookie) public static Cookie Clone(this Cookie cookie) { - s_cloneFunc ??= (Func)typeof(Cookie).GetMethod("Clone", BindingFlags.Instance | BindingFlags.NonPublic)?.CreateDelegate(typeof(Func))!; + s_cloneFunc ??= (Func)typeof(Cookie).GetMethod("Clone", BindingFlags.Instance | BindingFlags.NonPublic)!.CreateDelegate(typeof(Func)); Debug.Assert(s_cloneFunc != null, "Reflection failed for Cookie.Clone()."); return s_cloneFunc(cookie); } @@ -41,7 +41,7 @@ private enum CookieVariant public static bool IsRfc2965Variant(this Cookie cookie) { - s_getVariantFunc ??= (Func)typeof(Cookie).GetProperty("Variant", BindingFlags.Instance | BindingFlags.NonPublic)?.GetGetMethod(true)?.CreateDelegate(typeof(Func))!; + s_getVariantFunc ??= (Func)typeof(Cookie).GetProperty("Variant", BindingFlags.Instance | BindingFlags.NonPublic)!.GetGetMethod(true)!.CreateDelegate(typeof(Func)); Debug.Assert(s_getVariantFunc != null, "Reflection failed for Cookie.Variant."); return s_getVariantFunc(cookie) == CookieVariant.Rfc2965; } @@ -53,7 +53,7 @@ internal static class CookieCollectionExtensions public static int InternalAdd(this CookieCollection cookieCollection, Cookie cookie, bool isStrict) { - s_internalAddFunc ??= (Func)typeof(CookieCollection).GetMethod("InternalAdd", BindingFlags.Instance | BindingFlags.NonPublic)?.CreateDelegate(typeof(Func))!; + s_internalAddFunc ??= (Func)typeof(CookieCollection).GetMethod("InternalAdd", BindingFlags.Instance | BindingFlags.NonPublic)!.CreateDelegate(typeof(Func)); Debug.Assert(s_internalAddFunc != null, "Reflection failed for CookieCollection.InternalAdd()."); return s_internalAddFunc(cookieCollection, cookie, isStrict); } diff --git a/src/libraries/System.Net.Primitives/src/System/Net/IPAddress.cs b/src/libraries/System.Net.Primitives/src/System/Net/IPAddress.cs index 48f424407b400c..bdd381fc04901c 100644 --- a/src/libraries/System.Net.Primitives/src/System/Net/IPAddress.cs +++ b/src/libraries/System.Net.Primitives/src/System/Net/IPAddress.cs @@ -142,6 +142,7 @@ public IPAddress(ReadOnlySpan address, long scopeid) internal IPAddress(ReadOnlySpan numbers, uint scopeid) { + Debug.Assert(numbers != null); Debug.Assert(numbers.Length == NumberOfLabels); _numbers = numbers.ToArray(); diff --git a/src/libraries/System.Net.Primitives/src/System/Net/IPAddressParser.cs b/src/libraries/System.Net.Primitives/src/System/Net/IPAddressParser.cs index afe80e3cd5b829..964cd4308366ab 100644 --- a/src/libraries/System.Net.Primitives/src/System/Net/IPAddressParser.cs +++ b/src/libraries/System.Net.Primitives/src/System/Net/IPAddressParser.cs @@ -67,6 +67,7 @@ private static unsafe bool TryParseIpv4(ReadOnlySpan ipSpan, out long addr private static unsafe bool TryParseIPv6(ReadOnlySpan ipSpan, Span numbers, int numbersLength, out uint scope) { + Debug.Assert(numbers != null); Debug.Assert(numbersLength >= IPAddressParserStatics.IPv6AddressShorts); int end = ipSpan.Length; diff --git a/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj b/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj index aafca7f3e93b2a..16d83c57fe466d 100644 --- a/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj +++ b/src/libraries/System.Net.Quic/src/System.Net.Quic.csproj @@ -131,7 +131,6 @@ - @@ -172,7 +171,7 @@ GeneratePathProperty="true" /> diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.NativeMethods.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.NativeMethods.cs index 6906392f79ebee..206eac76ac7878 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.NativeMethods.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.NativeMethods.cs @@ -375,55 +375,4 @@ public int StreamReceiveSetEnabled(MsQuicSafeHandle stream, byte enabled) } } } - - public int DatagramSend(MsQuicSafeHandle connection, QUIC_BUFFER* buffers, uint buffersCount, QUIC_SEND_FLAGS flags, void* context) - { - bool success = false; - try - { - connection.DangerousAddRef(ref success); - return ApiTable->DatagramSend(connection.QuicHandle, buffers, buffersCount, flags, context); - } - finally - { - if (success) - { - connection.DangerousRelease(); - } - } - } - - public int ConnectionResumptionTicketValidationComplete(MsQuicSafeHandle connection, byte result) - { - bool success = false; - try - { - connection.DangerousAddRef(ref success); - return ApiTable->ConnectionResumptionTicketValidationComplete(connection.QuicHandle, result); - } - finally - { - if (success) - { - connection.DangerousRelease(); - } - } - } - - public int ConnectionCertificateValidationComplete(MsQuicSafeHandle connection, byte result, QUIC_TLS_ALERT_CODES alert) - { - bool success = false; - try - { - connection.DangerousAddRef(ref success); - return ApiTable->ConnectionCertificateValidationComplete(connection.QuicHandle, result, alert); - } - finally - { - if (success) - { - connection.DangerousRelease(); - } - } - } } diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs index 4b284284f5262c..e89119844c744c 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/Internal/MsQuicApi.cs @@ -54,16 +54,11 @@ private MsQuicApi(QUIC_API_TABLE* apiTable) private static readonly Lazy _api = new Lazy(AllocateMsQuicApi); internal static MsQuicApi Api => _api.Value; - internal static Version? Version { get; private set; } - internal static bool IsQuicSupported { get; } internal static string MsQuicLibraryVersion { get; } = "unknown"; internal static string? NotSupportedReason { get; } - // workaround for https://github.com/microsoft/msquic/issues/4132 - internal static bool SupportsAsyncCertValidation => Version >= new Version(2, 4, 0); - internal static bool UsesSChannelBackend { get; } internal static bool Tls13ServerMayBeDisabled { get; } @@ -74,7 +69,6 @@ static MsQuicApi() { bool loaded = false; IntPtr msQuicHandle; - Version = default; // MsQuic is using DualMode sockets and that will fail even for IPv4 if AF_INET6 is not available. if (!Socket.OSSupportsIPv6) @@ -141,7 +135,7 @@ static MsQuicApi() } return; } - Version = new Version((int)libVersion[0], (int)libVersion[1], (int)libVersion[2], (int)libVersion[3]); + Version version = new Version((int)libVersion[0], (int)libVersion[1], (int)libVersion[2], (int)libVersion[3]); paramSize = 64 * sizeof(sbyte); sbyte* libGitHash = stackalloc sbyte[64]; @@ -156,11 +150,11 @@ static MsQuicApi() } string? gitHash = Marshal.PtrToStringUTF8((IntPtr)libGitHash); - MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} {Version} ({gitHash})"; + MsQuicLibraryVersion = $"{Interop.Libraries.MsQuic} {version} ({gitHash})"; - if (Version < s_minMsQuicVersion) + if (version < s_minMsQuicVersion) { - NotSupportedReason = $"Incompatible MsQuic library version '{Version}', expecting higher than '{s_minMsQuicVersion}'."; + NotSupportedReason = $"Incompatible MsQuic library version '{version}', expecting higher than '{s_minMsQuicVersion}'."; if (NetEventSource.Log.IsEnabled()) { NetEventSource.Info(null, NotSupportedReason); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/msquic_generated.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/msquic_generated.cs index ad781e6ddd7c7e..b8c0b092df1df7 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/msquic_generated.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/Interop/msquic_generated.cs @@ -929,7 +929,6 @@ internal enum QUIC_PERFORMANCE_COUNTERS PATH_FAILURE, SEND_STATELESS_RESET, SEND_STATELESS_RETRY, - CONN_LOAD_REJECT, MAX, } diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/NetEventSource.Quic.Counters.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/NetEventSource.Quic.Counters.cs deleted file mode 100644 index 93ec7e7532c7f3..00000000000000 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/NetEventSource.Quic.Counters.cs +++ /dev/null @@ -1,249 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.Tracing; -using System.Diagnostics.Metrics; -using System.Net.Quic; - -using Microsoft.Quic; -using static Microsoft.Quic.MsQuic; - -namespace System.Net -{ - internal sealed partial class NetEventSource - { - private static Meter s_meter = new Meter("Private.InternalDiagnostics.System.Net.Quic.MsQuic"); - private static long s_countersLastFetched; - private static readonly long[] s_counters = new long[(int)QUIC_PERFORMANCE_COUNTERS.MAX]; - public static readonly ObservableCounter s_CONN_CREATED = s_meter.CreateObservableCounter( - name: "msquic.connection.created", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_CREATED), - unit: "{connection}", - description: "New connections allocated"); - - public static readonly ObservableCounter s_CONN_HANDSHAKE_FAIL = s_meter.CreateObservableCounter( - name: "msquic.connection.handshake_failures", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_HANDSHAKE_FAIL), - unit: "{connection}", - description: "Connections that failed during handshake"); - - public static readonly ObservableCounter s_CONN_APP_REJECT = s_meter.CreateObservableCounter( - name: "msquic.connection.app_rejected", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_APP_REJECT), - unit: "{connection}", - description: "Connections rejected by the application"); - - public static readonly ObservableCounter s_CONN_LOAD_REJECT = s_meter.CreateObservableCounter( - name: "msquic.connection.load_rejected", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_LOAD_REJECT), - unit: "{connection}", - description: "Connections rejected due to worker load."); - - public static readonly ObservableCounter s_CONN_RESUMED = s_meter.CreateObservableCounter( - name: "msquic.connection.resumed", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_RESUMED), - unit: "{connection}", - description: "Connections resumed"); - - public static readonly ObservableGauge s_CONN_ACTIVE = s_meter.CreateObservableGauge( - name: "msquic.connection.allocated", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_ACTIVE), - unit: "{connection}", - description: "Connections currently allocated"); - - public static readonly ObservableGauge s_CONN_CONNECTED = s_meter.CreateObservableGauge( - name: "msquic.connection.connected", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_CONNECTED), - unit: "{connection}", - description: "Connections currently in the connected state"); - - public static readonly ObservableCounter s_CONN_PROTOCOL_ERRORS = s_meter.CreateObservableCounter( - name: "msquic.connection.protocol_errors", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_PROTOCOL_ERRORS), - unit: "{connection}", - description: "Connections shutdown with a protocol error"); - - public static readonly ObservableCounter s_CONN_NO_ALPN = s_meter.CreateObservableCounter( - name: "msquic.connection.no_alpn", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_NO_ALPN), - unit: "{connection}", - description: "Connection attempts with no matching ALPN"); - - public static readonly ObservableGauge s_STRM_ACTIVE = s_meter.CreateObservableGauge( - name: "msquic.stream.allocated", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.STRM_ACTIVE), - unit: "{stream}", - description: "Current streams allocated"); - - public static readonly ObservableCounter s_PKTS_SUSPECTED_LOST = s_meter.CreateObservableCounter( - name: "msquic.packet.suspected_lost", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.PKTS_SUSPECTED_LOST), - unit: "{packet}", - description: "Packets suspected lost"); - - public static readonly ObservableCounter s_PKTS_DROPPED = s_meter.CreateObservableCounter( - name: "msquic.packet.dropped", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.PKTS_DROPPED), - unit: "{packet}", - description: "Packets dropped for any reason"); - - public static readonly ObservableCounter s_PKTS_DECRYPTION_FAIL = s_meter.CreateObservableCounter( - name: "msquic.packet.decryption_failures", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.PKTS_DECRYPTION_FAIL), - unit: "{packet}", - description: "Packets with decryption failures"); - - public static readonly ObservableCounter s_UDP_RECV = s_meter.CreateObservableCounter( - name: "msquic.udp.recv_datagrams", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.UDP_RECV), - unit: "{datagram}", - description: "UDP datagrams received"); - - public static readonly ObservableCounter s_UDP_SEND = s_meter.CreateObservableCounter( - name: "msquic.udp.send_datagrams", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.UDP_SEND), - unit: "{datagram}", - description: "UDP datagrams sent"); - - public static readonly ObservableCounter s_UDP_RECV_BYTES = s_meter.CreateObservableCounter( - name: "msquic.udp.recv_bytes", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.UDP_RECV_BYTES), - unit: "By", - description: "UDP payload bytes received"); - - public static readonly ObservableCounter s_UDP_SEND_BYTES = s_meter.CreateObservableCounter( - name: "msquic.udp.send_bytes", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.UDP_SEND_BYTES), - unit: "By", - description: "UDP payload bytes sent"); - - public static readonly ObservableCounter s_UDP_RECV_EVENTS = s_meter.CreateObservableCounter( - name: "msquic.udp.recv_events", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.UDP_RECV_EVENTS), - unit: "{event}", - description: "UDP receive events"); - - public static readonly ObservableCounter s_UDP_SEND_CALLS = s_meter.CreateObservableCounter( - name: "msquic.udp.send_calls", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.UDP_SEND_CALLS), - unit: "{call}", - description: "UDP send API calls"); - - public static readonly ObservableCounter s_APP_SEND_BYTES = s_meter.CreateObservableCounter( - name: "msquic.app.send_bytes", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.APP_SEND_BYTES), - unit: "By", - description: "Bytes sent by applications"); - - public static readonly ObservableCounter s_APP_RECV_BYTES = s_meter.CreateObservableCounter( - name: "msquic.app.recv_bytes", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.APP_RECV_BYTES), - unit: "By", - description: "Bytes received by applications"); - - public static readonly ObservableGauge s_CONN_QUEUE_DEPTH = s_meter.CreateObservableGauge( - name: "msquic.threadpool.conn_queue_depth", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_QUEUE_DEPTH), - unit: "{connection}", - description: "Current connections queued for processing"); - - public static readonly ObservableGauge s_CONN_OPER_QUEUE_DEPTH = s_meter.CreateObservableGauge( - name: "msquic.threadpool.conn_oper_queue_depth", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_OPER_QUEUE_DEPTH), - unit: "{operation}", - description: "Current connection operations queued"); - - public static readonly ObservableCounter s_CONN_OPER_QUEUED = s_meter.CreateObservableCounter( - name: "msquic.threadpool.conn_oper_queued", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_OPER_QUEUED), - unit: "{operation}", - description: "New connection operations queued"); - - public static readonly ObservableCounter s_CONN_OPER_COMPLETED = s_meter.CreateObservableCounter( - name: "msquic.threadpool.conn_oper_completed", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.CONN_OPER_COMPLETED), - unit: "{operation}", - description: "Connection operations processed"); - - public static readonly ObservableGauge s_WORK_OPER_QUEUE_DEPTH = s_meter.CreateObservableGauge( - name: "msquic.threadpool.work_oper_queue_depth", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.WORK_OPER_QUEUE_DEPTH), - unit: "{operation}", - description: "Current worker operations queued"); - - public static readonly ObservableCounter s_WORK_OPER_QUEUED = s_meter.CreateObservableCounter( - name: "msquic.threadpool.work_oper_queued", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.WORK_OPER_QUEUED), - unit: "{operation}", - description: "New worker operations queued"); - - public static readonly ObservableCounter s_WORK_OPER_COMPLETED = s_meter.CreateObservableCounter( - name: "msquic.threadpool.work_oper_completed", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.WORK_OPER_COMPLETED), - unit: "{operation}", - description: "Worker operations processed"); - - public static readonly ObservableCounter s_PATH_VALIDATED = s_meter.CreateObservableCounter( - name: "msquic.datapath.path_validated", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.PATH_VALIDATED), - unit: "{challenge}", - description: "Successful path challenges"); - - public static readonly ObservableCounter s_PATH_FAILURE = s_meter.CreateObservableCounter( - name: "msquic.datapath.path_failure", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.PATH_FAILURE), - unit: "{challenge}", - description: "Unsuccessful path challenges"); - - public static readonly ObservableCounter s_SEND_STATELESS_RESET = s_meter.CreateObservableCounter( - name: "msquic.datapath.send_stateless_reset", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.SEND_STATELESS_RESET), - unit: "{packet}", - description: "Stateless reset packets sent ever"); - - public static readonly ObservableCounter s_SEND_STATELESS_RETRY = s_meter.CreateObservableCounter( - name: "msquic.datapath.send_stateless_retry", - observeValue: () => GetCounterValue(QUIC_PERFORMANCE_COUNTERS.SEND_STATELESS_RETRY), - unit: "{packet}", - description: "Stateless retry packets sent"); - - [NonEvent] - private static void UpdateCounters() - { - if (!MsQuicApi.IsQuicSupported) - { - // Avoid calling into MsQuic if not supported (or not initialized yet) - return; - } - - unsafe - { - fixed (long* pCounters = s_counters) - { - uint size = (uint)s_counters.Length * sizeof(long); - MsQuicApi.Api.ApiTable->GetParam(null, QUIC_PARAM_GLOBAL_PERF_COUNTERS, &size, (byte*)pCounters); - } - } - } - - [NonEvent] - private static long GetCounterValue(QUIC_PERFORMANCE_COUNTERS counter) - { - // - // We wan't to avoid refreshing the counter values array for each counter callback, - // so we refresh the counters array only once every 50ms. This should be enough time - // for all the counters to be queried and at the same time but still low enough to not - // confuse any monitoring tool as their polling rate is usually in seconds. - // - if (s_countersLastFetched == 0 || Stopwatch.GetElapsedTime(s_countersLastFetched).TotalMilliseconds > 50) - { - UpdateCounters(); - s_countersLastFetched = Stopwatch.GetTimestamp(); - } - - return s_counters[(int)counter]; - } - } -} diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.SslConnectionOptions.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.SslConnectionOptions.cs index 1b352f10045404..dad23bfc342c8a 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.SslConnectionOptions.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.SslConnectionOptions.cs @@ -1,13 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; -using System.Diagnostics; using System.Net.Security; using System.Security.Authentication; using System.Security.Cryptography; using System.Security.Cryptography.X509Certificates; -using System.Threading.Tasks; using Microsoft.Quic; using static Microsoft.Quic.MsQuic; @@ -66,122 +63,18 @@ public SslConnectionOptions(QuicConnection connection, bool isClient, _certificateChainPolicy = certificateChainPolicy; } - internal async Task StartAsyncCertificateValidation(IntPtr certificatePtr, IntPtr chainPtr) - { - // - // The provided data pointers are valid only while still inside this function, so they need to be - // copied to separate buffers which are then handed off to threadpool. - // - - X509Certificate2? certificate = null; - - byte[]? certDataRented = null; - Memory certData = default; - byte[]? chainDataRented = null; - Memory chainData = default; - - if (certificatePtr != IntPtr.Zero) - { - if (MsQuicApi.UsesSChannelBackend) - { - // provided data is a pointer to a CERT_CONTEXT - certificate = new X509Certificate2(certificatePtr); - // TODO: what about chainPtr? - } - else - { - unsafe - { - // On non-SChannel backends we specify USE_PORTABLE_CERTIFICATES and the contents are buffers - // with DER encoded cert and chain. - QUIC_BUFFER* certificateBuffer = (QUIC_BUFFER*)certificatePtr; - QUIC_BUFFER* chainBuffer = (QUIC_BUFFER*)chainPtr; - - if (certificateBuffer->Length > 0) - { - certDataRented = ArrayPool.Shared.Rent((int)certificateBuffer->Length); - certData = certDataRented.AsMemory(0, (int)certificateBuffer->Length); - certificateBuffer->Span.CopyTo(certData.Span); - } - - if (chainBuffer->Length > 0) - { - chainDataRented = ArrayPool.Shared.Rent((int)chainBuffer->Length); - chainData = chainDataRented.AsMemory(0, (int)chainBuffer->Length); - chainBuffer->Span.CopyTo(chainData.Span); - } - } - } - } - - // We wan't to do the certificate validation asynchronously, but due to a bug in MsQuic, we need to call the callback synchronously on some versions - if (MsQuicApi.SupportsAsyncCertValidation) - { - // force yield to the thread pool to free up MsQuic worker thread. - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); - } - - // certificatePtr and chainPtr are invalid beyond this point - - QUIC_TLS_ALERT_CODES result; - try - { - if (certData.Length > 0) - { - Debug.Assert(certificate == null); - certificate = new X509Certificate2(certData.Span); - } - - result = _connection._sslConnectionOptions.ValidateCertificate(certificate, certData.Span, chainData.Span); - _connection._remoteCertificate = certificate; - } - catch (Exception ex) - { - certificate?.Dispose(); - _connection._connectedTcs.TrySetException(ex); - result = QUIC_TLS_ALERT_CODES.USER_CANCELED; - } - finally - { - if (certDataRented != null) - { - ArrayPool.Shared.Return(certDataRented); - } - - if (chainDataRented != null) - { - ArrayPool.Shared.Return(chainDataRented); - } - } - - if (MsQuicApi.SupportsAsyncCertValidation) - { - int status = MsQuicApi.Api.ConnectionCertificateValidationComplete( - _connection._handle, - result == QUIC_TLS_ALERT_CODES.SUCCESS ? (byte)1 : (byte)0, - result); - - if (MsQuic.StatusFailed(status)) - { - if (NetEventSource.Log.IsEnabled()) - { - NetEventSource.Error(_connection, $"{_connection} ConnectionCertificateValidationComplete failed with {ThrowHelper.GetErrorMessageForStatus(status)}"); - } - } - } - - return result == QUIC_TLS_ALERT_CODES.SUCCESS; - } - - private QUIC_TLS_ALERT_CODES ValidateCertificate(X509Certificate2? certificate, Span certData, Span chainData) + public unsafe int ValidateCertificate(QUIC_BUFFER* certificatePtr, QUIC_BUFFER* chainPtr, out X509Certificate2? certificate) { SslPolicyErrors sslPolicyErrors = SslPolicyErrors.None; + IntPtr certificateBuffer = 0; + int certificateLength = 0; bool wrapException = false; X509Chain? chain = null; + X509Certificate2? result = null; try { - if (certificate is not null) + if (certificatePtr is not null) { chain = new X509Chain(); if (_certificateChainPolicy != null) @@ -203,26 +96,43 @@ private QUIC_TLS_ALERT_CODES ValidateCertificate(X509Certificate2? certificate, chain.ChainPolicy.ApplicationPolicy.Add(_isClient ? s_serverAuthOid : s_clientAuthOid); } - if (chainData.Length > 0) + if (MsQuicApi.UsesSChannelBackend) { - X509Certificate2Collection additionalCertificates = new X509Certificate2Collection(); - additionalCertificates.Import(chainData); - chain.ChainPolicy.ExtraStore.AddRange(additionalCertificates); + result = new X509Certificate2((IntPtr)certificatePtr); } + else + { + if (certificatePtr->Length > 0) + { + certificateBuffer = (IntPtr)certificatePtr->Buffer; + certificateLength = (int)certificatePtr->Length; + result = new X509Certificate2(certificatePtr->Span); + } + if (chainPtr->Length > 0) + { + X509Certificate2Collection additionalCertificates = new X509Certificate2Collection(); + additionalCertificates.Import(chainPtr->Span); + chain.ChainPolicy.ExtraStore.AddRange(additionalCertificates); + } + } + } + + if (result is not null) + { bool checkCertName = !chain!.ChainPolicy!.VerificationFlags.HasFlag(X509VerificationFlags.IgnoreInvalidName); - sslPolicyErrors |= CertificateValidation.BuildChainAndVerifyProperties(chain!, certificate, checkCertName, !_isClient, TargetHostNameHelper.NormalizeHostName(_targetHost), certData); + sslPolicyErrors |= CertificateValidation.BuildChainAndVerifyProperties(chain!, result, checkCertName, !_isClient, TargetHostNameHelper.NormalizeHostName(_targetHost), certificateBuffer, certificateLength); } else if (_certificateRequired) { sslPolicyErrors |= SslPolicyErrors.RemoteCertificateNotAvailable; } - QUIC_TLS_ALERT_CODES result = QUIC_TLS_ALERT_CODES.SUCCESS; + int status = QUIC_STATUS_SUCCESS; if (_validationCallback is not null) { wrapException = true; - if (!_validationCallback(_connection, certificate, chain, sslPolicyErrors)) + if (!_validationCallback(_connection, result, chain, sslPolicyErrors)) { wrapException = false; if (_isClient) @@ -230,7 +140,7 @@ private QUIC_TLS_ALERT_CODES ValidateCertificate(X509Certificate2? certificate, throw new AuthenticationException(SR.net_quic_cert_custom_validation); } - result = QUIC_TLS_ALERT_CODES.BAD_CERTIFICATE; + status = QUIC_STATUS_USER_CANCELED; } } else if (sslPolicyErrors != SslPolicyErrors.None) @@ -240,13 +150,15 @@ private QUIC_TLS_ALERT_CODES ValidateCertificate(X509Certificate2? certificate, throw new AuthenticationException(SR.Format(SR.net_quic_cert_chain_validation, sslPolicyErrors)); } - result = QUIC_TLS_ALERT_CODES.BAD_CERTIFICATE; + status = QUIC_STATUS_HANDSHAKE_FAILURE; } - return result; + certificate = result; + return status; } catch (Exception ex) { + result?.Dispose(); if (wrapException) { throw new QuicException(QuicError.CallbackError, null, SR.net_quic_callback_error, ex); diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs index 5a4f626e2f5465..db3adf776d542c 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicConnection.cs @@ -571,20 +571,15 @@ private unsafe int HandleEventPeerStreamStarted(ref PEER_STREAM_STARTED_DATA dat } private unsafe int HandleEventPeerCertificateReceived(ref PEER_CERTIFICATE_RECEIVED_DATA data) { - // - // The certificate validation is an expensive operation and we don't want to delay MsQuic - // worker thread. So we offload the validation to the .NET threadpool. Incidentally, this - // also prevents potential user RemoteCertificateValidationCallback from blocking MsQuic - // worker threads. - // - - var task = _sslConnectionOptions.StartAsyncCertificateValidation((IntPtr)data.Certificate, (IntPtr)data.Chain); - if (task.IsCompletedSuccessfully) + try { - return task.Result ? QUIC_STATUS_SUCCESS : QUIC_STATUS_BAD_CERTIFICATE; + return _sslConnectionOptions.ValidateCertificate((QUIC_BUFFER*)data.Certificate, (QUIC_BUFFER*)data.Chain, out _remoteCertificate); + } + catch (Exception ex) + { + _connectedTcs.TrySetException(ex); + return QUIC_STATUS_HANDSHAKE_FAILURE; } - - return QUIC_STATUS_PENDING; } private unsafe int HandleConnectionEvent(ref QUIC_CONNECTION_EVENT connectionEvent) diff --git a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicListener.cs b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicListener.cs index 88ea309054a7db..6f0a0d8bb5b75f 100644 --- a/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicListener.cs +++ b/src/libraries/System.Net.Quic/src/System/Net/Quic/QuicListener.cs @@ -209,11 +209,6 @@ public async ValueTask AcceptConnectionAsync(CancellationToken c /// The TLS ClientHello data. private async void StartConnectionHandshake(QuicConnection connection, SslClientHelloInfo clientHello) { - // Yield to the threadpool immediately. This makes sure the connection options callback - // provided by the user is not invoked from the MsQuic thread and cannot delay acks - // or other operations on other connections. - await Task.CompletedTask.ConfigureAwait(ConfigureAwaitOptions.ForceYielding); - bool wrapException = false; CancellationToken cancellationToken = default; diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs index 45d65a0b7cdc91..459b9bce810dca 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicCipherSuitesPolicyTests.cs @@ -8,10 +8,9 @@ namespace System.Net.Quic.Tests { - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] [SkipOnPlatform(TestPlatforms.Windows, "CipherSuitesPolicy is not supported on Windows")] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public class MsQuicCipherSuitesPolicyTests : QuicTestBase { public MsQuicCipherSuitesPolicyTests(ITestOutputHelper output) : base(output) { } @@ -78,4 +77,4 @@ await Assert.ThrowsAsync(() => TestConnection( )); } } -} +} \ No newline at end of file diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs index 7c2511bbb6d1ba..16f267fc719535 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicPlatformDetectionTests.cs @@ -8,7 +8,6 @@ namespace System.Net.Quic.Tests { - [Collection(nameof(QuicTestCollection))] public class MsQuicPlatformDetectionTests : QuicTestBase { public MsQuicPlatformDetectionTests(ITestOutputHelper output) : base(output) { } @@ -60,7 +59,6 @@ public async Task SupportedLinuxPlatformsWithMsQuic_IsSupportedIsTrue() [ActiveIssue("https://github.com/dotnet/runtime/issues/82154", typeof(PlatformDetection), nameof(PlatformDetection.IsRaspbian10), nameof(PlatformDetection.IsArmv6Process), nameof(PlatformDetection.IsInContainer))] [ActiveIssue("https://github.com/dotnet/runtime/issues/82154", typeof(PlatformDetection), nameof(PlatformDetection.IsPpc64leProcess))] [ActiveIssue("https://github.com/dotnet/runtime/issues/82154", typeof(PlatformDetection), nameof(PlatformDetection.IsUbuntu2004), nameof(PlatformDetection.IsS390xProcess))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsInHelix))] [PlatformSpecific(TestPlatforms.Linux)] public void SupportedLinuxPlatforms_IsSupportedIsTrue() diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicRemoteExecutorTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicRemoteExecutorTests.cs index 882600acf6a2dd..051ead9b3bb2ec 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicRemoteExecutorTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicRemoteExecutorTests.cs @@ -12,9 +12,8 @@ namespace System.Net.Quic.Tests { - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public class MsQuicRemoteExecutorTests : QuicTestBase { public MsQuicRemoteExecutorTests() diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs index b432cfc0aba28c..4a11909d30bfef 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/MsQuicTests.cs @@ -46,9 +46,8 @@ public void Dispose() } } - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public class MsQuicTests : QuicTestBase, IClassFixture { private static byte[] s_data = "Hello world!"u8.ToArray(); @@ -357,10 +356,7 @@ public async Task UntrustedClientCertificateFails() } } - static bool SupportsAsyncCertValidation => QuicTestCollection.MsQuicVersion >= new Version(2, 4); - [Fact] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99074", typeof(MsQuicTests), nameof(SupportsAsyncCertValidation))] public async Task CertificateCallbackThrowPropagates() { using CancellationTokenSource cts = new CancellationTokenSource(PassingTestTimeout); diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs index 324a76f5e5693d..98d72124f0048c 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicConnectionTests.cs @@ -6,7 +6,6 @@ using System.Security.Cryptography.X509Certificates; using System.Threading; using System.Threading.Tasks; -using Microsoft.DotNet.XUnitExtensions; using Xunit; using Xunit.Abstractions; @@ -14,9 +13,8 @@ namespace System.Net.Quic.Tests { using Configuration = System.Net.Test.Common.Configuration; - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicConnectionTests : QuicTestBase { const int ExpectedErrorCode = 1234; @@ -24,7 +22,7 @@ public sealed class QuicConnectionTests : QuicTestBase public QuicConnectionTests(ITestOutputHelper output) : base(output) { } - [ConditionalTheory] + [Theory] [MemberData(nameof(LocalAddresses))] public async Task TestConnect(IPAddress address) { diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs index c9c394fcfb19ad..d9e27a9e394c4c 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicListenerTests.cs @@ -13,9 +13,8 @@ namespace System.Net.Quic.Tests { - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicListenerTests : QuicTestBase { public QuicListenerTests(ITestOutputHelper output) : base(output) { } diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs index 7b9257cdc26602..e224bf75c55312 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamConnectedStreamConformanceTests.cs @@ -14,9 +14,8 @@ namespace System.Net.Quic.Tests { - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicStreamConformanceTests : ConnectedStreamConformanceTests { protected override bool UsableAfterCanceledReads => false; diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs index e02851fac73ee9..72d0995823edc3 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicStreamTests.cs @@ -12,9 +12,8 @@ namespace System.Net.Quic.Tests { - [Collection(nameof(QuicTestCollection))] + [Collection(nameof(DisableParallelization))] [ConditionalClass(typeof(QuicTestBase), nameof(QuicTestBase.IsSupported), nameof(QuicTestBase.IsNotArm32CoreClrStressTest))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/91757", typeof(PlatformDetection), nameof(PlatformDetection.IsAlpine), nameof(PlatformDetection.IsArmProcess))] public sealed class QuicStreamTests : QuicTestBase { private static byte[] s_data = "Hello world!"u8.ToArray(); @@ -1212,16 +1211,16 @@ async ValueTask ReleaseOnReadsClosedAsync() private const int SmallestPayload = 1; private const int SmallPayload = 1024; - private const int BufferPayload = 64 * 1024; - private const int BufferPlusPayload = 64 * 1024 + 1; - private const int BigPayload = 1024 * 1024 * 1024; + private const int BufferPayload = 64*1024; + private const int BufferPlusPayload = 64*1024+1; + private const int BigPayload = 1024*1024*1024; public static IEnumerable PayloadSizeAndTwoBools() { - var boolValues = new[] { true, false }; + var boolValues = new [] { true, false }; var payloadValues = !PlatformDetection.IsInHelix ? - new[] { SmallestPayload, SmallPayload, BufferPayload, BufferPlusPayload, BigPayload } : - new[] { SmallestPayload, SmallPayload, BufferPayload, BufferPlusPayload }; + new [] { SmallestPayload, SmallPayload, BufferPayload, BufferPlusPayload, BigPayload } : + new [] { SmallestPayload, SmallPayload, BufferPayload, BufferPlusPayload }; return from payload in payloadValues from bool1 in boolValues diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs index c3e0e4e7372aba..79992aef5f16c7 100644 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs +++ b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestBase.cs @@ -18,8 +18,6 @@ namespace System.Net.Quic.Tests { - using Configuration = System.Net.Test.Common.Configuration; - public abstract class QuicTestBase : IDisposable { public const long DefaultStreamErrorCodeClient = 123456; @@ -33,7 +31,8 @@ public abstract class QuicTestBase : IDisposable public static bool IsSupported => QuicListener.IsSupported && QuicConnection.IsSupported; public static bool IsNotArm32CoreClrStressTest => !(CoreClrConfigurationDetection.IsStressTest && PlatformDetection.IsArmProcess); - public static bool IsIPv6Available => Configuration.Sockets.IsIPv6LoopbackAvailable; + private static readonly Lazy _isIPv6Available = new Lazy(GetIsIPv6Available); + public static bool IsIPv6Available => _isIPv6Available.Value; public static SslApplicationProtocol ApplicationProtocol { get; } = new SslApplicationProtocol("quictest"); @@ -44,7 +43,29 @@ public abstract class QuicTestBase : IDisposable public const int PassingTestTimeoutMilliseconds = 4 * 60 * 1000; public static TimeSpan PassingTestTimeout => TimeSpan.FromMilliseconds(PassingTestTimeoutMilliseconds); - public QuicTestBase(ITestOutputHelper output) + static unsafe QuicTestBase() + { + // If any of the reflection bellow breaks due to changes in "System.Net.Quic.MsQuicApi", also check and fix HttpStress project as it uses the same hack. + Type msQuicApiType = Type.GetType("System.Net.Quic.MsQuicApi, System.Net.Quic"); + + string msQuicLibraryVersion = (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); + Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{msQuicLibraryVersion}'."); + + if (IsSupported) + { + object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); + QUIC_API_TABLE* apiTable = (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty()))); + QUIC_SETTINGS settings = default(QUIC_SETTINGS); + settings.IsSet.MaxWorkerQueueDelayUs = 1; + settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default + if (MsQuic.StatusFailed(apiTable->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings))) + { + Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs."); + } + } + } + + public unsafe QuicTestBase(ITestOutputHelper output) { _output = output; } @@ -376,5 +397,19 @@ internal static async Task WriteForever(QuicStream stream, int size = 1) ArrayPool.Shared.Return(buffer); } } + + internal static bool GetIsIPv6Available() + { + try + { + using Socket s = new Socket(AddressFamily.InterNetworkV6, SocketType.Dgram, ProtocolType.Udp); + s.Bind(new IPEndPoint(IPAddress.IPv6Loopback, 0)); + return true; + } + catch (SocketException) + { + return false; + } + } } } diff --git a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestCollection.cs b/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestCollection.cs deleted file mode 100644 index f8dd160acb00b7..00000000000000 --- a/src/libraries/System.Net.Quic/tests/FunctionalTests/QuicTestCollection.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.DotNet.XUnitExtensions; -using System; -using System.Net.Quic; -using System.Reflection; -using System.Text; -using System.Linq; -using Xunit; -using Xunit.Abstractions; - -using Microsoft.Quic; -using static Microsoft.Quic.MsQuic; - -namespace System.Net.Quic.Tests; - -[CollectionDefinition(nameof(QuicTestCollection), DisableParallelization = true)] -public unsafe class QuicTestCollection : ICollectionFixture, IDisposable -{ - public static bool IsSupported => QuicListener.IsSupported && QuicConnection.IsSupported; - - public static Version MsQuicVersion { get; } = GetMsQuicVersion(); - - public QuicTestCollection() - { - string msQuicLibraryVersion = GetMsQuicLibraryVersion(); - // If any of the reflection bellow breaks due to changes in "System.Net.Quic.MsQuicApi", also check and fix HttpStress project as it uses the same hack. - Console.WriteLine($"MsQuic {(IsSupported ? "supported" : "not supported")} and using '{msQuicLibraryVersion}'."); - - if (IsSupported) - { - QUIC_SETTINGS settings = default(QUIC_SETTINGS); - settings.IsSet.MaxWorkerQueueDelayUs = 1; - settings.MaxWorkerQueueDelayUs = 2_500_000u; // 2.5s, 10x the default - if (MsQuic.StatusFailed(GetApiTable()->SetParam(null, MsQuic.QUIC_PARAM_GLOBAL_SETTINGS, (uint)sizeof(QUIC_SETTINGS), (byte*)&settings))) - { - Console.WriteLine($"Unable to set MsQuic MaxWorkerQueueDelayUs."); - } - } - } - - public unsafe void Dispose() - { - if (!IsSupported) - { - return; - } - - long[] counters = new long[(int)QUIC_PERFORMANCE_COUNTERS.MAX]; - int countersAvailable; - - int status; - fixed (long* pCounters = counters) - { - uint size = (uint)counters.Length * sizeof(long); - status = GetApiTable()->GetParam(null, QUIC_PARAM_GLOBAL_PERF_COUNTERS, &size, (byte*)pCounters); - countersAvailable = (int)size / sizeof(long); - } - - if (StatusFailed(status)) - { - System.Console.WriteLine($"Failed to read MsQuic counters: {status}"); - return; - } - - StringBuilder sb = new StringBuilder(); - sb.AppendLine("MsQuic Counters:"); - - int maxlen = Enum.GetNames(typeof(QUIC_PERFORMANCE_COUNTERS)).Max(s => s.Length); - void DumpCounter(QUIC_PERFORMANCE_COUNTERS counter) - { - var name = $"{counter}:".PadRight(maxlen + 1); - var index = (int)counter; - var value = index < countersAvailable ? counters[(int)counter].ToString() : "N/A"; - sb.AppendLine($" {counter} {value}"); - } - - DumpCounter(QUIC_PERFORMANCE_COUNTERS.CONN_CREATED); - DumpCounter(QUIC_PERFORMANCE_COUNTERS.CONN_HANDSHAKE_FAIL); - DumpCounter(QUIC_PERFORMANCE_COUNTERS.CONN_APP_REJECT); - DumpCounter(QUIC_PERFORMANCE_COUNTERS.CONN_LOAD_REJECT); - - System.Console.WriteLine(sb.ToString()); - } - - private static Version GetMsQuicVersion() - { - Type msQuicApiType = Type.GetType("System.Net.Quic.MsQuicApi, System.Net.Quic"); - - return (Version)msQuicApiType.GetProperty("Version", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); - } - - private static string? GetMsQuicLibraryVersion() - { - Type msQuicApiType = Type.GetType("System.Net.Quic.MsQuicApi, System.Net.Quic"); - - return (string)msQuicApiType.GetProperty("MsQuicLibraryVersion", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); - } - - private static QUIC_API_TABLE* GetApiTable() - { - Type msQuicApiType = Type.GetType("System.Net.Quic.MsQuicApi, System.Net.Quic"); - object msQuicApiInstance = msQuicApiType.GetProperty("Api", BindingFlags.NonPublic | BindingFlags.Static).GetGetMethod(true).Invoke(null, Array.Empty()); - return (QUIC_API_TABLE*)(Pointer.Unbox(msQuicApiType.GetProperty("ApiTable").GetGetMethod().Invoke(msQuicApiInstance, Array.Empty()))); - } -} diff --git a/src/libraries/System.Net.Requests/src/Resources/Strings.resx b/src/libraries/System.Net.Requests/src/Resources/Strings.resx index 4c0a7a45c146ff..b33f2a02440356 100644 --- a/src/libraries/System.Net.Requests/src/Resources/Strings.resx +++ b/src/libraries/System.Net.Requests/src/Resources/Strings.resx @@ -264,7 +264,4 @@ The ServicePointManager does not support proxies with the {0} scheme. - - Reached the maximum number of BindIPEndPointDelegate retries. - diff --git a/src/libraries/System.Net.Requests/src/System.Net.Requests.csproj b/src/libraries/System.Net.Requests/src/System.Net.Requests.csproj index 46bda299d9a644..397622b4806a03 100644 --- a/src/libraries/System.Net.Requests/src/System.Net.Requests.csproj +++ b/src/libraries/System.Net.Requests/src/System.Net.Requests.csproj @@ -29,7 +29,6 @@ - @@ -49,7 +48,6 @@ - @@ -112,7 +110,6 @@ - diff --git a/src/libraries/System.Net.Requests/src/System/Net/HttpWebRequest.cs b/src/libraries/System.Net.Requests/src/System/Net/HttpWebRequest.cs index 8f37002cf206b7..2a54bbb4d8d50d 100644 --- a/src/libraries/System.Net.Requests/src/System/Net/HttpWebRequest.cs +++ b/src/libraries/System.Net.Requests/src/System/Net/HttpWebRequest.cs @@ -1,19 +1,16 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; -using System.Net; using System.Net.Cache; using System.Net.Http; using System.Net.Http.Headers; using System.Net.Security; using System.Net.Sockets; -using System.Runtime.CompilerServices; using System.Runtime.Serialization; using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; @@ -42,10 +39,8 @@ public class HttpWebRequest : WebRequest, ISerializable private IWebProxy? _proxy = WebRequest.DefaultWebProxy; private Task? _sendRequestTask; - private HttpRequestMessage? _sendRequestMessage; private static int _defaultMaxResponseHeadersLength = HttpHandlerDefaults.DefaultMaxResponseHeadersLength; - private static int _defaultMaximumErrorResponseLength = -1; private int _beginGetRequestStreamCalled; private int _beginGetResponseCalled; @@ -65,7 +60,7 @@ public class HttpWebRequest : WebRequest, ISerializable private bool _hostHasPort; private Uri? _hostUri; - private Stream? _requestStream; + private RequestStream? _requestStream; private TaskCompletionSource? _requestStreamOperation; private TaskCompletionSource? _responseOperation; private AsyncCallback? _requestStreamCallback; @@ -81,8 +76,6 @@ public class HttpWebRequest : WebRequest, ISerializable private static readonly object s_syncRoot = new object(); private static volatile HttpClient? s_cachedHttpClient; private static HttpClientParameters? s_cachedHttpClientParameters; - private bool _disposeRequired; - private HttpClient? _httpClient; //these should be safe. [Flags] @@ -427,7 +420,11 @@ public string? Referer /// /// Sets the media type header /// - public string? MediaType { get; set; } + public string? MediaType + { + get; + set; + } /// /// @@ -680,22 +677,14 @@ public static int DefaultMaximumResponseHeadersLength } set { - ArgumentOutOfRangeException.ThrowIfLessThan(value, 0); _defaultMaxResponseHeadersLength = value; } } + // NOP public static int DefaultMaximumErrorResponseLength { - get - { - return _defaultMaximumErrorResponseLength; - } - set - { - ArgumentOutOfRangeException.ThrowIfLessThan(value, -1); - _defaultMaximumErrorResponseLength = value; - } + get; set; } private static RequestCachePolicy? _defaultCachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache); @@ -817,12 +806,10 @@ public Version ProtocolVersion if (value.Equals(HttpVersion.Version11)) { IsVersionHttp10 = false; - ServicePoint.ProtocolVersion = HttpVersion.Version11; } else if (value.Equals(HttpVersion.Version10)) { IsVersionHttp10 = true; - ServicePoint.ProtocolVersion = HttpVersion.Version10; } else { @@ -1008,17 +995,17 @@ public override void Abort() { _responseCallback(_responseOperation.Task); } + + // Cancel the underlying send operation. + Debug.Assert(_sendRequestCts != null); + _sendRequestCts.Cancel(); } - if (_requestStreamOperation != null) + else if (_requestStreamOperation != null) { if (_requestStreamOperation.TrySetCanceled() && _requestStreamCallback != null) { _requestStreamCallback(_requestStreamOperation.Task); } - - // Cancel the underlying send operation. - Debug.Assert(_sendRequestCts != null); - _sendRequestCts.Cancel(); } } @@ -1046,7 +1033,8 @@ public override WebResponse GetResponse() { try { - return HandleResponse(async: false).GetAwaiter().GetResult(); + _sendRequestCts = new CancellationTokenSource(); + return SendRequest(async: false).GetAwaiter().GetResult(); } catch (Exception ex) { @@ -1056,11 +1044,10 @@ public override WebResponse GetResponse() public override Stream GetRequestStream() { - CheckRequestStream(); return InternalGetRequestStream().Result; } - private void CheckRequestStream() + private Task InternalGetRequestStream() { CheckAbort(); @@ -1078,28 +1065,10 @@ private void CheckRequestStream() { throw new InvalidOperationException(SR.net_reqsubmitted); } - } - private async Task InternalGetRequestStream() - { - // If we aren't buffering we need to open the connection right away. - // Because we need to send the data as soon as possible when it's available from the RequestStream. - // Making this allows us to keep the sync send request path for buffering cases. - if (AllowWriteStreamBuffering is false) - { - // We're calling SendRequest with async, because we need to open the connection and send the request - // Otherwise, sync path will block the current thread until the request is sent. - TaskCompletionSource getStreamTcs = new(); - TaskCompletionSource completeTcs = new(); - _sendRequestTask = SendRequest(async: true, new RequestStreamContent(getStreamTcs, completeTcs)); - _requestStream = new RequestStream(await getStreamTcs.Task.ConfigureAwait(false), completeTcs); - } - else - { - _requestStream = new RequestBufferingStream(); - } + _requestStream = new RequestStream(); - return _requestStream; + return Task.FromResult((Stream)_requestStream); } public Stream EndGetRequestStream(IAsyncResult asyncResult, out TransportContext? context) @@ -1123,8 +1092,6 @@ public override IAsyncResult BeginGetRequestStream(AsyncCallback? callback, obje throw new InvalidOperationException(SR.net_repcall); } - CheckRequestStream(); - _requestStreamCallback = callback; _requestStreamOperation = InternalGetRequestStream().ToApm(callback, state); @@ -1158,95 +1125,78 @@ public override Stream EndGetRequestStream(IAsyncResult asyncResult) return stream; } - private Task SendRequest(bool async, HttpContent? content = null) + private async Task SendRequest(bool async) { if (RequestSubmitted) { throw new InvalidOperationException(SR.net_reqsubmitted); } - _sendRequestMessage = new HttpRequestMessage(HttpMethod.Parse(_originVerb), _requestUri); - _sendRequestCts = new CancellationTokenSource(); - _httpClient = GetCachedOrCreateHttpClient(async, out _disposeRequired); - - if (content is not null) - { - _sendRequestMessage.Content = content; - } + var request = new HttpRequestMessage(HttpMethod.Parse(_originVerb), _requestUri); - if (_hostUri is not null) - { - _sendRequestMessage.Headers.Host = Host; - } - - AddCacheControlHeaders(_sendRequestMessage); - - // Copy the HttpWebRequest request headers from the WebHeaderCollection into HttpRequestMessage.Headers and - // HttpRequestMessage.Content.Headers. - foreach (string headerName in _webHeaderCollection) + bool disposeRequired = false; + HttpClient? client = null; + try { - // The System.Net.Http APIs require HttpRequestMessage headers to be properly divided between the request headers - // collection and the request content headers collection for all well-known header names. And custom headers - // are only allowed in the request headers collection and not in the request content headers collection. - if (IsWellKnownContentHeader(headerName)) + client = GetCachedOrCreateHttpClient(async, out disposeRequired); + if (_requestStream != null) { - _sendRequestMessage.Content ??= new ByteArrayContent(Array.Empty()); - _sendRequestMessage.Content.Headers.TryAddWithoutValidation(headerName, _webHeaderCollection[headerName!]); + ArraySegment bytes = _requestStream.GetBuffer(); + request.Content = new ByteArrayContent(bytes.Array!, bytes.Offset, bytes.Count); } - else + + if (_hostUri != null) { - _sendRequestMessage.Headers.TryAddWithoutValidation(headerName, _webHeaderCollection[headerName!]); + request.Headers.Host = Host; } - } - if (_servicePoint?.Expect100Continue == true) - { - _sendRequestMessage.Headers.ExpectContinue = true; - } + AddCacheControlHeaders(request); - _sendRequestMessage.Headers.TransferEncodingChunked = SendChunked; + // Copy the HttpWebRequest request headers from the WebHeaderCollection into HttpRequestMessage.Headers and + // HttpRequestMessage.Content.Headers. + foreach (string headerName in _webHeaderCollection) + { + // The System.Net.Http APIs require HttpRequestMessage headers to be properly divided between the request headers + // collection and the request content headers collection for all well-known header names. And custom headers + // are only allowed in the request headers collection and not in the request content headers collection. + if (IsWellKnownContentHeader(headerName)) + { + // Create empty content so that we can send the entity-body header. + request.Content ??= new ByteArrayContent(Array.Empty()); - if (KeepAlive) - { - _sendRequestMessage.Headers.Connection.Add(HttpKnownHeaderNames.KeepAlive); - } - else - { - _sendRequestMessage.Headers.ConnectionClose = true; - } + request.Content.Headers.TryAddWithoutValidation(headerName, _webHeaderCollection[headerName!]); + } + else + { + request.Headers.TryAddWithoutValidation(headerName, _webHeaderCollection[headerName!]); + } + } - _sendRequestMessage.Version = ProtocolVersion; - HttpCompletionOption completionOption = _allowReadStreamBuffering ? HttpCompletionOption.ResponseContentRead : HttpCompletionOption.ResponseHeadersRead; - // If we're not buffering, there is no way to open the connection and not send the request without async. - // So we should use Async, if we're not buffering. - _sendRequestTask = async || !AllowWriteStreamBuffering ? - _httpClient.SendAsync(_sendRequestMessage, completionOption, _sendRequestCts.Token) : - Task.FromResult(_httpClient.Send(_sendRequestMessage, completionOption, _sendRequestCts.Token)); + request.Headers.TransferEncodingChunked = SendChunked; - return _sendRequestTask!; - } + if (KeepAlive) + { + request.Headers.Connection.Add(HttpKnownHeaderNames.KeepAlive); + } + else + { + request.Headers.ConnectionClose = true; + } - private async Task HandleResponse(bool async) - { - // If user code used requestStream and didn't dispose it - // We're completing it here. - if (_requestStream is RequestStream requestStream) - { - requestStream.Complete(); - } + if (_servicePoint?.Expect100Continue == true) + { + request.Headers.ExpectContinue = true; + } - if (_sendRequestTask is null && _requestStream is RequestBufferingStream requestBufferingStream) - { - ArraySegment buffer = requestBufferingStream.GetBuffer(); - _sendRequestTask = SendRequest(async, new ByteArrayContent(buffer.Array!, buffer.Offset, buffer.Count)); - } + request.Version = ProtocolVersion; - _sendRequestTask ??= SendRequest(async); + _sendRequestTask = async ? + client.SendAsync(request, _allowReadStreamBuffering ? HttpCompletionOption.ResponseContentRead : HttpCompletionOption.ResponseHeadersRead, _sendRequestCts!.Token) : + Task.FromResult(client.Send(request, _allowReadStreamBuffering ? HttpCompletionOption.ResponseContentRead : HttpCompletionOption.ResponseHeadersRead, _sendRequestCts!.Token)); - try - { HttpResponseMessage responseMessage = await _sendRequestTask.ConfigureAwait(false); - HttpWebResponse response = new(responseMessage, _requestUri, _cookieContainer); + + HttpWebResponse response = new HttpWebResponse(responseMessage, _requestUri, _cookieContainer); int maxSuccessStatusCode = AllowAutoRedirect ? 299 : 399; if ((int)response.StatusCode > maxSuccessStatusCode || (int)response.StatusCode < 200) @@ -1262,15 +1212,9 @@ private async Task HandleResponse(bool async) } finally { - _sendRequestMessage?.Dispose(); - if (_requestStream is RequestBufferingStream bufferStream) - { - bufferStream.GetMemoryStream().Dispose(); - } - - if (_disposeRequired) + if (disposeRequired) { - _httpClient?.Dispose(); + client?.Dispose(); } } } @@ -1396,8 +1340,9 @@ public override IAsyncResult BeginGetResponse(AsyncCallback? callback, object? s throw new InvalidOperationException(SR.net_repcall); } + _sendRequestCts = new CancellationTokenSource(); _responseCallback = callback; - _responseOperation = HandleResponse(async: true).ToApm(callback, state); + _responseOperation = SendRequest(async: true).ToApm(callback, state); return _responseOperation.Task; } @@ -1676,13 +1621,6 @@ private static HttpClient CreateHttpClient(HttpClientParameters parameters, Http handler.UseCookies = false; } - if (parameters.ServicePoint is { } servicePoint) - { - handler.MaxConnectionsPerServer = servicePoint.ConnectionLimit; - handler.PooledConnectionIdleTimeout = TimeSpan.FromMilliseconds(servicePoint.MaxIdleTime); - handler.PooledConnectionLifetime = TimeSpan.FromMilliseconds(servicePoint.ConnectionLeaseTimeout); - } - Debug.Assert(handler.UseProxy); // Default of handler.UseProxy is true. Debug.Assert(handler.Proxy == null); // Default of handler.Proxy is null. @@ -1700,7 +1638,7 @@ private static HttpClient CreateHttpClient(HttpClientParameters parameters, Http { handler.UseProxy = false; } - else if (!ReferenceEquals(parameters.Proxy, GetSystemWebProxy())) + else if (!object.ReferenceEquals(parameters.Proxy, WebRequest.GetSystemWebProxy())) { handler.Proxy = parameters.Proxy; } @@ -1721,20 +1659,10 @@ private static HttpClient CreateHttpClient(HttpClientParameters parameters, Http handler.SslOptions.EnabledSslProtocols = (SslProtocols)parameters.SslProtocols; handler.SslOptions.CertificateRevocationCheckMode = parameters.CheckCertificateRevocationList ? X509RevocationMode.Online : X509RevocationMode.NoCheck; RemoteCertificateValidationCallback? rcvc = parameters.ServerCertificateValidationCallback; - handler.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => + if (rcvc != null) { - if (parameters.ServicePoint is { } servicePoint) - { - servicePoint.Certificate = cert; - } - - if (rcvc is not null) - { - return rcvc(request!, cert, chain, errors); - } - - return errors == SslPolicyErrors.None; - }; + handler.SslOptions.RemoteCertificateValidationCallback = (message, cert, chain, errors) => rcvc(request!, cert, chain, errors); + } // Set up a ConnectCallback so that we can control Socket-specific settings, like ReadWriteTimeout => socket.Send/ReceiveTimeout. handler.ConnectCallback = async (context, cancellationToken) => @@ -1743,10 +1671,6 @@ private static HttpClient CreateHttpClient(HttpClientParameters parameters, Http try { - IPAddress[] addresses = parameters.Async ? - await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken).ConfigureAwait(false) : - Dns.GetHostAddresses(context.DnsEndPoint.Host); - if (parameters.ServicePoint is { } servicePoint) { if (servicePoint.ReceiveBufferSize != -1) @@ -1760,58 +1684,19 @@ await Dns.GetHostAddressesAsync(context.DnsEndPoint.Host, cancellationToken).Con socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveTime, keepAlive.Time); socket.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.TcpKeepAliveInterval, keepAlive.Interval); } - - BindHelper(servicePoint, ref addresses, socket, context.DnsEndPoint.Port); - static void BindHelper(ServicePoint servicePoint, ref IPAddress[] addresses, Socket socket, int port) - { - if (servicePoint.BindIPEndPointDelegate is null) - { - return; - } - - const int MaxRetries = 100; - foreach (IPAddress address in addresses) - { - int retryCount = 0; - for (; retryCount < MaxRetries; retryCount++) - { - IPEndPoint? endPoint = servicePoint.BindIPEndPointDelegate(servicePoint, new IPEndPoint(address, port), retryCount); - if (endPoint is null) // Get other address to try - { - break; - } - - try - { - socket.Bind(endPoint); - addresses = [address]; - return; // Bind successful, exit loops. - } - catch - { - continue; - } - } - - if (retryCount >= MaxRetries) - { - throw new OverflowException(SR.net_maximumbindretries); - } - } - } } - socket.NoDelay = !(parameters.ServicePoint?.UseNagleAlgorithm) ?? true; + socket.NoDelay = true; if (parameters.Async) { - await socket.ConnectAsync(addresses, context.DnsEndPoint.Port, cancellationToken).ConfigureAwait(false); + await socket.ConnectAsync(context.DnsEndPoint, cancellationToken).ConfigureAwait(false); } else { using (cancellationToken.UnsafeRegister(s => ((Socket)s!).Dispose(), socket)) { - socket.Connect(addresses, context.DnsEndPoint.Port); + socket.Connect(context.DnsEndPoint); } // Throw in case cancellation caused the socket to be disposed after the Connect completed diff --git a/src/libraries/System.Net.Requests/src/System/Net/HttpWebResponse.cs b/src/libraries/System.Net.Requests/src/System/Net/HttpWebResponse.cs index 7b0e9b90681fc9..f7fae7869b1e7f 100644 --- a/src/libraries/System.Net.Requests/src/System/Net/HttpWebResponse.cs +++ b/src/libraries/System.Net.Requests/src/System/Net/HttpWebResponse.cs @@ -8,8 +8,6 @@ using System.Net.Http; using System.Runtime.Serialization; using System.Text; -using System.Threading; -using System.Threading.Tasks; namespace System.Net { @@ -339,14 +337,7 @@ public override Stream GetResponseStream() CheckDisposed(); if (_httpResponseMessage.Content != null) { - Stream contentStream = _httpResponseMessage.Content.ReadAsStream(); - int maxErrorResponseLength = HttpWebRequest.DefaultMaximumErrorResponseLength; - if (maxErrorResponseLength < 0 || StatusCode < HttpStatusCode.BadRequest) - { - return contentStream; - } - - return new TruncatedReadStream(contentStream, maxErrorResponseLength); + return _httpResponseMessage.Content.ReadAsStream(); } return Stream.Null; @@ -380,56 +371,5 @@ private void CheckDisposed() } private static string GetHeaderValueAsString(IEnumerable values) => string.Join(", ", values); - - internal sealed class TruncatedReadStream(Stream innerStream, int maxSize) : Stream - { - public override bool CanRead => true; - public override bool CanSeek => false; - public override bool CanWrite => false; - - public override long Length => throw new NotSupportedException(); - public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } - - public override void Flush() => throw new NotSupportedException(); - - public override int Read(byte[] buffer, int offset, int count) - { - return Read(new Span(buffer, offset, count)); - } - - public override int Read(Span buffer) - { - int readBytes = innerStream.Read(buffer.Slice(0, Math.Min(buffer.Length, maxSize))); - maxSize -= readBytes; - return readBytes; - } - - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - return ReadAsync(new Memory(buffer, offset, count), cancellationToken).AsTask(); - } - - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) - { - int readBytes = await innerStream.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, maxSize)), cancellationToken) - .ConfigureAwait(false); - maxSize -= readBytes; - return readBytes; - } - - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotSupportedException(); - public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - - public override ValueTask DisposeAsync() => innerStream.DisposeAsync(); - - protected override void Dispose(bool disposing) - { - if (disposing) - { - innerStream.Dispose(); - } - } - } } } diff --git a/src/libraries/System.Net.Requests/src/System/Net/RequestBufferingStream.cs b/src/libraries/System.Net.Requests/src/System/Net/RequestBufferingStream.cs deleted file mode 100644 index 3a5bb170314e13..00000000000000 --- a/src/libraries/System.Net.Requests/src/System/Net/RequestBufferingStream.cs +++ /dev/null @@ -1,134 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.IO; -using System.Threading; -using System.Threading.Tasks; - -namespace System.Net -{ - // Cache the request stream into a MemoryStream. - internal sealed class RequestBufferingStream : Stream - { - private bool _disposed; - private readonly MemoryStream _buffer = new MemoryStream(); - - public RequestBufferingStream() - { - } - - public override bool CanRead => false; - public override bool CanSeek => false; - public override bool CanWrite => true; - - public override void Flush() => ThrowIfDisposed(); // Nothing to do. - - public override Task FlushAsync(CancellationToken cancellationToken) - { - ThrowIfDisposed(); - // Nothing to do. - return cancellationToken.IsCancellationRequested ? - Task.FromCanceled(cancellationToken) : - Task.CompletedTask; - } - - public override long Length - { - get - { - throw new NotSupportedException(); - } - } - - public override long Position - { - get - { - throw new NotSupportedException(); - } - set - { - throw new NotSupportedException(); - } - } - - public override int Read(byte[] buffer, int offset, int count) - { - throw new NotSupportedException(); - } - - public override long Seek(long offset, SeekOrigin origin) - { - throw new NotSupportedException(); - } - - public override void SetLength(long value) - { - throw new NotSupportedException(); - } - - public override void Write(byte[] buffer, int offset, int count) - { - ThrowIfDisposed(); - ValidateBufferArguments(buffer, offset, count); - _buffer.Write(buffer, offset, count); - } - - public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - { - ThrowIfDisposed(); - ValidateBufferArguments(buffer, offset, count); - return _buffer.WriteAsync(buffer, offset, count, cancellationToken); - } - - public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) - { - ThrowIfDisposed(); - return _buffer.WriteAsync(buffer, cancellationToken); - } - - public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) - { - ThrowIfDisposed(); - ValidateBufferArguments(buffer, offset, count); - return _buffer.BeginWrite(buffer, offset, count, asyncCallback, asyncState); - } - - public override void EndWrite(IAsyncResult asyncResult) - { - ThrowIfDisposed(); - _buffer.EndWrite(asyncResult); - } - - public ArraySegment GetBuffer() - { - ArraySegment bytes; - - bool success = _buffer.TryGetBuffer(out bytes); - Debug.Assert(success); // Buffer should always be visible since default MemoryStream constructor was used. - - return bytes; - } - - // We need this to dispose the MemoryStream. - public MemoryStream GetMemoryStream() - { - return _buffer; - } - - protected override void Dispose(bool disposing) - { - if (disposing && !_disposed) - { - _disposed = true; - } - base.Dispose(disposing); - } - - private void ThrowIfDisposed() - { - ObjectDisposedException.ThrowIf(_disposed, this); - } - } -} diff --git a/src/libraries/System.Net.Requests/src/System/Net/RequestStream.cs b/src/libraries/System.Net.Requests/src/System/Net/RequestStream.cs index 5961339576d306..5323c2ac836f04 100644 --- a/src/libraries/System.Net.Requests/src/System/Net/RequestStream.cs +++ b/src/libraries/System.Net.Requests/src/System/Net/RequestStream.cs @@ -3,22 +3,22 @@ using System.Diagnostics; using System.IO; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; namespace System.Net { + // Cache the request stream into a MemoryStream. This is the + // default behavior of Desktop HttpWebRequest.AllowWriteStreamBuffering (true). + // Unfortunately, this property is not exposed in .NET Core, so it can't be changed + // This will result in inefficient memory usage when sending (POST'ing) large + // amounts of data to the server such as from a file stream. internal sealed class RequestStream : Stream { - private bool _disposed; - private readonly TaskCompletionSource _completeTcs; - private readonly Stream _internalStream; + private readonly MemoryStream _buffer = new MemoryStream(); - public RequestStream(Stream internalStream, TaskCompletionSource completeTcs) + public RequestStream() { - _internalStream = internalStream; - _completeTcs = completeTcs; } public override bool CanRead @@ -47,14 +47,15 @@ public override bool CanWrite public override void Flush() { - ThrowIfDisposed(); - _internalStream.Flush(); + // Nothing to do. } public override Task FlushAsync(CancellationToken cancellationToken) { - ThrowIfDisposed(); - return _internalStream.FlushAsync(cancellationToken); + // Nothing to do. + return cancellationToken.IsCancellationRequested ? + Task.FromCanceled(cancellationToken) : + Task.CompletedTask; } public override long Length @@ -94,67 +95,40 @@ public override void SetLength(long value) public override void Write(byte[] buffer, int offset, int count) { - ThrowIfDisposed(); ValidateBufferArguments(buffer, offset, count); - _internalStream.Write(new(buffer, offset, count)); + _buffer.Write(buffer, offset, count); } public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - ThrowIfDisposed(); ValidateBufferArguments(buffer, offset, count); - return _internalStream.WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + return _buffer.WriteAsync(buffer, offset, count, cancellationToken); } public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) { - ThrowIfDisposed(); - return _internalStream.WriteAsync(buffer, cancellationToken); + return _buffer.WriteAsync(buffer, cancellationToken); } public override IAsyncResult BeginWrite(byte[] buffer, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) { - ThrowIfDisposed(); ValidateBufferArguments(buffer, offset, count); - return _internalStream.BeginWrite(buffer, offset, count, asyncCallback, asyncState); - } - - public void Complete() - { - _completeTcs.TrySetResult(); + return _buffer.BeginWrite(buffer, offset, count, asyncCallback, asyncState); } public override void EndWrite(IAsyncResult asyncResult) { - ThrowIfDisposed(); - _internalStream.EndWrite(asyncResult); + _buffer.EndWrite(asyncResult); } - protected override void Dispose(bool disposing) + public ArraySegment GetBuffer() { - if (disposing && !_disposed) - { - _disposed = true; - } - _internalStream.Flush(); - Complete(); - base.Dispose(disposing); - } + ArraySegment bytes; - public override async ValueTask DisposeAsync() - { - if (!_disposed) - { - _disposed = true; - } - await _internalStream.FlushAsync().ConfigureAwait(false); - Complete(); - await base.DisposeAsync().ConfigureAwait(false); - } + bool success = _buffer.TryGetBuffer(out bytes); + Debug.Assert(success); // Buffer should always be visible since default MemoryStream constructor was used. - private void ThrowIfDisposed() - { - ObjectDisposedException.ThrowIf(_disposed, this); + return bytes; } } } diff --git a/src/libraries/System.Net.Requests/src/System/Net/RequestStreamContent.cs b/src/libraries/System.Net.Requests/src/System/Net/RequestStreamContent.cs deleted file mode 100644 index b78829c22de721..00000000000000 --- a/src/libraries/System.Net.Requests/src/System/Net/RequestStreamContent.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.IO; -using System.Net.Http; -using System.Threading; -using System.Threading.Tasks; - -namespace System.Net -{ - internal sealed class RequestStreamContent(TaskCompletionSource getStreamTcs, TaskCompletionSource completeTcs) : HttpContent - { - protected override Task SerializeToStreamAsync(Stream stream, TransportContext? context) - { - return SerializeToStreamAsync(stream, context, default); - } - protected override async Task SerializeToStreamAsync(Stream stream, TransportContext? context, CancellationToken cancellationToken) - { - Debug.Assert(stream is not null); - - getStreamTcs.TrySetResult(stream); - await completeTcs.Task.WaitAsync(cancellationToken).ConfigureAwait(false); - } - protected override bool TryComputeLength(out long length) - { - length = -1; - return false; - } - } -} diff --git a/src/libraries/System.Net.Requests/src/System/Net/ServicePoint/ServicePointManager.cs b/src/libraries/System.Net.Requests/src/System/Net/ServicePoint/ServicePointManager.cs index bbf20b3e808b62..a0cf9dcece157f 100644 --- a/src/libraries/System.Net.Requests/src/System/Net/ServicePoint/ServicePointManager.cs +++ b/src/libraries/System.Net.Requests/src/System/Net/ServicePoint/ServicePointManager.cs @@ -78,7 +78,7 @@ public static int MaxServicePointIdleTime } } - public static bool UseNagleAlgorithm { get; set; } + public static bool UseNagleAlgorithm { get; set; } = true; public static bool Expect100Continue { get; set; } = true; @@ -156,8 +156,7 @@ public static ServicePoint FindServicePoint(Uri address, IWebProxy? proxy) IdleSince = DateTime.Now, Expect100Continue = Expect100Continue, UseNagleAlgorithm = UseNagleAlgorithm, - KeepAlive = KeepAlive, - MaxIdleTime = MaxServicePointIdleTime + KeepAlive = KeepAlive }; s_servicePointTable[tableKey] = new WeakReference(sp); diff --git a/src/libraries/System.Net.Requests/tests/HttpWebRequestTest.cs b/src/libraries/System.Net.Requests/tests/HttpWebRequestTest.cs index e88d8800f99f0f..45563ccc3dd0ed 100644 --- a/src/libraries/System.Net.Requests/tests/HttpWebRequestTest.cs +++ b/src/libraries/System.Net.Requests/tests/HttpWebRequestTest.cs @@ -258,7 +258,7 @@ public void Ctor_VerifyDefaults_Success(Uri remoteServer) Assert.Equal(64, HttpWebRequest.DefaultMaximumResponseHeadersLength); Assert.NotNull(HttpWebRequest.DefaultCachePolicy); Assert.Equal(RequestCacheLevel.BypassCache, HttpWebRequest.DefaultCachePolicy.Level); - Assert.Equal(-1, HttpWebRequest.DefaultMaximumErrorResponseLength); + Assert.Equal(0, HttpWebRequest.DefaultMaximumErrorResponseLength); Assert.NotNull(request.Proxy); Assert.Equal(remoteServer, request.RequestUri); Assert.True(request.SupportsCookieContainer); @@ -1914,7 +1914,7 @@ public void Abort_CreateRequestThenAbort_Success(Uri remoteServer) } [Theory] - [InlineData(HttpRequestCacheLevel.NoCacheNoStore, null, null, new string[] { "Pragma: no-cache", "Cache-Control: no-store, no-cache" })] + [InlineData(HttpRequestCacheLevel.NoCacheNoStore, null, null, new string[] { "Pragma: no-cache", "Cache-Control: no-store, no-cache"})] [InlineData(HttpRequestCacheLevel.Reload, null, null, new string[] { "Pragma: no-cache", "Cache-Control: no-cache" })] [InlineData(HttpRequestCacheLevel.CacheOrNextCacheOnly, null, null, new string[] { "Cache-Control: only-if-cached" })] [InlineData(HttpRequestCacheLevel.Default, HttpCacheAgeControl.MinFresh, 10, new string[] { "Cache-Control: min-fresh=10" })] @@ -2077,125 +2077,6 @@ await server.AcceptConnectionAsync(async connection => }); } - [Fact] - public async Task SendHttpPostRequest_BufferingDisabled_ConnectionShouldStartWithRequestStream() - { - await LoopbackServer.CreateClientAndServerAsync( - async (uri) => - { - HttpWebRequest request = WebRequest.CreateHttp(uri); - request.Method = "POST"; - request.AllowWriteStreamBuffering = false; - request.SendChunked = true; - var stream = await request.GetRequestStreamAsync(); - await Assert.ThrowsAnyAsync(() => request.GetResponseAsync()); - }, - async (server) => - { - await server.AcceptConnectionAsync(_ => - { - return Task.CompletedTask; - }); - } - ); - } - - [Theory] - [InlineData(true)] - [InlineData(false)] - public async Task SendHttpPostRequest_WhenBufferingChanges_Success(bool buffering) - { - byte[] randomData = Encoding.ASCII.GetBytes("Hello World!!!!\n"); - await LoopbackServer.CreateClientAndServerAsync( - async (uri) => - { - int size = randomData.Length * 100; - HttpWebRequest request = WebRequest.CreateHttp(uri); - request.Method = "POST"; - request.AllowWriteStreamBuffering = buffering; - using var stream = await request.GetRequestStreamAsync(); - for (int i = 0; i < size / randomData.Length; i++) - { - await stream.WriteAsync(new ReadOnlyMemory(randomData)); - } - await request.GetResponseAsync(); - }, - async (server) => - { - await server.AcceptConnectionAsync(async connection => - { - var data = await connection.ReadRequestDataAsync(); - for (int i = 0; i < data.Body.Length; i += randomData.Length) - { - Assert.Equal(randomData, data.Body[i..(i + randomData.Length)]); - } - await connection.SendResponseAsync(); - }); - } - ); - } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task SendHttpRequest_WhenNotBuffering_SendSuccess(bool isChunked) - { - byte[] firstBlock = "Hello"u8.ToArray(); - byte[] secondBlock = "WorlddD"u8.ToArray(); - SemaphoreSlim sem = new(0); - await LoopbackServer.CreateClientAndServerAsync( - async (uri) => - { - HttpWebRequest request = WebRequest.CreateHttp(uri); - request.Method = "POST"; - if (isChunked is false) - { - request.ContentLength = 5 + 7; - } - request.AllowWriteStreamBuffering = false; - - using (Stream requestStream = await request.GetRequestStreamAsync()) - { - requestStream.Write(firstBlock); - requestStream.Flush(); - await sem.WaitAsync(); - requestStream.Write(secondBlock); - requestStream.Flush(); - } - await request.GetResponseAsync(); - sem.Release(); - }, - async (server) => - { - await server.AcceptConnectionAsync(async (connection) => - { - byte[] buffer = new byte[1024]; - await connection.ReadRequestHeaderAsync(); - if (isChunked) - { - // Discard chunk length and CRLF. - await connection.ReadLineAsync(); - } - int readBytes = await connection.ReadBlockAsync(buffer, 0, firstBlock.Length); - Assert.Equal(firstBlock.Length, readBytes); - Assert.Equal(firstBlock, buffer[..readBytes]); - sem.Release(); - if (isChunked) - { - // Discard CRLF, chunk length and CRLF. - await connection.ReadLineAsync(); - await connection.ReadLineAsync(); - } - readBytes = await connection.ReadBlockAsync(buffer, 0, secondBlock.Length); - Assert.Equal(secondBlock.Length, readBytes); - Assert.Equal(secondBlock, buffer[..readBytes]); - await connection.SendResponseAsync(); - await sem.WaitAsync(); - }); - } - ); - } - [Fact] public async Task SendHttpPostRequest_WithContinueTimeoutAndBody_BodyIsDelayed() { @@ -2206,20 +2087,18 @@ await LoopbackServer.CreateClientAndServerAsync( request.Method = "POST"; request.ServicePoint.Expect100Continue = true; request.ContinueTimeout = 30000; - using (Stream requestStream = await request.GetRequestStreamAsync()) - { - requestStream.Write("aaaa\r\n\r\n"u8); - } - await GetResponseAsync(request); + Stream requestStream = await request.GetRequestStreamAsync(); + requestStream.Write("aaaa\r\n\r\n"u8); + await request.GetResponseAsync(); }, async (server) => { - await server.AcceptConnectionAsync(async (connection) => + await server.AcceptConnectionAsync(async (client) => { - await connection.ReadRequestHeaderAsync(); + await client.ReadRequestHeaderAsync(); // This should time out, because we're expecting the body itself but we'll get it after 30 sec. - await Assert.ThrowsAsync(() => connection.ReadLineAsync().WaitAsync(TimeSpan.FromMilliseconds(100))); - await connection.SendResponseAsync(); + await Assert.ThrowsAsync(() => client.ReadLineAsync().WaitAsync(TimeSpan.FromMilliseconds(100))); + await client.SendResponseAsync(); }); } ); @@ -2237,21 +2116,19 @@ await LoopbackServer.CreateClientAndServerAsync( request.Method = "POST"; request.ServicePoint.Expect100Continue = expect100Continue; request.ContinueTimeout = continueTimeout; - using (Stream requestStream = await request.GetRequestStreamAsync()) - { - requestStream.Write("aaaa\r\n\r\n"u8); - } - await GetResponseAsync(request); + Stream requestStream = await request.GetRequestStreamAsync(); + requestStream.Write("aaaa\r\n\r\n"u8); + await request.GetResponseAsync(); }, async (server) => { - await server.AcceptConnectionAsync(async (connection) => + await server.AcceptConnectionAsync(async (client) => { - await connection.ReadRequestHeaderAsync(); + await client.ReadRequestHeaderAsync(); // This should not time out, because we're expecting the body itself and we should get it after 1 sec. - string data = await connection.ReadLineAsync().WaitAsync(TimeSpan.FromSeconds(10)); + string data = await client.ReadLineAsync().WaitAsync(TimeSpan.FromSeconds(10)); Assert.StartsWith("aaaa", data); - await connection.SendResponseAsync(); + await client.SendResponseAsync(); }); }); } @@ -2267,14 +2144,14 @@ await LoopbackServer.CreateClientAndServerAsync( HttpWebRequest request = WebRequest.CreateHttp(uri); request.Method = "POST"; request.ServicePoint.Expect100Continue = expect100Continue; - await GetResponseAsync(request); + await request.GetResponseAsync(); }, async (server) => { await server.AcceptConnectionAsync( - async (connection) => + async (client) => { - List headers = await connection.ReadRequestHeaderAsync(); + List headers = await client.ReadRequestHeaderAsync(); if (expect100Continue) { Assert.Contains("Expect: 100-continue", headers); @@ -2283,129 +2160,13 @@ await server.AcceptConnectionAsync( { Assert.DoesNotContain("Expect: 100-continue", headers); } - await connection.SendResponseAsync(); + await client.SendResponseAsync(); } ); } ); } - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public void SendHttpRequest_WhenDefaultMaximumErrorResponseLengthSet_Success() - { - RemoteExecutor.Invoke(async (async) => - { - TaskCompletionSource tcs = new TaskCompletionSource(); - await LoopbackServer.CreateClientAndServerAsync( - async (uri) => - { - HttpWebRequest request = WebRequest.CreateHttp(uri); - HttpWebRequest.DefaultMaximumErrorResponseLength = 5; - var exception = - await Assert.ThrowsAsync(() => bool.Parse(async) ? request.GetResponseAsync() : Task.Run(() => request.GetResponse())); - tcs.SetResult(); - Assert.NotNull(exception.Response); - using (var responseStream = exception.Response.GetResponseStream()) - { - var buffer = new byte[10]; - int readLen = responseStream.Read(buffer, 0, buffer.Length); - Assert.Equal(5, readLen); - Assert.Equal(new string('a', 5), Encoding.UTF8.GetString(buffer[0..readLen])); - Assert.Equal(0, responseStream.Read(buffer)); - } - }, - async (server) => - { - await server.AcceptConnectionAsync( - async connection => - { - await connection.SendResponseAsync(statusCode: HttpStatusCode.BadRequest, content: new string('a', 10)); - await tcs.Task; - }); - }); - }, IsAsync.ToString()).Dispose(); - } - - [Fact] - public void HttpWebRequest_SetProtocolVersion_Success() - { - HttpWebRequest request = WebRequest.CreateHttp(Configuration.Http.RemoteEchoServer); - - request.ProtocolVersion = HttpVersion.Version10; - Assert.Equal(HttpVersion.Version10, request.ServicePoint.ProtocolVersion); - - request.ProtocolVersion = HttpVersion.Version11; - Assert.Equal(HttpVersion.Version11, request.ServicePoint.ProtocolVersion); - } - - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public void SendHttpRequest_BindIPEndPoint_Success() - { - RemoteExecutor.Invoke(async (async) => - { - TaskCompletionSource tcs = new TaskCompletionSource(); - await LoopbackServer.CreateClientAndServerAsync( - async (uri) => - { - HttpWebRequest request = WebRequest.CreateHttp(uri); - request.ServicePoint.BindIPEndPointDelegate = (_, _, _) => new IPEndPoint(IPAddress.Loopback, 27277); - var responseTask = bool.Parse(async) ? request.GetResponseAsync() : Task.Run(() => request.GetResponse()); - using (var response = (HttpWebResponse)await responseTask) - { - Assert.Equal(HttpStatusCode.OK, response.StatusCode); - } - tcs.SetResult(); - }, - async (server) => - { - await server.AcceptConnectionAsync( - async connection => - { - var ipEp = (IPEndPoint)connection.Socket.RemoteEndPoint; - Assert.Equal(27277, ipEp.Port); - await connection.SendResponseAsync(); - await tcs.Task; - }); - }); - }, IsAsync.ToString()).Dispose(); - } - - [ConditionalFact(typeof(RemoteExecutor), nameof(RemoteExecutor.IsSupported))] - public void SendHttpRequest_BindIPEndPoint_Throws() - { - RemoteExecutor.Invoke(async (async) => - { - Socket socket = new Socket(SocketType.Stream, ProtocolType.Tcp); - socket.Bind(new IPEndPoint(IPAddress.Loopback, 0)); - ValueTask? clientSocket = null; - CancellationTokenSource cts = new CancellationTokenSource(); - if (PlatformDetection.IsLinux) - { - socket.Listen(); - clientSocket = socket.AcceptAsync(cts.Token); - } - - try - { - // URI shouldn't matter because it should throw exception before connection open. - HttpWebRequest request = WebRequest.CreateHttp(Configuration.Http.RemoteEchoServer); - request.ServicePoint.BindIPEndPointDelegate = (_, _, _) => (IPEndPoint)socket.LocalEndPoint!; - var exception = await Assert.ThrowsAsync(() => - bool.Parse(async) ? request.GetResponseAsync() : Task.Run(() => request.GetResponse())); - Assert.IsType(exception.InnerException?.InnerException); - } - finally - { - if (clientSocket is not null) - { - await cts.CancelAsync(); - } - socket.Dispose(); - cts.Dispose(); - } - }, IsAsync.ToString()).Dispose(); - } - private void RequestStreamCallback(IAsyncResult asynchronousResult) { RequestState state = (RequestState)asynchronousResult.AsyncState; diff --git a/src/libraries/System.Net.Requests/tests/ServicePointTests/ServicePointManagerTest.cs b/src/libraries/System.Net.Requests/tests/ServicePointTests/ServicePointManagerTest.cs index ec64068ed456fc..c1230598a8d4e5 100644 --- a/src/libraries/System.Net.Requests/tests/ServicePointTests/ServicePointManagerTest.cs +++ b/src/libraries/System.Net.Requests/tests/ServicePointTests/ServicePointManagerTest.cs @@ -181,7 +181,7 @@ public static void ServerCertificateValidationCallback_Roundtrips() [Fact] public static void UseNagleAlgorithm_Roundtrips() { - Assert.False(ServicePointManager.UseNagleAlgorithm); + Assert.True(ServicePointManager.UseNagleAlgorithm); try { ServicePointManager.UseNagleAlgorithm = false; @@ -325,7 +325,7 @@ public static void FindServicePoint_ReturnedServicePointMatchesExpectedValues() Assert.Equal(new Version(1, 1), sp.ProtocolVersion); Assert.Equal(-1, sp.ReceiveBufferSize); Assert.True(sp.SupportsPipelining, "SupportsPipelining"); - Assert.False(sp.UseNagleAlgorithm, "UseNagleAlgorithm"); + Assert.True(sp.UseNagleAlgorithm, "UseNagleAlgorithm"); }).Dispose(); } diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.Protocol.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.Protocol.cs index a3b8b29144045b..8e6cb95ef77a61 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.Protocol.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStream.Protocol.cs @@ -16,39 +16,6 @@ namespace System.Net.Security { public partial class SslStream { - private const string DisableTlsResumeCtxSwitch = "System.Net.Security.DisableTlsResume"; - private const string DisableTlsResumeEnvironmentVariable = "DOTNET_SYSTEM_NET_SECURITY_DISABLETLSRESUME"; - - private static volatile int s_disableTlsResume = -1; - - internal static bool DisableTlsResume - { - get - { - int disableTlsResume = s_disableTlsResume; - if (disableTlsResume != -1) - { - return disableTlsResume != 0; - } - - // First check for the AppContext switch, giving it priority over the environment variable. - if (AppContext.TryGetSwitch(DisableTlsResumeCtxSwitch, out bool value)) - { - s_disableTlsResume = value ? 1 : 0; - } - else - { - // AppContext switch wasn't used. Check the environment variable. - s_disableTlsResume = - Environment.GetEnvironmentVariable(DisableTlsResumeEnvironmentVariable) is string envVar && - (envVar == "1" || envVar.Equals("true", StringComparison.OrdinalIgnoreCase)) ? 1 : 0; - } - - return s_disableTlsResume != 0; - } - } - - private SafeFreeCredentials? _credentialsHandle; private SafeDeleteSslContext? _securityContext; diff --git a/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs b/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs index 63b9aea0e4bd56..b04a0571f0a53d 100644 --- a/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs +++ b/src/libraries/System.Net.Security/src/System/Net/Security/SslStreamPal.Windows.cs @@ -178,9 +178,7 @@ public static ProtocolToken InitializeSecurityContext( token.Status = SecurityStatusAdapterPal.GetSecurityStatusPalFromNativeInt(errorCode); - bool allowTlsResume = sslAuthenticationOptions.AllowTlsResume && !SslStream.DisableTlsResume; - - if (!allowTlsResume && newContext && context != null) + if (!sslAuthenticationOptions.AllowTlsResume && newContext && context != null) { var securityBuffer = new SecurityBuffer(s_sessionTokenBuffer, SecurityBufferType.SECBUFFER_TOKEN); @@ -283,8 +281,6 @@ public static unsafe SafeFreeCredentials AcquireCredentialsHandleSchannelCred(Ss Interop.SspiCli.SCHANNEL_CRED.Flags flags; Interop.SspiCli.CredentialUse direction; - bool allowTlsResume = authOptions.AllowTlsResume && !SslStream.DisableTlsResume; - if (!isServer) { direction = Interop.SspiCli.CredentialUse.SECPKG_CRED_OUTBOUND; @@ -308,7 +304,7 @@ public static unsafe SafeFreeCredentials AcquireCredentialsHandleSchannelCred(Ss flags = Interop.SspiCli.SCHANNEL_CRED.Flags.SCH_SEND_AUX_RECORD | Interop.SspiCli.SCHANNEL_CRED.Flags.SCH_CRED_NO_SYSTEM_MAPPER; - if (!allowTlsResume) + if (!authOptions.AllowTlsResume) { // Works only on server flags |= Interop.SspiCli.SCHANNEL_CRED.Flags.SCH_CRED_DISABLE_RECONNECTS; @@ -333,7 +329,7 @@ public static unsafe SafeFreeCredentials AcquireCredentialsHandleSchannelCred(Ss protocolFlags, policy); - if (!isServer && !allowTlsResume) + if (!isServer && !authOptions.AllowTlsResume) { secureCredential.dwSessionLifespan = -1; } @@ -357,8 +353,6 @@ public static unsafe SafeFreeCredentials AcquireCredentialsHandleSchCredentials( Interop.SspiCli.SCH_CREDENTIALS.Flags flags; Interop.SspiCli.CredentialUse direction; - bool allowTlsResume = authOptions.AllowTlsResume && !SslStream.DisableTlsResume; - if (isServer) { direction = Interop.SspiCli.CredentialUse.SECPKG_CRED_INBOUND; @@ -367,7 +361,7 @@ public static unsafe SafeFreeCredentials AcquireCredentialsHandleSchCredentials( { flags |= Interop.SspiCli.SCH_CREDENTIALS.Flags.SCH_CRED_NO_SYSTEM_MAPPER; } - if (!allowTlsResume) + if (!authOptions.AllowTlsResume) { // Works only on server flags |= Interop.SspiCli.SCH_CREDENTIALS.Flags.SCH_CRED_DISABLE_RECONNECTS; @@ -416,7 +410,7 @@ public static unsafe SafeFreeCredentials AcquireCredentialsHandleSchCredentials( Interop.SspiCli.SCH_CREDENTIALS credential = default; credential.dwVersion = Interop.SspiCli.SCH_CREDENTIALS.CurrentVersion; credential.dwFlags = flags; - if (!isServer && !allowTlsResume) + if (!isServer && !authOptions.AllowTlsResume) { credential.dwSessionLifespan = -1; } diff --git a/src/libraries/System.Net.Security/tests/UnitTests/Fakes/FakeSslStream.Implementation.cs b/src/libraries/System.Net.Security/tests/UnitTests/Fakes/FakeSslStream.Implementation.cs index 3ad6be920392ac..593af087659179 100644 --- a/src/libraries/System.Net.Security/tests/UnitTests/Fakes/FakeSslStream.Implementation.cs +++ b/src/libraries/System.Net.Security/tests/UnitTests/Fakes/FakeSslStream.Implementation.cs @@ -14,8 +14,6 @@ namespace System.Net.Security { public partial class SslStream { - internal static bool DisableTlsResume { get; } - private class FakeOptions { public string TargetHost; diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs index 3ba24e90cf1019..e1891bef916f4d 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.Tasks.cs @@ -677,6 +677,7 @@ public ValueTask SendToAsync(ReadOnlyMemory buffer, SocketFlags socke Debug.Assert(saea.BufferList == null); saea.SetBuffer(MemoryMarshal.AsMemory(buffer)); saea.SocketFlags = socketFlags; + saea._socketAddress = null; saea.RemoteEndPoint = remoteEP; saea.WrapExceptionsForNetworkStream = false; return saea.SendToAsync(this, cancellationToken); @@ -708,17 +709,8 @@ public ValueTask SendToAsync(ReadOnlyMemory buffer, SocketFlags socke saea.SetBuffer(MemoryMarshal.AsMemory(buffer)); saea.SocketFlags = socketFlags; saea._socketAddress = socketAddress; - saea.RemoteEndPoint = null; saea.WrapExceptionsForNetworkStream = false; - try - { - return saea.SendToAsync(this, cancellationToken); - } - finally - { - // detach user provided SA so we do not accidentally stomp on it later. - saea._socketAddress = null; - } + return saea.SendToAsync(this, cancellationToken); } /// diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs index a8c95005154c93..11b8674d681f38 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/Socket.cs @@ -3095,22 +3095,14 @@ private bool SendToAsync(SocketAsyncEventArgs e, CancellationToken cancellationT ArgumentNullException.ThrowIfNull(e); EndPoint? endPointSnapshot = e.RemoteEndPoint; - - // RemoteEndPoint should be set unless somebody used SendTo with their own SA. - // In that case RemoteEndPoint will be null and we take provided SA as given. - if (endPointSnapshot == null && e._socketAddress == null) + if (e._socketAddress == null) { - throw new ArgumentException(SR.Format(SR.InvalidNullArgument, "e.RemoteEndPoint"), nameof(e)); - } + if (endPointSnapshot == null) + { + throw new ArgumentException(SR.Format(SR.InvalidNullArgument, "e.RemoteEndPoint"), nameof(e)); + } - if (e._socketAddress != null && endPointSnapshot is IPEndPoint ipep && e._socketAddress.Family == endPointSnapshot?.AddressFamily) - { - // we have matching SocketAddress. Since this is only used internally, it is ok to overwrite it without - ipep.Serialize(e._socketAddress.Buffer.Span); - } - else if (endPointSnapshot != null) - { - // Prepare new SocketAddress + // Prepare SocketAddress e._socketAddress = Serialize(ref endPointSnapshot); } diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs index 78dd22e5eda7bf..e94d862571a0f8 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketAsyncEventArgs.cs @@ -923,12 +923,7 @@ internal void FinishOperationSyncSuccess(int bytesTransferred, SocketFlags flags case SocketAsyncOperation.ReceiveFrom: // Deal with incoming address. UpdateReceivedSocketAddress(_socketAddress!); - if (_remoteEndPoint == null) - { - // detach user provided SA as it was updated in place. - _socketAddress = null; - } - else if (!SocketAddressExtensions.Equals(_socketAddress!, _remoteEndPoint)) + if (_remoteEndPoint != null && !SocketAddressExtensions.Equals(_socketAddress!, _remoteEndPoint)) { try { diff --git a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs index 837743dfa34478..d2ac959e0b4c73 100644 --- a/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs +++ b/src/libraries/System.Net.Sockets/src/System/Net/Sockets/SocketPal.Unix.cs @@ -528,6 +528,7 @@ private static unsafe int SysReceiveMessageFrom( out SocketFlags receivedFlags, out IPPacketInformation ipPacketInformation, out Interop.Error errno) { Debug.Assert(socket.IsSocket); + Debug.Assert(socketAddress != null, "Expected non-null socketAddress"); int buffersCount = buffers.Count; bool allocOnStack = buffersCount <= IovStackThreshold; @@ -809,6 +810,7 @@ public static unsafe bool TryCompleteReceiveFrom(SafeSocketHandle socket, Span { sent = buffers != null ? SysSend(socket, flags, buffers, ref bufferIndex, ref offset, socketAddress, out errno) : - socketAddress.IsEmpty ? SysSend(socket, flags, buffer, ref offset, ref count, out errno) : + socketAddress == null ? SysSend(socket, flags, buffer, ref offset, ref count, out errno) : SysSend(socket, flags, buffer, ref offset, ref count, socketAddress, out errno); } } diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs index 7a3c33b64bf796..bf0ad146588699 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendTo.cs @@ -173,35 +173,6 @@ public void SendToAsync_NullAsyncEventArgs_Throws_ArgumentNullException() public sealed class SendTo_Task : SendTo { public SendTo_Task(ITestOutputHelper output) : base(output) { } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task SendTo_DifferentEP_Success(bool ipv4) - { - IPAddress address = ipv4 ? IPAddress.Loopback : IPAddress.IPv6Loopback; - IPEndPoint remoteEp = new IPEndPoint(address, 0); - - using Socket receiver1 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - using Socket receiver2 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - using Socket sender = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - - receiver1.BindToAnonymousPort(address); - receiver2.BindToAnonymousPort(address); - - byte[] sendBuffer = new byte[32]; - var receiveInternalBuffer = new byte[sendBuffer.Length]; - ArraySegment receiveBuffer = new ArraySegment(receiveInternalBuffer, 0, receiveInternalBuffer.Length); - - - await sender.SendToAsync(sendBuffer, SocketFlags.None, receiver1.LocalEndPoint); - SocketReceiveFromResult result = await ReceiveFromAsync(receiver1, receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); - Assert.Equal(sendBuffer.Length, result.ReceivedBytes); - - await sender.SendToAsync(sendBuffer, SocketFlags.None, receiver2.LocalEndPoint); - result = await ReceiveFromAsync(receiver2, receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); - Assert.Equal(sendBuffer.Length, result.ReceivedBytes); - } } public sealed class SendTo_CancellableTask : SendTo diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs index 3d865cb864570f..ded34276f322fa 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SocketAsyncEventArgsTest.cs @@ -895,52 +895,5 @@ void CreateSocketAsyncEventArgs() // separated out so that JIT doesn't extend li return cwt.Count() == 0; // validate that the cwt becomes empty }, 30_000)); } - - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task SendTo_DifferentEP_Success(bool ipv4) - { - IPAddress address = ipv4 ? IPAddress.Loopback : IPAddress.IPv6Loopback; - IPEndPoint remoteEp = new IPEndPoint(address, 0); - - using Socket receiver1 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - using Socket receiver2 = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - using Socket sender = new Socket(address.AddressFamily, SocketType.Dgram, ProtocolType.Udp); - - receiver1.BindToAnonymousPort(address); - receiver2.BindToAnonymousPort(address); - - byte[] sendBuffer = new byte[32]; - var receiveInternalBuffer = new byte[sendBuffer.Length]; - ArraySegment receiveBuffer = new ArraySegment(receiveInternalBuffer, 0, receiveInternalBuffer.Length); - - using SocketAsyncEventArgs saea = new SocketAsyncEventArgs(); - ManualResetEventSlim mres = new ManualResetEventSlim(false); - - saea.SetBuffer(sendBuffer); - saea.RemoteEndPoint = receiver1.LocalEndPoint; - saea.Completed += delegate { mres.Set(); }; - if (sender.SendToAsync(saea)) - { - // did not finish synchronously. - mres.Wait(); - } - - SocketReceiveFromResult result = await receiver1.ReceiveFromAsync(receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); - Assert.Equal(sendBuffer.Length, result.ReceivedBytes); - mres.Reset(); - - - saea.RemoteEndPoint = receiver2.LocalEndPoint; - if (sender.SendToAsync(saea)) - { - // did not finish synchronously. - mres.Wait(); - } - - result = await receiver2.ReceiveFromAsync(receiveBuffer, remoteEp).WaitAsync(TestSettings.PassingTestTimeout); - Assert.Equal(sendBuffer.Length, result.ReceivedBytes); - } } } diff --git a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs index b18338d3e98664..53f43c3c8592f0 100644 --- a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs +++ b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserInterop.cs @@ -53,12 +53,14 @@ public static int GetReadyState(JSObject? webSocket) return -1; } - return BrowserInterop.WebSocketGetState(webSocket); - } + int? readyState = webSocket.GetPropertyAsInt32("readyState"); + if (!readyState.HasValue) + { + return -1; + } - [JSImport("INTERNAL.ws_get_state")] - public static partial int WebSocketGetState( - JSObject webSocket); + return readyState.Value; + } [JSImport("INTERNAL.ws_wasm_create")] public static partial JSObject WebSocketCreate( diff --git a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs index d21e80ba41fee5..6134d85656cc2c 100644 --- a/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs +++ b/src/libraries/System.Net.WebSockets.Client/src/System/Net/WebSockets/BrowserWebSockets/BrowserWebSocket.cs @@ -135,10 +135,9 @@ public override string? SubProtocol internal Task ConnectAsync(Uri uri, List? requestedSubProtocols, CancellationToken cancellationToken) { - AbortIfCancelationRequested(cancellationToken); - lock (_lockObject) { + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); if (FastState != WebSocketState.None) @@ -185,6 +184,7 @@ public override Task CloseOutputAsync(WebSocketCloseStatus closeStatus, string? // this validation should be synchronous WebSocketValidate.ValidateCloseStatus(closeStatus, statusDescription); + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); return CloseAsyncCore(closeStatus, statusDescription, false, cancellationToken); @@ -195,6 +195,7 @@ public override Task CloseAsync(WebSocketCloseStatus closeStatus, string? status // this validation should be synchronous WebSocketValidate.ValidateCloseStatus(closeStatus, statusDescription); + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); return CloseAsyncCore(closeStatus, statusDescription, true, cancellationToken); @@ -283,17 +284,6 @@ private void ThrowIfDisposed() } // lock } - private void AbortIfCancelationRequested(CancellationToken cancellationToken) - { - if (cancellationToken.IsCancellationRequested) - { - lock (_lockObject) - { - Abort(); - } // lock - cancellationToken.ThrowIfCancellationRequested(); - } - } private void CreateCore(Uri uri, List? requestedSubProtocols) { @@ -376,6 +366,7 @@ private async Task SendAsyncCore(ArraySegment buffer, WebSocketMessageType { lock (_lockObject) { + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); previousState = FastState; @@ -383,7 +374,6 @@ private async Task SendAsyncCore(ArraySegment buffer, WebSocketMessageType { throw new InvalidOperationException(SR.net_WebSockets_NotConnected); } - AbortIfCancelationRequested(cancellationToken); if (buffer.Count == 0) { @@ -426,6 +416,7 @@ private async Task ReceiveAsyncCore(ArraySegment b { lock (_lockObject) { + cancellationToken.ThrowIfCancellationRequested(); ThrowIfDisposed(); previousState = FastState; @@ -433,7 +424,6 @@ private async Task ReceiveAsyncCore(ArraySegment b { throw new WebSocketException(WebSocketError.InvalidState, SR.Format(SR.net_WebSockets_InvalidState, previousState, "Open, CloseSent")); } - AbortIfCancelationRequested(cancellationToken); Memory bufferMemory = buffer.AsMemory(); pinBuffer = bufferMemory.Pin(); @@ -512,22 +502,22 @@ private async Task CloseAsyncCore(WebSocketCloseStatus closeStatus, string? stat WebSocketState previousState; lock (_lockObject) { + cancellationToken.ThrowIfCancellationRequested(); + previousState = FastState; if (_aborted) { return; } - if (previousState == WebSocketState.None || previousState == WebSocketState.Closed) - { - throw new WebSocketException(WebSocketError.InvalidState, SR.Format(SR.net_WebSockets_InvalidState, previousState, "Connecting, Open, CloseSent, CloseReceived, Aborted")); - } - AbortIfCancelationRequested(cancellationToken); - if (!_closeReceived) { _closeStatus = closeStatus; _closeStatusDescription = statusDescription; } + if (previousState == WebSocketState.None || previousState == WebSocketState.Closed) + { + throw new WebSocketException(WebSocketError.InvalidState, SR.Format(SR.net_WebSockets_InvalidState, previousState, "Connecting, Open, CloseSent, Aborted")); + } _closeSent = true; @@ -554,6 +544,7 @@ private async Task CancellationHelper(Task promise, CancellationToken cancellati { try { + cancellationToken.ThrowIfCancellationRequested(); if (promise.IsCompletedSuccessfully) { disposable?.Dispose(); @@ -565,7 +556,6 @@ private async Task CancellationHelper(Task promise, CancellationToken cancellati await promise.ConfigureAwait(false); return; } - AbortIfCancelationRequested(cancellationToken); using (var receiveRegistration = cancellationToken.Register(static s => { diff --git a/src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs b/src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs deleted file mode 100644 index 0aa83697a9de74..00000000000000 --- a/src/libraries/System.Net.WebSockets.Client/tests/AbortTest.Loopback.cs +++ /dev/null @@ -1,246 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Xunit; -using Xunit.Abstractions; - -namespace System.Net.WebSockets.Client.Tests -{ - [ConditionalClass(typeof(ClientWebSocketTestBase), nameof(WebSocketsSupported))] - [SkipOnPlatform(TestPlatforms.Browser, "System.Net.Sockets are not supported on browser")] - public abstract class AbortTest_Loopback : ClientWebSocketTestBase - { - public AbortTest_Loopback(ITestOutputHelper output) : base(output) { } - - protected virtual Version HttpVersion => Net.HttpVersion.Version11; - - [Theory] - [MemberData(nameof(AbortClient_MemberData))] - public Task AbortClient_ServerGetsCorrectException(AbortType abortType, bool useSsl, bool verifySendReceive) - { - var clientMsg = new byte[] { 1, 2, 3, 4, 5, 6 }; - var serverMsg = new byte[] { 42 }; - var clientAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var serverAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - var timeoutCts = new CancellationTokenSource(TimeOutMilliseconds); - - return LoopbackWebSocketServer.RunAsync( - async (clientWebSocket, token) => - { - if (verifySendReceive) - { - await VerifySendReceiveAsync(clientWebSocket, clientMsg, serverMsg, clientAckTcs, serverAckTcs.Task, token); - } - - switch (abortType) - { - case AbortType.Abort: - clientWebSocket.Abort(); - break; - case AbortType.Dispose: - clientWebSocket.Dispose(); - break; - } - }, - async (serverWebSocket, token) => - { - if (verifySendReceive) - { - await VerifySendReceiveAsync(serverWebSocket, serverMsg, clientMsg, serverAckTcs, clientAckTcs.Task, token); - } - - var readBuffer = new byte[1]; - var exception = await Assert.ThrowsAsync(async () => - await serverWebSocket.ReceiveAsync(readBuffer, token)); - - Assert.Equal(WebSocketError.ConnectionClosedPrematurely, exception.WebSocketErrorCode); - Assert.Equal(WebSocketState.Aborted, serverWebSocket.State); - }, - new LoopbackWebSocketServer.Options(HttpVersion, useSsl, GetInvoker()), - timeoutCts.Token); - } - - [Theory] - [MemberData(nameof(ServerPrematureEos_MemberData))] - public Task ServerPrematureEos_ClientGetsCorrectException(ServerEosType serverEosType, bool useSsl) - { - var clientMsg = new byte[] { 1, 2, 3, 4, 5, 6 }; - var serverMsg = new byte[] { 42 }; - var clientAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var serverAckTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - var timeoutCts = new CancellationTokenSource(TimeOutMilliseconds); - - var globalOptions = new LoopbackWebSocketServer.Options(HttpVersion, useSsl, HttpInvoker: null) - { - DisposeServerWebSocket = false, - ManualServerHandshakeResponse = true - }; - - var serverReceivedEosTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - var clientReceivedEosTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - - return LoopbackWebSocketServer.RunAsync( - async uri => - { - var token = timeoutCts.Token; - var clientOptions = globalOptions with { HttpInvoker = GetInvoker() }; - var clientWebSocket = await LoopbackWebSocketServer.GetConnectedClientAsync(uri, clientOptions, token).ConfigureAwait(false); - - if (serverEosType == ServerEosType.AfterSomeData) - { - await VerifySendReceiveAsync(clientWebSocket, clientMsg, serverMsg, clientAckTcs, serverAckTcs.Task, token).ConfigureAwait(false); - } - - // only one side of the stream was closed. the other should work - await clientWebSocket.SendAsync(clientMsg, WebSocketMessageType.Binary, endOfMessage: true, token).ConfigureAwait(false); - - var exception = await Assert.ThrowsAsync(() => clientWebSocket.ReceiveAsync(new byte[1], token)); - Assert.Equal(WebSocketError.ConnectionClosedPrematurely, exception.WebSocketErrorCode); - - clientReceivedEosTcs.SetResult(); - clientWebSocket.Dispose(); - }, - async (requestData, token) => - { - WebSocket serverWebSocket = null!; - await SendServerResponseAndEosAsync( - requestData, - serverEosType, - (wsData, ct) => - { - var wsOptions = new WebSocketCreationOptions { IsServer = true }; - serverWebSocket = WebSocket.CreateFromStream(wsData.WebSocketStream, wsOptions); - - return serverEosType == ServerEosType.AfterSomeData - ? VerifySendReceiveAsync(serverWebSocket, serverMsg, clientMsg, serverAckTcs, clientAckTcs.Task, ct) - : Task.CompletedTask; - }, - token); - - Assert.NotNull(serverWebSocket); - - // only one side of the stream was closed. the other should work - var readBuffer = new byte[clientMsg.Length]; - var result = await serverWebSocket.ReceiveAsync(readBuffer, token); - Assert.Equal(WebSocketMessageType.Binary, result.MessageType); - Assert.Equal(clientMsg.Length, result.Count); - Assert.True(result.EndOfMessage); - Assert.Equal(clientMsg, readBuffer); - - await clientReceivedEosTcs.Task.WaitAsync(token).ConfigureAwait(false); - - var exception = await Assert.ThrowsAsync(() => serverWebSocket.ReceiveAsync(readBuffer, token)); - Assert.Equal(WebSocketError.ConnectionClosedPrematurely, exception.WebSocketErrorCode); - - serverWebSocket.Dispose(); - }, - globalOptions, - timeoutCts.Token); - } - - protected virtual Task SendServerResponseAndEosAsync(WebSocketRequestData requestData, ServerEosType serverEosType, Func serverFunc, CancellationToken cancellationToken) - => WebSocketHandshakeHelper.SendHttp11ServerResponseAndEosAsync(requestData, serverFunc, cancellationToken); // override for HTTP/2 - - private static readonly bool[] Bool_Values = new[] { false, true }; - private static readonly bool[] UseSsl_Values = PlatformDetection.SupportsAlpn ? Bool_Values : new[] { false }; - - public static IEnumerable AbortClient_MemberData() - { - foreach (var abortType in Enum.GetValues()) - { - foreach (var useSsl in UseSsl_Values) - { - foreach (var verifySendReceive in Bool_Values) - { - yield return new object[] { abortType, useSsl, verifySendReceive }; - } - } - } - } - - public static IEnumerable ServerPrematureEos_MemberData() - { - foreach (var serverEosType in Enum.GetValues()) - { - foreach (var useSsl in UseSsl_Values) - { - yield return new object[] { serverEosType, useSsl }; - } - } - } - - public enum AbortType - { - Abort, - Dispose - } - - public enum ServerEosType - { - WithHeaders, - RightAfterHeaders, - AfterSomeData - } - - private static async Task VerifySendReceiveAsync(WebSocket ws, byte[] localMsg, byte[] remoteMsg, - TaskCompletionSource localAckTcs, Task remoteAck, CancellationToken cancellationToken) - { - var sendTask = ws.SendAsync(localMsg, WebSocketMessageType.Binary, endOfMessage: true, cancellationToken); - - var recvBuf = new byte[remoteMsg.Length * 2]; - var recvResult = await ws.ReceiveAsync(recvBuf, cancellationToken).ConfigureAwait(false); - - Assert.Equal(WebSocketMessageType.Binary, recvResult.MessageType); - Assert.Equal(remoteMsg.Length, recvResult.Count); - Assert.True(recvResult.EndOfMessage); - Assert.Equal(remoteMsg, recvBuf[..recvResult.Count]); - - localAckTcs.SetResult(); - - await sendTask.ConfigureAwait(false); - await remoteAck.WaitAsync(cancellationToken).ConfigureAwait(false); - } - } - - // --- HTTP/1.1 WebSocket loopback tests --- - - public class AbortTest_Invoker_Loopback : AbortTest_Loopback - { - public AbortTest_Invoker_Loopback(ITestOutputHelper output) : base(output) { } - protected override bool UseCustomInvoker => true; - } - - public class AbortTest_HttpClient_Loopback : AbortTest_Loopback - { - public AbortTest_HttpClient_Loopback(ITestOutputHelper output) : base(output) { } - protected override bool UseHttpClient => true; - } - - public class AbortTest_SharedHandler_Loopback : AbortTest_Loopback - { - public AbortTest_SharedHandler_Loopback(ITestOutputHelper output) : base(output) { } - } - - // --- HTTP/2 WebSocket loopback tests --- - - public class AbortTest_Invoker_Http2 : AbortTest_Invoker_Loopback - { - public AbortTest_Invoker_Http2(ITestOutputHelper output) : base(output) { } - protected override Version HttpVersion => Net.HttpVersion.Version20; - protected override Task SendServerResponseAndEosAsync(WebSocketRequestData rd, ServerEosType eos, Func callback, CancellationToken ct) - => WebSocketHandshakeHelper.SendHttp2ServerResponseAndEosAsync(rd, eosInHeadersFrame: eos == ServerEosType.WithHeaders, callback, ct); - } - - public class AbortTest_HttpClient_Http2 : AbortTest_HttpClient_Loopback - { - public AbortTest_HttpClient_Http2(ITestOutputHelper output) : base(output) { } - protected override Version HttpVersion => Net.HttpVersion.Version20; - protected override Task SendServerResponseAndEosAsync(WebSocketRequestData rd, ServerEosType eos, Func callback, CancellationToken ct) - => WebSocketHandshakeHelper.SendHttp2ServerResponseAndEosAsync(rd, eosInHeadersFrame: eos == ServerEosType.WithHeaders, callback, ct); - } -} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs b/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs index f4690cf9a6a1c1..c768fab5972a8a 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs +++ b/src/libraries/System.Net.WebSockets.Client/tests/CloseTest.cs @@ -8,7 +8,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using System.Linq; using Xunit; using Xunit.Abstractions; @@ -363,43 +362,6 @@ await cws.SendAsync( } } - public static IEnumerable EchoServersSyncState => - EchoServers.SelectMany(server => new List - { - new object[] { server[0], true }, - new object[] { server[0], false } - }); - - [ActiveIssue("https://github.com/dotnet/runtime/issues/28957", typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] - [ConditionalTheory(nameof(WebSocketsSupported)), MemberData(nameof(EchoServersSyncState))] - public async Task CloseOutputAsync_ServerInitiated_CanReceiveAfterClose(Uri server, bool syncState) - { - using (ClientWebSocket cws = await GetConnectedWebSocket(server, TimeOutMilliseconds, _output)) - { - var cts = new CancellationTokenSource(TimeOutMilliseconds); - await cws.SendAsync( - WebSocketData.GetBufferFromText(".receiveMessageAfterClose"), - WebSocketMessageType.Text, - true, - cts.Token); - - await Task.Delay(2000); - - if (syncState) - { - var state = cws.State; - Assert.Equal(WebSocketState.Open, state); - // should be able to receive after this sync - } - - var recvBuffer = new ArraySegment(new byte[1024]); - WebSocketReceiveResult recvResult = await cws.ReceiveAsync(recvBuffer, cts.Token); - var message = Encoding.UTF8.GetString(recvBuffer.ToArray(), 0, recvResult.Count); - - Assert.Contains(".receiveMessageAfterClose", message); - } - } - [OuterLoop("Uses external servers", typeof(PlatformDetection), nameof(PlatformDetection.LocalEchoServerIsNotAvailable))] [ConditionalTheory(nameof(WebSocketsSupported)), MemberData(nameof(EchoServers))] public async Task CloseOutputAsync_CloseDescriptionIsNull_Success(Uri server) diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs index cee509ee068467..48d167b072f781 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs +++ b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackHelper.cs @@ -28,7 +28,14 @@ public static async Task> WebSocketHandshakeAsync(Loo if (headerName == "Sec-WebSocket-Key") { string headerValue = tokens[1].Trim(); - serverResponse = GetServerResponseString(headerValue, extensions); + string responseSecurityAcceptValue = ComputeWebSocketHandshakeSecurityAcceptValue(headerValue); + serverResponse = + "HTTP/1.1 101 Switching Protocols\r\n" + + "Content-Length: 0\r\n" + + "Upgrade: websocket\r\n" + + "Connection: Upgrade\r\n" + + (extensions is null ? null : $"Sec-WebSocket-Extensions: {extensions}\r\n") + + "Sec-WebSocket-Accept: " + responseSecurityAcceptValue + "\r\n\r\n"; } } } @@ -43,18 +50,6 @@ public static async Task> WebSocketHandshakeAsync(Loo return null; } - public static string GetServerResponseString(string secWebSocketKey, string? extensions = null) - { - var responseSecurityAcceptValue = ComputeWebSocketHandshakeSecurityAcceptValue(secWebSocketKey); - return - "HTTP/1.1 101 Switching Protocols\r\n" + - "Content-Length: 0\r\n" + - "Upgrade: websocket\r\n" + - "Connection: Upgrade\r\n" + - (extensions is null ? null : $"Sec-WebSocket-Extensions: {extensions}\r\n") + - "Sec-WebSocket-Accept: " + responseSecurityAcceptValue + "\r\n\r\n"; - } - private static string ComputeWebSocketHandshakeSecurityAcceptValue(string secWebSocketKey) { // GUID specified by RFC 6455. diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs deleted file mode 100644 index 1b3b51840ec995..00000000000000 --- a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/Http2LoopbackStream.cs +++ /dev/null @@ -1,100 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.IO; -using System.Net.Sockets; -using System.Threading; -using System.Threading.Tasks; -using Xunit; - -namespace System.Net.Test.Common -{ - public class Http2LoopbackStream : Stream - { - private readonly Http2LoopbackConnection _connection; - private readonly int _streamId; - private bool _readEnded; - private ReadOnlyMemory _leftoverReadData; - - public override bool CanRead => true; - public override bool CanSeek => false; - public override bool CanWrite => true; - - public Http2LoopbackConnection Connection => _connection; - public int StreamId => _streamId; - - public Http2LoopbackStream(Http2LoopbackConnection connection, int streamId) - { - _connection = connection; - _streamId = streamId; - } - - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) - { - if (!_leftoverReadData.IsEmpty) - { - int read = Math.Min(buffer.Length, _leftoverReadData.Length); - _leftoverReadData.Span.Slice(0, read).CopyTo(buffer.Span); - _leftoverReadData = _leftoverReadData.Slice(read); - return read; - } - - if (_readEnded) - { - return 0; - } - - DataFrame dataFrame = (DataFrame)await _connection.ReadFrameAsync(cancellationToken); - Assert.Equal(_streamId, dataFrame.StreamId); - _leftoverReadData = dataFrame.Data; - _readEnded = dataFrame.EndStreamFlag; - - return await ReadAsync(buffer, cancellationToken); - } - - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => - ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); - - public override async ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) - { - await _connection.SendResponseDataAsync(_streamId, buffer, endStream: false); - } - - public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => - WriteAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); - - protected override void Dispose(bool disposing) => DisposeAsync().GetAwaiter().GetResult(); - - public override async ValueTask DisposeAsync() - { - try - { - await _connection.SendResponseDataAsync(_streamId, Memory.Empty, endStream: true).ConfigureAwait(false); - - if (!_readEnded) - { - var rstFrame = new RstStreamFrame(FrameFlags.None, (int)ProtocolErrors.NO_ERROR, _streamId); - await _connection.WriteFrameAsync(rstFrame).ConfigureAwait(false); - } - } - catch (IOException) - { - // Ignore connection errors - } - catch (SocketException) - { - // Ignore connection errors - } - } - - public override void Flush() { } - public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; - - public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException(); - public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); - public override void SetLength(long value) => throw new NotImplementedException(); - public override void Write(byte[] buffer, int offset, int count) => throw new NotImplementedException(); - public override long Length => throw new NotImplementedException(); - public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - } -} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs deleted file mode 100644 index b24e2e20d40dfe..00000000000000 --- a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/LoopbackWebSocketServer.cs +++ /dev/null @@ -1,148 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Net.Http; -using System.Net.Test.Common; -using System.Threading; -using System.Threading.Tasks; -using Xunit; - -namespace System.Net.WebSockets.Client.Tests -{ - public static class LoopbackWebSocketServer - { - public static Task RunAsync( - Func clientWebSocketFunc, - Func serverWebSocketFunc, - Options options, - CancellationToken cancellationToken) - { - Assert.False(options.ManualServerHandshakeResponse, "Not supported in this overload"); - - return RunAsyncPrivate( - uri => RunClientAsync(uri, clientWebSocketFunc, options, cancellationToken), - (requestData, token) => RunServerAsync(requestData, serverWebSocketFunc, options, token), - options, - cancellationToken); - } - - public static Task RunAsync( - Func loopbackClientFunc, - Func loopbackServerFunc, - Options options, - CancellationToken cancellationToken) - { - Assert.False(options.DisposeClientWebSocket, "Not supported in this overload"); - Assert.False(options.DisposeServerWebSocket, "Not supported in this overload"); - Assert.False(options.DisposeHttpInvoker, "Not supported in this overload"); - Assert.Null(options.HttpInvoker); // Not supported in this overload - - return RunAsyncPrivate(loopbackClientFunc, loopbackServerFunc, options, cancellationToken); - } - - private static Task RunAsyncPrivate( - Func loopbackClientFunc, - Func loopbackServerFunc, - Options options, - CancellationToken cancellationToken) - { - bool sendDefaultServerHandshakeResponse = !options.ManualServerHandshakeResponse; - if (options.HttpVersion == HttpVersion.Version11) - { - return LoopbackServer.CreateClientAndServerAsync( - loopbackClientFunc, - async server => - { - await server.AcceptConnectionAsync(async connection => - { - var requestData = await WebSocketHandshakeHelper.ProcessHttp11RequestAsync(connection, sendDefaultServerHandshakeResponse, cancellationToken).ConfigureAwait(false); - await loopbackServerFunc(requestData, cancellationToken).ConfigureAwait(false); - }); - }, - new LoopbackServer.Options { WebSocketEndpoint = true, UseSsl = options.UseSsl }); - } - else if (options.HttpVersion == HttpVersion.Version20) - { - return Http2LoopbackServer.CreateClientAndServerAsync( - loopbackClientFunc, - async server => - { - var requestData = await WebSocketHandshakeHelper.ProcessHttp2RequestAsync(server, sendDefaultServerHandshakeResponse, cancellationToken).ConfigureAwait(false); - var http2Connection = requestData.Http2Connection!; - var http2StreamId = requestData.Http2StreamId.Value; - - await loopbackServerFunc(requestData, cancellationToken).ConfigureAwait(false); - - await http2Connection.DisposeAsync().ConfigureAwait(false); - }, - new Http2Options { WebSocketEndpoint = true, UseSsl = options.UseSsl }); - } - else - { - throw new ArgumentException(nameof(options.HttpVersion)); - } - } - - private static async Task RunServerAsync( - WebSocketRequestData requestData, - Func serverWebSocketFunc, - Options options, - CancellationToken cancellationToken) - { - var wsOptions = new WebSocketCreationOptions { IsServer = true }; - var serverWebSocket = WebSocket.CreateFromStream(requestData.WebSocketStream, wsOptions); - - await serverWebSocketFunc(serverWebSocket, cancellationToken).ConfigureAwait(false); - - if (options.DisposeServerWebSocket) - { - serverWebSocket.Dispose(); - } - } - - private static async Task RunClientAsync( - Uri uri, - Func clientWebSocketFunc, - Options options, - CancellationToken cancellationToken) - { - var clientWebSocket = await GetConnectedClientAsync(uri, options, cancellationToken).ConfigureAwait(false); - - await clientWebSocketFunc(clientWebSocket, cancellationToken).ConfigureAwait(false); - - if (options.DisposeClientWebSocket) - { - clientWebSocket.Dispose(); - } - - if (options.DisposeHttpInvoker) - { - options.HttpInvoker?.Dispose(); - } - } - - public static async Task GetConnectedClientAsync(Uri uri, Options options, CancellationToken cancellationToken) - { - var clientWebSocket = new ClientWebSocket(); - clientWebSocket.Options.HttpVersion = options.HttpVersion; - clientWebSocket.Options.HttpVersionPolicy = HttpVersionPolicy.RequestVersionExact; - - if (options.UseSsl && options.HttpInvoker is null) - { - clientWebSocket.Options.RemoteCertificateValidationCallback = delegate { return true; }; - } - - await clientWebSocket.ConnectAsync(uri, options.HttpInvoker, cancellationToken).ConfigureAwait(false); - - return clientWebSocket; - } - - public record class Options(Version HttpVersion, bool UseSsl, HttpMessageInvoker? HttpInvoker) - { - public bool DisposeServerWebSocket { get; set; } = true; - public bool DisposeClientWebSocket { get; set; } - public bool DisposeHttpInvoker { get; set; } - public bool ManualServerHandshakeResponse { get; set; } - } - } -} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs deleted file mode 100644 index 2a8c84e7de8eaf..00000000000000 --- a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketHandshakeHelper.cs +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Net.Sockets; -using System.Net.Test.Common; -using System.Threading; -using System.Threading.Tasks; -using Xunit; - -namespace System.Net.WebSockets.Client.Tests -{ - public static class WebSocketHandshakeHelper - { - public static async Task ProcessHttp11RequestAsync(LoopbackServer.Connection connection, bool sendServerResponse = true, CancellationToken cancellationToken = default) - { - List headers = await connection.ReadRequestHeaderAsync().WaitAsync(cancellationToken).ConfigureAwait(false); - - var data = new WebSocketRequestData() - { - HttpVersion = HttpVersion.Version11, - Http11Connection = connection - }; - - foreach (string header in headers.Skip(1)) - { - string[] tokens = header.Split(new char[] { ':' }, StringSplitOptions.RemoveEmptyEntries); - if (tokens.Length is 1 or 2) - { - data.Headers.Add( - tokens[0].Trim(), - tokens.Length == 2 ? tokens[1].Trim() : null); - } - } - - var isValidOpeningHandshake = data.Headers.TryGetValue("Sec-WebSocket-Key", out var secWebSocketKey); - Assert.True(isValidOpeningHandshake); - - if (sendServerResponse) - { - await SendHttp11ServerResponseAsync(connection, secWebSocketKey, cancellationToken).ConfigureAwait(false); - } - - data.WebSocketStream = connection.Stream; - return data; - } - - private static async Task SendHttp11ServerResponseAsync(LoopbackServer.Connection connection, string secWebSocketKey, CancellationToken cancellationToken) - { - var serverResponse = LoopbackHelper.GetServerResponseString(secWebSocketKey); - await connection.WriteStringAsync(serverResponse).WaitAsync(cancellationToken).ConfigureAwait(false); - } - - public static async Task ProcessHttp2RequestAsync(Http2LoopbackServer server, bool sendServerResponse = true, CancellationToken cancellationToken = default) - { - var connection = await server.EstablishConnectionAsync(new SettingsEntry { SettingId = SettingId.EnableConnect, Value = 1 }) - .WaitAsync(cancellationToken).ConfigureAwait(false); - connection.IgnoreWindowUpdates(); - - (int streamId, var httpRequestData) = await connection.ReadAndParseRequestHeaderAsync(readBody: false) - .WaitAsync(cancellationToken).ConfigureAwait(false); - - var data = new WebSocketRequestData - { - HttpVersion = HttpVersion.Version20, - Http2Connection = connection, - Http2StreamId = streamId - }; - - foreach (var header in httpRequestData.Headers) - { - Assert.NotNull(header.Name); - data.Headers.Add(header.Name, header.Value); - } - - var isValidOpeningHandshake = httpRequestData.Method == HttpMethod.Connect.ToString() && data.Headers.ContainsKey(":protocol"); - Assert.True(isValidOpeningHandshake); - - if (sendServerResponse) - { - await SendHttp2ServerResponseAsync(connection, streamId, cancellationToken: cancellationToken).ConfigureAwait(false); - } - - data.WebSocketStream = new Http2LoopbackStream(connection, streamId); - return data; - } - - private static async Task SendHttp2ServerResponseAsync(Http2LoopbackConnection connection, int streamId, bool endStream = false, CancellationToken cancellationToken = default) - { - // send status 200 OK to establish websocket - // we don't need to send anything additional as Sec-WebSocket-Key is not used for HTTP/2 - // note: endStream=true is abnormal and used for testing premature EOS scenarios only - await connection.SendResponseHeadersAsync(streamId, endStream: endStream).WaitAsync(cancellationToken).ConfigureAwait(false); - } - - public static async Task SendHttp11ServerResponseAndEosAsync(WebSocketRequestData requestData, Func? requestDataCallback, CancellationToken cancellationToken) - { - Assert.Equal(HttpVersion.Version11, requestData.HttpVersion); - - // sending default handshake response - await SendHttp11ServerResponseAsync(requestData.Http11Connection!, requestData.Headers["Sec-WebSocket-Key"], cancellationToken).ConfigureAwait(false); - - if (requestDataCallback is not null) - { - await requestDataCallback(requestData, cancellationToken).ConfigureAwait(false); - } - - // send server EOS (half-closing from server side) - requestData.Http11Connection!.Socket.Shutdown(SocketShutdown.Send); - } - - public static async Task SendHttp2ServerResponseAndEosAsync(WebSocketRequestData requestData, bool eosInHeadersFrame, Func? requestDataCallback, CancellationToken cancellationToken) - { - Assert.Equal(HttpVersion.Version20, requestData.HttpVersion); - - var connection = requestData.Http2Connection!; - var streamId = requestData.Http2StreamId!.Value; - - await SendHttp2ServerResponseAsync(connection, streamId, endStream: eosInHeadersFrame, cancellationToken).ConfigureAwait(false); - - if (requestDataCallback is not null) - { - await requestDataCallback(requestData, cancellationToken).ConfigureAwait(false); - } - - if (!eosInHeadersFrame) - { - // send server EOS (half-closing from server side) - await connection.SendResponseDataAsync(streamId, Array.Empty(), endStream: true).ConfigureAwait(false); - } - } - } -} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs b/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs deleted file mode 100644 index 799157a370f073..00000000000000 --- a/src/libraries/System.Net.WebSockets.Client/tests/LoopbackServer/WebSocketRequestData.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.IO; -using System.Net.Test.Common; - -namespace System.Net.WebSockets.Client.Tests -{ - public class WebSocketRequestData - { - public Dictionary Headers { get; set; } = new Dictionary(); - public Stream? WebSocketStream { get; set; } - - public Version HttpVersion { get; set; } - public LoopbackServer.Connection? Http11Connection { get; set; } - public Http2LoopbackConnection? Http2Connection { get; set; } - public int? Http2StreamId { get; set; } - } -} diff --git a/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj b/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj index 0292fd4c043532..7bb14097848cb5 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj +++ b/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj @@ -52,7 +52,6 @@ - @@ -62,10 +61,6 @@ - - - - diff --git a/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj b/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj index f5bb6c851dfb00..ae18ca1b2d9398 100644 --- a/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj +++ b/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj @@ -1,4 +1,4 @@ - + $(NetCoreAppCurrent);$(NetCoreAppPrevious);$(NetCoreAppMinimum);netstandard2.0;$(NetFrameworkMinimum) @@ -15,108 +15,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IAggregationOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IAggregationOperator.cs deleted file mode 100644 index b91d9d2038feb5..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IAggregationOperator.cs +++ /dev/null @@ -1,2490 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// that specializes horizontal aggregation of all elements in a vector. - private interface IAggregationOperator : IBinaryOperator - { - static abstract T Invoke(Vector128 x); - static abstract T Invoke(Vector256 x); - static abstract T Invoke(Vector512 x); - - static virtual T IdentityValue => throw new NotSupportedException(); - } - - /// Performs an aggregation over all elements in to produce a single-precision floating-point value. - /// The element type. - /// Specifies the transform operation that should be applied to each element loaded from . - /// - /// Specifies the aggregation binary operation that should be applied to multiple values to aggregate them into a single value. - /// The aggregation is applied after the transform is applied to each element. - /// - private static T Aggregate( - ReadOnlySpan x) - where TTransformOperator : struct, IUnaryOperator - where TAggregationOperator : struct, IAggregationOperator - { - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TTransformOperator.Vectorizable) - { - T result; - - if (remainder >= (uint)Vector512.Count) - { - result = Vectorized512(ref xRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - result = VectorizedSmall(ref xRef, remainder); - } - - return result; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TTransformOperator.Vectorizable) - { - T result; - - if (remainder >= (uint)Vector256.Count) - { - result = Vectorized256(ref xRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - result = VectorizedSmall(ref xRef, remainder); - } - - return result; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TTransformOperator.Vectorizable) - { - T result; - - if (remainder >= (uint)Vector128.Count) - { - result = Vectorized128(ref xRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - result = VectorizedSmall(ref xRef, remainder); - } - - return result; - } - - // This is the software fallback when no acceleration is available. - // It requires no branches to hit. - - return SoftwareFallback(ref xRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T SoftwareFallback(ref T xRef, nuint length) - { - T result = TAggregationOperator.IdentityValue; - - for (nuint i = 0; i < length; i++) - { - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, i))); - } - - return result; - } - - static T Vectorized128(ref T xRef, nuint remainder) - { - Vector128 vresult = Vector128.Create(TAggregationOperator.IdentityValue); - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - nuint misalignment = 0; - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - { - T* xPtr = px; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - misalignment = ((uint)sizeof(Vector128) - ((nuint)xPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - - Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - // We only need to load, so there isn't a lot of benefit to doing non-temporal operations - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); - vector2 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); - vector3 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); - vector4 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We load, process, and store the next four vectors - - vector1 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); - vector2 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); - vector3 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); - vector4 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - } - } - - // Store the first block. Handling this separately simplifies the latter code as we know - // they come after and so we can relegate it to full blocks or the trailing elements - - beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, beg); - - // Process the remaining [0, Count * 7] elements via a jump table - // - // We end up handling any trailing elements in case 0 and in the - // worst case end up just doing the identity operation here if there - // were no trailing elements. - - (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector128.Count); - blocks -= (misalignment == 0) ? 1u : 0u; - remainder -= trailing; - - switch (blocks) - { - case 7: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 6; - } - - case 6: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 5; - } - - case 5: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 4; - } - - case 4: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 3; - } - - case 3: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 2; - } - - case 2: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 1; - } - - case 1: - { - Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 1))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 0; - } - - case 0: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, end); - break; - } - } - - return TAggregationOperator.Invoke(vresult); - } - - static T Vectorized256(ref T xRef, nuint remainder) - { - Vector256 vresult = Vector256.Create(TAggregationOperator.IdentityValue); - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - nuint misalignment = 0; - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - { - T* xPtr = px; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - misalignment = ((uint)sizeof(Vector256) - ((nuint)xPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - - Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - // We only need to load, so there isn't a lot of benefit to doing non-temporal operations - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); - vector2 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); - vector3 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); - vector4 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We load, process, and store the next four vectors - - vector1 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); - vector2 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); - vector3 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); - vector4 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - } - } - - // Store the first block. Handling this separately simplifies the latter code as we know - // they come after and so we can relegate it to full blocks or the trailing elements - - beg = Vector256.ConditionalSelect(CreateAlignmentMaskVector256((int)misalignment), beg, Vector256.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, beg); - - // Process the remaining [0, Count * 7] elements via a jump table - // - // We end up handling any trailing elements in case 0 and in the - // worst case end up just doing the identity operation here if there - // were no trailing elements. - - (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector256.Count); - blocks -= (misalignment == 0) ? 1u : 0u; - remainder -= trailing; - - switch (blocks) - { - case 7: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 6; - } - - case 6: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 5; - } - - case 5: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 4; - } - - case 4: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 3; - } - - case 3: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 2; - } - - case 2: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 1; - } - - case 1: - { - Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 1))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 0; - } - - case 0: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)trailing), end, Vector256.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, end); - break; - } - } - - return TAggregationOperator.Invoke(vresult); - } - - static T Vectorized512(ref T xRef, nuint remainder) - { - Vector512 vresult = Vector512.Create(TAggregationOperator.IdentityValue); - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 beg = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef)); - Vector512 end = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)); - - nuint misalignment = 0; - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - { - T* xPtr = px; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - misalignment = ((uint)sizeof(Vector512) - ((nuint)xPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - - Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - // We only need to load, so there isn't a lot of benefit to doing non-temporal operations - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); - vector2 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); - vector3 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); - vector4 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We load, process, and store the next four vectors - - vector1 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); - vector2 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); - vector3 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); - vector4 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - } - } - - // Store the first block. Handling this separately simplifies the latter code as we know - // they come after and so we can relegate it to full blocks or the trailing elements - - beg = Vector512.ConditionalSelect(CreateAlignmentMaskVector512((int)misalignment), beg, Vector512.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, beg); - - // Process the remaining [0, Count * 7] elements via a jump table - // - // We end up handling any trailing elements in case 0 and in the - // worst case end up just doing the identity operation here if there - // were no trailing elements. - - (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector512.Count); - blocks -= (misalignment == 0) ? 1u : 0u; - remainder -= trailing; - - switch (blocks) - { - case 7: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 6; - } - - case 6: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 5; - } - - case 5: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 4; - } - - case 4: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 3; - } - - case 3: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 2; - } - - case 2: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 1; - } - - case 1: - { - Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 1))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 0; - } - - case 0: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end = Vector512.ConditionalSelect(CreateRemainderMaskVector512((int)trailing), end, Vector512.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, end); - break; - } - } - - return TAggregationOperator.Invoke(vresult); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall(ref T xRef, nuint remainder) - { - if (sizeof(T) == 1) - { - return VectorizedSmall1(ref xRef, remainder); - } - else if (sizeof(T) == 2) - { - return VectorizedSmall2(ref xRef, remainder); - } - else if (sizeof(T) == 4) - { - return VectorizedSmall4(ref xRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - return VectorizedSmall8(ref xRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall1(ref T xRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 14))); - goto case 14; - - case 14: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 13))); - goto case 13; - - case 13: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 12))); - goto case 12; - - case 12: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 11))); - goto case 11; - - case 11: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 10))); - goto case 10; - - case 10: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 9))); - goto case 9; - - case 9: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 8))); - goto case 8; - - case 8: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 7))); - goto case 7; - - case 7: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6))); - goto case 6; - - case 6: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5))); - goto case 5; - - case 5: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4))); - goto case 4; - - case 4: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3))); - goto case 3; - - case 3: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2))); - goto case 2; - - case 2: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1))); - goto case 1; - - case 1: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); - goto case 0; - - case 0: - break; - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall2(ref T xRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6))); - goto case 6; - - case 6: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5))); - goto case 5; - - case 5: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4))); - goto case 4; - - case 4: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3))); - goto case 3; - - case 3: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2))); - goto case 2; - - case 2: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1))); - goto case 1; - - case 1: - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); - goto case 0; - - case 0: - break; - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall4(ref T xRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 3: - { - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2))); - goto case 2; - } - - case 2: - { - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1))); - goto case 1; - } - - case 1: - { - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); - goto case 0; - } - - case 0: - { - break; - } - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall8(ref T xRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 1: - { - result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); - goto case 0; - } - - case 0: - { - break; - } - } - - return result; - } - } - - /// Performs an aggregation over all pair-wise elements in and to produce a single-precision floating-point value. - /// The element type. - /// Specifies the binary operation that should be applied to the pair-wise elements loaded from and . - /// - /// Specifies the aggregation binary operation that should be applied to multiple values to aggregate them into a single value. - /// The aggregation is applied to the results of the binary operations on the pair-wise values. - /// - private static T Aggregate( - ReadOnlySpan x, ReadOnlySpan y) - where TBinaryOperator : struct, IBinaryOperator - where TAggregationOperator : struct, IAggregationOperator - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TBinaryOperator.Vectorizable) - { - T result; - - if (remainder >= (uint)Vector512.Count) - { - result = Vectorized512(ref xRef, ref yRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - result = VectorizedSmall(ref xRef, ref yRef, remainder); - } - - return result; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TBinaryOperator.Vectorizable) - { - T result; - - if (remainder >= (uint)Vector256.Count) - { - result = Vectorized256(ref xRef, ref yRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - result = VectorizedSmall(ref xRef, ref yRef, remainder); - } - - return result; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TBinaryOperator.Vectorizable) - { - T result; - - if (remainder >= (uint)Vector128.Count) - { - result = Vectorized128(ref xRef, ref yRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - result = VectorizedSmall(ref xRef, ref yRef, remainder); - } - - return result; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - return SoftwareFallback(ref xRef, ref yRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T SoftwareFallback(ref T xRef, ref T yRef, nuint length) - { - T result = TAggregationOperator.IdentityValue; - - for (nuint i = 0; i < length; i++) - { - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, i), - Unsafe.Add(ref yRef, i))); - } - - return result; - } - - static T Vectorized128(ref T xRef, ref T yRef, nuint remainder) - { - Vector128 vresult = Vector128.Create(TAggregationOperator.IdentityValue); - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - nuint misalignment = 0; - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - { - T* xPtr = px; - T* yPtr = py; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - misalignment = ((uint)sizeof(Vector128) - ((nuint)xPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - - Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - // We only need to load, so there isn't a lot of benefit to doing non-temporal operations - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - } - } - - // Store the first block. Handling this separately simplifies the latter code as we know - // they come after and so we can relegate it to full blocks or the trailing elements - - beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, beg); - - // Process the remaining [0, Count * 7] elements via a jump table - // - // We end up handling any trailing elements in case 0 and in the - // worst case end up just doing the identity operation here if there - // were no trailing elements. - - (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector128.Count); - blocks -= (misalignment == 0) ? 1u : 0u; - remainder -= trailing; - - switch (blocks) - { - case 7: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 6; - } - - case 6: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 5; - } - - case 5: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 4; - } - - case 4: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 3; - } - - case 3: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 2; - } - - case 2: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 1; - } - - case 1: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 1)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 1))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 0; - } - - case 0: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, end); - break; - } - } - - return TAggregationOperator.Invoke(vresult); - } - - static T Vectorized256(ref T xRef, ref T yRef, nuint remainder) - { - Vector256 vresult = Vector256.Create(TAggregationOperator.IdentityValue); - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - nuint misalignment = 0; - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - { - T* xPtr = px; - T* yPtr = py; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - misalignment = ((uint)sizeof(Vector256) - ((nuint)xPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - - Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - // We only need to load, so there isn't a lot of benefit to doing non-temporal operations - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - } - } - - // Store the first block. Handling this separately simplifies the latter code as we know - // they come after and so we can relegate it to full blocks or the trailing elements - - beg = Vector256.ConditionalSelect(CreateAlignmentMaskVector256((int)misalignment), beg, Vector256.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, beg); - - // Process the remaining [0, Count * 7] elements via a jump table - // - // We end up handling any trailing elements in case 0 and in the - // worst case end up just doing the identity operation here if there - // were no trailing elements. - - (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector256.Count); - blocks -= (misalignment == 0) ? 1u : 0u; - remainder -= trailing; - - switch (blocks) - { - case 7: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 6; - } - - case 6: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 5; - } - - case 5: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 4; - } - - case 4: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 3; - } - - case 3: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 2; - } - - case 2: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 1; - } - - case 1: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 1)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 1))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 0; - } - - case 0: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)trailing), end, Vector256.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, end); - break; - } - } - - return TAggregationOperator.Invoke(vresult); - } - - static T Vectorized512(ref T xRef, ref T yRef, nuint remainder) - { - Vector512 vresult = Vector512.Create(TAggregationOperator.IdentityValue); - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 beg = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), - Vector512.LoadUnsafe(ref yRef)); - Vector512 end = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count)); - - nuint misalignment = 0; - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - { - T* xPtr = px; - T* yPtr = py; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - misalignment = ((uint)sizeof(Vector512) - ((nuint)xPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - - Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - // We only need to load, so there isn't a lot of benefit to doing non-temporal operations - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7))); - - vresult = TAggregationOperator.Invoke(vresult, vector1); - vresult = TAggregationOperator.Invoke(vresult, vector2); - vresult = TAggregationOperator.Invoke(vresult, vector3); - vresult = TAggregationOperator.Invoke(vresult, vector4); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - } - } - - // Store the first block. Handling this separately simplifies the latter code as we know - // they come after and so we can relegate it to full blocks or the trailing elements - - beg = Vector512.ConditionalSelect(CreateAlignmentMaskVector512((int)misalignment), beg, Vector512.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, beg); - - // Process the remaining [0, Count * 7] elements via a jump table - // - // We end up handling any trailing elements in case 0 and in the - // worst case end up just doing the identity operation here if there - // were no trailing elements. - - (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector512.Count); - blocks -= (misalignment == 0) ? 1u : 0u; - remainder -= trailing; - - switch (blocks) - { - case 7: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 6; - } - - case 6: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 5; - } - - case 5: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 4; - } - - case 4: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 3; - } - - case 3: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 2; - } - - case 2: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 1; - } - - case 1: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 1)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 1))); - vresult = TAggregationOperator.Invoke(vresult, vector); - goto case 0; - } - - case 0: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end = Vector512.ConditionalSelect(CreateRemainderMaskVector512((int)trailing), end, Vector512.Create(TAggregationOperator.IdentityValue)); - vresult = TAggregationOperator.Invoke(vresult, end); - break; - } - } - - return TAggregationOperator.Invoke(vresult); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall(ref T xRef, ref T yRef, nuint remainder) - { - if (sizeof(T) == 1) - { - return VectorizedSmall1(ref xRef, ref yRef, remainder); - } - else if (sizeof(T) == 2) - { - return VectorizedSmall2(ref xRef, ref yRef, remainder); - } - else if (sizeof(T) == 4) - { - return VectorizedSmall4(ref xRef, ref yRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - return VectorizedSmall8(ref xRef, ref yRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall1(ref T xRef, ref T yRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 14), Unsafe.Add(ref yRef, 14))); - goto case 14; - - case 14: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 13), Unsafe.Add(ref yRef, 13))); - goto case 13; - - case 13: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 12), Unsafe.Add(ref yRef, 12))); - goto case 12; - - case 12: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 11), Unsafe.Add(ref yRef, 11))); - goto case 11; - - case 11: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 10), Unsafe.Add(ref yRef, 10))); - goto case 10; - - case 10: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 9), Unsafe.Add(ref yRef, 9))); - goto case 9; - - case 9: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 8), Unsafe.Add(ref yRef, 8))); - goto case 8; - - case 8: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 7), Unsafe.Add(ref yRef, 7))); - goto case 7; - - case 7: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), Unsafe.Add(ref yRef, 6))); - goto case 6; - - case 6: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), Unsafe.Add(ref yRef, 5))); - goto case 5; - - case 5: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), Unsafe.Add(ref yRef, 4))); - goto case 4; - - case 4: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), Unsafe.Add(ref yRef, 3))); - goto case 3; - - case 3: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), Unsafe.Add(ref yRef, 2))); - goto case 2; - - case 2: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), Unsafe.Add(ref yRef, 1))); - goto case 1; - - case 1: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); - goto case 0; - - case 0: - break; - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall2(ref T xRef, ref T yRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), Unsafe.Add(ref yRef, 6))); - goto case 6; - - case 6: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), Unsafe.Add(ref yRef, 5))); - goto case 5; - - case 5: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), Unsafe.Add(ref yRef, 4))); - goto case 4; - - case 4: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), Unsafe.Add(ref yRef, 3))); - goto case 3; - - case 3: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), Unsafe.Add(ref yRef, 2))); - goto case 2; - - case 2: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), Unsafe.Add(ref yRef, 1))); - goto case 1; - - case 1: - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); - goto case 0; - - case 0: - break; - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall4(ref T xRef, ref T yRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 3: - { - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2))); - goto case 2; - } - - case 2: - { - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1))); - goto case 1; - } - - case 1: - { - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); - goto case 0; - } - - case 0: - { - break; - } - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static T VectorizedSmall8(ref T xRef, ref T yRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - T result = TAggregationOperator.IdentityValue; - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); - - result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - - result = TAggregationOperator.Invoke(beg); - break; - } - - case 1: - { - result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); - goto case 0; - } - - case 0: - { - break; - } - } - - return result; - } - } - - /// - /// Gets a vector mask that will be all-ones-set for the last elements - /// and zero for all other elements. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 CreateAlignmentMaskVector128(int count) - { - if (Unsafe.SizeOf() == 1) - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentByteMask_64x65)), - (uint)(count * 64)); - } - - if (Unsafe.SizeOf() == 2) - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt16Mask_32x33)), - (uint)(count * 32)); - } - - if (Unsafe.SizeOf() == 4) - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt32Mask_16x17)), - (uint)(count * 16)); - } - - Debug.Assert(Unsafe.SizeOf() == 8); - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt64Mask_8x9)), - (uint)(count * 8)); - } - } - - /// - /// Gets a vector mask that will be all-ones-set for the last elements - /// and zero for all other elements. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 CreateAlignmentMaskVector256(int count) - { - if (Unsafe.SizeOf() == 1) - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentByteMask_64x65)), - (uint)(count * 64)); - } - - if (Unsafe.SizeOf() == 2) - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt16Mask_32x33)), - (uint)(count * 32)); - } - - if (Unsafe.SizeOf() == 4) - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt32Mask_16x17)), - (uint)(count * 16)); - } - - Debug.Assert(Unsafe.SizeOf() == 8); - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt64Mask_8x9)), - (uint)(count * 8)); - } - } - - /// - /// Gets a vector mask that will be all-ones-set for the last elements - /// and zero for all other elements. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 CreateAlignmentMaskVector512(int count) - { - if (Unsafe.SizeOf() == 1) - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentByteMask_64x65)), - (uint)(count * 64)); - } - - if (Unsafe.SizeOf() == 2) - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt16Mask_32x33)), - (uint)(count * 32)); - } - - if (Unsafe.SizeOf() == 4) - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt32Mask_16x17)), - (uint)(count * 16)); - } - - Debug.Assert(Unsafe.SizeOf() == 8); - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt64Mask_8x9)), - (uint)(count * 8)); - } - } - - /// - /// Gets a vector mask that will be all-ones-set for the last elements - /// and zero for all other elements. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 CreateRemainderMaskVector128(int count) - { - if (Unsafe.SizeOf() == 1) - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderByteMask_64x65)), - (uint)(count * 64) + 48); // last 16 bytes in the row - } - - if (Unsafe.SizeOf() == 2) - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt16Mask_32x33)), - (uint)(count * 32) + 24); // last 8 shorts in the row - } - - if (Unsafe.SizeOf() == 4) - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt32Mask_16x17)), - (uint)(count * 16) + 12); // last 4 ints in the row - } - - Debug.Assert(Unsafe.SizeOf() == 8); - { - return Vector128.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt64Mask_8x9)), - (uint)(count * 8) + 6); // last 2 longs in the row - } - } - - /// - /// Gets a vector mask that will be all-ones-set for the last elements - /// and zero for all other elements. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 CreateRemainderMaskVector256(int count) - { - if (Unsafe.SizeOf() == 1) - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderByteMask_64x65)), - (uint)(count * 64) + 32); // last 32 bytes in the row - } - - if (Unsafe.SizeOf() == 2) - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt16Mask_32x33)), - (uint)(count * 32) + 16); // last 16 shorts in the row - } - - if (Unsafe.SizeOf() == 4) - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt32Mask_16x17)), - (uint)(count * 16) + 8); // last 8 ints in the row - } - - Debug.Assert(Unsafe.SizeOf() == 8); - { - return Vector256.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt64Mask_8x9)), - (uint)(count * 8) + 4); // last 4 longs in the row - } - } - - /// - /// Gets a vector mask that will be all-ones-set for the last elements - /// and zero for all other elements. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 CreateRemainderMaskVector512(int count) - { - if (Unsafe.SizeOf() == 1) - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderByteMask_64x65)), - (uint)(count * 64)); - } - - if (Unsafe.SizeOf() == 2) - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt16Mask_32x33)), - (uint)(count * 32)); - } - - if (Unsafe.SizeOf() == 4) - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt32Mask_16x17)), - (uint)(count * 16)); - } - - Debug.Assert(Unsafe.SizeOf() == 8); - { - return Vector512.LoadUnsafe( - ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt64Mask_8x9)), - (uint)(count * 8)); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBinaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBinaryOperator.cs deleted file mode 100644 index 0ec37a519ec241..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IBinaryOperator.cs +++ /dev/null @@ -1,2780 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Operator that takes two input values and returns a single value. - private interface IBinaryOperator - { - static abstract bool Vectorizable { get; } - static abstract T Invoke(T x, T y); - static abstract Vector128 Invoke(Vector128 x, Vector128 y); - static abstract Vector256 Invoke(Vector256 x, Vector256 y); - static abstract Vector512 Invoke(Vector512 x, Vector512 y); - } - - /// - /// Performs an element-wise operation on and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on each element loaded from with . - /// - private static void InvokeScalarSpanIntoSpan( - T x, ReadOnlySpan y, Span destination) - where TBinaryOperator : struct, IBinaryOperator => - InvokeSpanScalarIntoSpan, InvertedBinaryOperator>(y, x, destination); - - /// - /// Performs an element-wise operation on and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on each element loaded from with . - /// - private static void InvokeSpanScalarIntoSpan( - ReadOnlySpan x, T y, Span destination) - where TBinaryOperator : struct, IBinaryOperator => - InvokeSpanScalarIntoSpan, TBinaryOperator>(x, y, destination); - - /// - /// Performs an element-wise operation on and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on the pair-wise elements loaded from and . - /// - private static void InvokeSpanSpanIntoSpan( - ReadOnlySpan x, ReadOnlySpan y, Span destination) - where TBinaryOperator : struct, IBinaryOperator - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - ValidateInputOutputSpanNonOverlapping(y, destination); - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - ref T dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TBinaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, ref yRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, ref dRef, remainder); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TBinaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, ref yRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, ref dRef, remainder); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TBinaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, ref yRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, ref dRef, remainder); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, ref yRef, ref dRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref T xRef, ref T yRef, ref T dRef, nuint length) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, i), - Unsafe.Add(ref yRef, i)); - } - } - - static void Vectorized128(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 beg = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), - Vector512.LoadUnsafe(ref yRef)); - Vector512 end = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count)); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0))); - vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1))); - vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2))); - vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4))); - vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5))); - vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6))); - vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - if (sizeof(T) == 1) - { - VectorizedSmall1(ref xRef, ref yRef, ref dRef, remainder); - } - else if (sizeof(T) == 2) - { - VectorizedSmall2(ref xRef, ref yRef, ref dRef, remainder); - } - else if (sizeof(T) == 4) - { - VectorizedSmall4(ref xRef, ref yRef, ref dRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - VectorizedSmall8(ref xRef, ref yRef, ref dRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 14), - Unsafe.Add(ref yRef, 14)); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 13), - Unsafe.Add(ref yRef, 13)); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 12), - Unsafe.Add(ref yRef, 12)); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 11), - Unsafe.Add(ref yRef, 11)); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 10), - Unsafe.Add(ref yRef, 10)); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 9), - Unsafe.Add(ref yRef, 9)); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 8), - Unsafe.Add(ref yRef, 8)); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 7), - Unsafe.Add(ref yRef, 7)); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - Unsafe.Add(ref yRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - Unsafe.Add(ref yRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - Unsafe.Add(ref yRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - Unsafe.Add(ref yRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1)); - goto case 1; - - case 1: - dRef = TBinaryOperator.Invoke(xRef, yRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - Unsafe.Add(ref yRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - Unsafe.Add(ref yRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - Unsafe.Add(ref yRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - Unsafe.Add(ref yRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1)); - goto case 1; - - case 1: - dRef = TBinaryOperator.Invoke(xRef, yRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2)); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1)); - goto case 1; - } - - case 1: - { - dRef = TBinaryOperator.Invoke(xRef, yRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref T xRef, ref T yRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = TBinaryOperator.Invoke(xRef, yRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - - /// - /// Performs an element-wise operation on and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on each element loaded from . - /// It is not used with . - /// - /// - /// Specifies the operation to perform on the transformed value from with . - /// - private static void InvokeSpanScalarIntoSpan( - ReadOnlySpan x, T y, Span destination) - where TTransformOperator : struct, IUnaryOperator - where TBinaryOperator : struct, IBinaryOperator - { - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, y, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, y, ref dRef, remainder); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, y, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, y, ref dRef, remainder); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, y, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, y, ref dRef, remainder); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, y, ref dRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref T xRef, T y, ref T dRef, nuint length) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, i)), - y); - } - } - - static void Vectorized128(ref T xRef, T y, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - yVec); - Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), - yVec); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))), - yVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))), - yVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))), - yVec); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))), - yVec); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref T xRef, T y, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - yVec); - Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), - yVec); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))), - yVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))), - yVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))), - yVec); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))), - yVec); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref T xRef, T y, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 yVec = Vector512.Create(y); - - Vector512 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef)), - yVec); - Vector512 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)), - yVec); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))), - yVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))), - yVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))), - yVec); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))), - yVec); - vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))), - yVec); - vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))), - yVec); - vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))), - yVec); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))), - yVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref T xRef, T y, ref T dRef, nuint remainder) - { - if (sizeof(T) == 1) - { - VectorizedSmall1(ref xRef, y, ref dRef, remainder); - } - else if (sizeof(T) == 2) - { - VectorizedSmall2(ref xRef, y, ref dRef, remainder); - } - else if (sizeof(T) == 4) - { - VectorizedSmall4(ref xRef, y, ref dRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - VectorizedSmall8(ref xRef, y, ref dRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref T xRef, T y, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - yVec); - Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - Vector256.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - yVec); - Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - Vector128.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 14)), - y); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 13)), - y); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 12)), - y); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 11)), - y); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 10)), - y); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 9)), - y); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 8)), - y); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 7)), - y); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6)), - y); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5)), - y); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4)), - y); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3)), - y); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2)), - y); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1)), - y); - goto case 1; - - case 1: - dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref T xRef, T y, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - yVec); - Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - Vector256.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - yVec); - Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - Vector128.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6)), - y); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5)), - y); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4)), - y); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3)), - y); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2)), - y); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1)), - y); - goto case 1; - - case 1: - dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref T xRef, T y, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - yVec); - Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - Vector256.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - yVec); - Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - Vector128.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2)), - y); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1)), - y); - goto case 1; - } - - case 1: - { - dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref T xRef, T y, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - yVec); - Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), - Vector256.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - yVec); - Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), - yVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), - Vector128.Create(y)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - - /// Aggregates all of the elements in the into a single value. - /// The element type. - /// Specifies the operation to be performed on each pair of values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static T HorizontalAggregate(Vector256 x) where TAggregate : struct, IBinaryOperator => - HorizontalAggregate(TAggregate.Invoke(x.GetLower(), x.GetUpper())); - - /// Aggregates all of the elements in the into a single value. - /// The element type. - /// Specifies the operation to be performed on each pair of values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static T HorizontalAggregate(Vector512 x) where TAggregate : struct, IBinaryOperator => - HorizontalAggregate(TAggregate.Invoke(x.GetLower(), x.GetUpper())); - - /// Aggregates all of the elements in the into a single value. - /// The element type. - /// Specifies the operation to be performed on each pair of values. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static T HorizontalAggregate(Vector128 x) where TAggregate : struct, IBinaryOperator - { - // We need to do log2(count) operations to compute the total sum - - if (Unsafe.SizeOf() == 1) - { - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As()); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As()); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As()); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As()); - } - else if (Unsafe.SizeOf() == 2) - { - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As()); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As()); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As()); - } - else if (Unsafe.SizeOf() == 4) - { - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(2, 3, 0, 1)).As()); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(1, 0, 3, 2)).As()); - } - else - { - Debug.Assert(Unsafe.SizeOf() == 8); - x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt64(), Vector128.Create(1, 0)).As()); - } - - return x.ToScalar(); - } - - private readonly struct InvertedBinaryOperator : IBinaryOperator - where TOperator : IBinaryOperator - { - public static bool Vectorizable => TOperator.Vectorizable; - public static T Invoke(T x, T y) => TOperator.Invoke(y, x); - public static Vector128 Invoke(Vector128 x, Vector128 y) => TOperator.Invoke(y, x); - public static Vector256 Invoke(Vector256 x, Vector256 y) => TOperator.Invoke(y, x); - public static Vector512 Invoke(Vector512 x, Vector512 y) => TOperator.Invoke(y, x); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfOperator.cs deleted file mode 100644 index a90b8aaa2a33d4..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IIndexOfOperator.cs +++ /dev/null @@ -1,134 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - private interface IIndexOfOperator - { - static abstract int Invoke(ref T result, T current, int resultIndex, int currentIndex); - static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex); - static abstract void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex); - static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int IndexOfFinalAggregate(Vector128 result, Vector128 resultIndex) - where TIndexOfOperator : struct, IIndexOfOperator - { - Vector128 tmpResult; - Vector128 tmpIndex; - - if (sizeof(T) == 8) - { - // Compare 0 with 1 - tmpResult = Vector128.Shuffle(result.AsInt64(), Vector128.Create(1, 0)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsInt64(), Vector128.Create(1, 0)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Return 0 - return (int)resultIndex.As().ToScalar(); - } - - if (sizeof(T) == 4) - { - // Compare 0,1 with 2,3 - tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(2, 3, 0, 1)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(2, 3, 0, 1)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Compare 0 with 1 - tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(1, 0, 3, 2)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(1, 0, 3, 2)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Return 0 - return resultIndex.As().ToScalar(); - } - - if (sizeof(T) == 2) - { - // Compare 0,1,2,3 with 4,5,6,7 - tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Compare 0,1 with 2,3 - tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Compare 0 with 1 - tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Return 0 - return resultIndex.As().ToScalar(); - } - - Debug.Assert(sizeof(T) == 1); - { - // Compare 0,1,2,3,4,5,6,7 with 8,9,10,11,12,13,14,15 - tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Compare 0,1,2,3 with 4,5,6,7 - tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Compare 0,1 with 2,3 - tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Compare 0 with 1 - tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); - tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); - TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - - // Return 0 - return resultIndex.As().ToScalar(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int IndexOfFinalAggregate(Vector256 result, Vector256 resultIndex) - where TIndexOfOperator : struct, IIndexOfOperator - { - // Min the upper/lower halves of the Vector256 - Vector128 resultLower = result.GetLower(); - Vector128 indexLower = resultIndex.GetLower(); - - TIndexOfOperator.Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); - return IndexOfFinalAggregate(resultLower, indexLower); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int IndexOfFinalAggregate(Vector512 result, Vector512 resultIndex) - where TIndexOfOperator : struct, IIndexOfOperator - { - Vector256 resultLower = result.GetLower(); - Vector256 indexLower = resultIndex.GetLower(); - - TIndexOfOperator.Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); - return IndexOfFinalAggregate(resultLower, indexLower); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 IndexLessThan(Vector128 indices1, Vector128 indices2) => - sizeof(T) == sizeof(long) ? Vector128.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : - sizeof(T) == sizeof(int) ? Vector128.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : - sizeof(T) == sizeof(short) ? Vector128.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : - Vector128.LessThan(indices1.AsByte(), indices2.AsByte()).As(); - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IStatefulUnaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IStatefulUnaryOperator.cs deleted file mode 100644 index fc77a63a4c74a2..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IStatefulUnaryOperator.cs +++ /dev/null @@ -1,1211 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Operator that takes one input value and returns a single value. - private interface IStatefulUnaryOperator - { - static abstract bool Vectorizable { get; } - T Invoke(T x); - Vector128 Invoke(Vector128 x); - Vector256 Invoke(Vector256 x); - Vector512 Invoke(Vector512 x); - } - - /// Performs an element-wise operation on and writes the results to . - /// The element type. - /// Specifies the operation to perform on each element loaded from . - private static void InvokeSpanIntoSpan( - ReadOnlySpan x, TStatefulUnaryOperator op, Span destination) - where TStatefulUnaryOperator : struct, IStatefulUnaryOperator - { - // NOTE: This implementation is an exact copy of InvokeSpanIntoSpan, - // except it accepts an operator that carries state with it, using instance rather than - // static invocation methods. - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TStatefulUnaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, ref dRef, remainder, op); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref dRef, remainder, op); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TStatefulUnaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, ref dRef, remainder, op); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref dRef, remainder, op); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TStatefulUnaryOperator.Vectorizable) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, ref dRef, remainder, op); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref dRef, remainder, op); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, ref dRef, remainder, op); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref T xRef, ref T dRef, nuint length, TStatefulUnaryOperator op) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = op.Invoke(Unsafe.Add(ref xRef, i)); - } - } - - static void Vectorized128(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); - vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); - vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); - vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); - vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); - vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); - vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); - vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); - vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); - vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); - vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); - vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); - vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); - vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); - vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); - vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); - vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); - vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); - vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); - vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); - vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); - vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); - vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); - vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); - vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 beg = op.Invoke(Vector512.LoadUnsafe(ref xRef)); - Vector512 end = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); - vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); - vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); - vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); - vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); - vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); - vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); - vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); - vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); - vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); - vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); - vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); - vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - if (sizeof(T) == 1) - { - VectorizedSmall1(ref xRef, ref dRef, remainder, op); - } - else if (sizeof(T) == 2) - { - VectorizedSmall2(ref xRef, ref dRef, remainder, op); - } - else if (sizeof(T) == 4) - { - VectorizedSmall4(ref xRef, ref dRef, remainder, op); - } - else - { - Debug.Assert(sizeof(T) == 8); - VectorizedSmall8(ref xRef, ref dRef, remainder, op); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - Debug.Assert(sizeof(T) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = op.Invoke(Unsafe.Add(ref xRef, 14)); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = op.Invoke(Unsafe.Add(ref xRef, 13)); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = op.Invoke(Unsafe.Add(ref xRef, 12)); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = op.Invoke(Unsafe.Add(ref xRef, 11)); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = op.Invoke(Unsafe.Add(ref xRef, 10)); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = op.Invoke(Unsafe.Add(ref xRef, 9)); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = op.Invoke(Unsafe.Add(ref xRef, 8)); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = op.Invoke(Unsafe.Add(ref xRef, 7)); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = op.Invoke(Unsafe.Add(ref xRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = op.Invoke(Unsafe.Add(ref xRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = op.Invoke(Unsafe.Add(ref xRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = op.Invoke(Unsafe.Add(ref xRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = op.Invoke(Unsafe.Add(ref xRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = op.Invoke(Unsafe.Add(ref xRef, 1)); - goto case 1; - - case 1: - dRef = op.Invoke(xRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - Debug.Assert(sizeof(T) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = op.Invoke(Unsafe.Add(ref xRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = op.Invoke(Unsafe.Add(ref xRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = op.Invoke(Unsafe.Add(ref xRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = op.Invoke(Unsafe.Add(ref xRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = op.Invoke(Unsafe.Add(ref xRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = op.Invoke(Unsafe.Add(ref xRef, 1)); - goto case 1; - - case 1: - dRef = op.Invoke(xRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - Debug.Assert(sizeof(T) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = op.Invoke(Unsafe.Add(ref xRef, 2)); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = op.Invoke(Unsafe.Add(ref xRef, 1)); - goto case 1; - } - - case 1: - { - dRef = op.Invoke(xRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) - { - Debug.Assert(sizeof(T) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = op.Invoke(xRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.ITernaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.ITernaryOperator.cs deleted file mode 100644 index a36e52c108e6c6..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.ITernaryOperator.cs +++ /dev/null @@ -1,4459 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Operator that takes three input values and returns a single value. - private interface ITernaryOperator - { - static abstract T Invoke(T x, T y, T z); - static abstract Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z); - static abstract Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z); - static abstract Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z); - } - - /// - /// Performs an element-wise operation on , , and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on the pair-wise elements loaded from , , - /// and . - /// - private static void InvokeSpanSpanSpanIntoSpan( - ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan z, Span destination) - where TTernaryOperator : struct, ITernaryOperator - { - if (x.Length != y.Length || x.Length != z.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - ValidateInputOutputSpanNonOverlapping(y, destination); - ValidateInputOutputSpanNonOverlapping(z, destination); - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - ref T zRef = ref MemoryMarshal.GetReference(z); - ref T dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint length) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, i), - Unsafe.Add(ref yRef, i), - Unsafe.Add(ref zRef, i)); - } - } - - static void Vectorized128(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pz = &zRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* zPtr = pz; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - zPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - zPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - zPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - zRef = ref *zPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 8)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2)), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pz = &zRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* zPtr = pz; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (nuint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - zPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - zPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - zPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - zRef = ref *zPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 8)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2)), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 beg = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), - Vector512.LoadUnsafe(ref yRef), - Vector512.LoadUnsafe(ref zRef)); - Vector512 end = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)Vector512.Count)); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pz = &zRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* zPtr = pz; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - zPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - zPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - zPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - zRef = ref *zPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 8)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2)), - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - if (sizeof(T) == 1) - { - VectorizedSmall1(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - else if (sizeof(T) == 2) - { - VectorizedSmall2(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - else if (sizeof(T) == 4) - { - VectorizedSmall4(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - VectorizedSmall8(ref xRef, ref yRef, ref zRef, ref dRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 14), - Unsafe.Add(ref yRef, 14), - Unsafe.Add(ref zRef, 14)); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 13), - Unsafe.Add(ref yRef, 13), - Unsafe.Add(ref zRef, 13)); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 12), - Unsafe.Add(ref yRef, 12), - Unsafe.Add(ref zRef, 12)); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 11), - Unsafe.Add(ref yRef, 11), - Unsafe.Add(ref zRef, 11)); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 10), - Unsafe.Add(ref yRef, 10), - Unsafe.Add(ref zRef, 10)); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 9), - Unsafe.Add(ref yRef, 9), - Unsafe.Add(ref zRef, 9)); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 8), - Unsafe.Add(ref yRef, 8), - Unsafe.Add(ref zRef, 8)); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 7), - Unsafe.Add(ref yRef, 7), - Unsafe.Add(ref zRef, 7)); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - Unsafe.Add(ref yRef, 6), - Unsafe.Add(ref zRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - Unsafe.Add(ref yRef, 5), - Unsafe.Add(ref zRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - Unsafe.Add(ref yRef, 4), - Unsafe.Add(ref zRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - Unsafe.Add(ref yRef, 3), - Unsafe.Add(ref zRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2), - Unsafe.Add(ref zRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1), - Unsafe.Add(ref zRef, 1)); - goto case 1; - - case 1: - dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - Unsafe.Add(ref yRef, 6), - Unsafe.Add(ref zRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - Unsafe.Add(ref yRef, 5), - Unsafe.Add(ref zRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - Unsafe.Add(ref yRef, 4), - Unsafe.Add(ref zRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - Unsafe.Add(ref yRef, 3), - Unsafe.Add(ref zRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2), - Unsafe.Add(ref zRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1), - Unsafe.Add(ref zRef, 1)); - goto case 1; - - case 1: - dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2), - Unsafe.Add(ref zRef, 2)); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1), - Unsafe.Add(ref zRef, 1)); - goto case 1; - } - - case 1: - { - dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - - /// - /// Performs an element-wise operation on , , and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on the pair-wise elements loaded from and - /// with . - /// - private static void InvokeSpanSpanScalarIntoSpan( - ReadOnlySpan x, ReadOnlySpan y, T z, Span destination) - where TTernaryOperator : struct, ITernaryOperator - { - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - ValidateInputOutputSpanNonOverlapping(y, destination); - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - ref T dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, ref yRef, z, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, z, ref dRef, remainder); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, ref yRef, z, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, z, ref dRef, remainder); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, ref yRef, z, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref yRef, z, ref dRef, remainder); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, ref yRef, z, ref dRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref T xRef, ref T yRef, T z, ref T dRef, nuint length) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, i), - Unsafe.Add(ref yRef, i), - z); - } - } - - static void Vectorized128(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 zVec = Vector128.Create(z); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - zVec); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - zVec); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), - zVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), - zVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), - zVec); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), - zVec); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - yPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 8)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 zVec = Vector256.Create(z); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - zVec); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - zVec); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), - zVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), - zVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), - zVec); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), - zVec); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - yPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 8)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 zVec = Vector512.Create(z); - - Vector512 beg = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), - Vector512.LoadUnsafe(ref yRef), - zVec); - Vector512 end = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count), - zVec); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* py = &yRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* yPtr = py; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - yPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), - zVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), - zVec); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), - zVec); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), - zVec); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), - zVec); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), - zVec); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), - zVec); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - yPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - yRef = ref *yPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 8)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), - Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2)), - zVec); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - if (sizeof(T) == 1) - { - VectorizedSmall1(ref xRef, ref yRef, z, ref dRef, remainder); - } - else if (sizeof(T) == 2) - { - VectorizedSmall2(ref xRef, ref yRef, z, ref dRef, remainder); - } - else if (sizeof(T) == 4) - { - VectorizedSmall4(ref xRef, ref yRef, z, ref dRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - VectorizedSmall8(ref xRef, ref yRef, z, ref dRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 zVec = Vector256.Create(z); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - zVec); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 zVec = Vector128.Create(z); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - zVec); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 14), - Unsafe.Add(ref yRef, 14), - z); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 13), - Unsafe.Add(ref yRef, 13), - z); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 12), - Unsafe.Add(ref yRef, 12), - z); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 11), - Unsafe.Add(ref yRef, 11), - z); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 10), - Unsafe.Add(ref yRef, 10), - z); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 9), - Unsafe.Add(ref yRef, 9), - z); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 8), - Unsafe.Add(ref yRef, 8), - z); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 7), - Unsafe.Add(ref yRef, 7), - z); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - Unsafe.Add(ref yRef, 6), - z); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - Unsafe.Add(ref yRef, 5), - z); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - Unsafe.Add(ref yRef, 4), - z); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - Unsafe.Add(ref yRef, 3), - z); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2), - z); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1), - z); - goto case 1; - - case 1: - dRef = TTernaryOperator.Invoke(xRef, yRef, z); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 zVec = Vector256.Create(z); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - zVec); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 zVec = Vector128.Create(z); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - zVec); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - Unsafe.Add(ref yRef, 6), - z); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - Unsafe.Add(ref yRef, 5), - z); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - Unsafe.Add(ref yRef, 4), - z); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - Unsafe.Add(ref yRef, 3), - z); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2), - z); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1), - z); - goto case 1; - - case 1: - dRef = TTernaryOperator.Invoke(xRef, yRef, z); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 zVec = Vector256.Create(z); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - zVec); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 zVec = Vector128.Create(z); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - zVec); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - Unsafe.Add(ref yRef, 2), - z); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - Unsafe.Add(ref yRef, 1), - z); - goto case 1; - } - - case 1: - { - dRef = TTernaryOperator.Invoke(xRef, yRef, z); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 zVec = Vector256.Create(z); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - zVec); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.LoadUnsafe(ref yRef), - Vector256.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 zVec = Vector128.Create(z); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - zVec); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), - zVec); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.LoadUnsafe(ref yRef), - Vector128.Create(z)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = TTernaryOperator.Invoke(xRef, yRef, z); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - - /// - /// Performs an element-wise operation on , , and , - /// and writes the results to . - /// - /// The element type. - /// - /// Specifies the operation to perform on the pair-wise element loaded from , with , - /// and the element loaded from . - /// - private static void InvokeSpanScalarSpanIntoSpan( - ReadOnlySpan x, T y, ReadOnlySpan z, Span destination) - where TTernaryOperator : struct, ITernaryOperator - { - if (x.Length != z.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - ValidateInputOutputSpanNonOverlapping(z, destination); - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T zRef = ref MemoryMarshal.GetReference(z); - ref T dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, y, ref zRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, y, ref zRef, ref dRef, remainder); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, y, ref zRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, y, ref zRef, ref dRef, remainder); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, y, ref zRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, y, ref zRef, ref dRef, remainder); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, y, ref zRef, ref dRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref T xRef, T y, ref T zRef, ref T dRef, nuint length) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, i), - y, - Unsafe.Add(ref zRef, i)); - } - } - - static void Vectorized128(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - yVec, - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pz = &zRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* zPtr = pz; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); - - xPtr += misalignment; - zPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - zPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), - yVec, - Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - zPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - zRef = ref *zPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - yVec, - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pz = &zRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* zPtr = pz; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); - - xPtr += misalignment; - zPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - zPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), - yVec, - Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - zPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - zRef = ref *zPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - ref T dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 yVec = Vector512.Create(y); - - Vector512 beg = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), - yVec, - Vector512.LoadUnsafe(ref zRef)); - Vector512 end = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)Vector512.Count)); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (T* px = &xRef) - fixed (T* pz = &zRef) - fixed (T* pd = &dRef) - { - T* xPtr = px; - T* zPtr = pz; - T* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); - - xPtr += misalignment; - zPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - zPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); - vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); - vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); - vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), - yVec, - Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - zPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - zRef = ref *zPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), - yVec, - Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - if (sizeof(T) == 1) - { - VectorizedSmall1(ref xRef, y, ref zRef, ref dRef, remainder); - } - else if (sizeof(T) == 2) - { - VectorizedSmall2(ref xRef, y, ref zRef, ref dRef, remainder); - } - else if (sizeof(T) == 4) - { - VectorizedSmall4(ref xRef, y, ref zRef, ref dRef, remainder); - } - else - { - Debug.Assert(sizeof(T) == 8); - VectorizedSmall8(ref xRef, y, ref zRef, ref dRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - yVec, - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.Create(y), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - yVec, - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.Create(y), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 14), - y, - Unsafe.Add(ref zRef, 14)); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 13), - y, - Unsafe.Add(ref zRef, 13)); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 12), - y, - Unsafe.Add(ref zRef, 12)); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 11), - y, - Unsafe.Add(ref zRef, 11)); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 10), - y, - Unsafe.Add(ref zRef, 10)); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 9), - y, - Unsafe.Add(ref zRef, 9)); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 8), - y, - Unsafe.Add(ref zRef, 8)); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 7), - y, - Unsafe.Add(ref zRef, 7)); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - y, - Unsafe.Add(ref zRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - y, - Unsafe.Add(ref zRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - y, - Unsafe.Add(ref zRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - y, - Unsafe.Add(ref zRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - y, - Unsafe.Add(ref zRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - y, - Unsafe.Add(ref zRef, 1)); - goto case 1; - - case 1: - dRef = TTernaryOperator.Invoke(xRef, y, zRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - yVec, - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.Create(y), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - yVec, - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.Create(y), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), - y, - Unsafe.Add(ref zRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), - y, - Unsafe.Add(ref zRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), - y, - Unsafe.Add(ref zRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), - y, - Unsafe.Add(ref zRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - y, - Unsafe.Add(ref zRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - y, - Unsafe.Add(ref zRef, 1)); - goto case 1; - - case 1: - dRef = TTernaryOperator.Invoke(xRef, y, zRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - yVec, - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.Create(y), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - yVec, - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.Create(y), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), - y, - Unsafe.Add(ref zRef, 2)); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), - y, - Unsafe.Add(ref zRef, 1)); - goto case 1; - } - - case 1: - { - dRef = TTernaryOperator.Invoke(xRef, y, zRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) - { - Debug.Assert(sizeof(T) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 yVec = Vector256.Create(y); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - yVec, - Vector256.LoadUnsafe(ref zRef)); - Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), - yVec, - Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), - Vector256.Create(y), - Vector256.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 yVec = Vector128.Create(y); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - yVec, - Vector128.LoadUnsafe(ref zRef)); - Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), - yVec, - Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), - Vector128.Create(y), - Vector128.LoadUnsafe(ref zRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = TTernaryOperator.Invoke(xRef, y, zRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryInputBinaryOutput.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryInputBinaryOutput.cs deleted file mode 100644 index f84ccad005afa9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryInputBinaryOutput.cs +++ /dev/null @@ -1,146 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Operator that takes one input value and returns two output values. - private interface IUnaryInputBinaryOutput - { - static abstract bool Vectorizable { get; } - static abstract (T, T) Invoke(T x); - static abstract (Vector128 First, Vector128 Second) Invoke(Vector128 x); - static abstract (Vector256 First, Vector256 Second) Invoke(Vector256 x); - static abstract (Vector512 First, Vector512 Second) Invoke(Vector512 x); - } - - /// Performs an element-wise operation on and writes the results to and . - /// The element type. - /// Specifies the operation to perform on each element loaded from . - private static void InvokeSpanIntoSpan_TwoOutputs( - ReadOnlySpan x, Span destination1, Span destination2) - where TUnaryOperator : struct, IUnaryInputBinaryOutput - { - if (x.Length > destination1.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(nameof(destination1)); - } - - if (x.Length > destination2.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(nameof(destination2)); - } - - ValidateInputOutputSpanNonOverlapping(x, destination1); - ValidateInputOutputSpanNonOverlapping(x, destination2); - - ref T sourceRef = ref MemoryMarshal.GetReference(x); - ref T destination1Ref = ref MemoryMarshal.GetReference(destination1); - ref T destination2Ref = ref MemoryMarshal.GetReference(destination2); - int i = 0, oneVectorFromEnd; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TUnaryOperator.Vectorizable) - { - oneVectorFromEnd = x.Length - Vector512.Count; - if (i <= oneVectorFromEnd) - { - // Loop handling one input vector / two destination vectors at a time. - do - { - (Vector512 first, Vector512 second) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); - first.StoreUnsafe(ref destination1Ref, (uint)i); - second.StoreUnsafe(ref destination2Ref, (uint)i); - - i += Vector512.Count; - } - while (i <= oneVectorFromEnd); - - // Handle any remaining elements with a final input vector. - if (i != x.Length) - { - i = x.Length - Vector512.Count; - - (Vector512 first, Vector512 second) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); - first.StoreUnsafe(ref destination1Ref, (uint)i); - second.StoreUnsafe(ref destination2Ref, (uint)i); - } - - return; - } - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TUnaryOperator.Vectorizable) - { - oneVectorFromEnd = x.Length - Vector256.Count; - if (i <= oneVectorFromEnd) - { - // Loop handling one input vector / two destination vectors at a time. - do - { - (Vector256 first, Vector256 second) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); - first.StoreUnsafe(ref destination1Ref, (uint)i); - second.StoreUnsafe(ref destination2Ref, (uint)i); - - i += Vector256.Count; - } - while (i <= oneVectorFromEnd); - - // Handle any remaining elements with a final input vector. - if (i != x.Length) - { - i = x.Length - Vector256.Count; - - (Vector256 first, Vector256 second) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); - first.StoreUnsafe(ref destination1Ref, (uint)i); - second.StoreUnsafe(ref destination2Ref, (uint)i); - } - - return; - } - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TUnaryOperator.Vectorizable) - { - oneVectorFromEnd = x.Length - Vector128.Count; - if (i <= oneVectorFromEnd) - { - // Loop handling one input vector / two destination vectors at a time. - do - { - (Vector128 first, Vector128 second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); - first.StoreUnsafe(ref destination1Ref, (uint)i); - second.StoreUnsafe(ref destination2Ref, (uint)i); - - i += Vector128.Count; - } - while (i <= oneVectorFromEnd); - - // Handle any remaining elements with a final input vector. - if (i != x.Length) - { - i = x.Length - Vector128.Count; - - (Vector128 first, Vector128 second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); - first.StoreUnsafe(ref destination1Ref, (uint)i); - second.StoreUnsafe(ref destination2Ref, (uint)i); - } - - return; - } - } - - while (i < x.Length) - { - (T first, T second) = TUnaryOperator.Invoke(Unsafe.Add(ref sourceRef, i)); - Unsafe.Add(ref destination1Ref, i) = first; - Unsafe.Add(ref destination2Ref, i) = second; - i++; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryOneToTwoOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryOneToTwoOperator.cs deleted file mode 100644 index d8d906a603ee1d..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryOneToTwoOperator.cs +++ /dev/null @@ -1,152 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Operator that takes one input value and returns a single value. - /// The input type must be half the size of the output type. - private interface IUnaryOneToTwoOperator - { - static abstract bool Vectorizable { get; } - static abstract TOutput Invoke(TInput x); - static abstract (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x); - static abstract (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x); - static abstract (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x); - } - - /// Performs an element-wise operation on and writes the results to . - /// The element input type. - /// The element output type. Must be the same size as TInput if TInput and TOutput both support vectorization. - /// Specifies the operation to perform on each element loaded from . - /// This should only be used when it's known that TInput/TOutput are vectorizable and the size of TInput is half that of TOutput. - private static void InvokeSpanIntoSpan_1to2( - ReadOnlySpan x, Span destination) - where TUnaryOperator : struct, IUnaryOneToTwoOperator - { - Debug.Assert(sizeof(TInput) * 2 == sizeof(TOutput)); - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ref TInput sourceRef = ref MemoryMarshal.GetReference(x); - ref TOutput destinationRef = ref MemoryMarshal.GetReference(destination); - int i = 0, oneVectorFromEnd; - - if (Vector512.IsHardwareAccelerated && TUnaryOperator.Vectorizable) - { - Debug.Assert(Vector512.IsSupported); - Debug.Assert(Vector512.IsSupported); - - oneVectorFromEnd = x.Length - Vector512.Count; - if (i <= oneVectorFromEnd) - { - // Loop handling one input vector / two output vectors at a time. - do - { - (Vector512 lower, Vector512 upper) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); - lower.StoreUnsafe(ref destinationRef, (uint)i); - upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector512.Count)); - - i += Vector512.Count; - } - while (i <= oneVectorFromEnd); - - // Handle any remaining elements with a final input vector. - if (i != x.Length) - { - i = x.Length - Vector512.Count; - - (Vector512 lower, Vector512 upper) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); - lower.StoreUnsafe(ref destinationRef, (uint)i); - upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector512.Count)); - } - - return; - } - } - - if (Vector256.IsHardwareAccelerated && TUnaryOperator.Vectorizable) - { - Debug.Assert(Vector256.IsSupported); - Debug.Assert(Vector256.IsSupported); - - oneVectorFromEnd = x.Length - Vector256.Count; - if (i <= oneVectorFromEnd) - { - // Loop handling one input vector / two output vectors at a time. - do - { - (Vector256 lower, Vector256 upper) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); - lower.StoreUnsafe(ref destinationRef, (uint)i); - upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector256.Count)); - - i += Vector256.Count; - } - while (i <= oneVectorFromEnd); - - // Handle any remaining elements with a final input vector. - if (i != x.Length) - { - i = x.Length - Vector256.Count; - - (Vector256 lower, Vector256 upper) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); - lower.StoreUnsafe(ref destinationRef, (uint)i); - upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector256.Count)); - } - - return; - } - } - - if (Vector128.IsHardwareAccelerated && TUnaryOperator.Vectorizable) - { - Debug.Assert(Vector128.IsSupported); - Debug.Assert(Vector128.IsSupported); - - oneVectorFromEnd = x.Length - Vector128.Count; - if (i <= oneVectorFromEnd) - { - // Loop handling one input vector / two output vectors at a time. - do - { - (Vector128 lower, Vector128 upper) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); - lower.StoreUnsafe(ref destinationRef, (uint)i); - upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector128.Count)); - - i += Vector128.Count; - } - while (i <= oneVectorFromEnd); - - // Handle any remaining elements with a final input vector. - if (i != x.Length) - { - i = x.Length - Vector128.Count; - - (Vector128 lower, Vector128 upper) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); - lower.StoreUnsafe(ref destinationRef, (uint)i); - upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector128.Count)); - } - - return; - } - } - - while (i < x.Length) - { - Unsafe.Add(ref destinationRef, i) = TUnaryOperator.Invoke(Unsafe.Add(ref sourceRef, i)); - i++; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryOperator.cs deleted file mode 100644 index 20313a27eb77c0..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryOperator.cs +++ /dev/null @@ -1,1258 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Defines the threshold, in bytes, at which non-temporal stores will be used. - /// - /// A non-temporal store is one that allows the CPU to bypass the cache when writing to memory. - /// - /// This can be beneficial when working with large amounts of memory where the writes would otherwise - /// cause large amounts of repeated updates and evictions. The hardware optimization manuals recommend - /// the threshold to be roughly half the size of the last level of on-die cache -- that is, if you have approximately - /// 4MB of L3 cache per core, you'd want this to be approx. 1-2MB, depending on if hyperthreading was enabled. - /// - /// However, actually computing the amount of L3 cache per core can be tricky or error prone. Native memcpy - /// algorithms use a constant threshold that is typically around 256KB and we match that here for simplicity. This - /// threshold accounts for most processors in the last 10-15 years that had approx. 1MB L3 per core and support - /// hyperthreading, giving a per core last level cache of approx. 512KB. - /// - private const nuint NonTemporalByteThreshold = 256 * 1024; - - /// Operator that takes one input value and returns a single value. - /// The input and output type must be of the same size if vectorization is desired. - internal interface IUnaryOperator - { - static abstract bool Vectorizable { get; } - static abstract TOutput Invoke(TInput x); - static abstract Vector128 Invoke(Vector128 x); - static abstract Vector256 Invoke(Vector256 x); - static abstract Vector512 Invoke(Vector512 x); - } - - /// x - internal readonly struct IdentityOperator : IUnaryOperator - { - public static bool Vectorizable => true; - public static T Invoke(T x) => x; - public static Vector128 Invoke(Vector128 x) => x; - public static Vector256 Invoke(Vector256 x) => x; - public static Vector512 Invoke(Vector512 x) => x; - } - - /// Performs an element-wise operation on and writes the results to . - /// The element input type. - /// Specifies the operation to perform on each element loaded from . - private static void InvokeSpanIntoSpan( - ReadOnlySpan x, Span destination) - where TUnaryOperator : struct, IUnaryOperator => - InvokeSpanIntoSpan(x, destination); - - /// Performs an element-wise operation on and writes the results to . - /// The element input type. - /// The element output type. Must be the same size as TInput if TInput and TOutput both support vectorization. - /// Specifies the operation to perform on each element loaded from . - /// - /// This supports vectorizing the operation if and are the same size. - /// Otherwise, it'll fall back to scalar operations. - /// - private static void InvokeSpanIntoSpan( - ReadOnlySpan x, Span destination) - where TUnaryOperator : struct, IUnaryOperator - { - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - if (typeof(TInput) == typeof(TOutput)) - { - // This ignores the unsafe case where a developer passes in overlapping spans for distinct types. - ValidateInputOutputSpanNonOverlapping(x, Rename(destination)); - } - - // Since every branch has a cost and since that cost is - // essentially lost for larger inputs, we do branches - // in a way that allows us to have the minimum possible - // for small sizes - - ref TInput xRef = ref MemoryMarshal.GetReference(x); - ref TOutput dRef = ref MemoryMarshal.GetReference(destination); - - nuint remainder = (uint)x.Length; - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && Vector512.IsSupported && TUnaryOperator.Vectorizable && Unsafe.SizeOf() == Unsafe.SizeOf()) - { - if (remainder >= (uint)Vector512.Count) - { - Vectorized512(ref xRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref dRef, remainder); - } - - return; - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && Vector256.IsSupported && TUnaryOperator.Vectorizable && Unsafe.SizeOf() == Unsafe.SizeOf()) - { - if (remainder >= (uint)Vector256.Count) - { - Vectorized256(ref xRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref dRef, remainder); - } - - return; - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && Vector128.IsSupported && TUnaryOperator.Vectorizable && Unsafe.SizeOf() == Unsafe.SizeOf()) - { - if (remainder >= (uint)Vector128.Count) - { - Vectorized128(ref xRef, ref dRef, remainder); - } - else - { - // We have less than a vector and so we can only handle this as scalar. To do this - // efficiently, we simply have a small jump table and fallthrough. So we get a simple - // length check, single jump, and then linear execution. - - VectorizedSmall(ref xRef, ref dRef, remainder); - } - - return; - } - - // This is the software fallback when no acceleration is available - // It requires no branches to hit - - SoftwareFallback(ref xRef, ref dRef, remainder); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void SoftwareFallback(ref TInput xRef, ref TOutput dRef, nuint length) - { - for (nuint i = 0; i < length; i++) - { - Unsafe.Add(ref dRef, i) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, i)); - } - } - - static void Vectorized128(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - ref TOutput dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - if (remainder > (uint)(Vector128.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (TInput* px = &xRef) - fixed (TOutput* pd = &dRef) - { - TInput* xPtr = px; - TOutput* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(TInput)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(TInput); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); - - remainder -= misalignment; - } - - Vector128 vector1; - Vector128 vector2; - Vector128 vector3; - Vector128 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(TInput))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); - vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); - vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); - vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); - vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); - vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); - vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector128.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); - vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); - vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); - vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 0)); - vector2.Store(dPtr + (uint)(Vector128.Count * 1)); - vector3.Store(dPtr + (uint)(Vector128.Count * 2)); - vector4.Store(dPtr + (uint)(Vector128.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); - vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); - vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); - vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector128.Count * 4)); - vector2.Store(dPtr + (uint)(Vector128.Count * 5)); - vector3.Store(dPtr + (uint)(Vector128.Count * 6)); - vector4.Store(dPtr + (uint)(Vector128.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector128.Count * 8); - dPtr += (uint)(Vector128.Count * 8); - - remainder -= (uint)(Vector128.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); - - switch (remainder / (uint)Vector128.Count) - { - case 8: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); - goto case 7; - } - - case 7: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); - goto case 6; - } - - case 6: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); - goto case 5; - } - - case 5: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); - goto case 4; - } - - case 4: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); - goto case 3; - } - - case 3: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); - goto case 2; - } - - case 2: - { - Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized256(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - ref TOutput dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - if (remainder > (uint)(Vector256.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (TInput* px = &xRef) - fixed (TOutput* pd = &dRef) - { - TInput* xPtr = px; - TOutput* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(TInput)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(TInput); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); - - remainder -= misalignment; - } - - Vector256 vector1; - Vector256 vector2; - Vector256 vector3; - Vector256 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(TInput))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); - vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); - vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); - vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); - vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); - vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); - vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector256.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); - vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); - vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); - vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 0)); - vector2.Store(dPtr + (uint)(Vector256.Count * 1)); - vector3.Store(dPtr + (uint)(Vector256.Count * 2)); - vector4.Store(dPtr + (uint)(Vector256.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); - vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); - vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); - vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector256.Count * 4)); - vector2.Store(dPtr + (uint)(Vector256.Count * 5)); - vector3.Store(dPtr + (uint)(Vector256.Count * 6)); - vector4.Store(dPtr + (uint)(Vector256.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector256.Count * 8); - dPtr += (uint)(Vector256.Count * 8); - - remainder -= (uint)(Vector256.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); - - switch (remainder / (uint)Vector256.Count) - { - case 8: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); - goto case 7; - } - - case 7: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); - goto case 6; - } - - case 6: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); - goto case 5; - } - - case 5: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); - goto case 4; - } - - case 4: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); - goto case 3; - } - - case 3: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); - goto case 2; - } - - case 2: - { - Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - static void Vectorized512(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - ref TOutput dRefBeg = ref dRef; - - // Preload the beginning and end so that overlapping accesses don't negatively impact the data - - Vector512 beg = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef)); - Vector512 end = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)); - - if (remainder > (uint)(Vector512.Count * 8)) - { - // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful - // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. - - fixed (TInput* px = &xRef) - fixed (TOutput* pd = &dRef) - { - TInput* xPtr = px; - TOutput* dPtr = pd; - - // We need to the ensure the underlying data can be aligned and only align - // it if it can. It is possible we have an unaligned ref, in which case we - // can never achieve the required SIMD alignment. - - bool canAlign = ((nuint)dPtr % (nuint)sizeof(TInput)) == 0; - - if (canAlign) - { - // Compute by how many elements we're misaligned and adjust the pointers accordingly - // - // Noting that we are only actually aligning dPtr. This is because unaligned stores - // are more expensive than unaligned loads and aligning both is significantly more - // complex. - - nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(TInput); - - xPtr += misalignment; - dPtr += misalignment; - - Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); - - remainder -= misalignment; - } - - Vector512 vector1; - Vector512 vector2; - Vector512 vector3; - Vector512 vector4; - - if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(TInput))) && canAlign) - { - // This loop stores the data non-temporally, which benefits us when there - // is a large amount of data involved as it avoids polluting the cache. - - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); - vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); - vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); - vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); - vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); - vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); - vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); - - vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); - vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); - vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); - vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - else - { - while (remainder >= (uint)(Vector512.Count * 8)) - { - // We load, process, and store the first four vectors - - vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); - vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); - vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); - vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 0)); - vector2.Store(dPtr + (uint)(Vector512.Count * 1)); - vector3.Store(dPtr + (uint)(Vector512.Count * 2)); - vector4.Store(dPtr + (uint)(Vector512.Count * 3)); - - // We load, process, and store the next four vectors - - vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); - vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); - vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); - vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); - - vector1.Store(dPtr + (uint)(Vector512.Count * 4)); - vector2.Store(dPtr + (uint)(Vector512.Count * 5)); - vector3.Store(dPtr + (uint)(Vector512.Count * 6)); - vector4.Store(dPtr + (uint)(Vector512.Count * 7)); - - // We adjust the source and destination references, then update - // the count of remaining elements to process. - - xPtr += (uint)(Vector512.Count * 8); - dPtr += (uint)(Vector512.Count * 8); - - remainder -= (uint)(Vector512.Count * 8); - } - } - - // Adjusting the refs here allows us to avoid pinning for very small inputs - - xRef = ref *xPtr; - dRef = ref *dPtr; - } - } - - // Process the remaining [Count, Count * 8] elements via a jump table - // - // Unless the original length was an exact multiple of Count, then we'll - // end up reprocessing a couple elements in case 1 for end. We'll also - // potentially reprocess a few elements in case 0 for beg, to handle any - // data before the first aligned address. - - nuint endIndex = remainder; - remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); - - switch (remainder / (uint)Vector512.Count) - { - case 8: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); - goto case 7; - } - - case 7: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); - goto case 6; - } - - case 6: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); - goto case 5; - } - - case 5: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); - goto case 4; - } - - case 4: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); - goto case 3; - } - - case 3: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); - goto case 2; - } - - case 2: - { - Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))); - vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); - goto case 1; - } - - case 1: - { - // Store the last block, which includes any elements that wouldn't fill a full vector - end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); - goto case 0; - } - - case 0: - { - // Store the first block, which includes any elements preceding the first aligned block - beg.StoreUnsafe(ref dRefBeg); - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - if (sizeof(TInput) == 1) - { - VectorizedSmall1(ref xRef, ref dRef, remainder); - } - else if (sizeof(TInput) == 2) - { - VectorizedSmall2(ref xRef, ref dRef, remainder); - } - else if (sizeof(TInput) == 4) - { - VectorizedSmall4(ref xRef, ref dRef, remainder); - } - else - { - Debug.Assert(sizeof(TInput) == 8); - VectorizedSmall8(ref xRef, ref dRef, remainder); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall1(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - Debug.Assert(sizeof(TInput) == 1); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 63: - case 62: - case 61: - case 60: - case 59: - case 58: - case 57: - case 56: - case 55: - case 54: - case 53: - case 52: - case 51: - case 50: - case 49: - case 48: - case 47: - case 46: - case 45: - case 44: - case 43: - case 42: - case 41: - case 40: - case 39: - case 38: - case 37: - case 36: - case 35: - case 34: - case 33: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 32: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 16: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 15: - Unsafe.Add(ref dRef, 14) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 14)); - goto case 14; - - case 14: - Unsafe.Add(ref dRef, 13) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 13)); - goto case 13; - - case 13: - Unsafe.Add(ref dRef, 12) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 12)); - goto case 12; - - case 12: - Unsafe.Add(ref dRef, 11) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 11)); - goto case 11; - - case 11: - Unsafe.Add(ref dRef, 10) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 10)); - goto case 10; - - case 10: - Unsafe.Add(ref dRef, 9) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 9)); - goto case 9; - - case 9: - Unsafe.Add(ref dRef, 8) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 8)); - goto case 8; - - case 8: - Unsafe.Add(ref dRef, 7) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 7)); - goto case 7; - - case 7: - Unsafe.Add(ref dRef, 6) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 1)); - goto case 1; - - case 1: - dRef = TUnaryOperator.Invoke(xRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall2(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - Debug.Assert(sizeof(TInput) == 2); - - switch (remainder) - { - // Two Vector256's worth of data, with at least one element overlapping. - case 31: - case 30: - case 29: - case 28: - case 27: - case 26: - case 25: - case 24: - case 23: - case 22: - case 21: - case 20: - case 19: - case 18: - case 17: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - // One Vector256's worth of data. - case 16: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Two Vector128's worth of data, with at least one element overlapping. - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - // One Vector128's worth of data. - case 8: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each - // case to unroll the whole processing. - case 7: - Unsafe.Add(ref dRef, 6) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 6)); - goto case 6; - - case 6: - Unsafe.Add(ref dRef, 5) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 5)); - goto case 5; - - case 5: - Unsafe.Add(ref dRef, 4) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 4)); - goto case 4; - - case 4: - Unsafe.Add(ref dRef, 3) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 3)); - goto case 3; - - case 3: - Unsafe.Add(ref dRef, 2) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 2)); - goto case 2; - - case 2: - Unsafe.Add(ref dRef, 1) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 1)); - goto case 1; - - case 1: - dRef = TUnaryOperator.Invoke(xRef); - goto case 0; - - case 0: - break; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall4(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - Debug.Assert(sizeof(TInput) == 4); - - switch (remainder) - { - case 15: - case 14: - case 13: - case 12: - case 11: - case 10: - case 9: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 8: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 7: - case 6: - case 5: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Unsafe.Add(ref dRef, 2) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 2)); - goto case 2; - } - - case 2: - { - Unsafe.Add(ref dRef, 1) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 1)); - goto case 1; - } - - case 1: - { - dRef = TUnaryOperator.Invoke(xRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static void VectorizedSmall8(ref TInput xRef, ref TOutput dRef, nuint remainder) - { - Debug.Assert(sizeof(TInput) == 8); - - switch (remainder) - { - case 7: - case 6: - case 5: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); - - break; - } - - case 4: - { - Debug.Assert(Vector256.IsHardwareAccelerated); - - Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 3: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); - - beg.StoreUnsafe(ref dRef); - end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); - - break; - } - - case 2: - { - Debug.Assert(Vector128.IsHardwareAccelerated); - - Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); - beg.StoreUnsafe(ref dRef); - - break; - } - - case 1: - { - dRef = TUnaryOperator.Invoke(xRef); - goto case 0; - } - - case 0: - { - break; - } - } - } - } - - /// Creates a span of from a when they're the same type. - private static unsafe Span Rename(Span span) - { - Debug.Assert(sizeof(TFrom) == sizeof(TTo)); - return MemoryMarshal.CreateSpan(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryTwoToOneOperator.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryTwoToOneOperator.cs deleted file mode 100644 index e16c4ebca3a6af..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IUnaryTwoToOneOperator.cs +++ /dev/null @@ -1,152 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static unsafe partial class TensorPrimitives - { - /// Operator that takes one input value and returns a single value. - /// The input type must be twice the size of the output type. - private interface IUnaryTwoToOneOperator - { - static abstract bool Vectorizable { get; } - static abstract TOutput Invoke(TInput x); - static abstract Vector128 Invoke(Vector128 lower, Vector128 upper); - static abstract Vector256 Invoke(Vector256 lower, Vector256 upper); - static abstract Vector512 Invoke(Vector512 lower, Vector512 upper); - } - - /// Performs an element-wise operation on and writes the results to . - /// The element input type. - /// The element output type. Must be the same size as TInput if TInput and TOutput both support vectorization. - /// Specifies the operation to perform on each element loaded from . - /// This should only be used when it's known that TInput/TOutput are vectorizable and the size of TInput is twice that of TOutput. - private static void InvokeSpanIntoSpan_2to1( - ReadOnlySpan x, Span destination) - where TUnaryOperator : struct, IUnaryTwoToOneOperator - { - Debug.Assert(sizeof(TInput) == sizeof(TOutput) * 2); - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ref TInput xRef = ref MemoryMarshal.GetReference(x); - ref TOutput destinationRef = ref MemoryMarshal.GetReference(destination); - int i = 0, twoVectorsFromEnd; - - if (Vector512.IsHardwareAccelerated && TUnaryOperator.Vectorizable) - { - Debug.Assert(Vector512.IsSupported); - Debug.Assert(Vector512.IsSupported); - - twoVectorsFromEnd = x.Length - (Vector512.Count * 2); - if (i <= twoVectorsFromEnd) - { - // Loop handling two input vectors / one output vector at a time. - do - { - TUnaryOperator.Invoke( - Vector512.LoadUnsafe(ref xRef, (uint)i), - Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count))).StoreUnsafe(ref destinationRef, (uint)i); - - i += Vector512.Count * 2; - } - while (i <= twoVectorsFromEnd); - - // Handle any remaining elements with final vectors. - if (i != x.Length) - { - i = x.Length - (Vector512.Count * 2); - - TUnaryOperator.Invoke( - Vector512.LoadUnsafe(ref xRef, (uint)i), - Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count))).StoreUnsafe(ref destinationRef, (uint)i); - } - - return; - } - } - - if (Vector256.IsHardwareAccelerated && TUnaryOperator.Vectorizable) - { - Debug.Assert(Vector256.IsSupported); - Debug.Assert(Vector256.IsSupported); - - twoVectorsFromEnd = x.Length - (Vector256.Count * 2); - if (i <= twoVectorsFromEnd) - { - // Loop handling two input vectors / one output vector at a time. - do - { - TUnaryOperator.Invoke( - Vector256.LoadUnsafe(ref xRef, (uint)i), - Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count))).StoreUnsafe(ref destinationRef, (uint)i); - - i += Vector256.Count * 2; - } - while (i <= twoVectorsFromEnd); - - // Handle any remaining elements with final vectors. - if (i != x.Length) - { - i = x.Length - (Vector256.Count * 2); - - TUnaryOperator.Invoke( - Vector256.LoadUnsafe(ref xRef, (uint)i), - Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count))).StoreUnsafe(ref destinationRef, (uint)i); - } - - return; - } - } - - if (Vector128.IsHardwareAccelerated && TUnaryOperator.Vectorizable) - { - Debug.Assert(Vector128.IsSupported); - Debug.Assert(Vector128.IsSupported); - - twoVectorsFromEnd = x.Length - (Vector128.Count * 2); - if (i <= twoVectorsFromEnd) - { - // Loop handling two input vectors / one output vector at a time. - do - { - TUnaryOperator.Invoke( - Vector128.LoadUnsafe(ref xRef, (uint)i), - Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count))).StoreUnsafe(ref destinationRef, (uint)i); - - i += Vector128.Count * 2; - } - while (i <= twoVectorsFromEnd); - - // Handle any remaining elements with final vectors. - if (i != x.Length) - { - i = x.Length - (Vector128.Count * 2); - - TUnaryOperator.Invoke( - Vector128.LoadUnsafe(ref xRef, (uint)i), - Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count))).StoreUnsafe(ref destinationRef, (uint)i); - } - - return; - } - } - - while (i < x.Length) - { - Unsafe.Add(ref destinationRef, i) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, i)); - i++; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Abs.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Abs.cs deleted file mode 100644 index 4ea41759157936..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Abs.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise absolute value of each number in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is a signed integer type and contained a value equal to 's minimum value. - /// - /// - /// This method effectively computes [i] = .Abs([i]). - /// - /// - /// The absolute value of a is its numeric value without its sign. For example, the absolute value of both 1.2e-03 and -1.2e03 is 1.2e03. - /// - /// - /// If a value is equal to or , the result stored into the corresponding destination location is set to . - /// If a value is equal to , the result stored into the corresponding destination location is the original NaN value with the sign bit removed. - /// - /// - public static void Abs(ReadOnlySpan x, Span destination) - where T : INumberBase => - InvokeSpanIntoSpan>(x, destination); - - /// T.Abs(x) - internal readonly struct AbsoluteOperator : IUnaryOperator where T : INumberBase - { - public static bool Vectorizable => true; - - public static T Invoke(T x) => T.Abs(x); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(sbyte) || - typeof(T) == typeof(short) || - typeof(T) == typeof(int) || - typeof(T) == typeof(long) || - typeof(T) == typeof(nint)) - { - // Handle signed integers specially, in order to throw if any attempt is made to - // take the absolute value of the minimum value of the type, which doesn't have - // a positive absolute value representation. - Vector128 abs = Vector128.ConditionalSelect(Vector128.LessThan(x, Vector128.Zero), -x, x); - if (Vector128.LessThan(abs, Vector128.Zero) != Vector128.Zero) - { - ThrowNegateTwosCompOverflow(); - } - } - - return Vector128.Abs(x); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(sbyte) || - typeof(T) == typeof(short) || - typeof(T) == typeof(int) || - typeof(T) == typeof(long) || - typeof(T) == typeof(nint)) - { - // Handle signed integers specially, in order to throw if any attempt is made to - // take the absolute value of the minimum value of the type, which doesn't have - // a positive absolute value representation. - Vector256 abs = Vector256.ConditionalSelect(Vector256.LessThan(x, Vector256.Zero), -x, x); - if (Vector256.LessThan(abs, Vector256.Zero) != Vector256.Zero) - { - ThrowNegateTwosCompOverflow(); - } - } - - return Vector256.Abs(x); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(sbyte) || - typeof(T) == typeof(short) || - typeof(T) == typeof(int) || - typeof(T) == typeof(long) || - typeof(T) == typeof(nint)) - { - // Handle signed integers specially, in order to throw if any attempt is made to - // take the absolute value of the minimum value of the type, which doesn't have - // a positive absolute value representation. - Vector512 abs = Vector512.ConditionalSelect(Vector512.LessThan(x, Vector512.Zero), -x, x); - if (Vector512.LessThan(abs, Vector512.Zero) != Vector512.Zero) - { - ThrowNegateTwosCompOverflow(); - } - } - - return Vector512.Abs(x); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Acos.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Acos.cs deleted file mode 100644 index cdd36a41dc7f65..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Acos.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise angle in radians whose cosine is the specifed number. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Acos([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Acos(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Acos(x) - private readonly struct AcosOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T x) => T.Acos(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AcosPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AcosPi.cs deleted file mode 100644 index caf57f4e9d1644..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AcosPi.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise angle in radians whose cosine is the specifed number and divides the result by Pi. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .AcosPi([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void AcosPi(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.AcosPi(x) - private readonly struct AcosPiOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => AcosOperator.Vectorizable; - public static T Invoke(T x) => T.AcosPi(x); - public static Vector128 Invoke(Vector128 x) => AcosOperator.Invoke(x) / Vector128.Create(T.Pi); - public static Vector256 Invoke(Vector256 x) => AcosOperator.Invoke(x) / Vector256.Create(T.Pi); - public static Vector512 Invoke(Vector512 x) => AcosOperator.Invoke(x) / Vector512.Create(T.Pi); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Acosh.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Acosh.cs deleted file mode 100644 index 7313cfe15f2d5d..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Acosh.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hyperbolic arc-cosine of the specifed number. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Acosh([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Acosh(ReadOnlySpan x, Span destination) - where T : IHyperbolicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Acosh(x) - private readonly struct AcoshOperator : IUnaryOperator - where T : IHyperbolicFunctions - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T x) => T.Acosh(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Add.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Add.cs deleted file mode 100644 index 4c891bef8b04c0..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Add.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise addition of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IAdditionOperators, IAdditiveIdentity => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise addition of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] + . - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Add(ReadOnlySpan x, T y, Span destination) - where T : IAdditionOperators, IAdditiveIdentity => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// x + y - internal readonly struct AddOperator : IAggregationOperator where T : IAdditionOperators, IAdditiveIdentity - { - public static bool Vectorizable => true; - - public static T Invoke(T x, T y) => x + y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y; - - public static T Invoke(Vector128 x) => Vector128.Sum(x); - public static T Invoke(Vector256 x) => Vector256.Sum(x); - public static T Invoke(Vector512 x) => Vector512.Sum(x); - - public static T IdentityValue => T.AdditiveIdentity; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AddMultiply.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AddMultiply.cs deleted file mode 100644 index c69f379b4b59a5..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AddMultiply.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of ( + ) * for the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of and the length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] + [i]) * [i]. - /// - /// - /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void AddMultiply(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan multiplier, Span destination) - where T : IAdditionOperators, IMultiplyOperators => - InvokeSpanSpanSpanIntoSpan>(x, y, multiplier, destination); - - /// Computes the element-wise result of ( + ) * for the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] + [i]) * . - /// - /// - /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void AddMultiply(ReadOnlySpan x, ReadOnlySpan y, T multiplier, Span destination) - where T : IAdditionOperators, IMultiplyOperators => - InvokeSpanSpanScalarIntoSpan>(x, y, multiplier, destination); - - /// Computes the element-wise result of ( + ) * for the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] + ) * [i]. - /// - /// - /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void AddMultiply(ReadOnlySpan x, T y, ReadOnlySpan multiplier, Span destination) - where T : IAdditionOperators, IMultiplyOperators => - InvokeSpanScalarSpanIntoSpan>(x, y, multiplier, destination); - - /// (x + y) * z - internal readonly struct AddMultiplyOperator : ITernaryOperator where T : IAdditionOperators, IMultiplyOperators - { - public static T Invoke(T x, T y, T z) => (x + y) * z; - public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) => (x + y) * z; - public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) => (x + y) * z; - public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) => (x + y) * z; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Asin.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Asin.cs deleted file mode 100644 index be216eaffe6c43..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Asin.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise angle in radians whose sine is the specifed number. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Asin([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Asin(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Asin(x) - private readonly struct AsinOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T x) => T.Asin(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AsinPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AsinPi.cs deleted file mode 100644 index 710628f5806e1b..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AsinPi.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise angle in radians whose sine is the specifed number and divides the result by Pi. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .AsinPi([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void AsinPi(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.AsinPi(x) - private readonly struct AsinPiOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => AsinOperator.Vectorizable; - public static T Invoke(T x) => T.AsinPi(x); - public static Vector128 Invoke(Vector128 x) => AsinOperator.Invoke(x) / Vector128.Create(T.Pi); - public static Vector256 Invoke(Vector256 x) => AsinOperator.Invoke(x) / Vector256.Create(T.Pi); - public static Vector512 Invoke(Vector512 x) => AsinOperator.Invoke(x) / Vector512.Create(T.Pi); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Asinh.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Asinh.cs deleted file mode 100644 index d33168a42664d2..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Asinh.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hyperbolic arc-sine of the specifed number. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Asinh([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Asinh(ReadOnlySpan x, Span destination) - where T : IHyperbolicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Asinh(x) - internal readonly struct AsinhOperator : IUnaryOperator - where T : IHyperbolicFunctions - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T x) => T.Asinh(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan.cs deleted file mode 100644 index 023dcc03d7623b..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise angle in radians whose tangent is the specifed number. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Atan(x) - internal readonly struct AtanOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T x) => T.Atan(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan2.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan2.cs deleted file mode 100644 index 310738623184f4..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan2.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan2([i], [i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan2(ReadOnlySpan y, ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanIntoSpan>(y, x, destination); - - /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan2([i], ). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan2(ReadOnlySpan y, T x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanScalarIntoSpan>(y, x, destination); - - /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors. - /// The first tensor, represented as a scalar. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan2(, [i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan2(T y, ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeScalarSpanIntoSpan>(y, x, destination); - - /// T.Atan2(y, x) - private readonly struct Atan2Operator : IBinaryOperator - where T : IFloatingPointIeee754 - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T y, T x) => T.Atan2(y, x); - public static Vector128 Invoke(Vector128 y, Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 y, Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 y, Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan2Pi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan2Pi.cs deleted file mode 100644 index a055bdc08c21a1..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atan2Pi.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors and divides the result by Pi. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan2([i], [i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan2Pi(ReadOnlySpan y, ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanIntoSpan>(y, x, destination); - - /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors and divides the result by Pi. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan2([i], ). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan2Pi(ReadOnlySpan y, T x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanScalarIntoSpan>(y, x, destination); - - /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors and divides the result by Pi. - /// The first tensor, represented as a scalar. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atan2(, [i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atan2Pi(T y, ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeScalarSpanIntoSpan>(y, x, destination); - - /// T.Atan2Pi(y, x) - private readonly struct Atan2PiOperator : IBinaryOperator - where T : IFloatingPointIeee754 - { - public static bool Vectorizable => Atan2Operator.Vectorizable; - public static T Invoke(T y, T x) => T.Atan2Pi(y, x); - public static Vector128 Invoke(Vector128 y, Vector128 x) => Atan2Operator.Invoke(y, x) / Vector128.Create(T.Pi); - public static Vector256 Invoke(Vector256 y, Vector256 x) => Atan2Operator.Invoke(y, x) / Vector256.Create(T.Pi); - public static Vector512 Invoke(Vector512 y, Vector512 x) => Atan2Operator.Invoke(y, x) / Vector512.Create(T.Pi); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AtanPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AtanPi.cs deleted file mode 100644 index 67c350c50e0698..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.AtanPi.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise angle in radians whose tangent is the specifed number and divides the result by Pi. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .AtanPi([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void AtanPi(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.AtanPi(x) - internal readonly struct AtanPiOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => AtanOperator.Vectorizable; - public static T Invoke(T x) => T.AtanPi(x); - public static Vector128 Invoke(Vector128 x) => AtanOperator.Invoke(x) / Vector128.Create(T.Pi); - public static Vector256 Invoke(Vector256 x) => AtanOperator.Invoke(x) / Vector256.Create(T.Pi); - public static Vector512 Invoke(Vector512 x) => AtanOperator.Invoke(x) / Vector512.Create(T.Pi); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atanh.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atanh.cs deleted file mode 100644 index 83308d3d3a2290..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Atanh.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hyperbolic arc-tangent of the specifed number. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Atanh([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Atanh(ReadOnlySpan x, Span destination) - where T : IHyperbolicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Atanh(x) - internal readonly struct AtanhOperator : IUnaryOperator - where T : IHyperbolicFunctions - { - public static bool Vectorizable => false; // TODO: Vectorize - public static T Invoke(T x) => T.Atanh(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitwiseAnd.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitwiseAnd.cs deleted file mode 100644 index 90a94b4d5ff28c..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitwiseAnd.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise bitwise AND of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] & [i]. - /// - /// - public static void BitwiseAnd(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IBitwiseOperators => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise bitwise AND of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] & . - /// - /// - public static void BitwiseAnd(ReadOnlySpan x, T y, Span destination) - where T : IBitwiseOperators => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// x & y - private readonly struct BitwiseAndOperator : IBinaryOperator where T : IBitwiseOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x, T y) => x & y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x & y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x & y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x & y; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitwiseOr.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitwiseOr.cs deleted file mode 100644 index e052bcee899d19..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.BitwiseOr.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise bitwise OR of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] | [i]. - /// - /// - public static void BitwiseOr(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IBitwiseOperators => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise bitwise OR of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] | . - /// - /// - public static void BitwiseOr(ReadOnlySpan x, T y, Span destination) - where T : IBitwiseOperators => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// x | y - private readonly struct BitwiseOrOperator : IBinaryOperator where T : IBitwiseOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x, T y) => x | y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x | y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x | y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x | y; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cbrt.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cbrt.cs deleted file mode 100644 index 1bc8b85696c121..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cbrt.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise cube root of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Cbrt([i]). - /// - /// - public static void Cbrt(ReadOnlySpan x, Span destination) - where T : IRootFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Cbrt(x) - private readonly struct CbrtOperator : IUnaryOperator - where T : IRootFunctions - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Cbrt(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector128.Create(3f)).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector128.Create(3d)).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector256.Create(3f)).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector256.Create(3d)).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector512.Create(3f)).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector512.Create(3d)).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Ceiling.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Ceiling.cs deleted file mode 100644 index 17d2466bfad3eb..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Ceiling.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise ceiling of numbers in the specified tensor. - /// The first tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Ceiling([i]). - /// - /// - public static void Ceiling(ReadOnlySpan x, Span destination) - where T : IFloatingPoint => - InvokeSpanIntoSpan>(x, destination); - - private readonly struct CeilingOperator : IUnaryOperator where T : IFloatingPoint - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Ceiling(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return Vector128.Ceiling(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return Vector128.Ceiling(x.AsDouble()).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return Vector256.Ceiling(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return Vector256.Ceiling(x.AsDouble()).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return Vector512.Ceiling(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return Vector512.Ceiling(x.AsDouble()).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertChecked.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertChecked.cs deleted file mode 100644 index a8340b2d22ea00..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertChecked.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// - /// Copies to , converting each - /// value to a value. - /// - /// The source span from which to copy values. - /// The destination span into which the converted values should be written. - /// Destination is too short. - /// - /// - /// This method effectively computes [i] = TTo.CreateChecked([i]). - /// - /// - public static void ConvertChecked(ReadOnlySpan source, Span destination) - where TFrom : INumberBase - where TTo : INumberBase - { - if (!TryConvertUniversal(source, destination)) - { - InvokeSpanIntoSpan>(source, destination); - } - } - - /// T.CreateChecked(x) - internal readonly struct ConvertCheckedFallbackOperator : IUnaryOperator where TFrom : INumberBase where TTo : INumberBase - { - public static bool Vectorizable => false; - - public static TTo Invoke(TFrom x) => TTo.CreateChecked(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertHelpers.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertHelpers.cs deleted file mode 100644 index ca785bf257aa66..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertHelpers.cs +++ /dev/null @@ -1,665 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Performs conversions that are the same regardless of checked, truncating, or saturation. - [MethodImpl(MethodImplOptions.AggressiveInlining)] // at most one of the branches will be kept - private static bool TryConvertUniversal(ReadOnlySpan source, Span destination) - where TFrom : INumberBase - where TTo : INumberBase - { - if (typeof(TFrom) == typeof(TTo)) - { - if (source.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(source, Rename(destination)); - - source.CopyTo(Rename(destination)); - return true; - } - - if (IsInt32Like() && typeof(TTo) == typeof(float)) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return true; - } - - if (IsUInt32Like() && typeof(TTo) == typeof(float)) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return true; - } - - if (IsInt64Like() && typeof(TTo) == typeof(double)) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return true; - } - - if (IsUInt64Like() && typeof(TTo) == typeof(double)) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(float) && typeof(TTo) == typeof(Half)) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(Half) && typeof(TTo) == typeof(float)) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(float) && typeof(TTo) == typeof(double)) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(double) && typeof(TTo) == typeof(float)) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(byte) && typeof(TTo) == typeof(ushort)) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(sbyte) && typeof(TTo) == typeof(short)) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(ushort) && IsUInt32Like()) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (typeof(TFrom) == typeof(short) && IsInt32Like()) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (IsUInt32Like() && IsUInt64Like()) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - if (IsInt32Like() && IsInt64Like()) - { - InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); - return true; - } - - return false; - } - - /// (int)float - private readonly struct ConvertInt32ToSingle : IUnaryOperator - { - public static bool Vectorizable => true; - - public static float Invoke(int x) => x; - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToSingle(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToSingle(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToSingle(x); - } - - /// (uint)float - private readonly struct ConvertUInt32ToSingle : IUnaryOperator - { - public static bool Vectorizable => true; - - public static float Invoke(uint x) => x; - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToSingle(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToSingle(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToSingle(x); - } - - /// (double)ulong - private readonly struct ConvertUInt64ToDouble : IUnaryOperator - { - public static bool Vectorizable => true; - - public static double Invoke(ulong x) => x; - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToDouble(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToDouble(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToDouble(x); - } - - /// (double)long - private readonly struct ConvertInt64ToDouble : IUnaryOperator - { - public static bool Vectorizable => true; - - public static double Invoke(long x) => x; - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToDouble(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToDouble(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToDouble(x); - } - - /// (double)float - private readonly struct WidenSingleToDoubleOperator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static double Invoke(float x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - /// (float)double - private readonly struct NarrowDoubleToSingleOperator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static float Invoke(double x) => (float)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// (ushort)byte - private readonly struct WidenByteToUInt16Operator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static ushort Invoke(byte x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - /// (short)sbyte - private readonly struct WidenSByteToInt16Operator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static short Invoke(sbyte x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - /// (uint)ushort - private readonly struct WidenUInt16ToUInt32Operator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static uint Invoke(ushort x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - /// (int)short - private readonly struct WidenInt16ToInt32Operator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static int Invoke(short x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - /// (ulong)uint - private readonly struct WidenUInt32ToUInt64Operator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static ulong Invoke(uint x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - /// (long)int - private readonly struct WidenInt32ToInt64Operator : IUnaryOneToTwoOperator - { - public static bool Vectorizable => true; - - public static long Invoke(int x) => x; - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); - } - - private readonly struct WidenHalfAsInt16ToSingleOperator : IUnaryOneToTwoOperator - { - // This implements a vectorized version of the `explicit operator float(Half value) operator`. - // See detailed description of the algorithm used here: - // https://github.com/dotnet/runtime/blob/3bf40a378f00cb5bf18ff62796bc7097719b974c/src/libraries/System.Private.CoreLib/src/System/Half.cs#L1010-L1040 - // The cast operator converts a Half represented as uint to a float. This does the same, with an input VectorXx and an output VectorXx. - // The VectorXx is created by reading a vector of Halfs as a VectorXx then widened to two VectorXxs and cast to VectorXxs. - // We loop handling one input vector at a time, producing two output float vectors. - - private const uint ExponentLowerBound = 0x3880_0000u; // The smallest positive normal number in Half, converted to Single - private const uint ExponentOffset = 0x3800_0000u; // BitConverter.SingleToUInt32Bits(1.0f) - ((uint)BitConverter.HalfToUInt16Bits((Half)1.0f) << 13) - private const uint SingleSignMask = 0x8000_0000; // float.SignMask; // Mask for sign bit in Single - private const uint HalfExponentMask = 0x7C00; // Mask for exponent bits in Half - private const uint HalfToSingleBitsMask = 0x0FFF_E000; // Mask for bits in Single converted from Half - - public static bool Vectorizable => true; - - public static float Invoke(short x) => (float)Unsafe.BitCast(x); - - public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) - { - (Vector128 lowerInt32, Vector128 upperInt32) = Vector128.Widen(x); - return - (HalfAsWidenedUInt32ToSingle(lowerInt32.AsUInt32()), - HalfAsWidenedUInt32ToSingle(upperInt32.AsUInt32())); - - static Vector128 HalfAsWidenedUInt32ToSingle(Vector128 value) - { - // Extract sign bit of value - Vector128 sign = value & Vector128.Create(SingleSignMask); - - // Copy sign bit to upper bits - Vector128 bitValueInProcess = value; - - // Extract exponent bits of value (BiasedExponent is not for here as it performs unnecessary shift) - Vector128 offsetExponent = bitValueInProcess & Vector128.Create(HalfExponentMask); - - // ~0u when value is subnormal, 0 otherwise - Vector128 subnormalMask = Vector128.Equals(offsetExponent, Vector128.Zero); - - // ~0u when value is either Infinity or NaN, 0 otherwise - Vector128 infinityOrNaNMask = Vector128.Equals(offsetExponent, Vector128.Create(HalfExponentMask)); - - // 0x3880_0000u if value is subnormal, 0 otherwise - Vector128 maskedExponentLowerBound = subnormalMask & Vector128.Create(ExponentLowerBound); - - // 0x3880_0000u if value is subnormal, 0x3800_0000u otherwise - Vector128 offsetMaskedExponentLowerBound = Vector128.Create(ExponentOffset) | maskedExponentLowerBound; - - // Match the position of the boundary of exponent bits and fraction bits with IEEE 754 Binary32(Single) - bitValueInProcess = Vector128.ShiftLeft(bitValueInProcess, 13); - - // Double the offsetMaskedExponentLowerBound if value is either Infinity or NaN - offsetMaskedExponentLowerBound = Vector128.ConditionalSelect(Vector128.Equals(infinityOrNaNMask, Vector128.Zero), - offsetMaskedExponentLowerBound, - Vector128.ShiftLeft(offsetMaskedExponentLowerBound, 1)); - - // Extract exponent bits and fraction bits of value - bitValueInProcess &= Vector128.Create(HalfToSingleBitsMask); - - // Adjust exponent to match the range of exponent - bitValueInProcess += offsetMaskedExponentLowerBound; - - // If value is subnormal, remove unnecessary 1 on top of fraction bits. - Vector128 absoluteValue = (bitValueInProcess.AsSingle() - maskedExponentLowerBound.AsSingle()).AsUInt32(); - - // Merge sign bit with rest - return (absoluteValue | sign).AsSingle(); - } - } - - public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) - { - (Vector256 lowerInt32, Vector256 upperInt32) = Vector256.Widen(x); - return - (HalfAsWidenedUInt32ToSingle(lowerInt32.AsUInt32()), - HalfAsWidenedUInt32ToSingle(upperInt32.AsUInt32())); - - static Vector256 HalfAsWidenedUInt32ToSingle(Vector256 value) - { - // Extract sign bit of value - Vector256 sign = value & Vector256.Create(SingleSignMask); - - // Copy sign bit to upper bits - Vector256 bitValueInProcess = value; - - // Extract exponent bits of value (BiasedExponent is not for here as it performs unnecessary shift) - Vector256 offsetExponent = bitValueInProcess & Vector256.Create(HalfExponentMask); - - // ~0u when value is subnormal, 0 otherwise - Vector256 subnormalMask = Vector256.Equals(offsetExponent, Vector256.Zero); - - // ~0u when value is either Infinity or NaN, 0 otherwise - Vector256 infinityOrNaNMask = Vector256.Equals(offsetExponent, Vector256.Create(HalfExponentMask)); - - // 0x3880_0000u if value is subnormal, 0 otherwise - Vector256 maskedExponentLowerBound = subnormalMask & Vector256.Create(ExponentLowerBound); - - // 0x3880_0000u if value is subnormal, 0x3800_0000u otherwise - Vector256 offsetMaskedExponentLowerBound = Vector256.Create(ExponentOffset) | maskedExponentLowerBound; - - // Match the position of the boundary of exponent bits and fraction bits with IEEE 754 Binary32(Single) - bitValueInProcess = Vector256.ShiftLeft(bitValueInProcess, 13); - - // Double the offsetMaskedExponentLowerBound if value is either Infinity or NaN - offsetMaskedExponentLowerBound = Vector256.ConditionalSelect(Vector256.Equals(infinityOrNaNMask, Vector256.Zero), - offsetMaskedExponentLowerBound, - Vector256.ShiftLeft(offsetMaskedExponentLowerBound, 1)); - - // Extract exponent bits and fraction bits of value - bitValueInProcess &= Vector256.Create(HalfToSingleBitsMask); - - // Adjust exponent to match the range of exponent - bitValueInProcess += offsetMaskedExponentLowerBound; - - // If value is subnormal, remove unnecessary 1 on top of fraction bits. - Vector256 absoluteValue = (bitValueInProcess.AsSingle() - maskedExponentLowerBound.AsSingle()).AsUInt32(); - - // Merge sign bit with rest - return (absoluteValue | sign).AsSingle(); - } - } - - public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) - { - (Vector512 lowerInt32, Vector512 upperInt32) = Vector512.Widen(x); - return - (HalfAsWidenedUInt32ToSingle(lowerInt32.AsUInt32()), - HalfAsWidenedUInt32ToSingle(upperInt32.AsUInt32())); - - static Vector512 HalfAsWidenedUInt32ToSingle(Vector512 value) - { - // Extract sign bit of value - Vector512 sign = value & Vector512.Create(SingleSignMask); - - // Copy sign bit to upper bits - Vector512 bitValueInProcess = value; - - // Extract exponent bits of value (BiasedExponent is not for here as it performs unnecessary shift) - Vector512 offsetExponent = bitValueInProcess & Vector512.Create(HalfExponentMask); - - // ~0u when value is subnormal, 0 otherwise - Vector512 subnormalMask = Vector512.Equals(offsetExponent, Vector512.Zero); - - // ~0u when value is either Infinity or NaN, 0 otherwise - Vector512 infinityOrNaNMask = Vector512.Equals(offsetExponent, Vector512.Create(HalfExponentMask)); - - // 0x3880_0000u if value is subnormal, 0 otherwise - Vector512 maskedExponentLowerBound = subnormalMask & Vector512.Create(ExponentLowerBound); - - // 0x3880_0000u if value is subnormal, 0x3800_0000u otherwise - Vector512 offsetMaskedExponentLowerBound = Vector512.Create(ExponentOffset) | maskedExponentLowerBound; - - // Match the position of the boundary of exponent bits and fraction bits with IEEE 754 Binary32(Single) - bitValueInProcess = Vector512.ShiftLeft(bitValueInProcess, 13); - - // Double the offsetMaskedExponentLowerBound if value is either Infinity or NaN - offsetMaskedExponentLowerBound = Vector512.ConditionalSelect(Vector512.Equals(infinityOrNaNMask, Vector512.Zero), - offsetMaskedExponentLowerBound, - Vector512.ShiftLeft(offsetMaskedExponentLowerBound, 1)); - - // Extract exponent bits and fraction bits of value - bitValueInProcess &= Vector512.Create(HalfToSingleBitsMask); - - // Adjust exponent to match the range of exponent - bitValueInProcess += offsetMaskedExponentLowerBound; - - // If value is subnormal, remove unnecessary 1 on top of fraction bits. - Vector512 absoluteValue = (bitValueInProcess.AsSingle() - maskedExponentLowerBound.AsSingle()).AsUInt32(); - - // Merge sign bit with rest - return (absoluteValue | sign).AsSingle(); - } - } - } - - private readonly struct NarrowSingleToHalfAsUInt16Operator : IUnaryTwoToOneOperator - { - // This implements a vectorized version of the `explicit operator Half(float value) operator`. - // See detailed description of the algorithm used here: - // https://github.com/dotnet/runtime/blob/ca8d6f0420096831766ec11c7d400e4f7ccc7a34/src/libraries/System.Private.CoreLib/src/System/Half.cs#L606-L714 - // The cast operator converts a float to a Half represented as a UInt32, then narrows to a UInt16, and reinterpret casts to Half. - // This does the same, with an input VectorXx and an output VectorXx. - // Loop handling two input vectors at a time; each input float is double the size of each output Half, - // so we need two vectors of floats to produce one vector of Halfs. Half isn't supported in VectorXx, - // so we convert the VectorXx to a VectorXx, and the caller then uses this twice, narrows the combination - // into a VectorXx, and then saves that out to the destination `ref Half` reinterpreted as `ref ushort`. - - private const uint MinExp = 0x3880_0000u; // Minimum exponent for rounding - private const uint Exponent126 = 0x3f00_0000u; // Exponent displacement #1 - private const uint SingleBiasedExponentMask = 0x7F80_0000; // float.BiasedExponentMask; // Exponent mask - private const uint Exponent13 = 0x0680_0000u; // Exponent displacement #2 - private const float MaxHalfValueBelowInfinity = 65520.0f; // Maximum value that is not Infinity in Half - private const uint ExponentMask = 0x7C00; // Mask for exponent bits in Half - private const uint SingleSignMask = 0x8000_0000u; // float.SignMask; // Mask for sign bit in float - - public static bool Vectorizable => true; - - public static ushort Invoke(float x) => Unsafe.BitCast((Half)x); - - public static Vector128 Invoke(Vector128 lower, Vector128 upper) - { - return Vector128.Narrow( - SingleToHalfAsWidenedUInt32(lower), - SingleToHalfAsWidenedUInt32(upper)); - - static Vector128 SingleToHalfAsWidenedUInt32(Vector128 value) - { - Vector128 bitValue = value.AsUInt32(); - - // Extract sign bit - Vector128 sign = Vector128.ShiftRightLogical(bitValue & Vector128.Create(SingleSignMask), 16); - - // Detecting NaN (0u if value is NaN; otherwise, ~0u) - Vector128 realMask = Vector128.Equals(value, value).AsUInt32(); - - // Clear sign bit - value = Vector128.Abs(value); - - // Rectify values that are Infinity in Half. - value = Vector128.Min(Vector128.Create(MaxHalfValueBelowInfinity), value); - - // Rectify lower exponent - Vector128 exponentOffset0 = Vector128.Max(value, Vector128.Create(MinExp).AsSingle()).AsUInt32(); - - // Extract exponent - exponentOffset0 &= Vector128.Create(SingleBiasedExponentMask); - - // Add exponent by 13 - exponentOffset0 += Vector128.Create(Exponent13); - - // Round Single into Half's precision (NaN also gets modified here, just setting the MSB of fraction) - value += exponentOffset0.AsSingle(); - bitValue = value.AsUInt32(); - - // Only exponent bits will be modified if NaN - Vector128 maskedHalfExponentForNaN = ~realMask & Vector128.Create(ExponentMask); - - // Subtract exponent by 126 - bitValue -= Vector128.Create(Exponent126); - - // Shift bitValue right by 13 bits to match the boundary of exponent part and fraction part. - Vector128 newExponent = Vector128.ShiftRightLogical(bitValue, 13); - - // Clear the fraction parts if the value was NaN. - bitValue &= realMask; - - // Merge the exponent part with fraction part, and add the exponent part and fraction part's overflow. - bitValue += newExponent; - - // Clear exponents if value is NaN - bitValue &= ~maskedHalfExponentForNaN; - - // Merge sign bit with possible NaN exponent - Vector128 signAndMaskedExponent = maskedHalfExponentForNaN | sign; - - // Merge sign bit and possible NaN exponent - bitValue |= signAndMaskedExponent; - - // The final result - return bitValue; - } - } - - public static Vector256 Invoke(Vector256 lower, Vector256 upper) - { - return Vector256.Narrow( - SingleToHalfAsWidenedUInt32(lower), - SingleToHalfAsWidenedUInt32(upper)); - - static Vector256 SingleToHalfAsWidenedUInt32(Vector256 value) - { - Vector256 bitValue = value.AsUInt32(); - - // Extract sign bit - Vector256 sign = Vector256.ShiftRightLogical(bitValue & Vector256.Create(SingleSignMask), 16); - - // Detecting NaN (0u if value is NaN; otherwise, ~0u) - Vector256 realMask = Vector256.Equals(value, value).AsUInt32(); - - // Clear sign bit - value = Vector256.Abs(value); - - // Rectify values that are Infinity in Half. - value = Vector256.Min(Vector256.Create(MaxHalfValueBelowInfinity), value); - - // Rectify lower exponent - Vector256 exponentOffset0 = Vector256.Max(value, Vector256.Create(MinExp).AsSingle()).AsUInt32(); - - // Extract exponent - exponentOffset0 &= Vector256.Create(SingleBiasedExponentMask); - - // Add exponent by 13 - exponentOffset0 += Vector256.Create(Exponent13); - - // Round Single into Half's precision (NaN also gets modified here, just setting the MSB of fraction) - value += exponentOffset0.AsSingle(); - bitValue = value.AsUInt32(); - - // Only exponent bits will be modified if NaN - Vector256 maskedHalfExponentForNaN = ~realMask & Vector256.Create(ExponentMask); - - // Subtract exponent by 126 - bitValue -= Vector256.Create(Exponent126); - - // Shift bitValue right by 13 bits to match the boundary of exponent part and fraction part. - Vector256 newExponent = Vector256.ShiftRightLogical(bitValue, 13); - - // Clear the fraction parts if the value was NaN. - bitValue &= realMask; - - // Merge the exponent part with fraction part, and add the exponent part and fraction part's overflow. - bitValue += newExponent; - - // Clear exponents if value is NaN - bitValue &= ~maskedHalfExponentForNaN; - - // Merge sign bit with possible NaN exponent - Vector256 signAndMaskedExponent = maskedHalfExponentForNaN | sign; - - // Merge sign bit and possible NaN exponent - bitValue |= signAndMaskedExponent; - - // The final result - return bitValue; - } - } - - public static Vector512 Invoke(Vector512 lower, Vector512 upper) - { - return Vector512.Narrow( - SingleToHalfAsWidenedUInt32(lower), - SingleToHalfAsWidenedUInt32(upper)); - - static Vector512 SingleToHalfAsWidenedUInt32(Vector512 value) - { - Vector512 bitValue = value.AsUInt32(); - - // Extract sign bit - Vector512 sign = Vector512.ShiftRightLogical(bitValue & Vector512.Create(SingleSignMask), 16); - - // Detecting NaN (0u if value is NaN; otherwise, ~0u) - Vector512 realMask = Vector512.Equals(value, value).AsUInt32(); - - // Clear sign bit - value = Vector512.Abs(value); - - // Rectify values that are Infinity in Half. - value = Vector512.Min(Vector512.Create(MaxHalfValueBelowInfinity), value); - - // Rectify lower exponent - Vector512 exponentOffset0 = Vector512.Max(value, Vector512.Create(MinExp).AsSingle()).AsUInt32(); - - // Extract exponent - exponentOffset0 &= Vector512.Create(SingleBiasedExponentMask); - - // Add exponent by 13 - exponentOffset0 += Vector512.Create(Exponent13); - - // Round Single into Half's precision (NaN also gets modified here, just setting the MSB of fraction) - value += exponentOffset0.AsSingle(); - bitValue = value.AsUInt32(); - - // Only exponent bits will be modified if NaN - Vector512 maskedHalfExponentForNaN = ~realMask & Vector512.Create(ExponentMask); - - // Subtract exponent by 126 - bitValue -= Vector512.Create(Exponent126); - - // Shift bitValue right by 13 bits to match the boundary of exponent part and fraction part. - Vector512 newExponent = Vector512.ShiftRightLogical(bitValue, 13); - - // Clear the fraction parts if the value was NaN. - bitValue &= realMask; - - // Merge the exponent part with fraction part, and add the exponent part and fraction part's overflow. - bitValue += newExponent; - - // Clear exponents if value is NaN - bitValue &= ~maskedHalfExponentForNaN; - - // Merge sign bit with possible NaN exponent - Vector512 signAndMaskedExponent = maskedHalfExponentForNaN | sign; - - // Merge sign bit and possible NaN exponent - bitValue |= signAndMaskedExponent; - - // The final result - return bitValue; - } - } - } - - /// Creates a span of from a when they're the same type. - private static unsafe ReadOnlySpan Rename(ReadOnlySpan span) - { - Debug.Assert(sizeof(TFrom) == sizeof(TTo)); - return MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); - } - - /// Gets whether is or if in a 32-bit process. - private static bool IsUInt32Like() => typeof(T) == typeof(uint) || (IntPtr.Size == 4 && typeof(T) == typeof(nuint)); - - /// Gets whether is or if in a 32-bit process. - private static bool IsInt32Like() => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); - - /// Gets whether is or if in a 64-bit process. - private static bool IsUInt64Like() => typeof(T) == typeof(ulong) || (IntPtr.Size == 8 && typeof(T) == typeof(nuint)); - - /// Gets whether is or if in a 64-bit process. - private static bool IsInt64Like() => typeof(T) == typeof(long) || (IntPtr.Size == 8 && typeof(T) == typeof(nint)); - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertSaturating.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertSaturating.cs deleted file mode 100644 index 87a26c3c21643b..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertSaturating.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// - /// Copies to , converting each - /// value to a value. - /// - /// The source span from which to copy values. - /// The destination span into which the converted values should be written. - /// Destination is too short. - /// - /// - /// This method effectively computes [i] = TTo.CreateSaturating([i]). - /// - /// - public static void ConvertSaturating(ReadOnlySpan source, Span destination) - where TFrom : INumberBase - where TTo : INumberBase - { - if (!TryConvertUniversal(source, destination)) - { - InvokeSpanIntoSpan>(source, destination); - } - } - - /// T.CreateSaturating(x) - internal readonly struct ConvertSaturatingFallbackOperator : IUnaryOperator where TFrom : INumberBase where TTo : INumberBase - { - public static bool Vectorizable => false; - - public static TTo Invoke(TFrom x) => TTo.CreateSaturating(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertTruncating.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertTruncating.cs deleted file mode 100644 index 29edffd4bcec03..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ConvertTruncating.cs +++ /dev/null @@ -1,224 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// - /// Copies to , converting each - /// value to a value. - /// - /// The source span from which to copy values. - /// The destination span into which the converted values should be written. - /// Destination is too short. - /// - /// - /// This method effectively computes [i] = TTo.CreateTruncating([i]). - /// - /// - public static void ConvertTruncating(ReadOnlySpan source, Span destination) - where TFrom : INumberBase - where TTo : INumberBase - { - if (TryConvertUniversal(source, destination)) - { - return; - } - - if (((typeof(TFrom) == typeof(byte) || typeof(TFrom) == typeof(sbyte)) && (typeof(TTo) == typeof(byte) || typeof(TTo) == typeof(sbyte))) || - ((typeof(TFrom) == typeof(ushort) || typeof(TFrom) == typeof(short)) && (typeof(TTo) == typeof(ushort) || typeof(TTo) == typeof(short))) || - ((IsUInt32Like() || IsInt32Like()) && (IsUInt32Like() || IsInt32Like())) || - ((IsUInt64Like() || IsInt64Like()) && (IsUInt64Like() || IsInt64Like()))) - { - source.CopyTo(Rename(destination)); - return; - } - - if (typeof(TFrom) == typeof(float) && IsUInt32Like()) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return; - } - - if (typeof(TFrom) == typeof(float) && IsInt32Like()) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return; - } - - if (typeof(TFrom) == typeof(double) && IsUInt64Like()) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return; - } - - if (typeof(TFrom) == typeof(double) && IsInt64Like()) - { - InvokeSpanIntoSpan(Rename(source), Rename(destination)); - return; - } - - if (typeof(TFrom) == typeof(ushort) && typeof(TTo) == typeof(byte)) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return; - } - - if (typeof(TFrom) == typeof(short) && typeof(TTo) == typeof(sbyte)) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return; - } - - if (IsUInt32Like() && typeof(TTo) == typeof(ushort)) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return; - } - - if (IsInt32Like() && typeof(TTo) == typeof(short)) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return; - } - - if (IsUInt64Like() && IsUInt32Like()) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return; - } - - if (IsInt64Like() && IsInt32Like()) - { - InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); - return; - } - - InvokeSpanIntoSpan>(source, destination); - } - - /// (float)int - private readonly struct ConvertSingleToInt32 : IUnaryOperator - { - public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar - - public static int Invoke(float x) => int.CreateTruncating(x); - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToInt32(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToInt32(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToInt32(x); - } - - /// (float)uint - private readonly struct ConvertSingleToUInt32 : IUnaryOperator - { - public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar - - public static uint Invoke(float x) => uint.CreateTruncating(x); - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToUInt32(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToUInt32(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToUInt32(x); - } - - /// (ulong)double - private readonly struct ConvertDoubleToUInt64 : IUnaryOperator - { - public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar - - public static ulong Invoke(double x) => ulong.CreateTruncating(x); - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToUInt64(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToUInt64(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToUInt64(x); - } - - /// (long)double - private readonly struct ConvertDoubleToInt64 : IUnaryOperator - { - public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar - - public static long Invoke(double x) => long.CreateTruncating(x); - public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToInt64(x); - public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToInt64(x); - public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToInt64(x); - } - - /// (byte)ushort - private readonly struct NarrowUInt16ToByteOperator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static byte Invoke(ushort x) => (byte)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// (sbyte)short - private readonly struct NarrowInt16ToSByteOperator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static sbyte Invoke(short x) => (sbyte)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// (ushort)uint - private readonly struct NarrowUInt32ToUInt16Operator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static ushort Invoke(uint x) => (ushort)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// (short)int - private readonly struct NarrowInt32ToInt16Operator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static short Invoke(int x) => (short)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// (uint)ulong - private readonly struct NarrowUInt64ToUInt32Operator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static uint Invoke(ulong x) => (uint)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// (int)long - private readonly struct NarrowInt64ToInt32Operator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => true; - - public static int Invoke(long x) => (int)x; - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); - } - - /// T.CreateTruncating(x) - private readonly struct ConvertTruncatingFallbackOperator : IUnaryOperator where TFrom : INumberBase where TTo : INumberBase - { - public static bool Vectorizable => false; - - public static TTo Invoke(TFrom x) => TTo.CreateTruncating(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CopySign.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CopySign.cs deleted file mode 100644 index 0277c729a17aa4..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CopySign.cs +++ /dev/null @@ -1,133 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of copying the sign from one number to another number in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.CopySign([i], [i]). - /// - /// - public static void CopySign(ReadOnlySpan x, ReadOnlySpan sign, Span destination) - where T : INumber => - InvokeSpanSpanIntoSpan>(x, sign, destination); - - /// Computes the element-wise result of copying the sign from one number to another number in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.CopySign([i], [i]). - /// - /// - public static void CopySign(ReadOnlySpan x, T sign, Span destination) - where T : INumber => - InvokeSpanScalarIntoSpan>(x, sign, destination); - - private readonly struct CopySignOperator : IBinaryOperator where T : INumber - { - public static bool Vectorizable => true; - - public static T Invoke(T x, T y) => T.CopySign(x, y); - - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (typeof(T) == typeof(float)) - { - return Vector128.ConditionalSelect(Vector128.Create(-0.0f).As(), y, x); - } - - if (typeof(T) == typeof(double)) - { - return Vector128.ConditionalSelect(Vector128.Create(-0.0d).As(), y, x); - } - - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector128 absValue = Vector128.Abs(x); - Vector128 sign = Vector128.GreaterThanOrEqual(y, Vector128.Zero); - Vector128 error = sign & Vector128.LessThan(absValue, Vector128.Zero); - if (error != Vector128.Zero) - { - Math.Abs(int.MinValue); // throw OverflowException - } - - return Vector128.ConditionalSelect(sign, absValue, -absValue); - } - - return x; - } - - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float)) - { - return Vector256.ConditionalSelect(Vector256.Create(-0.0f).As(), y, x); - } - - if (typeof(T) == typeof(double)) - { - return Vector256.ConditionalSelect(Vector256.Create(-0.0d).As(), y, x); - } - - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector256 absValue = Vector256.Abs(x); - Vector256 sign = Vector256.GreaterThanOrEqual(y, Vector256.Zero); - Vector256 error = sign & Vector256.LessThan(absValue, Vector256.Zero); - if (error != Vector256.Zero) - { - Math.Abs(int.MinValue); // throw OverflowException - } - - return Vector256.ConditionalSelect(sign, absValue, -absValue); - } - - return x; - } - - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float)) - { - return Vector512.ConditionalSelect(Vector512.Create(-0.0f).As(), y, x); - } - - if (typeof(T) == typeof(double)) - { - return Vector512.ConditionalSelect(Vector512.Create(-0.0d).As(), y, x); - } - - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector512 absValue = Vector512.Abs(x); - Vector512 sign = Vector512.GreaterThanOrEqual(y, Vector512.Zero); - Vector512 error = sign & Vector512.LessThan(absValue, Vector512.Zero); - if (error != Vector512.Zero) - { - Math.Abs(int.MinValue); // throw OverflowException - } - - return Vector512.ConditionalSelect(sign, absValue, -absValue); - } - - return x; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cos.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cos.cs deleted file mode 100644 index 36bdcc82e337bb..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cos.cs +++ /dev/null @@ -1,351 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise cosine of the value in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Cos([i]). - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Cos(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Cos(x) - private readonly struct CosOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - // This code is based on `vrs4_cos` and `vrd2_cos` from amd/aocl-libm-ose - // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Implementation notes from amd/aocl-libm-ose: - // -------------------------------------------- - // To compute cosf(float x) - // Using the identity, - // cos(x) = sin(x + pi/2) (1) - // - // 1. Argument Reduction - // Now, let x be represented as, - // |x| = N * pi + f (2) | N is an integer, - // -pi/2 <= f <= pi/2 - // - // From (2), N = int( (x + pi/2) / pi) - 0.5 - // f = |x| - (N * pi) - // - // 2. Polynomial Evaluation - // From (1) and (2),sin(f) can be calculated using a polynomial - // sin(f) = f*(1 + C1*f^2 + C2*f^4 + C3*f^6 + c4*f^8) - // - // 3. Reconstruction - // Hence, cos(x) = sin(x + pi/2) = (-1)^N * sin(f) - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Cos(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return CosOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return CosOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return CosOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return CosOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return CosOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return CosOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - } - - /// float.Cos(x) - private readonly struct CosOperatorSingle : IUnaryOperator - { - internal const uint MaxVectorizedValue = 0x4A989680u; - internal const uint SignMask = 0x7FFFFFFFu; - private const float AlmHuge = 1.2582912e7f; - private const float Pi_Tail1 = 8.742278e-8f; - private const float Pi_Tail2 = 3.430249e-15f; - private const float C1 = -0.16666657f; - private const float C2 = 0.008332962f; - private const float C3 = -1.9801206e-4f; - private const float C4 = 2.5867037e-6f; - - public static bool Vectorizable => true; - - public static float Invoke(float x) => float.Cos(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 uxMasked = Vector128.Abs(x); - if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector128 almHuge = Vector128.Create(AlmHuge); - Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked + Vector128.Create(float.Pi / 2), Vector128.Create(1 / float.Pi), almHuge); - Vector128 odd = dn.AsUInt32() << 31; - dn = dn - almHuge - Vector128.Create(0.5f); - - Vector128 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-float.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); - - // POLY_EVAL_ODD_9 - Vector128 f2 = f * f; - Vector128 f4 = f2 * f2; - Vector128 a0 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); - Vector128 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector128.One); - Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C3), f2, Vector128.Create(C4) * f4); - Vector128 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector128 poly = f * a3; - - return (poly.AsUInt32() ^ odd).AsSingle(); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 uxMasked = Vector256.Abs(x); - if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector256 almHuge = Vector256.Create(AlmHuge); - Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked + Vector256.Create(float.Pi / 2), Vector256.Create(1 / float.Pi), almHuge); - Vector256 odd = dn.AsUInt32() << 31; - dn = dn - almHuge - Vector256.Create(0.5f); - - Vector256 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-float.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); - - // POLY_EVAL_ODD_9 - Vector256 f2 = f * f; - Vector256 f4 = f2 * f2; - Vector256 a0 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); - Vector256 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector256.One); - Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C3), f2, Vector256.Create(C4) * f4); - Vector256 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector256 poly = f * a3; - - return (poly.AsUInt32() ^ odd).AsSingle(); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 uxMasked = Vector512.Abs(x); - if (Vector512.GreaterThanAny(uxMasked.AsUInt32(), Vector512.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector512 almHuge = Vector512.Create(AlmHuge); - Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked + Vector512.Create(float.Pi / 2), Vector512.Create(1 / float.Pi), almHuge); - Vector512 odd = dn.AsUInt32() << 31; - dn = dn - almHuge - Vector512.Create(0.5f); - - Vector512 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-float.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); - - // POLY_EVAL_ODD_9 - Vector512 f2 = f * f; - Vector512 f4 = f2 * f2; - Vector512 a0 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); - Vector512 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector512.One); - Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C3), f2, Vector512.Create(C4) * f4); - Vector512 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector512 poly = f * a3; - - return (poly.AsUInt32() ^ odd).AsSingle(); - } - } - - /// double.Cos(x) - private readonly struct CosOperatorDouble : IUnaryOperator - { - internal const ulong SignMask = 0x7FFFFFFFFFFFFFFFul; - internal const ulong MaxVectorizedValue = 0x4160000000000000ul; - private const double AlmHuge = 6.755399441055744E15; - private const double Pi_Tail2 = -1.2246467991473532E-16; - private const double Pi_Tail3 = 2.9947698097183397E-33; - private const double C1 = -0.16666666666666666; - private const double C2 = 0.008333333333333165; - private const double C3 = -1.984126984120184E-4; - private const double C4 = 2.7557319210152756E-6; - private const double C5 = -2.5052106798274616E-8; - private const double C6 = 1.6058936490373254E-10; - private const double C7 = -7.642917806937501E-13; - private const double C8 = 2.7204790963151784E-15; - - public static bool Vectorizable => true; - - public static double Invoke(double x) => double.Cos(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 uxMasked = Vector128.Abs(x); - if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = int(x / pi + 1/2) - 1/2 - Vector128 almHuge = Vector128.Create(AlmHuge); - Vector128 half = Vector128.Create(0.5); - Vector128 dn = (uxMasked * Vector128.Create(1 / double.Pi)) + half + almHuge; - Vector128 odd = dn.AsUInt64() << 63; - dn = dn - almHuge - half; - - // f = x - (n*pi) - Vector128 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-double.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail3), f); - - // POLY_EVAL_ODD_17 - Vector128 f2 = f * f; - Vector128 f4 = f2 * f2; - Vector128 f6 = f4 * f2; - Vector128 f10 = f6 * f4; - Vector128 f14 = f10 * f4; - Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); - Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C4), f2, Vector128.Create(C3)); - Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C6), f2, Vector128.Create(C5)); - Vector128 a4 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C8), f2, Vector128.Create(C7)); - Vector128 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); - Vector128 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); - Vector128 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); - - return (poly.AsUInt64() ^ odd).AsDouble(); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 uxMasked = Vector256.Abs(x); - if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = int(x / pi + 1/2) - 1/2 - Vector256 almHuge = Vector256.Create(AlmHuge); - Vector256 half = Vector256.Create(0.5); - Vector256 dn = (uxMasked * Vector256.Create(1 / double.Pi)) + half + almHuge; - Vector256 odd = dn.AsUInt64() << 63; - dn = dn - almHuge - half; - - // f = x - (n*pi) - Vector256 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-double.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail3), f); - - // POLY_EVAL_ODD_17 - Vector256 f2 = f * f; - Vector256 f4 = f2 * f2; - Vector256 f6 = f4 * f2; - Vector256 f10 = f6 * f4; - Vector256 f14 = f10 * f4; - Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); - Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C4), f2, Vector256.Create(C3)); - Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C6), f2, Vector256.Create(C5)); - Vector256 a4 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C8), f2, Vector256.Create(C7)); - Vector256 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); - Vector256 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); - Vector256 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); - - return (poly.AsUInt64() ^ odd).AsDouble(); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 uxMasked = Vector512.Abs(x); - if (Vector512.GreaterThanAny(uxMasked.AsUInt64(), Vector512.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = int(x / pi + 1/2) - 1/2 - Vector512 almHuge = Vector512.Create(AlmHuge); - Vector512 half = Vector512.Create(0.5); - Vector512 dn = (uxMasked * Vector512.Create(1 / double.Pi)) + half + almHuge; - Vector512 odd = dn.AsUInt64() << 63; - dn = dn - almHuge - half; - - // f = x - (n*pi) - Vector512 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-double.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail3), f); - - // POLY_EVAL_ODD_17 - Vector512 f2 = f * f; - Vector512 f4 = f2 * f2; - Vector512 f6 = f4 * f2; - Vector512 f10 = f6 * f4; - Vector512 f14 = f10 * f4; - Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); - Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C4), f2, Vector512.Create(C3)); - Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C6), f2, Vector512.Create(C5)); - Vector512 a4 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C8), f2, Vector512.Create(C7)); - Vector512 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); - Vector512 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); - Vector512 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); - - return (poly.AsUInt64() ^ odd).AsDouble(); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CosPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CosPi.cs deleted file mode 100644 index b286a18d0f9424..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CosPi.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise cosine of the value in the specified tensor that has been multiplied by Pi. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .CosPi([i]). - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void CosPi(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.CosPi(x) - private readonly struct CosPiOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.CosPi(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 xpi = x * Vector128.Create(T.Pi); - if (typeof(T) == typeof(float)) - { - if (Vector128.GreaterThanAny(xpi.AsUInt32() & Vector128.Create(CosOperatorSingle.SignMask), Vector128.Create(CosOperatorSingle.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsSingle()).As(); - } - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - if (Vector128.GreaterThanAny(xpi.AsUInt64() & Vector128.Create(CosOperatorDouble.SignMask), Vector128.Create(CosOperatorDouble.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsDouble()).As(); - } - } - - return CosOperator.Invoke(xpi); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 xpi = x * Vector256.Create(T.Pi); - if (typeof(T) == typeof(float)) - { - if (Vector256.GreaterThanAny(xpi.AsUInt32() & Vector256.Create(CosOperatorSingle.SignMask), Vector256.Create(CosOperatorSingle.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsSingle()).As(); - } - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - if (Vector256.GreaterThanAny(xpi.AsUInt64() & Vector256.Create(CosOperatorDouble.SignMask), Vector256.Create(CosOperatorDouble.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsDouble()).As(); - } - } - - return CosOperator.Invoke(xpi); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 xpi = x * Vector512.Create(T.Pi); - if (typeof(T) == typeof(float)) - { - if (Vector512.GreaterThanAny(xpi.AsUInt32() & Vector512.Create(CosOperatorSingle.SignMask), Vector512.Create(CosOperatorSingle.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsSingle()).As(); - } - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - if (Vector512.GreaterThanAny(xpi.AsUInt64() & Vector512.Create(CosOperatorDouble.SignMask), Vector512.Create(CosOperatorDouble.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsDouble()).As(); - } - } - - return CosOperator.Invoke(xpi); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cosh.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cosh.cs deleted file mode 100644 index 2047ee6a26f5cb..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Cosh.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hyperbolic cosine of each radian angle in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Cosh([i]). - /// - /// - /// If a value is equal to or , the result stored into the corresponding destination location is set to . - /// If a value is equal to , the result stored into the corresponding destination location is also NaN. - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Cosh(ReadOnlySpan x, Span destination) - where T : IHyperbolicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Cosh(x) - internal readonly struct CoshOperator : IUnaryOperator - where T : IHyperbolicFunctions - { - // This code is based on `vrs4_coshf` from amd/aocl-libm-ose - // Copyright (C) 2008-2022 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Spec: - // coshf(|x| > 89.415985107421875) = Infinity - // coshf(Infinity) = infinity - // coshf(-Infinity) = infinity - // - // cosh(x) = (exp(x) + exp(-x))/2 - // cosh(-x) = +cosh(x) - // - // checks for special cases - // if ( asint(x) > infinity) return x with overflow exception and - // return x. - // if x is NaN then raise invalid FP operation exception and return x. - // - // coshf = v/2 * exp(x - log(v)) where v = 0x1.0000e8p-1 - - private const float Single_LOGV = 0.693161f; - private const float Single_HALFV = 1.0000138f; - private const float Single_INVV2 = 0.24999309f; - - private const double Double_LOGV = 0.6931471805599453; - private const double Double_HALFV = 1.0; - private const double Double_INVV2 = 0.25; - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Cosh(x); - - public static Vector128 Invoke(Vector128 t) - { - if (typeof(T) == typeof(float)) - { - Vector128 x = t.AsSingle(); - - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpOperator.Invoke(y - Vector128.Create((float)Single_LOGV)); - return (Vector128.Create((float)Single_HALFV) * (z + (Vector128.Create((float)Single_INVV2) / z))).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector128 x = t.AsDouble(); - - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpOperator.Invoke(y - Vector128.Create(Double_LOGV)); - return (Vector128.Create(Double_HALFV) * (z + (Vector128.Create(Double_INVV2) / z))).As(); - } - } - - public static Vector256 Invoke(Vector256 t) - { - if (typeof(T) == typeof(float)) - { - Vector256 x = t.AsSingle(); - - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpOperator.Invoke(y - Vector256.Create((float)Single_LOGV)); - return (Vector256.Create((float)Single_HALFV) * (z + (Vector256.Create((float)Single_INVV2) / z))).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector256 x = t.AsDouble(); - - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpOperator.Invoke(y - Vector256.Create(Double_LOGV)); - return (Vector256.Create(Double_HALFV) * (z + (Vector256.Create(Double_INVV2) / z))).As(); - } - } - - public static Vector512 Invoke(Vector512 t) - { - if (typeof(T) == typeof(float)) - { - Vector512 x = t.AsSingle(); - - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpOperator.Invoke(y - Vector512.Create((float)Single_LOGV)); - return (Vector512.Create((float)Single_HALFV) * (z + (Vector512.Create((float)Single_INVV2) / z))).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector512 x = t.AsDouble(); - - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpOperator.Invoke(y - Vector512.Create(Double_LOGV)); - return (Vector512.Create(Double_HALFV) * (z + (Vector512.Create(Double_INVV2) / z))).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CosineSimilarity.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CosineSimilarity.cs deleted file mode 100644 index cacbcab02a422c..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.CosineSimilarity.cs +++ /dev/null @@ -1,206 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public partial class TensorPrimitives - { - /// Computes the cosine similarity between the two specified non-empty, equal-length tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The cosine similarity of the two tensors. - /// Length of must be same as length of . - /// and must not be empty. - /// - /// - /// This method effectively computes TensorPrimitives.Dot(x, y) / (.Sqrt(TensorPrimitives.SumOfSquares(x)) * .Sqrt(TensorPrimitives.SumOfSquares(y)). - /// - /// - /// If any element in either input tensor is equal to , , or , - /// NaN is returned. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T CosineSimilarity(ReadOnlySpan x, ReadOnlySpan y) - where T : IRootFunctions => - CosineSimilarityCore(x, y); - - /// Computes the cosine similarity between the two specified non-empty, equal-length tensors of single-precision floating-point numbers. - /// Assumes arguments have already been validated to be non-empty and equal length. - private static T CosineSimilarityCore(ReadOnlySpan x, ReadOnlySpan y) where T : IRootFunctions - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - if (x.Length != y.Length) - { - ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); - } - - // Compute the same as: - // TensorPrimitives.Dot(x, y) / (Math.Sqrt(TensorPrimitives.SumOfSquares(x)) * Math.Sqrt(TensorPrimitives.SumOfSquares(y))) - // but only looping over each span once. - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) - { - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - - Vector512 dotProductVector = Vector512.Zero; - Vector512 xSumOfSquaresVector = Vector512.Zero; - Vector512 ySumOfSquaresVector = Vector512.Zero; - - // Process vectors, summing their dot products and squares, as long as there's a vector's worth remaining. - int oneVectorFromEnd = x.Length - Vector512.Count; - int i = 0; - do - { - Vector512 xVec = Vector512.LoadUnsafe(ref xRef, (uint)i); - Vector512 yVec = Vector512.LoadUnsafe(ref yRef, (uint)i); - - dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); - xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); - ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); - - i += Vector512.Count; - } - while (i <= oneVectorFromEnd); - - // Process the last vector in the span, masking off elements already processed. - if (i != x.Length) - { - Vector512 xVec = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); - Vector512 yVec = Vector512.LoadUnsafe(ref yRef, (uint)(x.Length - Vector512.Count)); - - Vector512 remainderMask = CreateRemainderMaskVector512(x.Length - i); - xVec &= remainderMask; - yVec &= remainderMask; - - dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); - xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); - ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); - } - - // Sum(X * Y) / (|X| * |Y|) - return - Vector512.Sum(dotProductVector) / - (T.Sqrt(Vector512.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector512.Sum(ySumOfSquaresVector))); - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) - { - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - - Vector256 dotProductVector = Vector256.Zero; - Vector256 xSumOfSquaresVector = Vector256.Zero; - Vector256 ySumOfSquaresVector = Vector256.Zero; - - // Process vectors, summing their dot products and squares, as long as there's a vector's worth remaining. - int oneVectorFromEnd = x.Length - Vector256.Count; - int i = 0; - do - { - Vector256 xVec = Vector256.LoadUnsafe(ref xRef, (uint)i); - Vector256 yVec = Vector256.LoadUnsafe(ref yRef, (uint)i); - - dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); - xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); - ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); - - i += Vector256.Count; - } - while (i <= oneVectorFromEnd); - - // Process the last vector in the span, masking off elements already processed. - if (i != x.Length) - { - Vector256 xVec = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); - Vector256 yVec = Vector256.LoadUnsafe(ref yRef, (uint)(x.Length - Vector256.Count)); - - Vector256 remainderMask = CreateRemainderMaskVector256(x.Length - i); - xVec &= remainderMask; - yVec &= remainderMask; - - dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); - xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); - ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); - } - - // Sum(X * Y) / (|X| * |Y|) - return - Vector256.Sum(dotProductVector) / - (T.Sqrt(Vector256.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector256.Sum(ySumOfSquaresVector))); - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) - { - ref T xRef = ref MemoryMarshal.GetReference(x); - ref T yRef = ref MemoryMarshal.GetReference(y); - - Vector128 dotProductVector = Vector128.Zero; - Vector128 xSumOfSquaresVector = Vector128.Zero; - Vector128 ySumOfSquaresVector = Vector128.Zero; - - // Process vectors, summing their dot products and squares, as long as there's a vector's worth remaining. - int oneVectorFromEnd = x.Length - Vector128.Count; - int i = 0; - do - { - Vector128 xVec = Vector128.LoadUnsafe(ref xRef, (uint)i); - Vector128 yVec = Vector128.LoadUnsafe(ref yRef, (uint)i); - - dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); - xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); - ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); - - i += Vector128.Count; - } - while (i <= oneVectorFromEnd); - - // Process the last vector in the span, masking off elements already processed. - if (i != x.Length) - { - Vector128 xVec = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); - Vector128 yVec = Vector128.LoadUnsafe(ref yRef, (uint)(x.Length - Vector128.Count)); - - Vector128 remainderMask = CreateRemainderMaskVector128(x.Length - i); - xVec &= remainderMask; - yVec &= remainderMask; - - dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); - xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); - ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); - } - - // Sum(X * Y) / (|X| * |Y|) - return - Vector128.Sum(dotProductVector) / - (T.Sqrt(Vector128.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector128.Sum(ySumOfSquaresVector))); - } - - // Vectorization isn't supported or there are too few elements to vectorize. - // Use a scalar implementation. - T dotProduct = T.Zero, xSumOfSquares = T.Zero, ySumOfSquares = T.Zero; - for (int i = 0; i < x.Length; i++) - { - dotProduct = MultiplyAddEstimateOperator.Invoke(x[i], y[i], dotProduct); - xSumOfSquares = MultiplyAddEstimateOperator.Invoke(x[i], x[i], xSumOfSquares); - ySumOfSquares = MultiplyAddEstimateOperator.Invoke(y[i], y[i], ySumOfSquares); - } - - // Sum(X * Y) / (|X| * |Y|) - return - dotProduct / - (T.Sqrt(xSumOfSquares) * T.Sqrt(ySumOfSquares)); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DegreesToRadians.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DegreesToRadians.cs deleted file mode 100644 index 68d7781f21cd02..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.DegreesToRadians.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise conversion of each number of degrees in the specified tensor to radiansx. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .DegreesToRadians([i]). - /// - /// - public static void DegreesToRadians(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.DegreesToRadians(x) - private readonly struct DegreesToRadiansOperator : IUnaryOperator where T : ITrigonometricFunctions - { - public static bool Vectorizable => true; - public static T Invoke(T x) => T.DegreesToRadians(x); - public static Vector128 Invoke(Vector128 x) => (x * T.Pi) / T.CreateChecked(180); - public static Vector256 Invoke(Vector256 x) => (x * T.Pi) / T.CreateChecked(180); - public static Vector512 Invoke(Vector512 x) => (x * T.Pi) / T.CreateChecked(180); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Distance.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Distance.cs deleted file mode 100644 index bf3481a9c7cead..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Distance.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the distance between two points, specified as non-empty, equal-length tensors of numbers, in Euclidean space. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The Euclidean distance. - /// Length of must be same as length of . - /// and must not be empty. - /// - /// - /// This method effectively computes the equivalent of: - /// - /// Span<T> difference = ...; - /// TensorPrimitives.Subtract(x, y, difference); - /// T result = .Sqrt(TensorPrimitives.SumOfSquares(difference)); - /// - /// but without requiring additional temporary storage for the intermediate differences. - /// - /// - /// If any element in either input tensor is equal to , NaN is returned. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Distance(ReadOnlySpan x, ReadOnlySpan y) - where T : IRootFunctions - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - return T.Sqrt(Aggregate, AddOperator>(x, y)); - } - - /// (x - y) * (x - y) - internal readonly struct SubtractSquaredOperator : IBinaryOperator where T : ISubtractionOperators, IMultiplyOperators - { - public static bool Vectorizable => true; - - public static T Invoke(T x, T y) - { - T tmp = x - y; - return tmp * tmp; - } - - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - Vector128 tmp = x - y; - return tmp * tmp; - } - - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - Vector256 tmp = x - y; - return tmp * tmp; - } - - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - Vector512 tmp = x - y; - return tmp * tmp; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Divide.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Divide.cs deleted file mode 100644 index 64a238face22e3..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Divide.cs +++ /dev/null @@ -1,79 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise division of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and an element in is equal to zero. - /// - /// - /// This method effectively computes [i] = [i] / [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Divide(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IDivisionOperators => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise division of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and is equal to zero. - /// - /// - /// This method effectively computes [i] = [i] / . - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Divide(ReadOnlySpan x, T y, Span destination) - where T : IDivisionOperators => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// Computes the element-wise division of numbers in the specified tensors. - /// The first tensor, represented as a scalar. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and an element in is equal to zero. - /// - /// - /// This method effectively computes [i] = / [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Divide(T x, ReadOnlySpan y, Span destination) - where T : IDivisionOperators => - InvokeScalarSpanIntoSpan>(x, y, destination); - - /// x / y - internal readonly struct DivideOperator : IBinaryOperator where T : IDivisionOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x, T y) => x / y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Dot.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Dot.cs deleted file mode 100644 index e66ec314ddbbe9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Dot.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the dot product of two tensors containing numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The dot product. - /// Length of must be same as length of . - /// - /// - /// This method effectively computes the equivalent of: - /// - /// Span<T> products = ...; - /// TensorPrimitives.Multiply(x, y, products); - /// T result = TensorPrimitives.Sum(products); - /// - /// but without requiring additional temporary storage for the intermediate products. It corresponds to the dot method defined by BLAS1. - /// - /// - /// If any of the input elements is equal to , the resulting value is also NaN. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Dot(ReadOnlySpan x, ReadOnlySpan y) - where T : IAdditionOperators, IAdditiveIdentity, IMultiplyOperators, IMultiplicativeIdentity => - Aggregate, AddOperator>(x, y); - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp.cs deleted file mode 100644 index 1147ec5bea5f78..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp.cs +++ /dev/null @@ -1,619 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of raising e to the number powers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Exp([i]). - /// - /// - /// If a value equals or , the result stored into the corresponding destination location is set to NaN. - /// If a value equals , the result stored into the corresponding destination location is set to 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Exp(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Exp(x) - internal readonly struct ExpOperator : IUnaryOperator - where T : IExponentialFunctions - { - public static bool Vectorizable => (typeof(T) == typeof(double)) - || (typeof(T) == typeof(float)); - - public static T Invoke(T x) => T.Exp(x); - - public static Vector128 Invoke(Vector128 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector128.Exp(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector128.Exp(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return ExpOperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return ExpOperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - - public static Vector256 Invoke(Vector256 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector256.Exp(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector256.Exp(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return ExpOperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return ExpOperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - - public static Vector512 Invoke(Vector512 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector512.Exp(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector512.Exp(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return ExpOperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return ExpOperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - } - -#if !NET9_0_OR_GREATER - /// double.Exp(x) - private readonly struct ExpOperatorDouble : IUnaryOperator - { - // This code is based on `vrd2_exp` from amd/aocl-libm-ose - // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Implementation Notes - // ---------------------- - // 1. Argument Reduction: - // e^x = 2^(x/ln2) = 2^(x*(64/ln(2))/64) --- (1) - // - // Choose 'n' and 'f', such that - // x * 64/ln2 = n + f --- (2) | n is integer - // | |f| <= 0.5 - // Choose 'm' and 'j' such that, - // n = (64 * m) + j --- (3) - // - // From (1), (2) and (3), - // e^x = 2^((64*m + j + f)/64) - // = (2^m) * (2^(j/64)) * 2^(f/64) - // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64)) - // - // 2. Table Lookup - // Values of (2^(j/64)) are precomputed, j = 0, 1, 2, 3 ... 63 - // - // 3. Polynomial Evaluation - // From (2), - // f = x*(64/ln(2)) - n - // Let, - // r = f*(ln(2)/64) = x - n*(ln(2)/64) - // - // 4. Reconstruction - // Thus, - // e^x = (2^m) * (2^(j/64)) * e^r - - private const ulong V_ARG_MAX = 0x40862000_00000000; - private const ulong V_DP64_BIAS = 1023; - - private const double V_EXPF_MIN = -709.782712893384; - private const double V_EXPF_MAX = +709.782712893384; - - private const double V_EXPF_HUGE = 6755399441055744; - private const double V_TBL_LN2 = 1.4426950408889634; - - private const double V_LN2_HEAD = +0.693359375; - private const double V_LN2_TAIL = -0.00021219444005469057; - - private const double C3 = 0.5000000000000018; - private const double C4 = 0.1666666666666617; - private const double C5 = 0.04166666666649277; - private const double C6 = 0.008333333333559272; - private const double C7 = 0.001388888895122404; - private const double C8 = 0.00019841269432677495; - private const double C9 = 2.4801486521374483E-05; - private const double C10 = 2.7557622532543023E-06; - private const double C11 = 2.7632293298250954E-07; - private const double C12 = 2.499430431958571E-08; - - public static bool Vectorizable => true; - - public static double Invoke(double x) => double.Exp(x); - - public static Vector128 Invoke(Vector128 x) - { - // x * (64.0 / ln(2)) - Vector128 z = x * Vector128.Create(V_TBL_LN2); - - Vector128 dn = z + Vector128.Create(V_EXPF_HUGE); - - // n = (int)z - Vector128 n = dn.AsUInt64(); - - // dn = (double)n - dn -= Vector128.Create(V_EXPF_HUGE); - - // r = x - (dn * (ln(2) / 64)) - // where ln(2) / 64 is split into Head and Tail values - Vector128 r = x - (dn * Vector128.Create(V_LN2_HEAD)) - (dn * Vector128.Create(V_LN2_TAIL)); - - Vector128 r2 = r * r; - Vector128 r4 = r2 * r2; - Vector128 r8 = r4 * r4; - - // Compute polynomial - Vector128 poly = ((Vector128.Create(C12) * r + Vector128.Create(C11)) * r2 + - Vector128.Create(C10) * r + Vector128.Create(C9)) * r8 + - ((Vector128.Create(C8) * r + Vector128.Create(C7)) * r2 + - (Vector128.Create(C6) * r + Vector128.Create(C5))) * r4 + - ((Vector128.Create(C4) * r + Vector128.Create(C3)) * r2 + (r + Vector128.One)); - - // m = (n - j) / 64 - // result = polynomial * 2^m - Vector128 ret = poly * ((n + Vector128.Create(V_DP64_BIAS)) << 52).AsDouble(); - - // Check if -709 < vx < 709 - if (Vector128.GreaterThanAny(Vector128.Abs(x).AsUInt64(), Vector128.Create(V_ARG_MAX))) - { - // (x > V_EXPF_MAX) ? double.PositiveInfinity : x - Vector128 infinityMask = Vector128.GreaterThan(x, Vector128.Create(V_EXPF_MAX)); - - ret = Vector128.ConditionalSelect( - infinityMask, - Vector128.Create(double.PositiveInfinity), - ret - ); - - // (x < V_EXPF_MIN) ? 0 : x - ret = Vector128.AndNot(ret, Vector128.LessThan(x, Vector128.Create(V_EXPF_MIN))); - } - - return ret; - } - - public static Vector256 Invoke(Vector256 x) - { - // x * (64.0 / ln(2)) - Vector256 z = x * Vector256.Create(V_TBL_LN2); - - Vector256 dn = z + Vector256.Create(V_EXPF_HUGE); - - // n = (int)z - Vector256 n = dn.AsUInt64(); - - // dn = (double)n - dn -= Vector256.Create(V_EXPF_HUGE); - - // r = x - (dn * (ln(2) / 64)) - // where ln(2) / 64 is split into Head and Tail values - Vector256 r = x - (dn * Vector256.Create(V_LN2_HEAD)) - (dn * Vector256.Create(V_LN2_TAIL)); - - Vector256 r2 = r * r; - Vector256 r4 = r2 * r2; - Vector256 r8 = r4 * r4; - - // Compute polynomial - Vector256 poly = ((Vector256.Create(C12) * r + Vector256.Create(C11)) * r2 + - Vector256.Create(C10) * r + Vector256.Create(C9)) * r8 + - ((Vector256.Create(C8) * r + Vector256.Create(C7)) * r2 + - (Vector256.Create(C6) * r + Vector256.Create(C5))) * r4 + - ((Vector256.Create(C4) * r + Vector256.Create(C3)) * r2 + (r + Vector256.One)); - - // m = (n - j) / 64 - // result = polynomial * 2^m - Vector256 ret = poly * ((n + Vector256.Create(V_DP64_BIAS)) << 52).AsDouble(); - - // Check if -709 < vx < 709 - if (Vector256.GreaterThanAny(Vector256.Abs(x).AsUInt64(), Vector256.Create(V_ARG_MAX))) - { - // (x > V_EXPF_MAX) ? double.PositiveInfinity : x - Vector256 infinityMask = Vector256.GreaterThan(x, Vector256.Create(V_EXPF_MAX)); - - ret = Vector256.ConditionalSelect( - infinityMask, - Vector256.Create(double.PositiveInfinity), - ret - ); - - // (x < V_EXPF_MIN) ? 0 : x - ret = Vector256.AndNot(ret, Vector256.LessThan(x, Vector256.Create(V_EXPF_MIN))); - } - - return ret; - } - - public static Vector512 Invoke(Vector512 x) - { - // x * (64.0 / ln(2)) - Vector512 z = x * Vector512.Create(V_TBL_LN2); - - Vector512 dn = z + Vector512.Create(V_EXPF_HUGE); - - // n = (int)z - Vector512 n = dn.AsUInt64(); - - // dn = (double)n - dn -= Vector512.Create(V_EXPF_HUGE); - - // r = x - (dn * (ln(2) / 64)) - // where ln(2) / 64 is split into Head and Tail values - Vector512 r = x - (dn * Vector512.Create(V_LN2_HEAD)) - (dn * Vector512.Create(V_LN2_TAIL)); - - Vector512 r2 = r * r; - Vector512 r4 = r2 * r2; - Vector512 r8 = r4 * r4; - - // Compute polynomial - Vector512 poly = ((Vector512.Create(C12) * r + Vector512.Create(C11)) * r2 + - Vector512.Create(C10) * r + Vector512.Create(C9)) * r8 + - ((Vector512.Create(C8) * r + Vector512.Create(C7)) * r2 + - (Vector512.Create(C6) * r + Vector512.Create(C5))) * r4 + - ((Vector512.Create(C4) * r + Vector512.Create(C3)) * r2 + (r + Vector512.One)); - - // m = (n - j) / 64 - // result = polynomial * 2^m - Vector512 ret = poly * ((n + Vector512.Create(V_DP64_BIAS)) << 52).AsDouble(); - - // Check if -709 < vx < 709 - if (Vector512.GreaterThanAny(Vector512.Abs(x).AsUInt64(), Vector512.Create(V_ARG_MAX))) - { - // (x > V_EXPF_MAX) ? double.PositiveInfinity : x - Vector512 infinityMask = Vector512.GreaterThan(x, Vector512.Create(V_EXPF_MAX)); - - ret = Vector512.ConditionalSelect( - infinityMask, - Vector512.Create(double.PositiveInfinity), - ret - ); - - // (x < V_EXPF_MIN) ? 0 : x - ret = Vector512.AndNot(ret, Vector512.LessThan(x, Vector512.Create(V_EXPF_MIN))); - } - - return ret; - } - } - - /// float.Exp(x) - private readonly struct ExpOperatorSingle : IUnaryOperator - { - // This code is based on `vrs4_expf` from amd/aocl-libm-ose - // Copyright (C) 2019-2022 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Implementation Notes: - // 1. Argument Reduction: - // e^x = 2^(x/ln2) --- (1) - // - // Let x/ln(2) = z --- (2) - // - // Let z = n + r , where n is an integer --- (3) - // |r| <= 1/2 - // - // From (1), (2) and (3), - // e^x = 2^z - // = 2^(N+r) - // = (2^N)*(2^r) --- (4) - // - // 2. Polynomial Evaluation - // From (4), - // r = z - N - // 2^r = C1 + C2*r + C3*r^2 + C4*r^3 + C5 *r^4 + C6*r^5 - // - // 4. Reconstruction - // Thus, - // e^x = (2^N) * (2^r) - - private const uint V_ARG_MAX = 0x42AE0000; - - private const float V_EXPF_MIN = -103.97208f; - private const float V_EXPF_MAX = +88.72284f; - - private const double V_EXPF_HUGE = 6755399441055744; - private const double V_TBL_LN2 = 1.4426950408889634; - - private const double C1 = 1.0000000754895704; - private const double C2 = 0.6931472254087585; - private const double C3 = 0.2402210737432219; - private const double C4 = 0.05550297297702539; - private const double C5 = 0.009676036358193323; - private const double C6 = 0.001341000536524434; - - public static bool Vectorizable => true; - - public static float Invoke(float x) => float.Exp(x); - - public static Vector128 Invoke(Vector128 x) - { - // Convert x to double precision - (Vector128 xl, Vector128 xu) = Vector128.Widen(x); - - // x * (64.0 / ln(2)) - Vector128 v_tbl_ln2 = Vector128.Create(V_TBL_LN2); - - Vector128 zl = xl * v_tbl_ln2; - Vector128 zu = xu * v_tbl_ln2; - - Vector128 v_expf_huge = Vector128.Create(V_EXPF_HUGE); - - Vector128 dnl = zl + v_expf_huge; - Vector128 dnu = zu + v_expf_huge; - - // n = (int)z - Vector128 nl = dnl.AsUInt64(); - Vector128 nu = dnu.AsUInt64(); - - // dn = (double)n - dnl -= v_expf_huge; - dnu -= v_expf_huge; - - // r = z - dn - Vector128 c1 = Vector128.Create(C1); - Vector128 c2 = Vector128.Create(C2); - Vector128 c3 = Vector128.Create(C3); - Vector128 c4 = Vector128.Create(C4); - Vector128 c5 = Vector128.Create(C5); - Vector128 c6 = Vector128.Create(C6); - - Vector128 rl = zl - dnl; - - Vector128 rl2 = rl * rl; - Vector128 rl4 = rl2 * rl2; - - Vector128 polyl = (c4 * rl + c3) * rl2 - + ((c6 * rl + c5) * rl4 - + (c2 * rl + c1)); - - - Vector128 ru = zu - dnu; - - Vector128 ru2 = ru * ru; - Vector128 ru4 = ru2 * ru2; - - Vector128 polyu = (c4 * ru + c3) * ru2 - + ((c6 * ru + c5) * ru4 - + (c2 * ru + c1)); - - // result = (float)(poly + (n << 52)) - Vector128 ret = Vector128.Narrow( - (polyl.AsUInt64() + (nl << 52)).AsDouble(), - (polyu.AsUInt64() + (nu << 52)).AsDouble() - ); - - // Check if -103 < |x| < 88 - if (Vector128.GreaterThanAny(Vector128.Abs(x).AsUInt32(), Vector128.Create(V_ARG_MAX))) - { - // (x > V_EXPF_MAX) ? float.PositiveInfinity : x - Vector128 infinityMask = Vector128.GreaterThan(x, Vector128.Create(V_EXPF_MAX)); - - ret = Vector128.ConditionalSelect( - infinityMask, - Vector128.Create(float.PositiveInfinity), - ret - ); - - // (x < V_EXPF_MIN) ? 0 : x - ret = Vector128.AndNot(ret, Vector128.LessThan(x, Vector128.Create(V_EXPF_MIN))); - } - - return ret; - } - - public static Vector256 Invoke(Vector256 x) - { - // Convert x to double precision - (Vector256 xl, Vector256 xu) = Vector256.Widen(x); - - // x * (64.0 / ln(2)) - Vector256 v_tbl_ln2 = Vector256.Create(V_TBL_LN2); - - Vector256 zl = xl * v_tbl_ln2; - Vector256 zu = xu * v_tbl_ln2; - - Vector256 v_expf_huge = Vector256.Create(V_EXPF_HUGE); - - Vector256 dnl = zl + v_expf_huge; - Vector256 dnu = zu + v_expf_huge; - - // n = (int)z - Vector256 nl = dnl.AsUInt64(); - Vector256 nu = dnu.AsUInt64(); - - // dn = (double)n - dnl -= v_expf_huge; - dnu -= v_expf_huge; - - // r = z - dn - Vector256 c1 = Vector256.Create(C1); - Vector256 c2 = Vector256.Create(C2); - Vector256 c3 = Vector256.Create(C3); - Vector256 c4 = Vector256.Create(C4); - Vector256 c5 = Vector256.Create(C5); - Vector256 c6 = Vector256.Create(C6); - - Vector256 rl = zl - dnl; - - Vector256 rl2 = rl * rl; - Vector256 rl4 = rl2 * rl2; - - Vector256 polyl = (c4 * rl + c3) * rl2 - + ((c6 * rl + c5) * rl4 - + (c2 * rl + c1)); - - - Vector256 ru = zu - dnu; - - Vector256 ru2 = ru * ru; - Vector256 ru4 = ru2 * ru2; - - Vector256 polyu = (c4 * ru + c3) * ru2 - + ((c6 * ru + c5) * ru4 - + (c2 * ru + c1)); - - // result = (float)(poly + (n << 52)) - Vector256 ret = Vector256.Narrow( - (polyl.AsUInt64() + (nl << 52)).AsDouble(), - (polyu.AsUInt64() + (nu << 52)).AsDouble() - ); - - // Check if -103 < |x| < 88 - if (Vector256.GreaterThanAny(Vector256.Abs(x).AsUInt32(), Vector256.Create(V_ARG_MAX))) - { - // (x > V_EXPF_MAX) ? float.PositiveInfinity : x - Vector256 infinityMask = Vector256.GreaterThan(x, Vector256.Create(V_EXPF_MAX)); - - ret = Vector256.ConditionalSelect( - infinityMask, - Vector256.Create(float.PositiveInfinity), - ret - ); - - // (x < V_EXPF_MIN) ? 0 : x - ret = Vector256.AndNot(ret, Vector256.LessThan(x, Vector256.Create(V_EXPF_MIN))); - } - - return ret; - } - - public static Vector512 Invoke(Vector512 x) - { - // Convert x to double precision - (Vector512 xl, Vector512 xu) = Vector512.Widen(x); - - // x * (64.0 / ln(2)) - Vector512 v_tbl_ln2 = Vector512.Create(V_TBL_LN2); - - Vector512 zl = xl * v_tbl_ln2; - Vector512 zu = xu * v_tbl_ln2; - - Vector512 v_expf_huge = Vector512.Create(V_EXPF_HUGE); - - Vector512 dnl = zl + v_expf_huge; - Vector512 dnu = zu + v_expf_huge; - - // n = (int)z - Vector512 nl = dnl.AsUInt64(); - Vector512 nu = dnu.AsUInt64(); - - // dn = (double)n - dnl -= v_expf_huge; - dnu -= v_expf_huge; - - // r = z - dn - Vector512 c1 = Vector512.Create(C1); - Vector512 c2 = Vector512.Create(C2); - Vector512 c3 = Vector512.Create(C3); - Vector512 c4 = Vector512.Create(C4); - Vector512 c5 = Vector512.Create(C5); - Vector512 c6 = Vector512.Create(C6); - - Vector512 rl = zl - dnl; - - Vector512 rl2 = rl * rl; - Vector512 rl4 = rl2 * rl2; - - Vector512 polyl = (c4 * rl + c3) * rl2 - + ((c6 * rl + c5) * rl4 - + (c2 * rl + c1)); - - - Vector512 ru = zu - dnu; - - Vector512 ru2 = ru * ru; - Vector512 ru4 = ru2 * ru2; - - Vector512 polyu = (c4 * ru + c3) * ru2 - + ((c6 * ru + c5) * ru4 - + (c2 * ru + c1)); - - // result = (float)(poly + (n << 52)) - Vector512 ret = Vector512.Narrow( - (polyl.AsUInt64() + (nl << 52)).AsDouble(), - (polyu.AsUInt64() + (nu << 52)).AsDouble() - ); - - // Check if -103 < |x| < 88 - if (Vector512.GreaterThanAny(Vector512.Abs(x).AsUInt32(), Vector512.Create(V_ARG_MAX))) - { - // (x > V_EXPF_MAX) ? float.PositiveInfinity : x - Vector512 infinityMask = Vector512.GreaterThan(x, Vector512.Create(V_EXPF_MAX)); - - ret = Vector512.ConditionalSelect( - infinityMask, - Vector512.Create(float.PositiveInfinity), - ret - ); - - // (x < V_EXPF_MIN) ? 0 : x - ret = Vector512.AndNot(ret, Vector512.LessThan(x, Vector512.Create(V_EXPF_MIN))); - } - - return ret; - } - } -#endif - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp10.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp10.cs deleted file mode 100644 index 6e228d7932f657..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp10.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of raising 10 to the number powers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Exp10([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Exp10(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Exp10(x) - private readonly struct Exp10Operator : IUnaryOperator - where T : IExponentialFunctions - { - private const double NaturalLog10 = 2.302585092994046; - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Exp10(x); - public static Vector128 Invoke(Vector128 x) => ExpOperator.Invoke(x * Vector128.Create(T.CreateTruncating(NaturalLog10))); - public static Vector256 Invoke(Vector256 x) => ExpOperator.Invoke(x * Vector256.Create(T.CreateTruncating(NaturalLog10))); - public static Vector512 Invoke(Vector512 x) => ExpOperator.Invoke(x * Vector512.Create(T.CreateTruncating(NaturalLog10))); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp10M1.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp10M1.cs deleted file mode 100644 index 5687a087af71d8..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp10M1.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of raising 10 to the number powers in the specified tensor, minus one. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Exp10M1([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Exp10M1(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Exp10M1(x) - private readonly struct Exp10M1Operator : IUnaryOperator - where T : IExponentialFunctions - { - public static bool Vectorizable => Exp2Operator.Vectorizable; - - public static T Invoke(T x) => T.Exp10M1(x); - public static Vector128 Invoke(Vector128 x) => Exp10Operator.Invoke(x) - Vector128.One; - public static Vector256 Invoke(Vector256 x) => Exp10Operator.Invoke(x) - Vector256.One; - public static Vector512 Invoke(Vector512 x) => Exp10Operator.Invoke(x) - Vector512.One; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp2.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp2.cs deleted file mode 100644 index a3aeacdefde981..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp2.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of raising 2 to the number powers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Exp2([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Exp2(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Exp2(x) - private readonly struct Exp2Operator : IUnaryOperator - where T : IExponentialFunctions - { - private const double NaturalLog2 = 0.6931471805599453; - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Exp2(x); - public static Vector128 Invoke(Vector128 x) => ExpOperator.Invoke(x * Vector128.Create(T.CreateTruncating(NaturalLog2))); - public static Vector256 Invoke(Vector256 x) => ExpOperator.Invoke(x * Vector256.Create(T.CreateTruncating(NaturalLog2))); - public static Vector512 Invoke(Vector512 x) => ExpOperator.Invoke(x * Vector512.Create(T.CreateTruncating(NaturalLog2))); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp2M1.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp2M1.cs deleted file mode 100644 index 1aebd4cc65c3bd..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Exp2M1.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of raising 2 to the number powers in the specified tensor, minus one. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Exp2M1([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Exp2M1(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Exp2M1(x) - private readonly struct Exp2M1Operator : IUnaryOperator - where T : IExponentialFunctions - { - public static bool Vectorizable => Exp2Operator.Vectorizable; - - public static T Invoke(T x) => T.Exp2M1(x); - public static Vector128 Invoke(Vector128 x) => Exp2Operator.Invoke(x) - Vector128.One; - public static Vector256 Invoke(Vector256 x) => Exp2Operator.Invoke(x) - Vector256.One; - public static Vector512 Invoke(Vector512 x) => Exp2Operator.Invoke(x) - Vector512.One; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ExpM1.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ExpM1.cs deleted file mode 100644 index 8cdc48b0947c45..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ExpM1.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of raising e to the number powers in the specified tensor, minus 1. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .ExpM1([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void ExpM1(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.ExpM1(x) - private readonly struct ExpM1Operator : IUnaryOperator - where T : IExponentialFunctions - { - public static bool Vectorizable => ExpOperator.Vectorizable; - - public static T Invoke(T x) => T.ExpM1(x); - public static Vector128 Invoke(Vector128 x) => ExpOperator.Invoke(x) - Vector128.One; - public static Vector256 Invoke(Vector256 x) => ExpOperator.Invoke(x) - Vector256.One; - public static Vector512 Invoke(Vector512 x) => ExpOperator.Invoke(x) - Vector512.One; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.FloatHelpers.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.FloatHelpers.cs deleted file mode 100644 index ec97b9a61af9a7..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.FloatHelpers.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - private static Vector128 ApplyScalar(Vector128 floats) where TOperator : IUnaryOperator => - Vector128.Create(TOperator.Invoke(floats[0]), TOperator.Invoke(floats[1]), TOperator.Invoke(floats[2]), TOperator.Invoke(floats[3])); - - private static Vector256 ApplyScalar(Vector256 floats) where TOperator : IUnaryOperator => - Vector256.Create(ApplyScalar(floats.GetLower()), ApplyScalar(floats.GetUpper())); - - private static Vector512 ApplyScalar(Vector512 floats) where TOperator : IUnaryOperator => - Vector512.Create(ApplyScalar(floats.GetLower()), ApplyScalar(floats.GetUpper())); - - private static Vector128 ApplyScalar(Vector128 doubles) where TOperator : IUnaryOperator => - Vector128.Create(TOperator.Invoke(doubles[0]), TOperator.Invoke(doubles[1])); - - private static Vector256 ApplyScalar(Vector256 doubles) where TOperator : IUnaryOperator => - Vector256.Create(ApplyScalar(doubles.GetLower()), ApplyScalar(doubles.GetUpper())); - - private static Vector512 ApplyScalar(Vector512 doubles) where TOperator : IUnaryOperator => - Vector512.Create(ApplyScalar(doubles.GetLower()), ApplyScalar(doubles.GetUpper())); - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Floor.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Floor.cs deleted file mode 100644 index ae33b05099eed2..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Floor.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise floor of numbers in the specified tensor. - /// The first tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Floor([i]). - /// - /// - public static void Floor(ReadOnlySpan x, Span destination) - where T : IFloatingPoint => - InvokeSpanIntoSpan>(x, destination); - - private readonly struct FloorOperator : IUnaryOperator where T : IFloatingPoint - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Floor(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return Vector128.Floor(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return Vector128.Floor(x.AsDouble()).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return Vector256.Floor(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return Vector256.Floor(x.AsDouble()).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return Vector512.Floor(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return Vector512.Floor(x.AsDouble()).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.FusedMultiplyAdd.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.FusedMultiplyAdd.cs deleted file mode 100644 index 57cc8f3a373abc..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.FusedMultiplyAdd.cs +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of and length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * [i]) + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - /// This computes ( * ) as if to infinite precision, adds to that result as if to - /// infinite precision, and finally rounds to the nearest representable value. This differs from the non-fused sequence which would compute - /// ( * ) as if to infinite precision, round the result to the nearest representable value, add to the - /// rounded result as if to infinite precision, and finally round to the nearest representable value. - /// - /// - public static void FusedMultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan addend, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanSpanIntoSpan>(x, y, addend, destination); - - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * [i]) + . - /// It corresponds to the axpy method defined by BLAS1. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - /// This computes ( * ) as if to infinite precision, adds to that result as if to - /// infinite precision, and finally rounds to the nearest representable value. This differs from the non-fused sequence which would compute - /// ( * ) as if to infinite precision, round the result to the nearest representable value, add to the - /// rounded result as if to infinite precision, and finally round to the nearest representable value. - /// - /// - public static void FusedMultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, T addend, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanScalarIntoSpan>(x, y, addend, destination); - - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * ) + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - /// This computes ( * ) as if to infinite precision, adds to that result as if to - /// infinite precision, and finally rounds to the nearest representable value. This differs from the non-fused sequence which would compute - /// ( * ) as if to infinite precision, round the result to the nearest representable value, add to the - /// rounded result as if to infinite precision, and finally round to the nearest representable value. - /// - /// - public static void FusedMultiplyAdd(ReadOnlySpan x, T y, ReadOnlySpan addend, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanScalarSpanIntoSpan>(x, y, addend, destination); - - /// (x * y) + z - private readonly struct FusedMultiplyAddOperator : ITernaryOperator where T : IFloatingPointIeee754 - { - public static T Invoke(T x, T y, T z) => T.FusedMultiplyAdd(x, y, z); - - public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) - { - if (Fma.IsSupported) - { - if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); - } - - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(float)) return AdvSimd.FusedMultiplyAdd(z.AsSingle(), x.AsSingle(), y.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.FusedMultiplyAdd(z.AsDouble(), x.AsDouble(), y.AsDouble()).As(); - } - - if (typeof(T) == typeof(float)) - { - Vector128 xFloats = x.AsSingle(); - Vector128 yFloats = y.AsSingle(); - Vector128 zFloats = z.AsSingle(); - return Vector128.Create( - float.FusedMultiplyAdd(xFloats[0], yFloats[0], zFloats[0]), - float.FusedMultiplyAdd(xFloats[1], yFloats[1], zFloats[1]), - float.FusedMultiplyAdd(xFloats[2], yFloats[2], zFloats[2]), - float.FusedMultiplyAdd(xFloats[3], yFloats[3], zFloats[3])).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector128 xDoubles = x.AsDouble(); - Vector128 yDoubles = y.AsDouble(); - Vector128 zDoubles = z.AsDouble(); - return Vector128.Create( - double.FusedMultiplyAdd(xDoubles[0], yDoubles[0], zDoubles[0]), - double.FusedMultiplyAdd(xDoubles[1], yDoubles[1], zDoubles[1])).As(); - } - } - - public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) - { - if (Fma.IsSupported) - { - if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); - } - - return Vector256.Create( - Invoke(x.GetLower(), y.GetLower(), z.GetLower()), - Invoke(x.GetUpper(), y.GetUpper(), z.GetUpper())); - } - - public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) - { - if (Avx512F.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx512F.FusedMultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Avx512F.FusedMultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); - } - - return Vector512.Create( - Invoke(x.GetLower(), y.GetLower(), z.GetLower()), - Invoke(x.GetUpper(), y.GetUpper(), z.GetUpper())); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Half.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Half.cs deleted file mode 100644 index c830a1f57f5bcf..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Half.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// - /// Copies to , converting each - /// value to its nearest representable half-precision floating-point value. - /// - /// The source span from which to copy values. - /// The destination span into which the converted values should be written. - /// Destination is too short. - /// - /// - /// This method effectively computes [i] = (Half)[i]. - /// - /// - /// and must not overlap. If they do, behavior is undefined. - /// - /// - public static void ConvertToHalf(ReadOnlySpan source, Span destination) => - ConvertTruncating(source, destination); - - /// - /// Copies to , converting each half-precision - /// floating-point value to its nearest representable value. - /// - /// The source span from which to copy values. - /// The destination span into which the converted values should be written. - /// Destination is too short. - /// - /// - /// This method effectively computes [i] = (float)[i]. - /// - /// - /// and must not overlap. If they do, behavior is undefined. - /// - /// - public static void ConvertToSingle(ReadOnlySpan source, Span destination) => - ConvertTruncating(source, destination); - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Hypot.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Hypot.cs deleted file mode 100644 index 30b9bc51785190..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Hypot.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hypotensue given values from two tensors representing the lengths of the shorter sides in a right-angled triangle. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Hypot([i], [i]). - /// - /// - public static void Hypot(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IRootFunctions => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// T.Hypot(x, y) - private readonly struct HypotOperator : IBinaryOperator - where T : IRootFunctions - { - public static bool Vectorizable => true; - public static T Invoke(T x, T y) => T.Hypot(x, y); - public static Vector128 Invoke(Vector128 x, Vector128 y) => Vector128.Sqrt((x * x) + (y * y)); - public static Vector256 Invoke(Vector256 x, Vector256 y) => Vector256.Sqrt((x * x) + (y * y)); - public static Vector512 Invoke(Vector512 x, Vector512 y) => Vector512.Sqrt((x * x) + (y * y)); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ILogB.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ILogB.cs deleted file mode 100644 index 36b0a5bd67c213..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ILogB.cs +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise integer logarithm of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.ILogB([i]). - /// - /// - public static void ILogB(ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 - { - if (typeof(T) == typeof(double)) - { - // Special-case double as the only vectorizable floating-point type whose size != sizeof(int). - InvokeSpanIntoSpan_2to1(Rename(x), destination); - } - else - { - InvokeSpanIntoSpan>(x, destination); - } - } - - /// T.ILogB(x) - private readonly struct ILogBOperator : IUnaryOperator where T : IFloatingPointIeee754 - { - public static bool Vectorizable => false; // TODO: vectorize for float - - public static int Invoke(T x) => T.ILogB(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - - /// double.ILogB(x) - private readonly struct ILogBDoubleOperator : IUnaryTwoToOneOperator - { - public static bool Vectorizable => false; // TODO: vectorize - - public static int Invoke(double x) => double.ILogB(x); - public static Vector128 Invoke(Vector128 lower, Vector128 upper) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 lower, Vector256 upper) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 lower, Vector512 upper) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Ieee754Remainder.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Ieee754Remainder.cs deleted file mode 100644 index 2aea50793a8942..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Ieee754Remainder.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise remainder of the numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Ieee754Remainder([i], [i]). - /// - /// - public static void Ieee754Remainder(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise remainder of the numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Ieee754Remainder([i], ). - /// - /// - public static void Ieee754Remainder(ReadOnlySpan x, T y, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// Computes the element-wise remainder of the numbers in the specified tensors. - /// The first tensor, represented as a scalar. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Ieee754Remainder(, [i]). - /// - /// - public static void Ieee754Remainder(T x, ReadOnlySpan y, Span destination) - where T : IFloatingPointIeee754 => - InvokeScalarSpanIntoSpan>(x, y, destination); - - /// T.Ieee754Remainder(x, y) - internal readonly struct Ieee754RemainderOperator : IBinaryOperator where T : IFloatingPointIeee754 - { - public static bool Vectorizable => false; - public static T Invoke(T x, T y) => T.Ieee754Remainder(x, y); - public static Vector128 Invoke(Vector128 x, Vector128 y) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x, Vector256 y) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x, Vector512 y) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs deleted file mode 100644 index 1b584d05a53748..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMax.cs +++ /dev/null @@ -1,520 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.X86; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the index of the largest number in the specified tensor. - /// The tensor, represented as a span. - /// The index of the maximum element in , or -1 if is empty. - /// - /// - /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If any value equal to NaN - /// is present, the index of the first is returned. Positive 0 is considered greater than negative 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static int IndexOfMax(ReadOnlySpan x) - where T : INumber => - IndexOfMinMaxCore>(x); - - /// Returns the index of MathF.Max(x, y) - internal readonly struct IndexOfMaxOperator : IIndexOfOperator where T : INumber - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) - { - Vector128 useResult = Vector128.GreaterThan(result, current); - Vector128 equalMask = Vector128.Equals(result, current); - - if (equalMask != Vector128.Zero) - { - Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector128 currentNegative = IsNegative(current); - Vector128 sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) - { - Vector256 useResult = Vector256.GreaterThan(result, current); - Vector256 equalMask = Vector256.Equals(result, current); - - if (equalMask != Vector256.Zero) - { - Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector256 currentNegative = IsNegative(current); - Vector256 sameSign = Vector256.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) - { - Vector512 useResult = Vector512.GreaterThan(result, current); - Vector512 equalMask = Vector512.Equals(result, current); - - if (equalMask != Vector512.Zero) - { - Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector512 currentNegative = IsNegative(current); - Vector512 sameSign = Vector512.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) - { - if (result == current) - { - bool resultNegative = IsNegative(result); - if ((resultNegative == IsNegative(current)) ? (currentIndex < resultIndex) : resultNegative) - { - result = current; - return currentIndex; - } - } - else if (current > result) - { - result = current; - return currentIndex; - } - - return resultIndex; - } - } - - private static unsafe int IndexOfMinMaxCore(ReadOnlySpan x) - where T : INumber - where TIndexOfMinMax : struct, IIndexOfOperator - { - if (x.IsEmpty) - { - return -1; - } - - // This matches the IEEE 754:2019 `maximum`/`minimum` functions. - // It propagates NaN inputs back to the caller and - // otherwise returns the index of the greater of the inputs. - // It treats +0 as greater than -0 as per the specification. - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) - { - Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static Vector512 CreateVector512T(int i) => - sizeof(T) == sizeof(long) ? Vector512.Create((long)i).As() : - sizeof(T) == sizeof(int) ? Vector512.Create(i).As() : - sizeof(T) == sizeof(short) ? Vector512.Create((short)i).As() : - Vector512.Create((byte)i).As(); - - ref T xRef = ref MemoryMarshal.GetReference(x); - Vector512 resultIndex = -#if NET9_0_OR_GREATER - sizeof(T) == sizeof(long) ? Vector512.Indices.As() : - sizeof(T) == sizeof(int) ? Vector512.Indices.As() : - sizeof(T) == sizeof(short) ? Vector512.Indices.As() : - Vector512.Indices.As(); -#else - sizeof(T) == sizeof(long) ? Vector512.Create(0L, 1, 2, 3, 4, 5, 6, 7).As() : - sizeof(T) == sizeof(int) ? Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).As() : - sizeof(T) == sizeof(short) ? Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31).As() : - Vector512.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63).As(); -#endif - Vector512 currentIndex = resultIndex; - Vector512 increment = CreateVector512T(Vector512.Count); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector512 result = Vector512.LoadUnsafe(ref xRef); - Vector512 current; - - Vector512 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector512.Equals(result, result); - if (nanMask != Vector512.Zero) - { - return IndexOfFirstMatch(nanMask); - } - } - - int oneVectorFromEnd = x.Length - Vector512.Count; - int i = Vector512.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector512.LoadUnsafe(ref xRef, (uint)i); - currentIndex += increment; - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - return i + IndexOfFirstMatch(nanMask); - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - - i += Vector512.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); - currentIndex += CreateVector512T(x.Length - i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); - return typeof(T) == typeof(double) ? - (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : - (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return IndexOfFinalAggregate(result, resultIndex); - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) - { - Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static Vector256 CreateVector256T(int i) => - sizeof(T) == sizeof(long) ? Vector256.Create((long)i).As() : - sizeof(T) == sizeof(int) ? Vector256.Create(i).As() : - sizeof(T) == sizeof(short) ? Vector256.Create((short)i).As() : - Vector256.Create((byte)i).As(); - - ref T xRef = ref MemoryMarshal.GetReference(x); - Vector256 resultIndex = -#if NET9_0_OR_GREATER - sizeof(T) == sizeof(long) ? Vector256.Indices.As() : - sizeof(T) == sizeof(int) ? Vector256.Indices.As() : - sizeof(T) == sizeof(short) ? Vector256.Indices.As() : - Vector256.Indices.As(); -#else - sizeof(T) == sizeof(long) ? Vector256.Create(0L, 1, 2, 3).As() : - sizeof(T) == sizeof(int) ? Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7).As() : - sizeof(T) == sizeof(short) ? Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).As() : - Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31).As(); -#endif - Vector256 currentIndex = resultIndex; - Vector256 increment = CreateVector256T(Vector256.Count); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector256 result = Vector256.LoadUnsafe(ref xRef); - Vector256 current; - - Vector256 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector256.Equals(result, result); - if (nanMask != Vector256.Zero) - { - return IndexOfFirstMatch(nanMask); - } - } - - int oneVectorFromEnd = x.Length - Vector256.Count; - int i = Vector256.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector256.LoadUnsafe(ref xRef, (uint)i); - currentIndex += increment; - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - return i + IndexOfFirstMatch(nanMask); - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - - i += Vector256.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); - currentIndex += CreateVector256T(x.Length - i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); - return typeof(T) == typeof(double) ? - (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : - (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return IndexOfFinalAggregate(result, resultIndex); - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) - { - Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static Vector128 CreateVector128T(int i) => - sizeof(T) == sizeof(long) ? Vector128.Create((long)i).As() : - sizeof(T) == sizeof(int) ? Vector128.Create(i).As() : - sizeof(T) == sizeof(short) ? Vector128.Create((short)i).As() : - Vector128.Create((byte)i).As(); - - ref T xRef = ref MemoryMarshal.GetReference(x); - Vector128 resultIndex = -#if NET9_0_OR_GREATER - sizeof(T) == sizeof(long) ? Vector128.Indices.As() : - sizeof(T) == sizeof(int) ? Vector128.Indices.As() : - sizeof(T) == sizeof(short) ? Vector128.Indices.As() : - Vector128.Indices.As(); -#else - sizeof(T) == sizeof(long) ? Vector128.Create(0L, 1).As() : - sizeof(T) == sizeof(int) ? Vector128.Create(0, 1, 2, 3).As() : - sizeof(T) == sizeof(short) ? Vector128.Create(0, 1, 2, 3, 4, 5, 6, 7).As() : - Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).As(); -#endif - Vector128 currentIndex = resultIndex; - Vector128 increment = CreateVector128T(Vector128.Count); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector128 result = Vector128.LoadUnsafe(ref xRef); - Vector128 current; - - Vector128 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector128.Equals(result, result); - if (nanMask != Vector128.Zero) - { - return IndexOfFirstMatch(nanMask); - } - } - - int oneVectorFromEnd = x.Length - Vector128.Count; - int i = Vector128.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector128.LoadUnsafe(ref xRef, (uint)i); - currentIndex += increment; - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) - { - return i + IndexOfFirstMatch(nanMask); - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - - i += Vector128.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); - currentIndex += CreateVector128T(x.Length - i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) - { - int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); - return typeof(T) == typeof(double) ? - (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : - (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; - } - } - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return IndexOfFinalAggregate(result, resultIndex); - } - - // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - T curResult = x[0]; - int curIn = 0; - if (T.IsNaN(curResult)) - { - return curIn; - } - - for (int i = 1; i < x.Length; i++) - { - T current = x[i]; - if (T.IsNaN(current)) - { - return i; - } - - curIn = TIndexOfMinMax.Invoke(ref curResult, current, curIn, i); - } - - return curIn; - } - - private static int IndexOfFirstMatch(Vector128 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - - private static int IndexOfFirstMatch(Vector256 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - - private static int IndexOfFirstMatch(Vector512 mask) => - BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => - sizeof(T) == sizeof(long) ? Vector256.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : - sizeof(T) == sizeof(int) ? Vector256.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : - sizeof(T) == sizeof(short) ? Vector256.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : - Vector256.LessThan(indices1.AsByte(), indices2.AsByte()).As(); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector512 IndexLessThan(Vector512 indices1, Vector512 indices2) => - sizeof(T) == sizeof(long) ? Vector512.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : - sizeof(T) == sizeof(int) ? Vector512.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : - sizeof(T) == sizeof(short) ? Vector512.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : - Vector512.LessThan(indices1.AsByte(), indices2.AsByte()).As(); - - /// Gets whether the specified is negative. - private static bool IsNegative(T f) where T : INumberBase => T.IsNegative(f); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) - { - if (Sse41.IsSupported) - { - if (typeof(T) == typeof(float)) return Sse41.BlendVariable(left.AsSingle(), right.AsSingle(), (~mask).AsSingle()).As(); - if (typeof(T) == typeof(double)) return Sse41.BlendVariable(left.AsDouble(), right.AsDouble(), (~mask).AsDouble()).As(); - - if (sizeof(T) == 1) return Sse41.BlendVariable(left.AsByte(), right.AsByte(), (~mask).AsByte()).As(); - if (sizeof(T) == 2) return Sse41.BlendVariable(left.AsUInt16(), right.AsUInt16(), (~mask).AsUInt16()).As(); - if (sizeof(T) == 4) return Sse41.BlendVariable(left.AsUInt32(), right.AsUInt32(), (~mask).AsUInt32()).As(); - if (sizeof(T) == 8) return Sse41.BlendVariable(left.AsUInt64(), right.AsUInt64(), (~mask).AsUInt64()).As(); - } - - return Vector128.ConditionalSelect(mask, left, right); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) - { - if (Avx2.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx2.BlendVariable(left.AsSingle(), right.AsSingle(), (~mask).AsSingle()).As(); - if (typeof(T) == typeof(double)) return Avx2.BlendVariable(left.AsDouble(), right.AsDouble(), (~mask).AsDouble()).As(); - - if (sizeof(T) == 1) return Avx2.BlendVariable(left.AsByte(), right.AsByte(), (~mask).AsByte()).As(); - if (sizeof(T) == 2) return Avx2.BlendVariable(left.AsUInt16(), right.AsUInt16(), (~mask).AsUInt16()).As(); - if (sizeof(T) == 4) return Avx2.BlendVariable(left.AsUInt32(), right.AsUInt32(), (~mask).AsUInt32()).As(); - if (sizeof(T) == 8) return Avx2.BlendVariable(left.AsUInt64(), right.AsUInt64(), (~mask).AsUInt64()).As(); - } - - return Vector256.ConditionalSelect(mask, left, right); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe Vector512 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) - { - if (Avx512F.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx512F.BlendVariable(left.AsSingle(), right.AsSingle(), (~mask).AsSingle()).As(); - if (typeof(T) == typeof(double)) return Avx512F.BlendVariable(left.AsDouble(), right.AsDouble(), (~mask).AsDouble()).As(); - - if (sizeof(T) == 4) return Avx512F.BlendVariable(left.AsUInt32(), right.AsUInt32(), (~mask).AsUInt32()).As(); - if (sizeof(T) == 8) return Avx512F.BlendVariable(left.AsUInt64(), right.AsUInt64(), (~mask).AsUInt64()).As(); - } - - return Vector512.ConditionalSelect(mask, left, right); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMaxMagnitude.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMaxMagnitude.cs deleted file mode 100644 index f1f5016a86b13e..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMaxMagnitude.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the index of the number with the largest magnitude in the specified tensor. - /// The tensor, represented as a span. - /// The index of the element in with the largest magnitude (absolute value), or -1 if is empty. - /// - /// - /// The determination of the maximum magnitude matches the IEEE 754:2019 `maximumMagnitude` function. If any value equal to NaN - /// is present, the index of the first is returned. If two values have the same magnitude and one is positive and the other is negative, - /// the positive value is considered to have the larger magnitude. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static int IndexOfMaxMagnitude(ReadOnlySpan x) - where T : INumber => - IndexOfMinMaxCore>(x); - - internal readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator where T : INumber - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) - { - Vector128 resultMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); - Vector128 useResult = Vector128.GreaterThan(resultMag, currentMag); - Vector128 equalMask = Vector128.Equals(resultMag, currentMag); - - if (equalMask != Vector128.Zero) - { - Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector128 currentNegative = IsNegative(current); - Vector128 sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) - { - Vector256 resultMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); - Vector256 useResult = Vector256.GreaterThan(resultMag, currentMag); - Vector256 equalMask = Vector256.Equals(resultMag, currentMag); - - if (equalMask != Vector256.Zero) - { - Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector256 currentNegative = IsNegative(current); - Vector256 sameSign = Vector256.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) - { - Vector512 resultMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); - Vector512 useResult = Vector512.GreaterThan(resultMag, currentMag); - Vector512 equalMask = Vector512.Equals(resultMag, currentMag); - - if (equalMask != Vector512.Zero) - { - Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); - Vector512 currentNegative = IsNegative(current); - Vector512 sameSign = Vector512.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) - { - T resultMag = T.Abs(result); - T currentMag = T.Abs(current); - - if (resultMag == currentMag) - { - bool resultNegative = IsNegative(result); - if ((resultNegative == IsNegative(current)) ? (currentIndex < resultIndex) : resultNegative) - { - result = current; - return currentIndex; - } - } - else if (currentMag > resultMag) - { - result = current; - return currentIndex; - } - - return resultIndex; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMin.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMin.cs deleted file mode 100644 index 011021b6c0015f..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMin.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the index of the smallest number in the specified tensor. - /// The tensor, represented as a span. - /// The index of the minimum element in , or -1 if is empty. - /// - /// - /// The determination of the minimum element matches the IEEE 754:2019 `minimum` function. If any value equal to NaN - /// is present, the index of the first is returned. Negative 0 is considered smaller than positive 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static int IndexOfMin(ReadOnlySpan x) - where T : INumber => - IndexOfMinMaxCore>(x); - - /// Returns the index of MathF.Min(x, y) - internal readonly struct IndexOfMinOperator : IIndexOfOperator where T : INumber - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) - { - Vector128 useResult = Vector128.LessThan(result, current); - Vector128 equalMask = Vector128.Equals(result, current); - - if (equalMask != Vector128.Zero) - { - Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); - Vector128 resultNegative = IsNegative(result); - Vector128 sameSign = Vector128.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) - { - Vector256 useResult = Vector256.LessThan(result, current); - Vector256 equalMask = Vector256.Equals(result, current); - - if (equalMask != Vector256.Zero) - { - Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); - Vector256 resultNegative = IsNegative(result); - Vector256 sameSign = Vector256.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) - { - Vector512 useResult = Vector512.LessThan(result, current); - Vector512 equalMask = Vector512.Equals(result, current); - - if (equalMask != Vector512.Zero) - { - Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); - Vector512 resultNegative = IsNegative(result); - Vector512 sameSign = Vector512.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) - { - if (result == current) - { - bool currentNegative = IsNegative(current); - if ((IsNegative(result) == currentNegative) ? (currentIndex < resultIndex) : currentNegative) - { - result = current; - return currentIndex; - } - } - else if (current < result) - { - result = current; - return currentIndex; - } - - return resultIndex; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMinMagnitude.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMinMagnitude.cs deleted file mode 100644 index 813bcf4637dd12..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.IndexOfMinMagnitude.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the index of the number with the smallest magnitude in the specified tensor. - /// The tensor, represented as a span. - /// The index of the element in with the smallest magnitude (absolute value), or -1 if is empty. - /// - /// - /// The determination of the minimum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If any value equal to NaN - /// is present, the index of the first is returned. If two values have the same magnitude and one is positive and the other is negative, - /// the negative value is considered to have the smaller magnitude. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static int IndexOfMinMagnitude(ReadOnlySpan x) - where T : INumber => - IndexOfMinMaxCore>(x); - - internal readonly struct IndexOfMinMagnitudeOperator : IIndexOfOperator where T : INumber - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) - { - Vector128 resultMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); - Vector128 useResult = Vector128.LessThan(resultMag, currentMag); - Vector128 equalMask = Vector128.Equals(resultMag, currentMag); - - if (equalMask != Vector128.Zero) - { - Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); - Vector128 resultNegative = IsNegative(result); - Vector128 sameSign = Vector128.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) - { - Vector256 resultMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); - Vector256 useResult = Vector256.LessThan(resultMag, currentMag); - Vector256 equalMask = Vector256.Equals(resultMag, currentMag); - - if (equalMask != Vector256.Zero) - { - Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); - Vector256 resultNegative = IsNegative(result); - Vector256 sameSign = Vector256.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) - { - Vector512 resultMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); - Vector512 useResult = Vector512.LessThan(resultMag, currentMag); - Vector512 equalMask = Vector512.Equals(resultMag, currentMag); - - if (equalMask != Vector512.Zero) - { - Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); - Vector512 resultNegative = IsNegative(result); - Vector512 sameSign = Vector512.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); - useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); - } - else - { - useResult |= equalMask & lessThanIndexMask; - } - } - - result = ElementWiseSelect(useResult, result, current); - resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) - { - T resultMag = T.Abs(result); - T currentMag = T.Abs(current); - - if (resultMag == currentMag) - { - bool currentNegative = IsNegative(current); - if ((IsNegative(result) == currentNegative) ? (currentIndex < resultIndex) : currentNegative) - { - result = current; - return currentIndex; - } - } - else if (currentMag < resultMag) - { - result = current; - return currentIndex; - } - - return resultIndex; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.LeadingZeroCount.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.LeadingZeroCount.cs deleted file mode 100644 index ab51042925f058..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.LeadingZeroCount.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise leading zero count of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.LeadingZeroCount([i]). - /// - /// - public static void LeadingZeroCount(ReadOnlySpan x, Span destination) - where T : IBinaryInteger => - InvokeSpanIntoSpan>(x, destination); - - /// T.LeadingZeroCount(x) - internal readonly unsafe struct LeadingZeroCountOperator : IUnaryOperator where T : IBinaryInteger - { - public static bool Vectorizable => - (Avx512CD.VL.IsSupported && (sizeof(T) == 4 || sizeof(T) == 8)) || - (AdvSimd.IsSupported && (sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4)); - - public static T Invoke(T x) => T.LeadingZeroCount(x); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x) - { - if (Avx512CD.VL.IsSupported) - { - if (sizeof(T) == 4) return Avx512CD.VL.LeadingZeroCount(x.AsUInt32()).As(); - if (sizeof(T) == 8) return Avx512CD.VL.LeadingZeroCount(x.AsUInt64()).As(); - } - - Debug.Assert(AdvSimd.IsSupported); - { - if (sizeof(T) == 1) return AdvSimd.LeadingZeroCount(x.AsByte()).As(); - if (sizeof(T) == 2) return AdvSimd.LeadingZeroCount(x.AsUInt16()).As(); - - Debug.Assert(sizeof(T) == 4); - return AdvSimd.LeadingZeroCount(x.AsUInt32()).As(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x) - { - if (Avx512CD.VL.IsSupported) - { - if (sizeof(T) == 4) return Avx512CD.VL.LeadingZeroCount(x.AsUInt32()).As(); - if (sizeof(T) == 8) return Avx512CD.VL.LeadingZeroCount(x.AsUInt64()).As(); - } - - return Vector256.Create(Invoke(x.GetLower()), Invoke(x.GetUpper())); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x) - { - if (Avx512CD.IsSupported) - { - if (sizeof(T) == 4) return Avx512CD.LeadingZeroCount(x.AsUInt32()).As(); - if (sizeof(T) == 8) return Avx512CD.LeadingZeroCount(x.AsUInt64()).As(); - } - - return Vector512.Create(Invoke(x.GetLower()), Invoke(x.GetUpper())); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Lerp.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Lerp.cs deleted file mode 100644 index a605b62430d3fa..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Lerp.cs +++ /dev/null @@ -1,83 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise linear interpolation between two values based on the given weight in the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of and length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Lerp([i], [i], [i]). - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Lerp(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan amount, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanSpanIntoSpan>(x, y, amount, destination); - - /// Computes the element-wise linear interpolation between two values based on the given weight in the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Lerp([i], [i], ). - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Lerp(ReadOnlySpan x, ReadOnlySpan y, T amount, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanSpanScalarIntoSpan>(x, y, amount, destination); - - /// Computes the element-wise linear interpolation between two values based on the given weight in the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Lerp([i], , [i]). - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Lerp(ReadOnlySpan x, T y, ReadOnlySpan amount, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanScalarSpanIntoSpan>(x, y, amount, destination); - - /// (x * (1 - z)) + (y * z) - private readonly struct LerpOperator : ITernaryOperator where T : IFloatingPointIeee754 - { - public static T Invoke(T x, T y, T amount) => T.Lerp(x, y, amount); - public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 amount) => (x * (Vector128.One - amount)) + (y * amount); - public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 amount) => (x * (Vector256.One - amount)) + (y * amount); - public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 amount) => (x * (Vector512.One - amount)) + (y * amount); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log.cs deleted file mode 100644 index 74d17f036d9404..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log.cs +++ /dev/null @@ -1,765 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise natural (base e) logarithm of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log([i]). - /// - /// - /// If a value equals 0, the result stored into the corresponding destination location is set to . - /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. - /// If a value is positive infinity, the result stored into the corresponding destination location is set to . - /// Otherwise, if a value is positive, its natural logarithm is stored into the corresponding destination location. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log(ReadOnlySpan x, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// Computes the element-wise logarithm of the numbers in a specified tensor to the specified base in another specified tensor. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log([i], [i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise logarithm of the numbers in a specified tensor to the specified base in another specified tensor. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log([i], ). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log(ReadOnlySpan x, T y, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// T.Log(x) - internal readonly struct LogOperator : IUnaryOperator - where T : ILogarithmicFunctions - { - public static bool Vectorizable => (typeof(T) == typeof(double)) - || (typeof(T) == typeof(float)); - - public static T Invoke(T x) => T.Log(x); - - public static Vector128 Invoke(Vector128 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector128.Log(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector128.Log(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return LogOperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return LogOperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - - public static Vector256 Invoke(Vector256 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector256.Log(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector256.Log(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return LogOperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return LogOperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - - public static Vector512 Invoke(Vector512 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector512.Log(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector512.Log(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return LogOperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return LogOperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - } - - /// T.Log(x, y) - private readonly struct LogBaseOperator : IBinaryOperator - where T : ILogarithmicFunctions - { - public static bool Vectorizable => LogOperator.Vectorizable; - public static T Invoke(T x, T y) => T.Log(x, y); - public static Vector128 Invoke(Vector128 x, Vector128 y) => LogOperator.Invoke(x) / LogOperator.Invoke(y); - public static Vector256 Invoke(Vector256 x, Vector256 y) => LogOperator.Invoke(x) / LogOperator.Invoke(y); - public static Vector512 Invoke(Vector512 x, Vector512 y) => LogOperator.Invoke(x) / LogOperator.Invoke(y); - } - -#if !NET9_0_OR_GREATER - /// double.Log(x) - private readonly struct LogOperatorDouble : IUnaryOperator - { - // This code is based on `vrd2_log` from amd/aocl-libm-ose - // Copyright (C) 2018-2020 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Reduce x into the form: - // x = (-1)^s*2^n*m - // s will be always zero, as log is defined for positive numbers - // n is an integer known as the exponent - // m is mantissa - // - // x is reduced such that the mantissa, m lies in [2/3,4/3] - // x = 2^n*m where m is in [2/3,4/3] - // log(x) = log(2^n*m) We have log(a*b) = log(a)+log(b) - // = log(2^n) + log(m) We have log(a^n) = n*log(a) - // = n*log(2) + log(m) - // = n*log(2) + log(1+(m-1)) - // = n*log(2) + log(1+f) Where f = m-1 - // = n*log(2) + log1p(f) f lies in [-1/3,+1/3] - // - // Thus we have : - // log(x) = n*log(2) + log1p(f) - // In the above, the first term n*log(2), n can be calculated by using right shift operator and the value of log(2) - // is known and is stored as a constant - // The second term log1p(F) is approximated by using a polynomial - - private const ulong V_MIN = 0x00100000_00000000; // SmallestNormal - private const ulong V_MAX = 0x7FF00000_00000000; // +Infinity - private const ulong V_MSK = 0x000FFFFF_FFFFFFFF; // (1 << 52) - 1 - private const ulong V_OFF = 0x3FE55555_55555555; // 2.0 / 3.0 - - private const double LN2_HEAD = 0.693359375; - private const double LN2_TAIL = -0.00021219444005469057; - - private const double C02 = -0.499999999999999560; - private const double C03 = +0.333333333333414750; - private const double C04 = -0.250000000000297430; - private const double C05 = +0.199999999975985220; - private const double C06 = -0.166666666608919500; - private const double C07 = +0.142857145600277100; - private const double C08 = -0.125000005127831270; - private const double C09 = +0.111110952357159440; - private const double C10 = -0.099999750495501240; - private const double C11 = +0.090914349823462390; - private const double C12 = -0.083340600527551860; - private const double C13 = +0.076817603328311300; - private const double C14 = -0.071296718946287310; - private const double C15 = +0.067963465211535730; - private const double C16 = -0.063995035098960040; - private const double C17 = +0.049370587082412105; - private const double C18 = -0.045370170994891980; - private const double C19 = +0.088970636003577750; - private const double C20 = -0.086906174116908760; - - public static bool Vectorizable => true; - - public static double Invoke(double x) => double.Log(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 specialResult = x; - - // x is zero, subnormal, infinity, or NaN - Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt64() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); - - if (specialMask != Vector128.Zero) - { - Vector128 xBits = x.AsInt64(); - - // (x < 0) ? float.NaN : x - Vector128 lessThanZeroMask = Vector128.LessThan(xBits, Vector128.Zero).AsDouble(); - - specialResult = Vector128.ConditionalSelect( - lessThanZeroMask, - Vector128.Create(double.NaN), - specialResult - ); - - // double.IsZero(x) ? double.NegativeInfinity : x - Vector128 zeroMask = Vector128.Equals(xBits << 1, Vector128.Zero).AsDouble(); - - specialResult = Vector128.ConditionalSelect( - zeroMask, - Vector128.Create(double.NegativeInfinity), - specialResult - ); - - // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) - Vector128 temp = zeroMask - | lessThanZeroMask - | Vector128.GreaterThanOrEqual(xBits, Vector128.Create(double.PositiveInfinity).AsInt64()).AsDouble(); - - // subnormal - Vector128 subnormalMask = Vector128.AndNot(specialMask.AsDouble(), temp); - - // multiply by 2^52, then normalize - x = Vector128.ConditionalSelect( - subnormalMask, - ((x * 4503599627370496.0).AsUInt64() - Vector128.Create(52ul << 52)).AsDouble(), - x - ); - - specialMask = temp.AsUInt64(); - } - - // Reduce the mantissa to [+2/3, +4/3] - Vector128 vx = x.AsUInt64() - Vector128.Create(V_OFF); - Vector128 n = Vector128.ConvertToDouble(vx.AsInt64() >> 52); - vx = (vx & Vector128.Create(V_MSK)) + Vector128.Create(V_OFF); - - // Adjust the mantissa to [-1/3, +1/3] - Vector128 r = vx.AsDouble() - Vector128.One; - - Vector128 r02 = r * r; - Vector128 r04 = r02 * r02; - Vector128 r08 = r04 * r04; - Vector128 r16 = r08 * r08; - - // Compute log(x + 1) using Polynomial approximation - // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) - - Vector128 poly = (((r04 * C20) - + ((((r * C19) + Vector128.Create(C18)) * r02) - + ((r * C17) + Vector128.Create(C16)))) * r16) - + (((((((r * C15) + Vector128.Create(C14)) * r02) - + ((r * C13) + Vector128.Create(C12))) * r04) - + ((((r * C11) + Vector128.Create(C10)) * r02) - + ((r * C09) + Vector128.Create(C08)))) * r08) - + (((((r * C07) + Vector128.Create(C06)) * r02) - + ((r * C05) + Vector128.Create(C04))) * r04) - + ((((r * C03) + Vector128.Create(C02)) * r02) + r); - - return Vector128.ConditionalSelect( - specialMask.AsDouble(), - specialResult, - (n * LN2_HEAD) + ((n * LN2_TAIL) + poly) - ); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 specialResult = x; - - // x is zero, subnormal, infinity, or NaN - Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt64() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); - - if (specialMask != Vector256.Zero) - { - Vector256 xBits = x.AsInt64(); - - // (x < 0) ? float.NaN : x - Vector256 lessThanZeroMask = Vector256.LessThan(xBits, Vector256.Zero).AsDouble(); - - specialResult = Vector256.ConditionalSelect( - lessThanZeroMask, - Vector256.Create(double.NaN), - specialResult - ); - - // double.IsZero(x) ? double.NegativeInfinity : x - Vector256 zeroMask = Vector256.Equals(xBits << 1, Vector256.Zero).AsDouble(); - - specialResult = Vector256.ConditionalSelect( - zeroMask, - Vector256.Create(double.NegativeInfinity), - specialResult - ); - - // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) - Vector256 temp = zeroMask - | lessThanZeroMask - | Vector256.GreaterThanOrEqual(xBits, Vector256.Create(double.PositiveInfinity).AsInt64()).AsDouble(); - - // subnormal - Vector256 subnormalMask = Vector256.AndNot(specialMask.AsDouble(), temp); - - // multiply by 2^52, then normalize - x = Vector256.ConditionalSelect( - subnormalMask, - ((x * 4503599627370496.0).AsUInt64() - Vector256.Create(52ul << 52)).AsDouble(), - x - ); - - specialMask = temp.AsUInt64(); - } - - // Reduce the mantissa to [+2/3, +4/3] - Vector256 vx = x.AsUInt64() - Vector256.Create(V_OFF); - Vector256 n = Vector256.ConvertToDouble(vx.AsInt64() >> 52); - vx = (vx & Vector256.Create(V_MSK)) + Vector256.Create(V_OFF); - - // Adjust the mantissa to [-1/3, +1/3] - Vector256 r = vx.AsDouble() - Vector256.One; - - Vector256 r02 = r * r; - Vector256 r04 = r02 * r02; - Vector256 r08 = r04 * r04; - Vector256 r16 = r08 * r08; - - // Compute log(x + 1) using Polynomial approximation - // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) - - Vector256 poly = (((r04 * C20) - + ((((r * C19) + Vector256.Create(C18)) * r02) - + ((r * C17) + Vector256.Create(C16)))) * r16) - + (((((((r * C15) + Vector256.Create(C14)) * r02) - + ((r * C13) + Vector256.Create(C12))) * r04) - + ((((r * C11) + Vector256.Create(C10)) * r02) - + ((r * C09) + Vector256.Create(C08)))) * r08) - + (((((r * C07) + Vector256.Create(C06)) * r02) - + ((r * C05) + Vector256.Create(C04))) * r04) - + ((((r * C03) + Vector256.Create(C02)) * r02) + r); - - return Vector256.ConditionalSelect( - specialMask.AsDouble(), - specialResult, - (n * LN2_HEAD) + ((n * LN2_TAIL) + poly) - ); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 specialResult = x; - - // x is zero, subnormal, infinity, or NaN - Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt64() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); - - if (specialMask != Vector512.Zero) - { - Vector512 xBits = x.AsInt64(); - - // (x < 0) ? float.NaN : x - Vector512 lessThanZeroMask = Vector512.LessThan(xBits, Vector512.Zero).AsDouble(); - - specialResult = Vector512.ConditionalSelect( - lessThanZeroMask, - Vector512.Create(double.NaN), - specialResult - ); - - // double.IsZero(x) ? double.NegativeInfinity : x - Vector512 zeroMask = Vector512.Equals(xBits << 1, Vector512.Zero).AsDouble(); - - specialResult = Vector512.ConditionalSelect( - zeroMask, - Vector512.Create(double.NegativeInfinity), - specialResult - ); - - // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) - Vector512 temp = zeroMask - | lessThanZeroMask - | Vector512.GreaterThanOrEqual(xBits, Vector512.Create(double.PositiveInfinity).AsInt64()).AsDouble(); - - // subnormal - Vector512 subnormalMask = Vector512.AndNot(specialMask.AsDouble(), temp); - - // multiply by 2^52, then normalize - x = Vector512.ConditionalSelect( - subnormalMask, - ((x * 4503599627370496.0).AsUInt64() - Vector512.Create(52ul << 52)).AsDouble(), - x - ); - - specialMask = temp.AsUInt64(); - } - - // Reduce the mantissa to [+2/3, +4/3] - Vector512 vx = x.AsUInt64() - Vector512.Create(V_OFF); - Vector512 n = Vector512.ConvertToDouble(vx.AsInt64() >> 52); - vx = (vx & Vector512.Create(V_MSK)) + Vector512.Create(V_OFF); - - // Adjust the mantissa to [-1/3, +1/3] - Vector512 r = vx.AsDouble() - Vector512.One; - - Vector512 r02 = r * r; - Vector512 r04 = r02 * r02; - Vector512 r08 = r04 * r04; - Vector512 r16 = r08 * r08; - - // Compute log(x + 1) using Polynomial approximation - // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) - - Vector512 poly = (((r04 * C20) - + ((((r * C19) + Vector512.Create(C18)) * r02) - + ((r * C17) + Vector512.Create(C16)))) * r16) - + (((((((r * C15) + Vector512.Create(C14)) * r02) - + ((r * C13) + Vector512.Create(C12))) * r04) - + ((((r * C11) + Vector512.Create(C10)) * r02) - + ((r * C09) + Vector512.Create(C08)))) * r08) - + (((((r * C07) + Vector512.Create(C06)) * r02) - + ((r * C05) + Vector512.Create(C04))) * r04) - + ((((r * C03) + Vector512.Create(C02)) * r02) + r); - - return Vector512.ConditionalSelect( - specialMask.AsDouble(), - specialResult, - (n * LN2_HEAD) + ((n * LN2_TAIL) + poly) - ); - } - } - - /// float.Log(x) - private readonly struct LogOperatorSingle : IUnaryOperator - { - // This code is based on `vrs4_logf` from amd/aocl-libm-ose - // Copyright (C) 2018-2019 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Spec: - // logf(x) - // = logf(x) if x ∈ F and x > 0 - // = x if x = qNaN - // = 0 if x = 1 - // = -inf if x = (-0, 0} - // = NaN otherwise - // - // Assumptions/Expectations - // - ULP is derived to be << 4 (always) - // - Some FPU Exceptions may not be available - // - Performance is at least 3x - // - // Implementation Notes: - // 1. Range Reduction: - // x = 2^n*(1+f) .... (1) - // where n is exponent and is an integer - // (1+f) is mantissa ∈ [1,2). i.e., 1 ≤ 1+f < 2 .... (2) - // - // From (1), taking log on both sides - // log(x) = log(2^n * (1+f)) - // = log(2^n) + log(1+f) - // = n*log(2) + log(1+f) .... (3) - // - // let z = 1 + f - // log(z) = log(k) + log(z) - log(k) - // log(z) = log(kz) - log(k) - // - // From (2), range of z is [1, 2) - // by simply dividing range by 'k', z is in [1/k, 2/k) .... (4) - // Best choice of k is the one which gives equal and opposite values - // at extrema +- -+ - // 1 | 2 | - // --- - 1 = - |--- - 1 | - // k | k | .... (5) - // +- -+ - // - // Solving for k, k = 3/2, - // From (4), using 'k' value, range is therefore [-0.3333, 0.3333] - // - // 2. Polynomial Approximation: - // More information refer to tools/sollya/vrs4_logf.sollya - // - // 7th Deg - Error abs: 0x1.04c4ac98p-22 rel: 0x1.2216e6f8p-19 - // 6th Deg - Error abs: 0x1.179e97d8p-19 rel: 0x1.db676c1p-17 - - private const uint V_MIN = 0x00800000; - private const uint V_MAX = 0x7F800000; - private const uint V_MASK = 0x007FFFFF; - private const uint V_OFF = 0x3F2AAAAB; - - private const float V_LN2 = 0.6931472f; - - private const float C0 = 0.0f; - private const float C1 = 1.0f; - private const float C2 = -0.5000001f; - private const float C3 = 0.33332965f; - private const float C4 = -0.24999046f; - private const float C5 = 0.20018855f; - private const float C6 = -0.16700386f; - private const float C7 = 0.13902695f; - private const float C8 = -0.1197452f; - private const float C9 = 0.14401625f; - private const float C10 = -0.13657966f; - - public static bool Vectorizable => true; - - public static float Invoke(float x) => float.Log(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 specialResult = x; - - // x is subnormal or infinity or NaN - Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt32() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); - - if (specialMask != Vector128.Zero) - { - // float.IsZero(x) ? float.NegativeInfinity : x - Vector128 zeroMask = Vector128.Equals(x, Vector128.Zero); - - specialResult = Vector128.ConditionalSelect( - zeroMask, - Vector128.Create(float.NegativeInfinity), - specialResult - ); - - // (x < 0) ? float.NaN : x - Vector128 lessThanZeroMask = Vector128.LessThan(x, Vector128.Zero); - - specialResult = Vector128.ConditionalSelect( - lessThanZeroMask, - Vector128.Create(float.NaN), - specialResult - ); - - // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) - Vector128 temp = zeroMask - | lessThanZeroMask - | ~Vector128.Equals(x, x) - | Vector128.Equals(x, Vector128.Create(float.PositiveInfinity)); - - // subnormal - Vector128 subnormalMask = Vector128.AndNot(specialMask.AsSingle(), temp); - - x = Vector128.ConditionalSelect( - subnormalMask, - ((x * 8388608.0f).AsUInt32() - Vector128.Create(23u << 23)).AsSingle(), - x - ); - - specialMask = temp.AsUInt32(); - } - - Vector128 vx = x.AsUInt32() - Vector128.Create(V_OFF); - Vector128 n = Vector128.ConvertToSingle(Vector128.ShiftRightArithmetic(vx.AsInt32(), 23)); - - vx = (vx & Vector128.Create(V_MASK)) + Vector128.Create(V_OFF); - - Vector128 r = vx.AsSingle() - Vector128.One; - - Vector128 r2 = r * r; - Vector128 r4 = r2 * r2; - Vector128 r8 = r4 * r4; - - Vector128 q = (Vector128.Create(C10) * r2 + (Vector128.Create(C9) * r + Vector128.Create(C8))) - * r8 + (((Vector128.Create(C7) * r + Vector128.Create(C6)) - * r2 + (Vector128.Create(C5) * r + Vector128.Create(C4))) - * r4 + ((Vector128.Create(C3) * r + Vector128.Create(C2)) - * r2 + (Vector128.Create(C1) * r + Vector128.Create(C0)))); - - return Vector128.ConditionalSelect( - specialMask.AsSingle(), - specialResult, - n * Vector128.Create(V_LN2) + q - ); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 specialResult = x; - - // x is subnormal or infinity or NaN - Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt32() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); - - if (specialMask != Vector256.Zero) - { - // float.IsZero(x) ? float.NegativeInfinity : x - Vector256 zeroMask = Vector256.Equals(x, Vector256.Zero); - - specialResult = Vector256.ConditionalSelect( - zeroMask, - Vector256.Create(float.NegativeInfinity), - specialResult - ); - - // (x < 0) ? float.NaN : x - Vector256 lessThanZeroMask = Vector256.LessThan(x, Vector256.Zero); - - specialResult = Vector256.ConditionalSelect( - lessThanZeroMask, - Vector256.Create(float.NaN), - specialResult - ); - - // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) - Vector256 temp = zeroMask - | lessThanZeroMask - | ~Vector256.Equals(x, x) - | Vector256.Equals(x, Vector256.Create(float.PositiveInfinity)); - - // subnormal - Vector256 subnormalMask = Vector256.AndNot(specialMask.AsSingle(), temp); - - x = Vector256.ConditionalSelect( - subnormalMask, - ((x * 8388608.0f).AsUInt32() - Vector256.Create(23u << 23)).AsSingle(), - x - ); - - specialMask = temp.AsUInt32(); - } - - Vector256 vx = x.AsUInt32() - Vector256.Create(V_OFF); - Vector256 n = Vector256.ConvertToSingle(Vector256.ShiftRightArithmetic(vx.AsInt32(), 23)); - - vx = (vx & Vector256.Create(V_MASK)) + Vector256.Create(V_OFF); - - Vector256 r = vx.AsSingle() - Vector256.One; - - Vector256 r2 = r * r; - Vector256 r4 = r2 * r2; - Vector256 r8 = r4 * r4; - - Vector256 q = (Vector256.Create(C10) * r2 + (Vector256.Create(C9) * r + Vector256.Create(C8))) - * r8 + (((Vector256.Create(C7) * r + Vector256.Create(C6)) - * r2 + (Vector256.Create(C5) * r + Vector256.Create(C4))) - * r4 + ((Vector256.Create(C3) * r + Vector256.Create(C2)) - * r2 + (Vector256.Create(C1) * r + Vector256.Create(C0)))); - - return Vector256.ConditionalSelect( - specialMask.AsSingle(), - specialResult, - n * Vector256.Create(V_LN2) + q - ); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 specialResult = x; - - // x is subnormal or infinity or NaN - Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt32() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); - - if (specialMask != Vector512.Zero) - { - // float.IsZero(x) ? float.NegativeInfinity : x - Vector512 zeroMask = Vector512.Equals(x, Vector512.Zero); - - specialResult = Vector512.ConditionalSelect( - zeroMask, - Vector512.Create(float.NegativeInfinity), - specialResult - ); - - // (x < 0) ? float.NaN : x - Vector512 lessThanZeroMask = Vector512.LessThan(x, Vector512.Zero); - - specialResult = Vector512.ConditionalSelect( - lessThanZeroMask, - Vector512.Create(float.NaN), - specialResult - ); - - // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) - Vector512 temp = zeroMask - | lessThanZeroMask - | ~Vector512.Equals(x, x) - | Vector512.Equals(x, Vector512.Create(float.PositiveInfinity)); - - // subnormal - Vector512 subnormalMask = Vector512.AndNot(specialMask.AsSingle(), temp); - - x = Vector512.ConditionalSelect( - subnormalMask, - ((x * 8388608.0f).AsUInt32() - Vector512.Create(23u << 23)).AsSingle(), - x - ); - - specialMask = temp.AsUInt32(); - } - - Vector512 vx = x.AsUInt32() - Vector512.Create(V_OFF); - Vector512 n = Vector512.ConvertToSingle(Vector512.ShiftRightArithmetic(vx.AsInt32(), 23)); - - vx = (vx & Vector512.Create(V_MASK)) + Vector512.Create(V_OFF); - - Vector512 r = vx.AsSingle() - Vector512.One; - - Vector512 r2 = r * r; - Vector512 r4 = r2 * r2; - Vector512 r8 = r4 * r4; - - Vector512 q = (Vector512.Create(C10) * r2 + (Vector512.Create(C9) * r + Vector512.Create(C8))) - * r8 + (((Vector512.Create(C7) * r + Vector512.Create(C6)) - * r2 + (Vector512.Create(C5) * r + Vector512.Create(C4))) - * r4 + ((Vector512.Create(C3) * r + Vector512.Create(C2)) - * r2 + (Vector512.Create(C1) * r + Vector512.Create(C0)))); - - return Vector512.ConditionalSelect( - specialMask.AsSingle(), - specialResult, - n * Vector512.Create(V_LN2) + q - ); - } - } -#endif - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log10.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log10.cs deleted file mode 100644 index 7f82a41e473db0..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log10.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise base 10 logarithm of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log10([i]). - /// - /// - /// If a value equals 0, the result stored into the corresponding destination location is set to . - /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. - /// If a value is positive infinity, the result stored into the corresponding destination location is set to . - /// Otherwise, if a value is positive, its base 10 logarithm is stored into the corresponding destination location. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log10(ReadOnlySpan x, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Log10(x) - private readonly struct Log10Operator : IUnaryOperator - where T : ILogarithmicFunctions - { - private const double NaturalLog10 = 2.302585092994046; - public static bool Vectorizable => LogOperator.Vectorizable; - public static T Invoke(T x) => T.Log10(x); - public static Vector128 Invoke(Vector128 x) => LogOperator.Invoke(x) / Vector128.Create(T.CreateTruncating(NaturalLog10)); - public static Vector256 Invoke(Vector256 x) => LogOperator.Invoke(x) / Vector256.Create(T.CreateTruncating(NaturalLog10)); - public static Vector512 Invoke(Vector512 x) => LogOperator.Invoke(x) / Vector512.Create(T.CreateTruncating(NaturalLog10)); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log10P1.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log10P1.cs deleted file mode 100644 index c1a55018312a32..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log10P1.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise base 10 logarithm of numbers in the specified tensor plus 1. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log10P1([i]). - /// - /// - /// If a value equals 0, the result stored into the corresponding destination location is set to . - /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. - /// If a value is positive infinity, the result stored into the corresponding destination location is set to . - /// Otherwise, if a value is positive, its base 10 logarithm plus 1 is stored into the corresponding destination location. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log10P1(ReadOnlySpan x, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Log10P1(x) - private readonly struct Log10P1Operator : IUnaryOperator - where T : ILogarithmicFunctions - { - public static bool Vectorizable => Log10Operator.Vectorizable; - public static T Invoke(T x) => T.Log10P1(x); - public static Vector128 Invoke(Vector128 x) => Log10Operator.Invoke(x + Vector128.One); - public static Vector256 Invoke(Vector256 x) => Log10Operator.Invoke(x + Vector256.One); - public static Vector512 Invoke(Vector512 x) => Log10Operator.Invoke(x + Vector512.One); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log2.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log2.cs deleted file mode 100644 index b8679ea9faee08..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log2.cs +++ /dev/null @@ -1,707 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise base 2 logarithm of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log2([i]). - /// - /// - /// If a value equals 0, the result stored into the corresponding destination location is set to . - /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. - /// If a value is positive infinity, the result stored into the corresponding destination location is set to . - /// Otherwise, if a value is positive, its base 2 logarithm is stored into the corresponding destination location. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log2(ReadOnlySpan x, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Log2(x) - internal readonly struct Log2Operator : IUnaryOperator - where T : ILogarithmicFunctions - { - public static bool Vectorizable => (typeof(T) == typeof(double)) - || (typeof(T) == typeof(float)); - - public static T Invoke(T x) => T.Log2(x); - - public static Vector128 Invoke(Vector128 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector128.Log2(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector128.Log2(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return Log2OperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Log2OperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - - public static Vector256 Invoke(Vector256 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector256.Log2(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector256.Log2(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return Log2OperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Log2OperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - - public static Vector512 Invoke(Vector512 x) - { -#if NET9_0_OR_GREATER - if (typeof(T) == typeof(double)) - { - return Vector512.Log2(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Vector512.Log2(x.AsSingle()).As(); - } -#else - if (typeof(T) == typeof(double)) - { - return Log2OperatorDouble.Invoke(x.AsDouble()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(float)); - return Log2OperatorSingle.Invoke(x.AsSingle()).As(); - } -#endif - } - } - -#if !NET9_0_OR_GREATER - /// double.Log2(x) - private readonly struct Log2OperatorDouble : IUnaryOperator - { - // This code is based on `vrd2_log2` from amd/aocl-libm-ose - // Copyright (C) 2021-2022 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Reduce x into the form: - // x = (-1)^s*2^n*m - // s will be always zero, as log is defined for positive numbers - // n is an integer known as the exponent - // m is mantissa - // - // x is reduced such that the mantissa, m lies in [2/3,4/3] - // x = 2^n*m where m is in [2/3,4/3] - // log2(x) = log2(2^n*m) We have log(a*b) = log(a)+log(b) - // = log2(2^n) + log2(m) We have log(a^n) = n*log(a) - // = n + log2(m) - // = n + log2(1+(m-1)) - // = n + ln(1+f) * log2(e) Where f = m-1 - // = n + log1p(f) * log2(e) f lies in [-1/3,+1/3] - // - // Thus we have : - // log(x) = n + log1p(f) * log2(e) - // The second term log1p(F) is approximated by using a polynomial - - private const ulong V_MIN = 0x00100000_00000000; // SmallestNormal - private const ulong V_MAX = 0x7FF00000_00000000; // +Infinity - private const ulong V_MSK = 0x000FFFFF_FFFFFFFF; // (1 << 52) - 1 - private const ulong V_OFF = 0x3FE55555_55555555; // 2.0 / 3.0 - - private const double LN2_HEAD = 1.44269180297851562500E+00; - private const double LN2_TAIL = 3.23791044778235969970E-06; - - private const double C02 = -0.499999999999999560; - private const double C03 = +0.333333333333414750; - private const double C04 = -0.250000000000297430; - private const double C05 = +0.199999999975985220; - private const double C06 = -0.166666666608919500; - private const double C07 = +0.142857145600277100; - private const double C08 = -0.125000005127831270; - private const double C09 = +0.111110952357159440; - private const double C10 = -0.099999750495501240; - private const double C11 = +0.090914349823462390; - private const double C12 = -0.083340600527551860; - private const double C13 = +0.076817603328311300; - private const double C14 = -0.071296718946287310; - private const double C15 = +0.067963465211535730; - private const double C16 = -0.063995035098960040; - private const double C17 = +0.049370587082412105; - private const double C18 = -0.045370170994891980; - private const double C19 = +0.088970636003577750; - private const double C20 = -0.086906174116908760; - - public static bool Vectorizable => true; - - public static double Invoke(double x) => double.Log2(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 specialResult = x; - - // x is zero, subnormal, infinity, or NaN - Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt64() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); - - if (specialMask != Vector128.Zero) - { - Vector128 xBits = x.AsInt64(); - - // (x < 0) ? float.NaN : x - Vector128 lessThanZeroMask = Vector128.LessThan(xBits, Vector128.Zero).AsDouble(); - - specialResult = Vector128.ConditionalSelect( - lessThanZeroMask, - Vector128.Create(double.NaN), - specialResult - ); - - // double.IsZero(x) ? double.NegativeInfinity : x - Vector128 zeroMask = Vector128.Equals(xBits << 1, Vector128.Zero).AsDouble(); - - specialResult = Vector128.ConditionalSelect( - zeroMask, - Vector128.Create(double.NegativeInfinity), - specialResult - ); - - // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) - Vector128 temp = zeroMask - | lessThanZeroMask - | Vector128.GreaterThanOrEqual(xBits, Vector128.Create(double.PositiveInfinity).AsInt64()).AsDouble(); - - // subnormal - Vector128 subnormalMask = Vector128.AndNot(specialMask.AsDouble(), temp); - - // multiply by 2^52, then normalize - x = Vector128.ConditionalSelect( - subnormalMask, - ((x * 4503599627370496.0).AsUInt64() - Vector128.Create(52ul << 52)).AsDouble(), - x - ); - - specialMask = temp.AsUInt64(); - } - - // Reduce the mantissa to [+2/3, +4/3] - Vector128 vx = x.AsUInt64() - Vector128.Create(V_OFF); - Vector128 n = Vector128.ConvertToDouble(vx.AsInt64() >> 52); - vx = (vx & Vector128.Create(V_MSK)) + Vector128.Create(V_OFF); - - // Adjust the mantissa to [-1/3, +1/3] - Vector128 r = vx.AsDouble() - Vector128.One; - - Vector128 r02 = r * r; - Vector128 r04 = r02 * r02; - Vector128 r08 = r04 * r04; - Vector128 r16 = r08 * r08; - - // Compute log(x + 1) using polynomial approximation - // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) - - Vector128 poly = (((r04 * C20) - + ((((r * C19) + Vector128.Create(C18)) * r02) - + ((r * C17) + Vector128.Create(C16)))) * r16) - + (((((((r * C15) + Vector128.Create(C14)) * r02) - + ((r * C13) + Vector128.Create(C12))) * r04) - + ((((r * C11) + Vector128.Create(C10)) * r02) - + ((r * C09) + Vector128.Create(C08)))) * r08) - + (((((r * C07) + Vector128.Create(C06)) * r02) - + ((r * C05) + Vector128.Create(C04))) * r04) - + ((((r * C03) + Vector128.Create(C02)) * r02) + r); - - return Vector128.ConditionalSelect( - specialMask.AsDouble(), - specialResult, - (poly * LN2_HEAD) + ((poly * LN2_TAIL) + n) - ); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 specialResult = x; - - // x is zero, subnormal, infinity, or NaN - Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt64() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); - - if (specialMask != Vector256.Zero) - { - Vector256 xBits = x.AsInt64(); - - // (x < 0) ? float.NaN : x - Vector256 lessThanZeroMask = Vector256.LessThan(xBits, Vector256.Zero).AsDouble(); - - specialResult = Vector256.ConditionalSelect( - lessThanZeroMask, - Vector256.Create(double.NaN), - specialResult - ); - - // double.IsZero(x) ? double.NegativeInfinity : x - Vector256 zeroMask = Vector256.Equals(xBits << 1, Vector256.Zero).AsDouble(); - - specialResult = Vector256.ConditionalSelect( - zeroMask, - Vector256.Create(double.NegativeInfinity), - specialResult - ); - - // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) - Vector256 temp = zeroMask - | lessThanZeroMask - | Vector256.GreaterThanOrEqual(xBits, Vector256.Create(double.PositiveInfinity).AsInt64()).AsDouble(); - - // subnormal - Vector256 subnormalMask = Vector256.AndNot(specialMask.AsDouble(), temp); - - // multiply by 2^52, then normalize - x = Vector256.ConditionalSelect( - subnormalMask, - ((x * 4503599627370496.0).AsUInt64() - Vector256.Create(52ul << 52)).AsDouble(), - x - ); - - specialMask = temp.AsUInt64(); - } - - // Reduce the mantissa to [+2/3, +4/3] - Vector256 vx = x.AsUInt64() - Vector256.Create(V_OFF); - Vector256 n = Vector256.ConvertToDouble(vx.AsInt64() >> 52); - vx = (vx & Vector256.Create(V_MSK)) + Vector256.Create(V_OFF); - - // Adjust the mantissa to [-1/3, +1/3] - Vector256 r = vx.AsDouble() - Vector256.One; - - Vector256 r02 = r * r; - Vector256 r04 = r02 * r02; - Vector256 r08 = r04 * r04; - Vector256 r16 = r08 * r08; - - // Compute log(x + 1) using polynomial approximation - // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) - - Vector256 poly = (((r04 * C20) - + ((((r * C19) + Vector256.Create(C18)) * r02) - + ((r * C17) + Vector256.Create(C16)))) * r16) - + (((((((r * C15) + Vector256.Create(C14)) * r02) - + ((r * C13) + Vector256.Create(C12))) * r04) - + ((((r * C11) + Vector256.Create(C10)) * r02) - + ((r * C09) + Vector256.Create(C08)))) * r08) - + (((((r * C07) + Vector256.Create(C06)) * r02) - + ((r * C05) + Vector256.Create(C04))) * r04) - + ((((r * C03) + Vector256.Create(C02)) * r02) + r); - - return Vector256.ConditionalSelect( - specialMask.AsDouble(), - specialResult, - (poly * LN2_HEAD) + ((poly * LN2_TAIL) + n) - ); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 specialResult = x; - - // x is zero, subnormal, infinity, or NaN - Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt64() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); - - if (specialMask != Vector512.Zero) - { - Vector512 xBits = x.AsInt64(); - - // (x < 0) ? float.NaN : x - Vector512 lessThanZeroMask = Vector512.LessThan(xBits, Vector512.Zero).AsDouble(); - - specialResult = Vector512.ConditionalSelect( - lessThanZeroMask, - Vector512.Create(double.NaN), - specialResult - ); - - // double.IsZero(x) ? double.NegativeInfinity : x - Vector512 zeroMask = Vector512.Equals(xBits << 1, Vector512.Zero).AsDouble(); - - specialResult = Vector512.ConditionalSelect( - zeroMask, - Vector512.Create(double.NegativeInfinity), - specialResult - ); - - // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) - Vector512 temp = zeroMask - | lessThanZeroMask - | Vector512.GreaterThanOrEqual(xBits, Vector512.Create(double.PositiveInfinity).AsInt64()).AsDouble(); - - // subnormal - Vector512 subnormalMask = Vector512.AndNot(specialMask.AsDouble(), temp); - - // multiply by 2^52, then normalize - x = Vector512.ConditionalSelect( - subnormalMask, - ((x * 4503599627370496.0).AsUInt64() - Vector512.Create(52ul << 52)).AsDouble(), - x - ); - - specialMask = temp.AsUInt64(); - } - - // Reduce the mantissa to [+2/3, +4/3] - Vector512 vx = x.AsUInt64() - Vector512.Create(V_OFF); - Vector512 n = Vector512.ConvertToDouble(vx.AsInt64() >> 52); - vx = (vx & Vector512.Create(V_MSK)) + Vector512.Create(V_OFF); - - // Adjust the mantissa to [-1/3, +1/3] - Vector512 r = vx.AsDouble() - Vector512.One; - - Vector512 r02 = r * r; - Vector512 r04 = r02 * r02; - Vector512 r08 = r04 * r04; - Vector512 r16 = r08 * r08; - - // Compute log(x + 1) using polynomial approximation - // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) - - Vector512 poly = (((r04 * C20) - + ((((r * C19) + Vector512.Create(C18)) * r02) - + ((r * C17) + Vector512.Create(C16)))) * r16) - + (((((((r * C15) + Vector512.Create(C14)) * r02) - + ((r * C13) + Vector512.Create(C12))) * r04) - + ((((r * C11) + Vector512.Create(C10)) * r02) - + ((r * C09) + Vector512.Create(C08)))) * r08) - + (((((r * C07) + Vector512.Create(C06)) * r02) - + ((r * C05) + Vector512.Create(C04))) * r04) - + ((((r * C03) + Vector512.Create(C02)) * r02) + r); - - return Vector512.ConditionalSelect( - specialMask.AsDouble(), - specialResult, - (poly * LN2_HEAD) + ((poly * LN2_TAIL) + n) - ); - } - } - - /// float.Log2(x) - private readonly struct Log2OperatorSingle : IUnaryOperator - { - // This code is based on `vrs4_log2f` from amd/aocl-libm-ose - // Copyright (C) 2021-2022 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Spec: - // log2f(x) - // = log2f(x) if x ∈ F and x > 0 - // = x if x = qNaN - // = 0 if x = 1 - // = -inf if x = (-0, 0} - // = NaN otherwise - // - // Assumptions/Expectations - // - Maximum ULP is observed to be at 4 - // - Some FPU Exceptions may not be available - // - Performance is at least 3x - // - // Implementation Notes: - // 1. Range Reduction: - // x = 2^n*(1+f) .... (1) - // where n is exponent and is an integer - // (1+f) is mantissa ∈ [1,2). i.e., 1 ≤ 1+f < 2 .... (2) - // - // From (1), taking log on both sides - // log2(x) = log2(2^n * (1+f)) - // = n + log2(1+f) .... (3) - // - // let z = 1 + f - // log2(z) = log2(k) + log2(z) - log2(k) - // log2(z) = log2(kz) - log2(k) - // - // From (2), range of z is [1, 2) - // by simply dividing range by 'k', z is in [1/k, 2/k) .... (4) - // Best choice of k is the one which gives equal and opposite values - // at extrema +- -+ - // 1 | 2 | - // --- - 1 = - |--- - 1 | - // k | k | .... (5) - // +- -+ - // - // Solving for k, k = 3/2, - // From (4), using 'k' value, range is therefore [-0.3333, 0.3333] - // - // 2. Polynomial Approximation: - // More information refer to tools/sollya/vrs4_logf.sollya - // - // 7th Deg - Error abs: 0x1.04c4ac98p-22 rel: 0x1.2216e6f8p-19 - - private const uint V_MIN = 0x00800000; - private const uint V_MAX = 0x7F800000; - private const uint V_MASK = 0x007FFFFF; - private const uint V_OFF = 0x3F2AAAAB; - - private const float C0 = 0.0f; - private const float C1 = 1.4426951f; - private const float C2 = -0.72134554f; - private const float C3 = 0.48089063f; - private const float C4 = -0.36084408f; - private const float C5 = 0.2888971f; - private const float C6 = -0.23594281f; - private const float C7 = 0.19948183f; - private const float C8 = -0.22616665f; - private const float C9 = 0.21228963f; - - public static bool Vectorizable => true; - - public static float Invoke(float x) => float.Log2(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 specialResult = x; - - // x is subnormal or infinity or NaN - Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt32() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); - - if (specialMask != Vector128.Zero) - { - // float.IsZero(x) ? float.NegativeInfinity : x - Vector128 zeroMask = Vector128.Equals(x, Vector128.Zero); - - specialResult = Vector128.ConditionalSelect( - zeroMask, - Vector128.Create(float.NegativeInfinity), - specialResult - ); - - // (x < 0) ? float.NaN : x - Vector128 lessThanZeroMask = Vector128.LessThan(x, Vector128.Zero); - - specialResult = Vector128.ConditionalSelect( - lessThanZeroMask, - Vector128.Create(float.NaN), - specialResult - ); - - // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) - Vector128 temp = zeroMask - | lessThanZeroMask - | ~Vector128.Equals(x, x) - | Vector128.Equals(x, Vector128.Create(float.PositiveInfinity)); - - // subnormal - Vector128 subnormalMask = Vector128.AndNot(specialMask.AsSingle(), temp); - - x = Vector128.ConditionalSelect( - subnormalMask, - ((x * 8388608.0f).AsUInt32() - Vector128.Create(23u << 23)).AsSingle(), - x - ); - - specialMask = temp.AsUInt32(); - } - - Vector128 vx = x.AsUInt32() - Vector128.Create(V_OFF); - Vector128 n = Vector128.ConvertToSingle(Vector128.ShiftRightArithmetic(vx.AsInt32(), 23)); - - vx = (vx & Vector128.Create(V_MASK)) + Vector128.Create(V_OFF); - - Vector128 r = vx.AsSingle() - Vector128.One; - - Vector128 r2 = r * r; - Vector128 r4 = r2 * r2; - Vector128 r8 = r4 * r4; - - Vector128 poly = (Vector128.Create(C9) * r + Vector128.Create(C8)) * r8 - + (((Vector128.Create(C7) * r + Vector128.Create(C6)) * r2 - + (Vector128.Create(C5) * r + Vector128.Create(C4))) * r4 - + ((Vector128.Create(C3) * r + Vector128.Create(C2)) * r2 - + (Vector128.Create(C1) * r + Vector128.Create(C0)))); - - return Vector128.ConditionalSelect( - specialMask.AsSingle(), - specialResult, - n + poly - ); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 specialResult = x; - - // x is subnormal or infinity or NaN - Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt32() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); - - if (specialMask != Vector256.Zero) - { - // float.IsZero(x) ? float.NegativeInfinity : x - Vector256 zeroMask = Vector256.Equals(x, Vector256.Zero); - - specialResult = Vector256.ConditionalSelect( - zeroMask, - Vector256.Create(float.NegativeInfinity), - specialResult - ); - - // (x < 0) ? float.NaN : x - Vector256 lessThanZeroMask = Vector256.LessThan(x, Vector256.Zero); - - specialResult = Vector256.ConditionalSelect( - lessThanZeroMask, - Vector256.Create(float.NaN), - specialResult - ); - - // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) - Vector256 temp = zeroMask - | lessThanZeroMask - | ~Vector256.Equals(x, x) - | Vector256.Equals(x, Vector256.Create(float.PositiveInfinity)); - - // subnormal - Vector256 subnormalMask = Vector256.AndNot(specialMask.AsSingle(), temp); - - x = Vector256.ConditionalSelect( - subnormalMask, - ((x * 8388608.0f).AsUInt32() - Vector256.Create(23u << 23)).AsSingle(), - x - ); - - specialMask = temp.AsUInt32(); - } - - Vector256 vx = x.AsUInt32() - Vector256.Create(V_OFF); - Vector256 n = Vector256.ConvertToSingle(Vector256.ShiftRightArithmetic(vx.AsInt32(), 23)); - - vx = (vx & Vector256.Create(V_MASK)) + Vector256.Create(V_OFF); - - Vector256 r = vx.AsSingle() - Vector256.One; - - Vector256 r2 = r * r; - Vector256 r4 = r2 * r2; - Vector256 r8 = r4 * r4; - - Vector256 poly = (Vector256.Create(C9) * r + Vector256.Create(C8)) * r8 - + (((Vector256.Create(C7) * r + Vector256.Create(C6)) * r2 - + (Vector256.Create(C5) * r + Vector256.Create(C4))) * r4 - + ((Vector256.Create(C3) * r + Vector256.Create(C2)) * r2 - + (Vector256.Create(C1) * r + Vector256.Create(C0)))); - - return Vector256.ConditionalSelect( - specialMask.AsSingle(), - specialResult, - n + poly - ); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 specialResult = x; - - // x is subnormal or infinity or NaN - Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt32() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); - - if (specialMask != Vector512.Zero) - { - // float.IsZero(x) ? float.NegativeInfinity : x - Vector512 zeroMask = Vector512.Equals(x, Vector512.Zero); - - specialResult = Vector512.ConditionalSelect( - zeroMask, - Vector512.Create(float.NegativeInfinity), - specialResult - ); - - // (x < 0) ? float.NaN : x - Vector512 lessThanZeroMask = Vector512.LessThan(x, Vector512.Zero); - - specialResult = Vector512.ConditionalSelect( - lessThanZeroMask, - Vector512.Create(float.NaN), - specialResult - ); - - // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) - Vector512 temp = zeroMask - | lessThanZeroMask - | ~Vector512.Equals(x, x) - | Vector512.Equals(x, Vector512.Create(float.PositiveInfinity)); - - // subnormal - Vector512 subnormalMask = Vector512.AndNot(specialMask.AsSingle(), temp); - - x = Vector512.ConditionalSelect( - subnormalMask, - ((x * 8388608.0f).AsUInt32() - Vector512.Create(23u << 23)).AsSingle(), - x - ); - - specialMask = temp.AsUInt32(); - } - - Vector512 vx = x.AsUInt32() - Vector512.Create(V_OFF); - Vector512 n = Vector512.ConvertToSingle(Vector512.ShiftRightArithmetic(vx.AsInt32(), 23)); - - vx = (vx & Vector512.Create(V_MASK)) + Vector512.Create(V_OFF); - - Vector512 r = vx.AsSingle() - Vector512.One; - - Vector512 r2 = r * r; - Vector512 r4 = r2 * r2; - Vector512 r8 = r4 * r4; - - Vector512 poly = (Vector512.Create(C9) * r + Vector512.Create(C8)) * r8 - + (((Vector512.Create(C7) * r + Vector512.Create(C6)) * r2 - + (Vector512.Create(C5) * r + Vector512.Create(C4))) * r4 - + ((Vector512.Create(C3) * r + Vector512.Create(C2)) * r2 - + (Vector512.Create(C1) * r + Vector512.Create(C0)))); - - return Vector512.ConditionalSelect( - specialMask.AsSingle(), - specialResult, - n + poly - ); - } - } -#endif - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log2P1.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log2P1.cs deleted file mode 100644 index 7fa38203addd39..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Log2P1.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise base 2 logarithm of numbers in the specified tensor plus 1. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Log2P1([i]). - /// - /// - /// If a value equals 0, the result stored into the corresponding destination location is set to . - /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. - /// If a value is positive infinity, the result stored into the corresponding destination location is set to . - /// Otherwise, if a value is positive, its base 2 logarithm plus 1 is stored into the corresponding destination location. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Log2P1(ReadOnlySpan x, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Log2P1(x) - private readonly struct Log2P1Operator : IUnaryOperator - where T : ILogarithmicFunctions - { - public static bool Vectorizable => Log2Operator.Vectorizable; - public static T Invoke(T x) => T.Log2P1(x); - public static Vector128 Invoke(Vector128 x) => Log2Operator.Invoke(x + Vector128.One); - public static Vector256 Invoke(Vector256 x) => Log2Operator.Invoke(x + Vector256.One); - public static Vector512 Invoke(Vector512 x) => Log2Operator.Invoke(x + Vector512.One); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.LogP1.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.LogP1.cs deleted file mode 100644 index 2985f57de25cdf..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.LogP1.cs +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise natural (base e) logarithm of numbers in the specified tensor plus 1. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .LogP1([i]). - /// - /// - /// If a value equals 0, the result stored into the corresponding destination location is set to . - /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. - /// If a value is positive infinity, the result stored into the corresponding destination location is set to . - /// Otherwise, if a value is positive, its natural logarithm plus 1 is stored into the corresponding destination location. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void LogP1(ReadOnlySpan x, Span destination) - where T : ILogarithmicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.LogP1(x) - private readonly struct LogP1Operator : IUnaryOperator - where T : ILogarithmicFunctions - { - public static bool Vectorizable => LogOperator.Vectorizable; - public static T Invoke(T x) => T.LogP1(x); - public static Vector128 Invoke(Vector128 x) => LogOperator.Invoke(x + Vector128.One); - public static Vector256 Invoke(Vector256 x) => LogOperator.Invoke(x + Vector256.One); - public static Vector512 Invoke(Vector512 x) => LogOperator.Invoke(x + Vector512.One); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs deleted file mode 100644 index 0e0566f2d935a5..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Max.cs +++ /dev/null @@ -1,549 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the largest number in the specified tensor. - /// The tensor, represented as a span. - /// The maximum element in . - /// Length of must be greater than zero. - /// - /// - /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If any value equal to - /// is present, the first is returned. Positive 0 is considered greater than negative 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Max(ReadOnlySpan x) - where T : INumber => - MinMaxCore>(x); - - /// Computes the element-wise maximum of the numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Max([i], [i]). - /// - /// - /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , - /// that value is stored as the result. Positive 0 is considered greater than negative 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Max(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : INumber => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise maximum of the numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Max([i], ). - /// - /// - /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , - /// that value is stored as the result. Positive 0 is considered greater than negative 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Max(ReadOnlySpan x, T y, Span destination) - where T : INumber => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// T.Max(x, y) (but NaNs may not be propagated) - internal readonly struct MaxOperator : IAggregationOperator where T : INumber - { - public static bool Vectorizable => true; - - public static T Invoke(T x, T y) - { - if (typeof(T) == typeof(Half) || typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return x == y ? - (IsNegative(x) ? y : x) : - (y > x ? y : x); - } - - return T.Max(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(byte)) return AdvSimd.Max(x.AsByte(), y.AsByte()).As(); - if (typeof(T) == typeof(sbyte)) return AdvSimd.Max(x.AsSByte(), y.AsSByte()).As(); - if (typeof(T) == typeof(short)) return AdvSimd.Max(x.AsInt16(), y.AsInt16()).As(); - if (typeof(T) == typeof(ushort)) return AdvSimd.Max(x.AsUInt16(), y.AsUInt16()).As(); - if (typeof(T) == typeof(int)) return AdvSimd.Max(x.AsInt32(), y.AsInt32()).As(); - if (typeof(T) == typeof(uint)) return AdvSimd.Max(x.AsUInt32(), y.AsUInt32()).As(); - if (typeof(T) == typeof(float)) return AdvSimd.Max(x.AsSingle(), y.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Max(x.AsDouble(), y.AsDouble()).As(); - } - - if (typeof(T) == typeof(float)) - { - return - Vector128.ConditionalSelect(Vector128.Equals(x, y), - Vector128.ConditionalSelect(IsNegative(x.AsSingle()).As(), y, x), - Vector128.Max(x, y)); - } - - if (typeof(T) == typeof(double)) - { - return - Vector128.ConditionalSelect(Vector128.Equals(x, y), - Vector128.ConditionalSelect(IsNegative(x.AsDouble()).As(), y, x), - Vector128.Max(x, y)); - } - - return Vector128.Max(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float)) - { - return - Vector256.ConditionalSelect(Vector256.Equals(x, y), - Vector256.ConditionalSelect(IsNegative(x.AsSingle()).As(), y, x), - Vector256.Max(x, y)); - } - - if (typeof(T) == typeof(double)) - { - return - Vector256.ConditionalSelect(Vector256.Equals(x, y), - Vector256.ConditionalSelect(IsNegative(x.AsDouble()).As(), y, x), - Vector256.Max(x, y)); - } - - return Vector256.Max(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float)) - { - return - Vector512.ConditionalSelect(Vector512.Equals(x, y), - Vector512.ConditionalSelect(IsNegative(x.AsSingle()).As(), y, x), - Vector512.Max(x, y)); - } - - if (typeof(T) == typeof(double)) - { - return - Vector512.ConditionalSelect(Vector512.Equals(x, y), - Vector512.ConditionalSelect(IsNegative(x.AsDouble()).As(), y, x), - Vector512.Max(x, y)); - } - - return Vector512.Max(x, y); - } - - public static T Invoke(Vector128 x) => HorizontalAggregate>(x); - public static T Invoke(Vector256 x) => HorizontalAggregate>(x); - public static T Invoke(Vector512 x) => HorizontalAggregate>(x); - } - - /// Max(x, y) - internal readonly struct MaxPropagateNaNOperator : IBinaryOperator - where T : INumber - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.Max(x, y); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(byte)) return AdvSimd.Max(x.AsByte(), y.AsByte()).As(); - if (typeof(T) == typeof(sbyte)) return AdvSimd.Max(x.AsSByte(), y.AsSByte()).As(); - if (typeof(T) == typeof(ushort)) return AdvSimd.Max(x.AsUInt16(), y.AsUInt16()).As(); - if (typeof(T) == typeof(short)) return AdvSimd.Max(x.AsInt16(), y.AsInt16()).As(); - if (typeof(T) == typeof(uint)) return AdvSimd.Max(x.AsUInt32(), y.AsUInt32()).As(); - if (typeof(T) == typeof(int)) return AdvSimd.Max(x.AsInt32(), y.AsInt32()).As(); - if (typeof(T) == typeof(float)) return AdvSimd.Max(x.AsSingle(), y.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Max(x.AsDouble(), y.AsDouble()).As(); - } - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector128.ConditionalSelect(Vector128.Equals(x, x), - Vector128.ConditionalSelect(Vector128.Equals(y, y), - Vector128.ConditionalSelect(Vector128.Equals(x, y), - Vector128.ConditionalSelect(IsNegative(x), y, x), - Vector128.Max(x, y)), - y), - x); - } - - return Vector128.Max(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector256.ConditionalSelect(Vector256.Equals(x, x), - Vector256.ConditionalSelect(Vector256.Equals(y, y), - Vector256.ConditionalSelect(Vector256.Equals(x, y), - Vector256.ConditionalSelect(IsNegative(x), y, x), - Vector256.Max(x, y)), - y), - x); - } - - return Vector256.Max(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector512.ConditionalSelect(Vector512.Equals(x, x), - Vector512.ConditionalSelect(Vector512.Equals(y, y), - Vector512.ConditionalSelect(Vector512.Equals(x, y), - Vector512.ConditionalSelect(IsNegative(x), y, x), - Vector512.Max(x, y)), - y), - x); - } - - return Vector512.Max(x, y); - } - } - - /// Gets whether each specified is negative. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector128 IsNegative(Vector128 vector) - { - if (typeof(T) == typeof(float)) - { - return Vector128.LessThan(vector.AsInt32(), Vector128.Zero).As(); - } - - if (typeof(T) == typeof(double)) - { - return Vector128.LessThan(vector.AsInt64(), Vector128.Zero).As(); - } - - return Vector128.LessThan(vector, Vector128.Zero); - } - - /// Gets whether each specified is negative. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector256 IsNegative(Vector256 vector) - { - if (typeof(T) == typeof(float)) - { - return Vector256.LessThan(vector.AsInt32(), Vector256.Zero).As(); - } - - if (typeof(T) == typeof(double)) - { - return Vector256.LessThan(vector.AsInt64(), Vector256.Zero).As(); - } - - return Vector256.LessThan(vector, Vector256.Zero); - } - - /// Gets whether each specified is negative. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static Vector512 IsNegative(Vector512 vector) - { - if (typeof(T) == typeof(float)) - { - return Vector512.LessThan(vector.AsInt32(), Vector512.Zero).As(); - } - - if (typeof(T) == typeof(double)) - { - return Vector512.LessThan(vector.AsInt64(), Vector512.Zero).As(); - } - - return Vector512.LessThan(vector, Vector512.Zero); - } - - /// - /// This is the same as - /// with an identity transform, except it early exits on NaN. - /// - private static T MinMaxCore(ReadOnlySpan x) - where T : INumberBase - where TMinMaxOperator : struct, IAggregationOperator - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - // This matches the IEEE 754:2019 `maximum`/`minimum` functions. - // It propagates NaN inputs back to the caller and - // otherwise returns the greater of the inputs. - // It treats +0 as greater than -0 as per the specification. - - if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) - { - ref T xRef = ref MemoryMarshal.GetReference(x); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector512 result = Vector512.LoadUnsafe(ref xRef, 0); - Vector512 current; - - Vector512 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector512.Equals(result, result); - if (nanMask != Vector512.Zero) - { - return result.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - int oneVectorFromEnd = x.Length - Vector512.Count; - int i = Vector512.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector512.LoadUnsafe(ref xRef, (uint)i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector512.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - result = TMinMaxOperator.Invoke(result, current); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); - } - - if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) - { - ref T xRef = ref MemoryMarshal.GetReference(x); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector256 result = Vector256.LoadUnsafe(ref xRef, 0); - Vector256 current; - - Vector256 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector256.Equals(result, result); - if (nanMask != Vector256.Zero) - { - return result.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - int oneVectorFromEnd = x.Length - Vector256.Count; - int i = Vector256.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector256.LoadUnsafe(ref xRef, (uint)i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector256.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); - - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - result = TMinMaxOperator.Invoke(result, current); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); - } - - if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) - { - ref T xRef = ref MemoryMarshal.GetReference(x); - - // Load the first vector as the initial set of results, and bail immediately - // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector128 result = Vector128.LoadUnsafe(ref xRef, 0); - Vector128 current; - - Vector128 nanMask; - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector128.Equals(result, result); - if (nanMask != Vector128.Zero) - { - return result.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - int oneVectorFromEnd = x.Length - Vector128.Count; - int i = Vector128.Count; - - // Aggregate additional vectors into the result as long as there's at least one full vector left to process. - while (i <= oneVectorFromEnd) - { - // Load the next vector, and early exit on NaN. - current = Vector128.LoadUnsafe(ref xRef, (uint)i); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - result = TMinMaxOperator.Invoke(result, current); - i += Vector128.Count; - } - - // If any elements remain, handle them in one final vector. - if (i != x.Length) - { - current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - // Check for NaNs - nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) - { - return current.GetElement(IndexOfFirstMatch(nanMask)); - } - } - - result = TMinMaxOperator.Invoke(result, current); - } - - // Aggregate the lanes in the vector to create the final scalar result. - return TMinMaxOperator.Invoke(result); - } - - // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - T curResult = x[0]; - if (T.IsNaN(curResult)) - { - return curResult; - } - - for (int i = 1; i < x.Length; i++) - { - T current = x[i]; - if (T.IsNaN(current)) - { - return current; - } - - curResult = TMinMaxOperator.Invoke(curResult, current); - } - - return curResult; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MaxMagnitude.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MaxMagnitude.cs deleted file mode 100644 index eb28249ed1ea43..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MaxMagnitude.cs +++ /dev/null @@ -1,243 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the number with the largest magnitude in the specified tensor. - /// The tensor, represented as a span. - /// The element in with the largest magnitude (absolute value). - /// Length of must be greater than zero. - /// - /// - /// The determination of the maximum magnitude matches the IEEE 754:2019 `maximumMagnitude` function. If any value equal to - /// is present, the first is returned. If two values have the same magnitude and one is positive and the other is negative, - /// the positive value is considered to have the larger magnitude. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T MaxMagnitude(ReadOnlySpan x) - where T : INumberBase => - MinMaxCore>(x); - - /// Computes the element-wise number with the largest magnitude in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// This method effectively computes [i] = .MaxMagnitude([i], [i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void MaxMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : INumberBase => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise number with the largest magnitude in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// This method effectively computes [i] = .MaxMagnitude([i], ). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void MaxMagnitude(ReadOnlySpan x, T y, Span destination) - where T : INumberBase => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// Searches for the smallest number in the specified tensor. - /// The tensor, represented as a span. - /// The minimum element in . - /// Length of must be greater than zero. - /// - /// - /// The determination of the minimum element matches the IEEE 754:2019 `minimum` function. If any value is equal to - /// is present, the first is returned. Negative 0 is considered smaller than positive 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Min(ReadOnlySpan x) - where T : INumber => - MinMaxCore>(x); - - /// Operator to get x or y based on which has the larger MathF.Abs (but NaNs may not be propagated) - internal readonly struct MaxMagnitudeOperator : IAggregationOperator - where T : INumberBase - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.MaxMagnitude(x, y); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); - - Vector128 result = - Vector128.ConditionalSelect(Vector128.Equals(xMag, yMag), - Vector128.ConditionalSelect(IsNegative(x), y, x), - Vector128.ConditionalSelect(Vector128.GreaterThan(xMag, yMag), x, y)); - - // Handle minimum signed value that should have the largest magnitude - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector128 negativeMagnitudeX = Vector128.LessThan(xMag, Vector128.Zero); - Vector128 negativeMagnitudeY = Vector128.LessThan(yMag, Vector128.Zero); - result = Vector128.ConditionalSelect(negativeMagnitudeX, - x, - Vector128.ConditionalSelect(negativeMagnitudeY, - y, - result)); - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); - - Vector256 result = - Vector256.ConditionalSelect(Vector256.Equals(xMag, yMag), - Vector256.ConditionalSelect(IsNegative(x), y, x), - Vector256.ConditionalSelect(Vector256.GreaterThan(xMag, yMag), x, y)); - - // Handle minimum signed value that should have the largest magnitude - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector256 negativeMagnitudeX = Vector256.LessThan(xMag, Vector256.Zero); - Vector256 negativeMagnitudeY = Vector256.LessThan(yMag, Vector256.Zero); - result = Vector256.ConditionalSelect(negativeMagnitudeX, - x, - Vector256.ConditionalSelect(negativeMagnitudeY, - y, - result)); - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); - - Vector512 result = - Vector512.ConditionalSelect(Vector512.Equals(xMag, yMag), - Vector512.ConditionalSelect(IsNegative(x), y, x), - Vector512.ConditionalSelect(Vector512.GreaterThan(xMag, yMag), x, y)); - - // Handle minimum signed value that should have the largest magnitude - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector512 negativeMagnitudeX = Vector512.LessThan(xMag, Vector512.Zero); - Vector512 negativeMagnitudeY = Vector512.LessThan(yMag, Vector512.Zero); - result = Vector512.ConditionalSelect(negativeMagnitudeX, - x, - Vector512.ConditionalSelect(negativeMagnitudeY, - y, - result)); - } - - return result; - } - - public static T Invoke(Vector128 x) => HorizontalAggregate>(x); - public static T Invoke(Vector256 x) => HorizontalAggregate>(x); - public static T Invoke(Vector512 x) => HorizontalAggregate>(x); - } - - /// Operator to get x or y based on which has the larger MathF.Abs - internal readonly struct MaxMagnitudePropagateNaNOperator : IBinaryOperator - where T : INumberBase - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.MaxMagnitude(x, y); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - // Handle NaNs - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); - return - Vector128.ConditionalSelect(Vector128.Equals(x, x), - Vector128.ConditionalSelect(Vector128.Equals(y, y), - Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), - Vector128.ConditionalSelect(IsNegative(x), y, x), - Vector128.ConditionalSelect(Vector128.GreaterThan(yMag, xMag), y, x)), - y), - x); - } - - return MaxMagnitudeOperator.Invoke(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - // Handle NaNs - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); - return - Vector256.ConditionalSelect(Vector256.Equals(x, x), - Vector256.ConditionalSelect(Vector256.Equals(y, y), - Vector256.ConditionalSelect(Vector256.Equals(xMag, yMag), - Vector256.ConditionalSelect(IsNegative(x), y, x), - Vector256.ConditionalSelect(Vector256.GreaterThan(xMag, yMag), x, y)), - y), - x); - } - - return MaxMagnitudeOperator.Invoke(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - // Handle NaNs - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); - return - Vector512.ConditionalSelect(Vector512.Equals(x, x), - Vector512.ConditionalSelect(Vector512.Equals(y, y), - Vector512.ConditionalSelect(Vector512.Equals(xMag, yMag), - Vector512.ConditionalSelect(IsNegative(x), y, x), - Vector512.ConditionalSelect(Vector512.GreaterThan(xMag, yMag), x, y)), - y), - x); - } - - return MaxMagnitudeOperator.Invoke(x, y); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Min.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Min.cs deleted file mode 100644 index faea478057982c..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Min.cs +++ /dev/null @@ -1,220 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise minimum of the numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Max([i], [i]). - /// - /// - /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , - /// that value is stored as the result. Positive 0 is considered greater than negative 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Min(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : INumber => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise minimum of the numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Max([i], ). - /// - /// - /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , - /// that value is stored as the result. Positive 0 is considered greater than negative 0. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Min(ReadOnlySpan x, T y, Span destination) - where T : INumber => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// T.Min(x, y) (but NaNs may not be propagated) - internal readonly struct MinOperator : IAggregationOperator - where T : INumber - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) - { - if (typeof(T) == typeof(Half) || typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return x == y ? - (IsNegative(y) ? y : x) : - (y < x ? y : x); - } - - return T.Min(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(byte)) return AdvSimd.Min(x.AsByte(), y.AsByte()).As(); - if (typeof(T) == typeof(sbyte)) return AdvSimd.Min(x.AsSByte(), y.AsSByte()).As(); - if (typeof(T) == typeof(short)) return AdvSimd.Min(x.AsInt16(), y.AsInt16()).As(); - if (typeof(T) == typeof(ushort)) return AdvSimd.Min(x.AsUInt16(), y.AsUInt16()).As(); - if (typeof(T) == typeof(int)) return AdvSimd.Min(x.AsInt32(), y.AsInt32()).As(); - if (typeof(T) == typeof(uint)) return AdvSimd.Min(x.AsUInt32(), y.AsUInt32()).As(); - if (typeof(T) == typeof(float)) return AdvSimd.Min(x.AsSingle(), y.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Min(x.AsDouble(), y.AsDouble()).As(); - } - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector128.ConditionalSelect(Vector128.Equals(x, y), - Vector128.ConditionalSelect(IsNegative(y), y, x), - Vector128.Min(x, y)); - } - - return Vector128.Min(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return Vector256.ConditionalSelect(Vector256.Equals(x, y), - Vector256.ConditionalSelect(IsNegative(y), y, x), - Vector256.Min(x, y)); - } - - return Vector256.Min(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return Vector512.ConditionalSelect(Vector512.Equals(x, y), - Vector512.ConditionalSelect(IsNegative(y), y, x), - Vector512.Min(x, y)); - } - - return Vector512.Min(x, y); - } - - public static T Invoke(Vector128 x) => HorizontalAggregate>(x); - public static T Invoke(Vector256 x) => HorizontalAggregate>(x); - public static T Invoke(Vector512 x) => HorizontalAggregate>(x); - } - - /// T.Min(x, y) - internal readonly struct MinPropagateNaNOperator : IBinaryOperator - where T : INumber - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.Min(x, y); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(byte)) return AdvSimd.Min(x.AsByte(), y.AsByte()).As(); - if (typeof(T) == typeof(sbyte)) return AdvSimd.Min(x.AsSByte(), y.AsSByte()).As(); - if (typeof(T) == typeof(short)) return AdvSimd.Min(x.AsInt16(), y.AsInt16()).As(); - if (typeof(T) == typeof(ushort)) return AdvSimd.Min(x.AsUInt16(), y.AsUInt16()).As(); - if (typeof(T) == typeof(int)) return AdvSimd.Min(x.AsInt32(), y.AsInt32()).As(); - if (typeof(T) == typeof(uint)) return AdvSimd.Min(x.AsUInt32(), y.AsUInt32()).As(); - if (typeof(T) == typeof(float)) return AdvSimd.Min(x.AsSingle(), y.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Min(x.AsDouble(), y.AsDouble()).As(); - } - - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector128.ConditionalSelect(Vector128.Equals(x, x), - Vector128.ConditionalSelect(Vector128.Equals(y, y), - Vector128.ConditionalSelect(Vector128.Equals(x, y), - Vector128.ConditionalSelect(IsNegative(x), x, y), - Vector128.Min(x, y)), - y), - x); - } - - return Vector128.Min(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector256.ConditionalSelect(Vector256.Equals(x, x), - Vector256.ConditionalSelect(Vector256.Equals(y, y), - Vector256.ConditionalSelect(Vector256.Equals(x, y), - Vector256.ConditionalSelect(IsNegative(x), x, y), - Vector256.Min(x, y)), - y), - x); - } - - return Vector256.Min(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - return - Vector512.ConditionalSelect(Vector512.Equals(x, x), - Vector512.ConditionalSelect(Vector512.Equals(y, y), - Vector512.ConditionalSelect(Vector512.Equals(x, y), - Vector512.ConditionalSelect(IsNegative(x), x, y), - Vector512.Min(x, y)), - y), - x); - } - - return Vector512.Min(x, y); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MinMagnitude.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MinMagnitude.cs deleted file mode 100644 index 47b492eaffb8a4..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MinMagnitude.cs +++ /dev/null @@ -1,232 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Searches for the number with the smallest magnitude in the specified tensor. - /// The tensor, represented as a span. - /// The element in with the smallest magnitude (absolute value). - /// Length of must be greater than zero. - /// - /// - /// The determination of the minimum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If any value equal to - /// is present, the first is returned. If two values have the same magnitude and one is positive and the other is negative, - /// the negative value is considered to have the smaller magnitude. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T MinMagnitude(ReadOnlySpan x) - where T : INumberBase => - MinMaxCore>(x); - - /// Computes the element-wise number with the smallest magnitude in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// This method effectively computes [i] = .MinMagnitude([i], [i]). - /// - /// - /// The determination of the maximum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If either value is equal to , - /// that value is stored as the result. If the two values have the same magnitude and one is positive and the other is negative, - /// the negative value is considered to have the smaller magnitude. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void MinMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : INumberBase => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise number with the smallest magnitude in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// This method effectively computes [i] = .MinMagnitude([i], ). - /// - /// - /// The determination of the maximum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If either value is equal to , - /// that value is stored as the result. If the two values have the same magnitude and one is positive and the other is negative, - /// the negative value is considered to have the smaller magnitude. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void MinMagnitude(ReadOnlySpan x, T y, Span destination) - where T : INumberBase => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// Operator to get x or y based on which has the smaller MathF.Abs (but NaNs may not be propagated) - internal readonly struct MinMagnitudeOperator : IAggregationOperator - where T : INumberBase - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.MinMagnitude(x, y); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); - - Vector128 result = - Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), - Vector128.ConditionalSelect(IsNegative(y), y, x), - Vector128.ConditionalSelect(Vector128.LessThan(yMag, xMag), y, x)); - - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector128 negativeMagnitudeX = Vector128.LessThan(xMag, Vector128.Zero); - Vector128 negativeMagnitudeY = Vector128.LessThan(yMag, Vector128.Zero); - result = Vector128.ConditionalSelect(negativeMagnitudeX, - y, - Vector128.ConditionalSelect(negativeMagnitudeY, - x, - result)); - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); - - Vector256 result = - Vector256.ConditionalSelect(Vector256.Equals(yMag, xMag), - Vector256.ConditionalSelect(IsNegative(y), y, x), - Vector256.ConditionalSelect(Vector256.LessThan(yMag, xMag), y, x)); - - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector256 negativeMagnitudeX = Vector256.LessThan(xMag, Vector256.Zero); - Vector256 negativeMagnitudeY = Vector256.LessThan(yMag, Vector256.Zero); - result = Vector256.ConditionalSelect(negativeMagnitudeX, - y, - Vector256.ConditionalSelect(negativeMagnitudeY, - x, - result)); - } - - return result; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); - - Vector512 result = - Vector512.ConditionalSelect(Vector512.Equals(yMag, xMag), - Vector512.ConditionalSelect(IsNegative(y), y, x), - Vector512.ConditionalSelect(Vector512.LessThan(yMag, xMag), y, x)); - - if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) - { - Vector512 negativeMagnitudeX = Vector512.LessThan(xMag, Vector512.Zero); - Vector512 negativeMagnitudeY = Vector512.LessThan(yMag, Vector512.Zero); - result = Vector512.ConditionalSelect(negativeMagnitudeX, - y, - Vector512.ConditionalSelect(negativeMagnitudeY, - x, - result)); - } - - return result; - } - - public static T Invoke(Vector128 x) => HorizontalAggregate>(x); - public static T Invoke(Vector256 x) => HorizontalAggregate>(x); - public static T Invoke(Vector512 x) => HorizontalAggregate>(x); - } - - /// Operator to get x or y based on which has the smaller MathF.Abs - internal readonly struct MinMagnitudePropagateNaNOperator : IBinaryOperator - where T : INumberBase - { - public static bool Vectorizable => true; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y) => T.MinMagnitude(x, y); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - // Handle NaNs - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); - return - Vector128.ConditionalSelect(Vector128.Equals(x, x), - Vector128.ConditionalSelect(Vector128.Equals(y, y), - Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), - Vector128.ConditionalSelect(IsNegative(x), x, y), - Vector128.ConditionalSelect(Vector128.LessThan(xMag, yMag), x, y)), - y), - x); - } - - return MinMagnitudeOperator.Invoke(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - // Handle NaNs - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); - return - Vector256.ConditionalSelect(Vector256.Equals(x, x), - Vector256.ConditionalSelect(Vector256.Equals(y, y), - Vector256.ConditionalSelect(Vector256.Equals(yMag, xMag), - Vector256.ConditionalSelect(IsNegative(x), x, y), - Vector256.ConditionalSelect(Vector256.LessThan(xMag, yMag), x, y)), - y), - x); - } - - return MinMagnitudeOperator.Invoke(x, y); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - // Handle NaNs - if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) - { - Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); - return - Vector512.ConditionalSelect(Vector512.Equals(x, x), - Vector512.ConditionalSelect(Vector512.Equals(y, y), - Vector512.ConditionalSelect(Vector512.Equals(yMag, xMag), - Vector512.ConditionalSelect(IsNegative(x), x, y), - Vector512.ConditionalSelect(Vector512.LessThan(xMag, yMag), x, y)), - y), - x); - } - - return MinMagnitudeOperator.Invoke(x, y); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Multiply.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Multiply.cs deleted file mode 100644 index 80d0f488fd978f..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Multiply.cs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise product of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] * [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Multiply(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IMultiplyOperators, IMultiplicativeIdentity => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise product of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] * . - /// It corresponds to the scal method defined by BLAS1. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Multiply(ReadOnlySpan x, T y, Span destination) - where T : IMultiplyOperators, IMultiplicativeIdentity => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// x * y - internal readonly struct MultiplyOperator : IAggregationOperator where T : IMultiplyOperators, IMultiplicativeIdentity - { - public static bool Vectorizable => true; - - public static T Invoke(T x, T y) => x * y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x * y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x * y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; - - public static T Invoke(Vector128 x) => HorizontalAggregate>(x); - public static T Invoke(Vector256 x) => HorizontalAggregate>(x); - public static T Invoke(Vector512 x) => HorizontalAggregate>(x); - - public static T IdentityValue => T.MultiplicativeIdentity; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MultiplyAdd.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MultiplyAdd.cs deleted file mode 100644 index 07042b32c37936..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MultiplyAdd.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of and length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * [i]) + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void MultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan addend, Span destination) - where T : IAdditionOperators, IMultiplyOperators => - InvokeSpanSpanSpanIntoSpan>(x, y, addend, destination); - - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * [i]) + . - /// It corresponds to the axpy method defined by BLAS1. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void MultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, T addend, Span destination) - where T : IAdditionOperators, IMultiplyOperators => - InvokeSpanSpanScalarIntoSpan>(x, y, addend, destination); - - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * ) + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void MultiplyAdd(ReadOnlySpan x, T y, ReadOnlySpan addend, Span destination) - where T : IAdditionOperators, IMultiplyOperators => - InvokeSpanScalarSpanIntoSpan>(x, y, addend, destination); - - /// (x * y) + z - internal readonly struct MultiplyAddOperator : ITernaryOperator where T : IAdditionOperators, IMultiplyOperators - { - public static T Invoke(T x, T y, T z) => (x * y) + z; - public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) => (x * y) + z; - public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) => (x * y) + z; - public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) => (x * y) + z; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MultiplyAddEstimate.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MultiplyAddEstimate.cs deleted file mode 100644 index 3bd615e54701a6..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.MultiplyAddEstimate.cs +++ /dev/null @@ -1,169 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of and length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * [i]) + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - /// Behaves the same as either or - /// depending on the current machine's capabilities. - /// - /// - public static void MultiplyAddEstimate(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan addend, Span destination) - where T : INumberBase => - InvokeSpanSpanSpanIntoSpan>(x, y, addend, destination); - - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The third tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * [i]) + . - /// It corresponds to the axpy method defined by BLAS1. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - /// Behaves the same as either or - /// depending on the current machine's capabilities. - /// - /// - public static void MultiplyAddEstimate(ReadOnlySpan x, ReadOnlySpan y, T addend, Span destination) - where T : INumberBase => - InvokeSpanSpanScalarIntoSpan>(x, y, addend, destination); - - /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The third tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ([i] * ) + [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - /// Behaves the same as either or - /// depending on the current machine's capabilities. - /// - /// - public static void MultiplyAddEstimate(ReadOnlySpan x, T y, ReadOnlySpan addend, Span destination) - where T : INumberBase => - InvokeSpanScalarSpanIntoSpan>(x, y, addend, destination); - - /// (x * y) + z - private readonly struct MultiplyAddEstimateOperator : ITernaryOperator where T : INumberBase - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Invoke(T x, T y, T z) - { - // TODO https://github.com/dotnet/runtime/issues/98053: Use T.MultiplyAddEstimate when it's available. - - if (Fma.IsSupported || AdvSimd.IsSupported) - { - if (typeof(T) == typeof(Half)) - { - Half result = Half.FusedMultiplyAdd(Unsafe.As(ref x), Unsafe.As(ref y), Unsafe.As(ref z)); - return Unsafe.As(ref result); - } - - if (typeof(T) == typeof(float)) - { - float result = float.FusedMultiplyAdd(Unsafe.As(ref x), Unsafe.As(ref y), Unsafe.As(ref z)); - return Unsafe.As(ref result); - } - - if (typeof(T) == typeof(double)) - { - double result = double.FusedMultiplyAdd(Unsafe.As(ref x), Unsafe.As(ref y), Unsafe.As(ref z)); - return Unsafe.As(ref result); - } - } - - return (x * y) + z; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) - { - if (Fma.IsSupported) - { - if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); - } - - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(float)) return AdvSimd.FusedMultiplyAdd(z.AsSingle(), x.AsSingle(), y.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.FusedMultiplyAdd(z.AsDouble(), x.AsDouble(), y.AsDouble()).As(); - } - - return (x * y) + z; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) - { - if (Fma.IsSupported) - { - if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); - } - - return (x * y) + z; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) - { - if (Avx512F.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx512F.FusedMultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Avx512F.FusedMultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); - } - - return (x * y) + z; - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Negate.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Negate.cs deleted file mode 100644 index 15e34947591895..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Negate.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise negation of each number in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = -[i]. - /// - /// - /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Negate(ReadOnlySpan x, Span destination) - where T : IUnaryNegationOperators => - InvokeSpanIntoSpan>(x, destination); - - /// -x - internal readonly struct NegateOperator : IUnaryOperator where T : IUnaryNegationOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x) => -x; - public static Vector128 Invoke(Vector128 x) => -x; - public static Vector256 Invoke(Vector256 x) => -x; - public static Vector512 Invoke(Vector512 x) => -x; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Norm.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Norm.cs deleted file mode 100644 index 4f12b833800857..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Norm.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the Euclidean norm of the specified tensor of numbers. - /// The first tensor, represented as a span. - /// The norm. - /// - /// - /// This method effectively computes .Sqrt(TensorPrimitives.SumOfSquares(x)). - /// This is often referred to as the Euclidean norm or L2 norm. - /// It corresponds to the nrm2 method defined by BLAS1. - /// - /// - /// If any of the input values is equal to , the result value is also NaN. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Norm(ReadOnlySpan x) - where T : IRootFunctions => - T.Sqrt(SumOfSquares(x)); - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.OnesComplement.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.OnesComplement.cs deleted file mode 100644 index fc9dea6c420be4..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.OnesComplement.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise one's complement of numbers in the specified tensor. - /// The first tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = ~[i]. - /// - /// - public static void OnesComplement(ReadOnlySpan x, Span destination) - where T : IBitwiseOperators => - InvokeSpanIntoSpan>(x, destination); - - /// ~x - private readonly struct OnesComplementOperator : IUnaryOperator where T : IBitwiseOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x) => ~x; - public static Vector128 Invoke(Vector128 x) => ~x; - public static Vector256 Invoke(Vector256 x) => ~x; - public static Vector512 Invoke(Vector512 x) => ~x; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.PopCount.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.PopCount.cs deleted file mode 100644 index 8bc90f3c6968c9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.PopCount.cs +++ /dev/null @@ -1,200 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise population count of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.PopCount([i]). - /// - /// - public static void PopCount(ReadOnlySpan x, Span destination) - where T : IBinaryInteger => - InvokeSpanIntoSpan>(x, destination); - - /// T.PopCount(x) - private readonly unsafe struct PopCountOperator : IUnaryOperator where T : IBinaryInteger - { - // TODO https://github.com/dotnet/runtime/issues/96162: Use AVX512 popcount operations when available - - public static bool Vectorizable => - // The implementation uses a vectorized version of the BitOperations.PopCount software fallback: - // https://github.com/dotnet/runtime/blob/aff061bab1b6d9ccd5731bd16fa8e89ad82ab75a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs#L496-L508 - // This relies on 64-bit shifts for sizeof(T) == 8, and such shifts aren't accelerated on today's hardware. - // Alternative approaches, such as doing two 32-bit operations and combining them were observed to not - // provide any meaningfuls speedup over scalar. So for now, we don't vectorize when sizeof(T) == 8. - sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4; - - public static T Invoke(T x) => T.PopCount(x); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x) - { - if (sizeof(T) == 1) - { - if (AdvSimd.IsSupported) - { - return AdvSimd.PopCount(x.AsByte()).As(); - } - - if (PackedSimd.IsSupported) - { - return PackedSimd.PopCount(x.AsByte()).As(); - } - - Vector128 c1 = Vector128.Create((byte)0x55); - Vector128 c2 = Vector128.Create((byte)0x33); - Vector128 c3 = Vector128.Create((byte)0x0F); - - // We don't have a per element shuffle for byte on some platforms. - // However, we do currently always have a 16-bit shift available and - // due to how the algorithm works, we don't need to worry about - // any bits that shift into the lower 8-bits from the upper 8-bits. - Vector128 tmp = x.AsByte(); - tmp -= (x.AsUInt16() >> 1).AsByte() & c1; - tmp = (tmp & c2) + ((tmp.AsUInt16() >> 2).AsByte() & c2); - return ((tmp + (tmp.AsUInt16() >> 4).AsByte()) & c3).As(); - } - - if (sizeof(T) == 2) - { - Vector128 c1 = Vector128.Create((ushort)0x5555); - Vector128 c2 = Vector128.Create((ushort)0x3333); - Vector128 c3 = Vector128.Create((ushort)0x0F0F); - Vector128 c4 = Vector128.Create((ushort)0x0101); - - Vector128 tmp = x.AsUInt16(); - tmp -= (tmp >> 1) & c1; - tmp = (tmp & c2) + ((tmp >> 2) & c2); - tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 8; - return tmp.As(); - } - - Debug.Assert(sizeof(T) == 4); - { - Vector128 c1 = Vector128.Create(0x55555555u); - Vector128 c2 = Vector128.Create(0x33333333u); - Vector128 c3 = Vector128.Create(0x0F0F0F0Fu); - Vector128 c4 = Vector128.Create(0x01010101u); - - Vector128 tmp = x.AsUInt32(); - tmp -= (tmp >> 1) & c1; - tmp = (tmp & c2) + ((tmp >> 2) & c2); - tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 24; - return tmp.As(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x) - { - if (sizeof(T) == 1) - { - Vector256 c1 = Vector256.Create((byte)0x55); - Vector256 c2 = Vector256.Create((byte)0x33); - Vector256 c3 = Vector256.Create((byte)0x0F); - - // We don't have a per element shuffle for byte on some platforms. - // However, we do currently always have a 16-bit shift available and - // due to how the algorithm works, we don't need to worry about - // any bits that shift into the lower 8-bits from the upper 8-bits. - Vector256 tmp = x.AsByte(); - tmp -= (x.AsUInt16() >> 1).AsByte() & c1; - tmp = (tmp & c2) + ((tmp.AsUInt16() >> 2).AsByte() & c2); - return ((tmp + (tmp.AsUInt16() >> 4).AsByte()) & c3).As(); - } - - if (sizeof(T) == 2) - { - Vector256 c1 = Vector256.Create((ushort)0x5555); - Vector256 c2 = Vector256.Create((ushort)0x3333); - Vector256 c3 = Vector256.Create((ushort)0x0F0F); - Vector256 c4 = Vector256.Create((ushort)0x0101); - - Vector256 tmp = x.AsUInt16(); - tmp -= (tmp >> 1) & c1; - tmp = (tmp & c2) + ((tmp >> 2) & c2); - tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 8; - return tmp.As(); - } - - Debug.Assert(sizeof(T) == 4); - { - Vector256 c1 = Vector256.Create(0x55555555u); - Vector256 c2 = Vector256.Create(0x33333333u); - Vector256 c3 = Vector256.Create(0x0F0F0F0Fu); - Vector256 c4 = Vector256.Create(0x01010101u); - - Vector256 tmp = x.AsUInt32(); - tmp -= (tmp >> 1) & c1; - tmp = (tmp & c2) + ((tmp >> 2) & c2); - tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 24; - return tmp.As(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x) - { - if (sizeof(T) == 1) - { - Vector512 c1 = Vector512.Create((byte)0x55); - Vector512 c2 = Vector512.Create((byte)0x33); - Vector512 c3 = Vector512.Create((byte)0x0F); - - // We don't have a per element shuffle for byte on some platforms. - // However, we do currently always have a 16-bit shift available and - // due to how the algorithm works, we don't need to worry about - // any bits that shift into the lower 8-bits from the upper 8-bits. - Vector512 tmp = x.AsByte(); - tmp -= (x.AsUInt16() >> 1).AsByte() & c1; - tmp = (tmp & c2) + ((tmp.AsUInt16() >> 2).AsByte() & c2); - return ((tmp + (tmp.AsUInt16() >> 4).AsByte()) & c3).As(); - } - - if (sizeof(T) == 2) - { - Vector512 c1 = Vector512.Create((ushort)0x5555); - Vector512 c2 = Vector512.Create((ushort)0x3333); - Vector512 c3 = Vector512.Create((ushort)0x0F0F); - Vector512 c4 = Vector512.Create((ushort)0x0101); - - Vector512 tmp = x.AsUInt16(); - tmp -= (tmp >> 1) & c1; - tmp = (tmp & c2) + ((tmp >> 2) & c2); - tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 8; - return tmp.As(); - } - - Debug.Assert(sizeof(T) == 4); - { - Vector512 c1 = Vector512.Create(0x55555555u); - Vector512 c2 = Vector512.Create(0x33333333u); - Vector512 c3 = Vector512.Create(0x0F0F0F0Fu); - Vector512 c4 = Vector512.Create(0x01010101u); - - Vector512 tmp = x.AsUInt32(); - tmp -= (tmp >> 1) & c1; - tmp = (tmp & c2) + ((tmp >> 2) & c2); - tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 24; - return tmp.As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Pow.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Pow.cs deleted file mode 100644 index 72d35ed5be779f..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Pow.cs +++ /dev/null @@ -1,106 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise power of a number in a specified tensor raised to a number in another specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Pow([i], [i]). - /// - /// - public static void Pow(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IPowerFunctions => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise power of a number in a specified tensor raised to a number in another specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Pow([i], ). - /// - /// - public static void Pow(ReadOnlySpan x, T y, Span destination) - where T : IPowerFunctions => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// Computes the element-wise power of a number in a specified tensor raised to a number in another specified tensors. - /// The first tensor, represented as a scalar. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Pow(, [i]). - /// - /// - public static void Pow(T x, ReadOnlySpan y, Span destination) - where T : IPowerFunctions => - InvokeScalarSpanIntoSpan>(x, y, destination); - - /// T.Pow(x, y) - private readonly struct PowOperator : IBinaryOperator - where T : IPowerFunctions - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x, T y) => T.Pow(x, y); - - public static Vector128 Invoke(Vector128 x, Vector128 y) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(y.AsSingle() * LogOperator.Invoke(x.AsSingle())).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(y.AsDouble() * LogOperator.Invoke(x.AsDouble())).As(); - } - } - - public static Vector256 Invoke(Vector256 x, Vector256 y) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(y.AsSingle() * LogOperator.Invoke(x.AsSingle())).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(y.AsDouble() * LogOperator.Invoke(x.AsDouble())).As(); - } - } - - public static Vector512 Invoke(Vector512 x, Vector512 y) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(y.AsSingle() * LogOperator.Invoke(x.AsSingle())).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(y.AsDouble() * LogOperator.Invoke(x.AsDouble())).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Product.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Product.cs deleted file mode 100644 index a43d2c45c3876d..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Product.cs +++ /dev/null @@ -1,96 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the product of all elements in the specified non-empty tensor of numbers. - /// The tensor, represented as a span. - /// The result of multiplying all elements in . - /// Length of must be greater than zero. - /// - /// - /// If any of the input values is equal to , the result value is also NaN. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Product(ReadOnlySpan x) - where T : IMultiplyOperators, IMultiplicativeIdentity - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - return Aggregate, MultiplyOperator>(x); - } - - /// Computes the product of the element-wise differences of the numbers in the specified non-empty tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The result of multiplying the element-wise subtraction of the elements in the second tensor from the first tensor. - /// Length of both input spans must be greater than zero. - /// and must have the same length. - /// - /// - /// This method effectively computes: - /// - /// Span<T> differences = ...; - /// TensorPrimitives.Subtract(x, y, differences); - /// T result = TensorPrimitives.Product(differences); - /// - /// but without requiring additional temporary storage for the intermediate differences. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T ProductOfDifferences(ReadOnlySpan x, ReadOnlySpan y) - where T : ISubtractionOperators, IMultiplyOperators, IMultiplicativeIdentity - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - return Aggregate, MultiplyOperator>(x, y); - } - - /// Computes the product of the element-wise sums of the numbers in the specified non-empty tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The result of multiplying the element-wise additions of the elements in each tensor. - /// Length of both input spans must be greater than zero. - /// and must have the same length. - /// - /// - /// This method effectively computes: - /// - /// Span<T> sums = ...; - /// TensorPrimitives.Add(x, y, sums); - /// T result = TensorPrimitives.Product(sums); - /// - /// but without requiring additional temporary storage for the intermediate sums. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T ProductOfSums(ReadOnlySpan x, ReadOnlySpan y) - where T : IAdditionOperators, IAdditiveIdentity, IMultiplyOperators, IMultiplicativeIdentity - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - return Aggregate, MultiplyOperator>(x, y); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.RadiansToDegrees.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.RadiansToDegrees.cs deleted file mode 100644 index 53298f5cf3c0e5..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.RadiansToDegrees.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise conversion of each number of radians in the specified tensor to degrees. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .RadiansToDegrees([i]). - /// - /// - public static void RadiansToDegrees(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.RadiansToDegrees(x) - private readonly struct RadiansToDegreesOperator : IUnaryOperator where T : ITrigonometricFunctions - { - public static bool Vectorizable => true; - public static T Invoke(T x) => T.RadiansToDegrees(x); - public static Vector128 Invoke(Vector128 x) => (x * T.CreateChecked(180)) / T.Pi; - public static Vector256 Invoke(Vector256 x) => (x * T.CreateChecked(180)) / T.Pi; - public static Vector512 Invoke(Vector512 x) => (x * T.CreateChecked(180)) / T.Pi; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Reciprocal.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Reciprocal.cs deleted file mode 100644 index 50ef635a21addc..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Reciprocal.cs +++ /dev/null @@ -1,186 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise reciprocal of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and an element in is equal to zero. - /// - /// - /// This method effectively computes [i] = 1 / [i]. - /// - /// - public static void Reciprocal(ReadOnlySpan x, Span destination) - where T : IFloatingPoint => - InvokeSpanIntoSpan>(x, destination); - - /// Computes the element-wise reciprocal of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and an element in is equal to zero. - /// - /// - /// This method effectively computes [i] = 1 / [i]. - /// - /// - public static void ReciprocalEstimate(ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanIntoSpan>(x, destination); - - /// Computes the element-wise reciprocal of the square root of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and an element in is equal to zero. - /// - /// - /// This method effectively computes [i] = 1 / [i]. - /// - /// - public static void ReciprocalSqrt(ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanIntoSpan>(x, destination); - - /// Computes the element-wise reciprocal of the square root of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is an integer type and an element in is equal to zero. - /// - /// - /// This method effectively computes [i] = 1 / [i]. - /// - /// - public static void ReciprocalSqrtEstimate(ReadOnlySpan x, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanIntoSpan>(x, destination); - - private readonly struct ReciprocalOperator : IUnaryOperator where T : IFloatingPoint - { - public static bool Vectorizable => true; - public static T Invoke(T x) => T.One / x; - public static Vector128 Invoke(Vector128 x) => Vector128.One / x; - public static Vector256 Invoke(Vector256 x) => Vector256.One / x; - public static Vector512 Invoke(Vector512 x) => Vector512.One / x; - } - - private readonly struct ReciprocalSqrtOperator : IUnaryOperator where T : IFloatingPointIeee754 - { - public static bool Vectorizable => true; - public static T Invoke(T x) => T.One / T.Sqrt(x); - public static Vector128 Invoke(Vector128 x) => Vector128.One / Vector128.Sqrt(x); - public static Vector256 Invoke(Vector256 x) => Vector256.One / Vector256.Sqrt(x); - public static Vector512 Invoke(Vector512 x) => Vector512.One / Vector512.Sqrt(x); - } - - private readonly struct ReciprocalEstimateOperator : IUnaryOperator where T : IFloatingPointIeee754 - { - public static bool Vectorizable => true; - - public static T Invoke(T x) => T.ReciprocalEstimate(x); - - public static Vector128 Invoke(Vector128 x) - { - if (Sse.IsSupported) - { - if (typeof(T) == typeof(float)) return Sse.Reciprocal(x.AsSingle()).As(); - } - - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(float)) return AdvSimd.ReciprocalEstimate(x.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.ReciprocalEstimate(x.AsDouble()).As(); - } - - return Vector128.One / x; - } - - public static Vector256 Invoke(Vector256 x) - { - if (Avx.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx.Reciprocal(x.AsSingle()).As(); - } - - return Vector256.One / x; - } - - public static Vector512 Invoke(Vector512 x) - { - if (Avx512F.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx512F.Reciprocal14(x.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Avx512F.Reciprocal14(x.AsDouble()).As(); - } - - return Vector512.One / x; - } - } - - private readonly struct ReciprocalSqrtEstimateOperator : IUnaryOperator where T : IFloatingPointIeee754 - { - public static bool Vectorizable => true; - - public static T Invoke(T x) => T.ReciprocalSqrtEstimate(x); - - public static Vector128 Invoke(Vector128 x) - { - if (Sse.IsSupported) - { - if (typeof(T) == typeof(float)) return Sse.ReciprocalSqrt(x.AsSingle()).As(); - } - - if (AdvSimd.IsSupported) - { - if (typeof(T) == typeof(float)) return AdvSimd.ReciprocalSquareRootEstimate(x.AsSingle()).As(); - } - - if (AdvSimd.Arm64.IsSupported) - { - if (typeof(T) == typeof(double)) return AdvSimd.Arm64.ReciprocalSquareRootEstimate(x.AsDouble()).As(); - } - - return Vector128.One / Vector128.Sqrt(x); - } - - public static Vector256 Invoke(Vector256 x) - { - if (Avx.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx.ReciprocalSqrt(x.AsSingle()).As(); - } - - return Vector256.One / Vector256.Sqrt(x); - } - - public static Vector512 Invoke(Vector512 x) - { - if (Avx512F.IsSupported) - { - if (typeof(T) == typeof(float)) return Avx512F.ReciprocalSqrt14(x.AsSingle()).As(); - if (typeof(T) == typeof(double)) return Avx512F.ReciprocalSqrt14(x.AsDouble()).As(); - } - - return Vector512.One / Vector512.Sqrt(x); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.RootN.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.RootN.cs deleted file mode 100644 index e7c394892950a9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.RootN.cs +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise n-th root of the values in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// The degree of the root to be computed, represented as a scalar. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.RootN([i], ). - /// - /// - public static void RootN(ReadOnlySpan x, int n, Span destination) - where T : IRootFunctions => - InvokeSpanIntoSpan(x, new RootNOperator(n), destination); - - /// T.RootN(x, n) - private readonly struct RootNOperator(int n) : IStatefulUnaryOperator where T : IRootFunctions - { - private readonly int _n = n; - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public T Invoke(T x) => T.RootN(x, _n); - - public Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector128.Create((float)_n)).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector128.Create((double)_n)).As(); - } - } - - public Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector256.Create((float)_n)).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector256.Create((double)_n)).As(); - } - } - - public Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector512.Create((float)_n)).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector512.Create((double)_n)).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Rotate.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Rotate.cs deleted file mode 100644 index e22ee978c3a62a..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Rotate.cs +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise rotation left of numbers in the specified tensor by the specified rotation amount. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// The number of bits to rotate, represented as a scalar. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.RotateLeft([i], ). - /// - /// - public static void RotateLeft(ReadOnlySpan x, int rotateAmount, Span destination) - where T : IBinaryInteger => - InvokeSpanIntoSpan(x, new RotateLeftOperator(rotateAmount), destination); - - /// Computes the element-wise rotation right of numbers in the specified tensor by the specified rotation amount. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// The number of bits to rotate, represented as a scalar. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.RotateRight([i], ). - /// - /// - public static void RotateRight(ReadOnlySpan x, int rotateAmount, Span destination) - where T : IBinaryInteger => - InvokeSpanIntoSpan(x, new RotateRightOperator(rotateAmount), destination); - - /// T.RotateLeft(amount) - private readonly unsafe struct RotateLeftOperator(int amount) : IStatefulUnaryOperator where T : IBinaryInteger - { - private readonly int _amount = amount; - - public static bool Vectorizable => true; - - public T Invoke(T x) => T.RotateLeft(x, _amount); - public Vector128 Invoke(Vector128 x) => (x << _amount) | (x >>> ((sizeof(T) * 8) - _amount)); - public Vector256 Invoke(Vector256 x) => (x << _amount) | (x >>> ((sizeof(T) * 8) - _amount)); - public Vector512 Invoke(Vector512 x) => (x << _amount) | (x >>> ((sizeof(T) * 8) - _amount)); - } - - /// T.RotateRight(amount) - private readonly unsafe struct RotateRightOperator(int amount) : IStatefulUnaryOperator where T : IBinaryInteger - { - private readonly int _amount = amount; - - public static bool Vectorizable => true; - - public T Invoke(T x) => T.RotateRight(x, _amount); - public Vector128 Invoke(Vector128 x) => (x >>> _amount) | (x << ((sizeof(T) * 8) - _amount)); - public Vector256 Invoke(Vector256 x) => (x >>> _amount) | (x << ((sizeof(T) * 8) - _amount)); - public Vector512 Invoke(Vector512 x) => (x >>> _amount) | (x << ((sizeof(T) * 8) - _amount)); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Round.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Round.cs deleted file mode 100644 index 575ce0f523278b..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Round.cs +++ /dev/null @@ -1,323 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise rounding of the numbers in the specified tensor - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Round([i]). - /// - /// - public static void Round(ReadOnlySpan x, Span destination) - where T : IFloatingPoint => - InvokeSpanIntoSpan>(x, destination); - - /// Computes the element-wise rounding of the numbers in the specified tensor - /// The tensor, represented as a span. - /// The mode under which should be rounded. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Round([i], ). - /// - /// - public static void Round(ReadOnlySpan x, MidpointRounding mode, Span destination) - where T : IFloatingPoint - { - switch (mode) - { - case MidpointRounding.ToEven: - Round(x, destination); - return; - - case MidpointRounding.AwayFromZero: - InvokeSpanIntoSpan>(x, destination); - break; - - case MidpointRounding.ToZero: - Truncate(x, destination); - return; - - case MidpointRounding.ToNegativeInfinity: - Floor(x, destination); - return; - - case MidpointRounding.ToPositiveInfinity: - Ceiling(x, destination); - return; - - default: - throw new ArgumentException(SR.Format(SR.Argument_InvalidEnumValue, mode, typeof(MidpointRounding)), nameof(mode)); - } - } - - /// Computes the element-wise rounding of the numbers in the specified tensor - /// The tensor, represented as a span. - /// The number of fractional digits to which the numbers in should be rounded. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Round([i], ). - /// - /// - public static void Round(ReadOnlySpan x, int digits, Span destination) where T : IFloatingPoint => - Round(x, digits, MidpointRounding.ToEven, destination); - - /// Computes the element-wise rounding of the numbers in the specified tensor - /// The tensor, represented as a span. - /// The number of fractional digits to which the numbers in should be rounded. - /// The mode under which should be rounded. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// is invalid. - /// is invalid. - /// - /// - /// This method effectively computes [i] = T.Round([i], , ). - /// - /// - public static void Round(ReadOnlySpan x, int digits, MidpointRounding mode, Span destination) - where T : IFloatingPoint - { - if (digits == 0) - { - Round(x, mode, destination); - } - - ReadOnlySpan roundPower10; - if (typeof(T) == typeof(float)) - { - ReadOnlySpan roundPower10Single = [1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f]; - roundPower10 = MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(roundPower10Single)), roundPower10Single.Length); - } - else if (typeof(T) == typeof(double)) - { - Debug.Assert(typeof(T) == typeof(double)); - ReadOnlySpan roundPower10Double = [1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15]; - roundPower10 = MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(roundPower10Double)), roundPower10Double.Length); - } - else - { - if ((uint)mode > (uint)MidpointRounding.ToPositiveInfinity) - { - throw new ArgumentException(SR.Format(SR.Argument_InvalidEnumValue, mode, typeof(MidpointRounding)), nameof(mode)); - } - - InvokeSpanIntoSpan(x, new RoundFallbackOperator(digits, mode), destination); - return; - } - - if ((uint)digits >= (uint)roundPower10.Length) - { - throw new ArgumentOutOfRangeException(nameof(digits)); - } - - T power10 = roundPower10[digits]; - switch (mode) - { - case MidpointRounding.ToEven: - InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); - return; - - case MidpointRounding.AwayFromZero: - InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); - break; - - case MidpointRounding.ToZero: - InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); - return; - - case MidpointRounding.ToNegativeInfinity: - InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); - return; - - case MidpointRounding.ToPositiveInfinity: - InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); - return; - - default: - throw new ArgumentException(SR.Format(SR.Argument_InvalidEnumValue, mode, typeof(MidpointRounding)), nameof(mode)); - } - } - - /// T.Round(x) - private readonly struct RoundToEvenOperator : IUnaryOperator where T : IFloatingPoint - { - // This code is based on `nearbyint` from amd/aocl-libm-ose - // Copyright (C) 2008-2022 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Round(x); - - private const float SingleBoundary = 8388608.0f; // 2^23 - private const double DoubleBoundary = 4503599627370496.0; // 2^52 - - public static Vector128 Invoke(Vector128 x) - { - Vector128 boundary = Vector128.Create(typeof(T) == typeof(float) ? T.CreateTruncating(SingleBoundary) : T.CreateTruncating(DoubleBoundary)); - Vector128 temp = CopySignOperator.Invoke(boundary, x); - return Vector128.ConditionalSelect(Vector128.GreaterThan(Vector128.Abs(x), boundary), x, CopySignOperator.Invoke((x + temp) - temp, x)); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 boundary = Vector256.Create(typeof(T) == typeof(float) ? T.CreateTruncating(SingleBoundary) : T.CreateTruncating(DoubleBoundary)); - Vector256 temp = CopySignOperator.Invoke(boundary, x); - return Vector256.ConditionalSelect(Vector256.GreaterThan(Vector256.Abs(x), boundary), x, CopySignOperator.Invoke((x + temp) - temp, x)); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 boundary = Vector512.Create(typeof(T) == typeof(float) ? T.CreateTruncating(SingleBoundary) : T.CreateTruncating(DoubleBoundary)); - Vector512 temp = CopySignOperator.Invoke(boundary, x); - return Vector512.ConditionalSelect(Vector512.GreaterThan(Vector512.Abs(x), boundary), x, CopySignOperator.Invoke((x + temp) - temp, x)); - } - } - - /// T.Round(x, MidpointRounding.AwayFromZero) - private readonly struct RoundAwayFromZeroOperator : IUnaryOperator where T : IFloatingPoint - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Round(x, MidpointRounding.AwayFromZero); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - if (AdvSimd.IsSupported) - { - return AdvSimd.RoundAwayFromZero(x.AsSingle()).As(); - } - - return TruncateOperator.Invoke(x.AsSingle() + CopySignOperator.Invoke(Vector128.Create(0.49999997f), x.AsSingle())).As(); - } - else - { - if (AdvSimd.Arm64.IsSupported) - { - return AdvSimd.Arm64.RoundAwayFromZero(x.AsDouble()).As(); - } - - Debug.Assert(typeof(T) == typeof(double)); - return TruncateOperator.Invoke(x.AsDouble() + CopySignOperator.Invoke(Vector128.Create(0.49999999999999994), x.AsDouble())).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return TruncateOperator.Invoke(x.AsSingle() + CopySignOperator.Invoke(Vector256.Create(0.49999997f), x.AsSingle())).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return TruncateOperator.Invoke(x.AsDouble() + CopySignOperator.Invoke(Vector256.Create(0.49999999999999994), x.AsDouble())).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return TruncateOperator.Invoke(x.AsSingle() + CopySignOperator.Invoke(Vector512.Create(0.49999997f), x.AsSingle())).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return TruncateOperator.Invoke(x.AsDouble() + CopySignOperator.Invoke(Vector512.Create(0.49999999999999994), x.AsDouble())).As(); - } - } - } - - /// (T.Round(x * power10, digits, mode)) / power10 - private readonly struct MultiplyRoundDivideOperator : IStatefulUnaryOperator - where T : IFloatingPoint - where TDelegatedRound : IUnaryOperator - { - private readonly T _factor; - - public MultiplyRoundDivideOperator(T factor) - { - Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); - _factor = factor; - } - - public static bool Vectorizable => true; - - private const float Single_RoundLimit = 1e8f; - private const double Double_RoundLimit = 1e16d; - - public T Invoke(T x) - { - T limit = typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit); - return T.Abs(x) < limit ? - TDelegatedRound.Invoke(x * _factor) / _factor : - x; - } - - public Vector128 Invoke(Vector128 x) - { - Vector128 limit = Vector128.Create(typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit)); - return Vector128.ConditionalSelect(Vector128.LessThan(Vector128.Abs(x), limit), - TDelegatedRound.Invoke(x * _factor) / _factor, - x); - } - - public Vector256 Invoke(Vector256 x) - { - Vector256 limit = Vector256.Create(typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit)); - return Vector256.ConditionalSelect(Vector256.LessThan(Vector256.Abs(x), limit), - TDelegatedRound.Invoke(x * _factor) / _factor, - x); - } - - public Vector512 Invoke(Vector512 x) - { - Vector512 limit = Vector512.Create(typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit)); - return Vector512.ConditionalSelect(Vector512.LessThan(Vector512.Abs(x), limit), - TDelegatedRound.Invoke(x * _factor) / _factor, - x); - } - } - - /// T.Round(x, digits, mode) - private readonly struct RoundFallbackOperator(int digits, MidpointRounding mode) : IStatefulUnaryOperator - where T : IFloatingPoint - { - private readonly int _digits = digits; - private readonly MidpointRounding _mode = mode; - - public static bool Vectorizable => false; - - public T Invoke(T x) => T.Round(x, _digits, _mode); - - public Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ScaleB.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ScaleB.cs deleted file mode 100644 index 34c84782d40ab0..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ScaleB.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise product of numbers in the specified tensor and their base-radix raised to the specified power. - /// The tensor, represented as a span. - /// The value to which base-radix is raised before multipliying x, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.ILogB([i]). - /// - /// - public static void ScaleB(ReadOnlySpan x, int n, Span destination) - where T : IFloatingPointIeee754 => - InvokeSpanIntoSpan(x, new ScaleBOperator(n), destination); - - /// T.ScaleB(x, n) - private readonly struct ScaleBOperator(int n) : IStatefulUnaryOperator where T : IFloatingPointIeee754 - { - private readonly int _n = n; - private readonly T _pow2n = typeof(T) == typeof(float) || typeof(T) == typeof(double) ? T.Pow(T.CreateTruncating(2), T.CreateTruncating(n)) : default!; - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public T Invoke(T x) => T.ScaleB(x, _n); - public Vector128 Invoke(Vector128 x) => x * Vector128.Create(_pow2n); - public Vector256 Invoke(Vector256 x) => x * Vector256.Create(_pow2n); - public Vector512 Invoke(Vector512 x) => x * Vector512.Create(_pow2n); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ShiftLeft.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ShiftLeft.cs deleted file mode 100644 index 22d5ea4c268679..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.ShiftLeft.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise shifting left of numbers in the specified tensor by the specified shift amount. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// The number of bits to shift, represented as a scalar. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] << . - /// - /// - public static void ShiftLeft(ReadOnlySpan x, int shiftAmount, Span destination) - where T : IShiftOperators => - InvokeSpanIntoSpan(x, new ShiftLeftOperator(shiftAmount), destination); - - /// Computes the element-wise arithmetic (signed) shifting right of numbers in the specified tensor by the specified shift amount. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// The number of bits to shift, represented as a scalar. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] >> . - /// - /// - public static void ShiftRightArithmetic(ReadOnlySpan x, int shiftAmount, Span destination) - where T : IShiftOperators => - InvokeSpanIntoSpan(x, new ShiftRightArithmeticOperator(shiftAmount), destination); - - /// Computes the element-wise logical (unsigned) shifting right of numbers in the specified tensor by the specified shift amount. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// The number of bits to shift, represented as a scalar. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] >>> . - /// - /// - public static void ShiftRightLogical(ReadOnlySpan x, int shiftAmount, Span destination) - where T : IShiftOperators => - InvokeSpanIntoSpan(x, new ShiftRightLogicalOperator(shiftAmount), destination); - - /// T << amount - private readonly struct ShiftLeftOperator(int amount) : IStatefulUnaryOperator where T : IShiftOperators - { - private readonly int _amount = amount; - - public static bool Vectorizable => true; - - public T Invoke(T x) => x << _amount; - public Vector128 Invoke(Vector128 x) => x << _amount; - public Vector256 Invoke(Vector256 x) => x << _amount; - public Vector512 Invoke(Vector512 x) => x << _amount; - } - - /// T >> amount - private readonly struct ShiftRightArithmeticOperator(int amount) : IStatefulUnaryOperator where T : IShiftOperators - { - private readonly int _amount = amount; - - public static bool Vectorizable => true; - - public T Invoke(T x) => x >> _amount; - public Vector128 Invoke(Vector128 x) => x >> _amount; - public Vector256 Invoke(Vector256 x) => x >> _amount; - public Vector512 Invoke(Vector512 x) => x >> _amount; - } - - /// T >>> amount - private readonly struct ShiftRightLogicalOperator(int amount) : IStatefulUnaryOperator where T : IShiftOperators - { - private readonly int _amount = amount; - - public static bool Vectorizable => true; - - public T Invoke(T x) => x >>> _amount; - public Vector128 Invoke(Vector128 x) => x >>> _amount; - public Vector256 Invoke(Vector256 x) => x >>> _amount; - public Vector512 Invoke(Vector512 x) => x >>> _amount; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sigmoid.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sigmoid.cs deleted file mode 100644 index 2fdcf9f11e3ed9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sigmoid.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise sigmoid function on the specified non-empty tensor of numbers. - /// The tensor, represented as a span. - /// The destination tensor. - /// Destination is too short. - /// must not be empty. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = 1f / (1f + .Exp(-[i])). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Sigmoid(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - InvokeSpanIntoSpan>(x, destination); - } - - /// 1 / (1 + T.Exp(-x)) - internal readonly struct SigmoidOperator : IUnaryOperator where T : IExponentialFunctions - { - public static bool Vectorizable => ExpOperator.Vectorizable; - public static T Invoke(T x) => T.One / (T.One + T.Exp(-x)); - public static Vector128 Invoke(Vector128 x) => Vector128.Create(T.One) / (Vector128.Create(T.One) + ExpOperator.Invoke(-x)); - public static Vector256 Invoke(Vector256 x) => Vector256.Create(T.One) / (Vector256.Create(T.One) + ExpOperator.Invoke(-x)); - public static Vector512 Invoke(Vector512 x) => Vector512.Create(T.One) / (Vector512.Create(T.One) + ExpOperator.Invoke(-x)); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sin.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sin.cs deleted file mode 100644 index 6976a35b3d23a3..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sin.cs +++ /dev/null @@ -1,338 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise sine of the value in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Sin([i]). - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Sin(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Sin(x) - internal readonly struct SinOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - // This code is based on `vrs4_sin` and `vrd2_sin` from amd/aocl-libm-ose - // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Implementation notes from amd/aocl-libm-ose: - // ----------------------------------------------------------------- - // Convert given x into the form - // |x| = N * pi + f where N is an integer and f lies in [-pi/2,pi/2] - // N is obtained by : N = round(x/pi) - // f is obtained by : f = abs(x)-N*pi - // sin(x) = sin(N * pi + f) = sin(N * pi)*cos(f) + cos(N*pi)*sin(f) - // sin(x) = sign(x)*sin(f)*(-1)**N - // - // The term sin(f) can be approximated by using a polynomial - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Sin(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return SinOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return SinOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return SinOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return SinOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return SinOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return SinOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - } - - /// float.Sin(x) - private readonly struct SinOperatorSingle : IUnaryOperator - { - internal const uint MaxVectorizedValue = 0x49800000u; - internal const uint SignMask = 0x7FFFFFFFu; - private const float AlmHuge = 1.2582912e7f; - private const float Pi_Tail1 = 8.742278e-8f; - private const float Pi_Tail2 = 3.430249e-15f; - private const float C1 = -0.16666657f; - private const float C2 = 0.0083330255f; - private const float C3 = -1.980742e-4f; - private const float C4 = 2.6019031e-6f; - - public static bool Vectorizable => true; - - public static float Invoke(float x) => float.Sin(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 uxMasked = Vector128.Abs(x); - if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector128 almHuge = Vector128.Create(AlmHuge); - Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(1 / float.Pi), almHuge); - Vector128 odd = dn.AsUInt32() << 31; - dn -= almHuge; - - Vector128 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-float.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); - - // POLY_EVAL_ODD_9 - Vector128 f2 = f * f; - Vector128 f4 = f2 * f2; - Vector128 a0 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); - Vector128 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector128.One); - Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C3), f2, Vector128.Create(C4) * f4); - Vector128 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector128 poly = f * a3; - - return (poly.AsUInt32() ^ (x.AsUInt32() & Vector128.Create(~SignMask)) ^ odd).AsSingle(); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 uxMasked = Vector256.Abs(x); - if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector256 almHuge = Vector256.Create(AlmHuge); - Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(1 / float.Pi), almHuge); - Vector256 odd = dn.AsUInt32() << 31; - dn -= almHuge; - - Vector256 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-float.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); - - // POLY_EVAL_ODD_9 - Vector256 f2 = f * f; - Vector256 f4 = f2 * f2; - Vector256 a0 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); - Vector256 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector256.One); - Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C3), f2, Vector256.Create(C4) * f4); - Vector256 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector256 poly = f * a3; - - return (poly.AsUInt32() ^ (x.AsUInt32() & Vector256.Create(~SignMask)) ^ odd).AsSingle(); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 uxMasked = Vector512.Abs(x); - if (Vector512.GreaterThanAny(uxMasked.AsUInt32(), Vector512.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector512 almHuge = Vector512.Create(AlmHuge); - Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(1 / float.Pi), almHuge); - Vector512 odd = dn.AsUInt32() << 31; - dn -= almHuge; - - Vector512 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-float.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); - - // POLY_EVAL_ODD_9 - Vector512 f2 = f * f; - Vector512 f4 = f2 * f2; - Vector512 a0 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); - Vector512 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector512.One); - Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C3), f2, Vector512.Create(C4) * f4); - Vector512 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector512 poly = f * a3; - - return (poly.AsUInt32() ^ (x.AsUInt32() & Vector512.Create(~SignMask)) ^ odd).AsSingle(); - } - } - - /// double.Sin(x) - private readonly struct SinOperatorDouble : IUnaryOperator - { - internal const ulong SignMask = 0x7FFFFFFFFFFFFFFFul; - internal const ulong MaxVectorizedValue = 0x4160000000000000ul; - private const double AlmHuge = 6.755399441055744e15; - private const double Pi_Tail1 = 1.224646799147353e-16; - private const double Pi_Tail2 = 2.165713347843828e-32; - private const double C0 = -0.16666666666666666; - private const double C2 = 0.008333333333333165; - private const double C4 = -1.984126984120184e-4; - private const double C6 = 2.7557319210152756e-6; - private const double C8 = -2.5052106798274583e-8; - private const double C10 = 1.605893649037159e-10; - private const double C12 = -7.642917806891047e-13; - private const double C14 = 2.7204790957888847e-15; - - public static bool Vectorizable => true; - - public static double Invoke(double x) => double.Sin(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 uxMasked = Vector128.Abs(x); - if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = |x| * (1 / π) - Vector128 almHuge = Vector128.Create(AlmHuge); - Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(1 / double.Pi), almHuge); - Vector128 odd = dn.AsUInt64() << 63; - dn -= almHuge; - - // f = |x| - (dn * π) - Vector128 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-double.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-Pi_Tail2), f); - - // POLY_EVAL_ODD_17 - Vector128 f2 = f * f; - Vector128 f4 = f2 * f2; - Vector128 f6 = f4 * f2; - Vector128 f10 = f6 * f4; - Vector128 f14 = f10 * f4; - Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C0)); - Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C6), f2, Vector128.Create(C4)); - Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C10), f2, Vector128.Create(C8)); - Vector128 a4 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C14), f2, Vector128.Create(C12)); - Vector128 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); - Vector128 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); - Vector128 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); - - return (poly.AsUInt64() ^ (x.AsUInt64() & Vector128.Create(~SignMask)) ^ odd).AsDouble(); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 uxMasked = Vector256.Abs(x); - if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = |x| * (1 / π) - Vector256 almHuge = Vector256.Create(AlmHuge); - Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(1 / double.Pi), almHuge); - Vector256 odd = dn.AsUInt64() << 63; - dn -= almHuge; - - // f = |x| - (dn * π) - Vector256 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-double.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-Pi_Tail2), f); - - // POLY_EVAL_ODD_17 - Vector256 f2 = f * f; - Vector256 f4 = f2 * f2; - Vector256 f6 = f4 * f2; - Vector256 f10 = f6 * f4; - Vector256 f14 = f10 * f4; - Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C0)); - Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C6), f2, Vector256.Create(C4)); - Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C10), f2, Vector256.Create(C8)); - Vector256 a4 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C14), f2, Vector256.Create(C12)); - Vector256 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); - Vector256 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); - Vector256 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); - - return (poly.AsUInt64() ^ (x.AsUInt64() & Vector256.Create(~SignMask)) ^ odd).AsDouble(); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 uxMasked = Vector512.Abs(x); - if (Vector512.GreaterThanAny(uxMasked.AsUInt64(), Vector512.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = |x| * (1 / π) - Vector512 almHuge = Vector512.Create(AlmHuge); - Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(1 / double.Pi), almHuge); - Vector512 odd = dn.AsUInt64() << 63; - dn -= almHuge; - - // f = |x| - (dn * π) - Vector512 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-double.Pi), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-Pi_Tail1), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-Pi_Tail2), f); - - // POLY_EVAL_ODD_17 - Vector512 f2 = f * f; - Vector512 f4 = f2 * f2; - Vector512 f6 = f4 * f2; - Vector512 f10 = f6 * f4; - Vector512 f14 = f10 * f4; - Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C0)); - Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C6), f2, Vector512.Create(C4)); - Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C10), f2, Vector512.Create(C8)); - Vector512 a4 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C14), f2, Vector512.Create(C12)); - Vector512 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); - Vector512 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); - Vector512 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); - - return (poly.AsUInt64() ^ (x.AsUInt64() & Vector512.Create(~SignMask)) ^ odd).AsDouble(); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinCos.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinCos.cs deleted file mode 100644 index 766269957a2e75..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinCos.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise sine and cosine of the value in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor for the element-wise sine result, represented as a span. - /// The destination tensor for the element-wise cosine result, represented as a span. - /// Destination is too short. - /// and or reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes ([i], [i]) = .SinCos([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void SinCos(ReadOnlySpan x, Span sinDestination, Span cosDestination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan_TwoOutputs>(x, sinDestination, cosDestination); - - /// T.SinCos(x) - private readonly struct SinCosOperator : IUnaryInputBinaryOutput where T : ITrigonometricFunctions - { - public static bool Vectorizable => false; // TODO: vectorize - - public static (T, T) Invoke(T x) => T.SinCos(x); - public static (Vector128 First, Vector128 Second) Invoke(Vector128 x) => throw new NotSupportedException(); - public static (Vector256 First, Vector256 Second) Invoke(Vector256 x) => throw new NotSupportedException(); - public static (Vector512 First, Vector512 Second) Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinCosPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinCosPi.cs deleted file mode 100644 index 574db7667be00f..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinCosPi.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise sine and cosine of the value in the specified tensor that has been multiplied by Pi. - /// The tensor, represented as a span. - /// The destination tensor for the element-wise sine result, represented as a span. - /// The destination tensor for the element-wise cosine result, represented as a span. - /// Destination is too short. - /// and or reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes ([i], [i]) = .SinCos([i]). - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void SinCosPi(ReadOnlySpan x, Span sinPiDestination, Span cosPiDestination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan_TwoOutputs>(x, sinPiDestination, cosPiDestination); - - /// T.SinCosPi(x) - private readonly struct SinCosPiOperator : IUnaryInputBinaryOutput where T : ITrigonometricFunctions - { - public static bool Vectorizable => false; // TODO: vectorize - - public static (T, T) Invoke(T x) => T.SinCosPi(x); - public static (Vector128 First, Vector128 Second) Invoke(Vector128 x) => throw new NotSupportedException(); - public static (Vector256 First, Vector256 Second) Invoke(Vector256 x) => throw new NotSupportedException(); - public static (Vector512 First, Vector512 Second) Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinPi.cs deleted file mode 100644 index 3ee43ecd58c0a9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SinPi.cs +++ /dev/null @@ -1,107 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise sine of the value in the specified tensor that has been multiplied by Pi. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .SinPi([i]). - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void SinPi(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.SinPi(x) - private readonly struct SinPiOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.SinPi(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 xpi = x * Vector128.Create(T.Pi); - if (typeof(T) == typeof(float)) - { - if (Vector128.GreaterThanAny(xpi.AsUInt32() & Vector128.Create(SinOperatorSingle.SignMask), Vector128.Create(SinOperatorSingle.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsSingle()).As(); - } - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - if (Vector128.GreaterThanAny(xpi.AsUInt64() & Vector128.Create(SinOperatorDouble.SignMask), Vector128.Create(SinOperatorDouble.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsDouble()).As(); - } - } - - return SinOperator.Invoke(xpi); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 xpi = x * Vector256.Create(T.Pi); - if (typeof(T) == typeof(float)) - { - if (Vector256.GreaterThanAny(xpi.AsUInt32() & Vector256.Create(SinOperatorSingle.SignMask), Vector256.Create(SinOperatorSingle.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsSingle()).As(); - } - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - if (Vector256.GreaterThanAny(xpi.AsUInt64() & Vector256.Create(SinOperatorDouble.SignMask), Vector256.Create(SinOperatorDouble.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsDouble()).As(); - } - } - - return SinOperator.Invoke(xpi); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 xpi = x * Vector512.Create(T.Pi); - if (typeof(T) == typeof(float)) - { - if (Vector512.GreaterThanAny(xpi.AsUInt32() & Vector512.Create(SinOperatorSingle.SignMask), Vector512.Create(SinOperatorSingle.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsSingle()).As(); - } - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - if (Vector512.GreaterThanAny(xpi.AsUInt64() & Vector512.Create(SinOperatorDouble.SignMask), Vector512.Create(SinOperatorDouble.MaxVectorizedValue))) - { - return ApplyScalar>(x.AsDouble()).As(); - } - } - - return SinOperator.Invoke(xpi); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Single.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Single.netcore.cs index b1f6309e6dbca7..32723d2b3c3c5e 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Single.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Single.netcore.cs @@ -39,7 +39,7 @@ namespace System.Numerics.Tensors { - public static partial class TensorPrimitives + public static unsafe partial class TensorPrimitives { private static void InvokeSpanIntoSpan( ReadOnlySpan x, Span destination) @@ -54,7 +54,7 @@ private static void InvokeSpanSpanIntoSpan( private static void InvokeSpanScalarIntoSpan( ReadOnlySpan x, float y, Span destination) where TSingleBinaryOperator : struct, IBinaryOperator => - InvokeSpanScalarIntoSpan(x, y, destination); + InvokeSpanScalarIntoSpan, TSingleBinaryOperator>(x, y, destination); private static unsafe void InvokeSpanScalarIntoSpan( ReadOnlySpan x, float y, Span destination) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sinh.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sinh.cs deleted file mode 100644 index a154e3d9edce9d..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sinh.cs +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hyperbolic sine of each radian angle in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Sinh([i]). - /// - /// - /// If a value is equal to , , or , - /// the corresponding destination location is set to that value. - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi / 180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Sinh(ReadOnlySpan x, Span destination) - where T : IHyperbolicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Sinh(x) - internal readonly struct SinhOperator : IUnaryOperator - where T : IHyperbolicFunctions - { - // Same as cosh, but with `z -` rather than `z +`, and with the sign - // flipped on the result based on the sign of the input. - - private const float Single_LOGV = 0.693161f; - private const float Single_HALFV = 1.0000138f; - private const float Single_INVV2 = 0.24999309f; - - private const double Double_LOGV = 0.6931471805599453; - private const double Double_HALFV = 1.0; - private const double Double_INVV2 = 0.25; - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Sinh(x); - - public static Vector128 Invoke(Vector128 t) - { - if (typeof(T) == typeof(float)) - { - Vector128 x = t.AsSingle(); - - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpOperator.Invoke(y - Vector128.Create((float)Single_LOGV)); - Vector128 result = Vector128.Create((float)Single_HALFV) * (z - (Vector128.Create((float)Single_INVV2) / z)); - Vector128 sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue); - return (sign ^ result.AsUInt32()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector128 x = t.AsDouble(); - - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpOperator.Invoke(y - Vector128.Create(Double_LOGV)); - Vector128 result = Vector128.Create(Double_HALFV) * (z - (Vector128.Create(Double_INVV2) / z)); - Vector128 sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue); - return (sign ^ result.AsUInt64()).As(); - } - } - - public static Vector256 Invoke(Vector256 t) - { - if (typeof(T) == typeof(float)) - { - Vector256 x = t.AsSingle(); - - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpOperator.Invoke(y - Vector256.Create((float)Single_LOGV)); - Vector256 result = Vector256.Create((float)Single_HALFV) * (z - (Vector256.Create((float)Single_INVV2) / z)); - Vector256 sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue); - return (sign ^ result.AsUInt32()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector256 x = t.AsDouble(); - - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpOperator.Invoke(y - Vector256.Create(Double_LOGV)); - Vector256 result = Vector256.Create(Double_HALFV) * (z - (Vector256.Create(Double_INVV2) / z)); - Vector256 sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue); - return (sign ^ result.AsUInt64()).As(); - } - } - - public static Vector512 Invoke(Vector512 t) - { - if (typeof(T) == typeof(float)) - { - Vector512 x = t.AsSingle(); - - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpOperator.Invoke(y - Vector512.Create((float)Single_LOGV)); - Vector512 result = Vector512.Create((float)Single_HALFV) * (z - (Vector512.Create((float)Single_INVV2) / z)); - Vector512 sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue); - return (sign ^ result.AsUInt32()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - Vector512 x = t.AsDouble(); - - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpOperator.Invoke(y - Vector512.Create(Double_LOGV)); - Vector512 result = Vector512.Create(Double_HALFV) * (z - (Vector512.Create(Double_INVV2) / z)); - Vector512 sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue); - return (sign ^ result.AsUInt64()).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SoftMax.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SoftMax.cs deleted file mode 100644 index 429f7baf59c436..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.SoftMax.cs +++ /dev/null @@ -1,44 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the softmax function over the specified non-empty tensor of numbers. - /// The tensor, represented as a span. - /// The destination tensor. - /// Destination is too short. - /// must not be empty. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes a sum of .Exp(x[i]) for all elements in . - /// It then effectively computes [i] = .Exp([i]) / sum. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void SoftMax(ReadOnlySpan x, Span destination) - where T : IExponentialFunctions - { - if (x.IsEmpty) - { - ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); - } - - if (x.Length > destination.Length) - { - ThrowHelper.ThrowArgument_DestinationTooShort(); - } - - ValidateInputOutputSpanNonOverlapping(x, destination); - - T expSum = Aggregate, AddOperator>(x); - - InvokeSpanScalarIntoSpan, DivideOperator>(x, expSum, destination); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sqrt.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sqrt.cs deleted file mode 100644 index f5c9525e7e92c8..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sqrt.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise square root of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Sqrt([i]). - /// - /// - public static void Sqrt(ReadOnlySpan x, Span destination) - where T : IRootFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Sqrt(x) - private readonly struct SqrtOperator : IUnaryOperator - where T : IRootFunctions - { - public static bool Vectorizable => true; - public static T Invoke(T x) => T.Sqrt(x); - public static Vector128 Invoke(Vector128 x) => Vector128.Sqrt(x); - public static Vector256 Invoke(Vector256 x) => Vector256.Sqrt(x); - public static Vector512 Invoke(Vector512 x) => Vector512.Sqrt(x); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Subtract.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Subtract.cs deleted file mode 100644 index 2130745cee7bde..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Subtract.cs +++ /dev/null @@ -1,76 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise difference between numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] - [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Subtract(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : ISubtractionOperators => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise difference between numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] - . - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Subtract(ReadOnlySpan x, T y, Span destination) - where T : ISubtractionOperators => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// Computes the element-wise difference between numbers in the specified tensors. - /// The first tensor, represented as a scalar. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = - [i]. - /// - /// - /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. - /// - /// - public static void Subtract(T x, ReadOnlySpan y, Span destination) - where T : ISubtractionOperators => - InvokeScalarSpanIntoSpan>(x, y, destination); - - /// x - y - internal readonly struct SubtractOperator : IBinaryOperator where T : ISubtractionOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x, T y) => x - y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x - y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x - y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x - y; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sum.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sum.cs deleted file mode 100644 index 1abd51b2def890..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Sum.cs +++ /dev/null @@ -1,81 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the sum of all elements in the specified tensor of numbers. - /// The tensor, represented as a span. - /// The result of adding all elements in , or zero if is empty. - /// - /// - /// If any of the values in the input is equal to , the result is also NaN. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T Sum(ReadOnlySpan x) - where T : IAdditionOperators, IAdditiveIdentity => - Aggregate, AddOperator>(x); - - /// Computes the sum of the absolute values of every element in the specified tensor of numbers. - /// The tensor, represented as a span. - /// The result of adding the absolute value of every element in , or zero if is empty. - /// is a signed integer type and contained a value equal to 's minimum value. - /// - /// - /// This method effectively computes: - /// - /// Span<T> absoluteValues = ...; - /// TensorPrimitives.Abs(x, absoluteValues); - /// T result = TensorPrimitives.Sum(absoluteValues); - /// - /// but without requiring intermediate storage for the absolute values. It corresponds to the asum method defined by BLAS1. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T SumOfMagnitudes(ReadOnlySpan x) - where T : INumberBase => - Aggregate, AddOperator>(x); - - /// Computes the sum of the square of every element in the specified tensor of numbers. - /// The tensor, represented as a span. - /// The result of adding the square of every element in , or zero if is empty. - /// - /// - /// This method effectively computes: - /// - /// Span<T> squaredValues = ...; - /// TensorPrimitives.Multiply(x, x, squaredValues); - /// T result = TensorPrimitives.Sum(squaredValues); - /// - /// but without requiring intermediate storage for the squared values. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static T SumOfSquares(ReadOnlySpan x) - where T : IAdditionOperators, IAdditiveIdentity, IMultiplyOperators => - Aggregate, AddOperator>(x); - - /// x * x - internal readonly struct SquaredOperator : IUnaryOperator where T : IMultiplyOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x) => x * x; - public static Vector128 Invoke(Vector128 x) => x * x; - public static Vector256 Invoke(Vector256 x) => x * x; - public static Vector512 Invoke(Vector512 x) => x * x; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.T.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.T.cs new file mode 100644 index 00000000000000..0604768ba9fc49 --- /dev/null +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.T.cs @@ -0,0 +1,3025 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace System.Numerics.Tensors +{ + /// Performs primitive tensor operations over spans of memory. + public static partial class TensorPrimitives + { + /// Computes the element-wise absolute value of each number in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is a signed integer type and contained a value equal to 's minimum value. + /// + /// + /// This method effectively computes [i] = .Abs([i]). + /// + /// + /// The absolute value of a is its numeric value without its sign. For example, the absolute value of both 1.2e-03 and -1.2e03 is 1.2e03. + /// + /// + /// If a value is equal to or , the result stored into the corresponding destination location is set to . + /// If a value is equal to , the result stored into the corresponding destination location is the original NaN value with the sign bit removed. + /// + /// + public static void Abs(ReadOnlySpan x, Span destination) + where T : INumberBase => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise angle in radians whose cosine is the specifed number. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Acos([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Acos(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hyperbolic arc-cosine of the specifed number. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Acosh([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Acosh(ReadOnlySpan x, Span destination) + where T : IHyperbolicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise angle in radians whose cosine is the specifed number and divides the result by Pi. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .AcosPi([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void AcosPi(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise angle in radians whose sine is the specifed number. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Asin([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Asin(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hyperbolic arc-sine of the specifed number. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Asinh([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Asinh(ReadOnlySpan x, Span destination) + where T : IHyperbolicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise angle in radians whose sine is the specifed number and divides the result by Pi. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .AsinPi([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void AsinPi(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise angle in radians whose tangent is the specifed number. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hyperbolic arc-tangent of the specifed number. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atanh([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atanh(ReadOnlySpan x, Span destination) + where T : IHyperbolicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise angle in radians whose tangent is the specifed number and divides the result by Pi. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .AtanPi([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void AtanPi(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan2([i], [i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan2(ReadOnlySpan y, ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanIntoSpan>(y, x, destination); + + /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan2([i], ). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan2(ReadOnlySpan y, T x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanScalarIntoSpan>(y, x, destination); + + /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors. + /// The first tensor, represented as a scalar. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan2(, [i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan2(T y, ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeScalarSpanIntoSpan>(y, x, destination); + + /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors and divides the result by Pi. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan2([i], [i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan2Pi(ReadOnlySpan y, ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanIntoSpan>(y, x, destination); + + /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors and divides the result by Pi. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan2([i], ). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan2Pi(ReadOnlySpan y, T x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanScalarIntoSpan>(y, x, destination); + + /// Computes the element-wise arc-tangent for the quotient of two values in the specified tensors and divides the result by Pi. + /// The first tensor, represented as a scalar. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Atan2(, [i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Atan2Pi(T y, ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeScalarSpanIntoSpan>(y, x, destination); + + /// Computes the element-wise addition of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IAdditionOperators, IAdditiveIdentity => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise addition of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] + . + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Add(ReadOnlySpan x, T y, Span destination) + where T : IAdditionOperators, IAdditiveIdentity => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise result of ( + ) * for the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of and the length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] + [i]) * [i]. + /// + /// + /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void AddMultiply(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan multiplier, Span destination) + where T : IAdditionOperators, IMultiplyOperators => + InvokeSpanSpanSpanIntoSpan>(x, y, multiplier, destination); + + /// Computes the element-wise result of ( + ) * for the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] + [i]) * . + /// + /// + /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void AddMultiply(ReadOnlySpan x, ReadOnlySpan y, T multiplier, Span destination) + where T : IAdditionOperators, IMultiplyOperators => + InvokeSpanSpanScalarIntoSpan>(x, y, multiplier, destination); + + /// Computes the element-wise result of ( + ) * for the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] + ) * [i]. + /// + /// + /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void AddMultiply(ReadOnlySpan x, T y, ReadOnlySpan multiplier, Span destination) + where T : IAdditionOperators, IMultiplyOperators => + InvokeSpanScalarSpanIntoSpan>(x, y, multiplier, destination); + + /// Computes the element-wise bitwise AND of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] & [i]. + /// + /// + public static void BitwiseAnd(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IBitwiseOperators => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise bitwise AND of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] & . + /// + /// + public static void BitwiseAnd(ReadOnlySpan x, T y, Span destination) + where T : IBitwiseOperators => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise bitwise OR of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] | [i]. + /// + /// + public static void BitwiseOr(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IBitwiseOperators => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise bitwise OR of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] | . + /// + /// + public static void BitwiseOr(ReadOnlySpan x, T y, Span destination) + where T : IBitwiseOperators => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise ceiling of numbers in the specified tensor. + /// The first tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Ceiling([i]). + /// + /// + public static void Ceiling(ReadOnlySpan x, Span destination) + where T : IFloatingPoint => + InvokeSpanIntoSpan>(x, destination); + + /// + /// Copies to , converting each + /// value to a value. + /// + /// The source span from which to copy values. + /// The destination span into which the converted values should be written. + /// Destination is too short. + /// + /// + /// This method effectively computes [i] = TTo.CreateChecked([i]). + /// + /// + public static void ConvertChecked(ReadOnlySpan source, Span destination) + where TFrom : INumberBase + where TTo : INumberBase + { + if (!TryConvertUniversal(source, destination)) + { + InvokeSpanIntoSpan>(source, destination); + } + } + + /// + /// Copies to , converting each + /// value to a value. + /// + /// The source span from which to copy values. + /// The destination span into which the converted values should be written. + /// Destination is too short. + /// + /// + /// This method effectively computes [i] = TTo.CreateSaturating([i]). + /// + /// + public static void ConvertSaturating(ReadOnlySpan source, Span destination) + where TFrom : INumberBase + where TTo : INumberBase + { + if (!TryConvertUniversal(source, destination)) + { + InvokeSpanIntoSpan>(source, destination); + } + } + + /// + /// Copies to , converting each + /// value to a value. + /// + /// The source span from which to copy values. + /// The destination span into which the converted values should be written. + /// Destination is too short. + /// + /// + /// This method effectively computes [i] = TTo.CreateTruncating([i]). + /// + /// + public static void ConvertTruncating(ReadOnlySpan source, Span destination) + where TFrom : INumberBase + where TTo : INumberBase + { + if (TryConvertUniversal(source, destination)) + { + return; + } + + if (((typeof(TFrom) == typeof(byte) || typeof(TFrom) == typeof(sbyte)) && (typeof(TTo) == typeof(byte) || typeof(TTo) == typeof(sbyte))) || + ((typeof(TFrom) == typeof(ushort) || typeof(TFrom) == typeof(short)) && (typeof(TTo) == typeof(ushort) || typeof(TTo) == typeof(short))) || + ((IsUInt32Like() || IsInt32Like()) && (IsUInt32Like() || IsInt32Like())) || + ((IsUInt64Like() || IsInt64Like()) && (IsUInt64Like() || IsInt64Like()))) + { + source.CopyTo(Rename(destination)); + return; + } + + if (typeof(TFrom) == typeof(float) && IsUInt32Like()) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return; + } + + if (typeof(TFrom) == typeof(float) && IsInt32Like()) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return; + } + + if (typeof(TFrom) == typeof(double) && IsUInt64Like()) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return; + } + + if (typeof(TFrom) == typeof(double) && IsInt64Like()) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return; + } + + if (typeof(TFrom) == typeof(ushort) && typeof(TTo) == typeof(byte)) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return; + } + + if (typeof(TFrom) == typeof(short) && typeof(TTo) == typeof(sbyte)) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return; + } + + if (IsUInt32Like() && typeof(TTo) == typeof(ushort)) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return; + } + + if (IsInt32Like() && typeof(TTo) == typeof(short)) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return; + } + + if (IsUInt64Like() && IsUInt32Like()) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return; + } + + if (IsInt64Like() && IsInt32Like()) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return; + } + + InvokeSpanIntoSpan>(source, destination); + } + + /// Performs conversions that are the same regardless of checked, truncating, or saturation. + [MethodImpl(MethodImplOptions.AggressiveInlining)] // at most one of the branches will be kept + private static bool TryConvertUniversal(ReadOnlySpan source, Span destination) + where TFrom : INumberBase + where TTo : INumberBase + { + if (typeof(TFrom) == typeof(TTo)) + { + if (source.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(source, Rename(destination)); + + source.CopyTo(Rename(destination)); + return true; + } + + if (IsInt32Like() && typeof(TTo) == typeof(float)) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return true; + } + + if (IsUInt32Like() && typeof(TTo) == typeof(float)) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return true; + } + + if (IsInt64Like() && typeof(TTo) == typeof(double)) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return true; + } + + if (IsUInt64Like() && typeof(TTo) == typeof(double)) + { + InvokeSpanIntoSpan(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(float) && typeof(TTo) == typeof(Half)) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(Half) && typeof(TTo) == typeof(float)) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(float) && typeof(TTo) == typeof(double)) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(double) && typeof(TTo) == typeof(float)) + { + InvokeSpanIntoSpan_2to1(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(byte) && typeof(TTo) == typeof(ushort)) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(sbyte) && typeof(TTo) == typeof(short)) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(ushort) && IsUInt32Like()) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (typeof(TFrom) == typeof(short) && IsInt32Like()) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (IsUInt32Like() && IsUInt64Like()) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + if (IsInt32Like() && IsInt64Like()) + { + InvokeSpanIntoSpan_1to2(Rename(source), Rename(destination)); + return true; + } + + return false; + } + + /// Computes the element-wise result of copying the sign from one number to another number in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.CopySign([i], [i]). + /// + /// + public static void CopySign(ReadOnlySpan x, ReadOnlySpan sign, Span destination) + where T : INumber => + InvokeSpanSpanIntoSpan>(x, sign, destination); + + /// Computes the element-wise result of copying the sign from one number to another number in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.CopySign([i], [i]). + /// + /// + public static void CopySign(ReadOnlySpan x, T sign, Span destination) + where T : INumber => + InvokeSpanScalarIntoSpan>(x, sign, destination); + + /// Computes the element-wise cosine of the value in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Cos([i]). + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Cos(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise cosine of the value in the specified tensor that has been multiplied by Pi. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .CosPi([i]). + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void CosPi(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hyperbolic cosine of each radian angle in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Cosh([i]). + /// + /// + /// If a value is equal to or , the result stored into the corresponding destination location is set to . + /// If a value is equal to , the result stored into the corresponding destination location is also NaN. + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Cosh(ReadOnlySpan x, Span destination) + where T : IHyperbolicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the cosine similarity between the two specified non-empty, equal-length tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The cosine similarity of the two tensors. + /// Length of must be same as length of . + /// and must not be empty. + /// + /// + /// This method effectively computes TensorPrimitives.Dot(x, y) / (.Sqrt(TensorPrimitives.SumOfSquares(x)) * .Sqrt(TensorPrimitives.SumOfSquares(y)). + /// + /// + /// If any element in either input tensor is equal to , , or , + /// NaN is returned. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T CosineSimilarity(ReadOnlySpan x, ReadOnlySpan y) + where T : IRootFunctions => + CosineSimilarityCore(x, y); + + /// Computes the element-wise cube root of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Cbrt([i]). + /// + /// + public static void Cbrt(ReadOnlySpan x, Span destination) + where T : IRootFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise conversion of each number of degrees in the specified tensor to radiansx. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .DegreesToRadians([i]). + /// + /// + public static void DegreesToRadians(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the distance between two points, specified as non-empty, equal-length tensors of numbers, in Euclidean space. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The Euclidean distance. + /// Length of must be same as length of . + /// and must not be empty. + /// + /// + /// This method effectively computes the equivalent of: + /// + /// Span<T> difference = ...; + /// TensorPrimitives.Subtract(x, y, difference); + /// T result = .Sqrt(TensorPrimitives.SumOfSquares(difference)); + /// + /// but without requiring additional temporary storage for the intermediate differences. + /// + /// + /// If any element in either input tensor is equal to , NaN is returned. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Distance(ReadOnlySpan x, ReadOnlySpan y) + where T : IRootFunctions + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + return T.Sqrt(Aggregate, AddOperator>(x, y)); + } + + /// Computes the element-wise division of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and an element in is equal to zero. + /// + /// + /// This method effectively computes [i] = [i] / [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Divide(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IDivisionOperators => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise division of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and is equal to zero. + /// + /// + /// This method effectively computes [i] = [i] / . + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Divide(ReadOnlySpan x, T y, Span destination) + where T : IDivisionOperators => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise division of numbers in the specified tensors. + /// The first tensor, represented as a scalar. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and an element in is equal to zero. + /// + /// + /// This method effectively computes [i] = / [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Divide(T x, ReadOnlySpan y, Span destination) + where T : IDivisionOperators => + InvokeScalarSpanIntoSpan>(x, y, destination); + + /// Computes the dot product of two tensors containing numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The dot product. + /// Length of must be same as length of . + /// + /// + /// This method effectively computes the equivalent of: + /// + /// Span<T> products = ...; + /// TensorPrimitives.Multiply(x, y, products); + /// T result = TensorPrimitives.Sum(products); + /// + /// but without requiring additional temporary storage for the intermediate products. It corresponds to the dot method defined by BLAS1. + /// + /// + /// If any of the input elements is equal to , the resulting value is also NaN. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Dot(ReadOnlySpan x, ReadOnlySpan y) + where T : IAdditionOperators, IAdditiveIdentity, IMultiplyOperators, IMultiplicativeIdentity => + Aggregate, AddOperator>(x, y); + + /// Computes the element-wise result of raising e to the number powers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Exp([i]). + /// + /// + /// If a value equals or , the result stored into the corresponding destination location is set to NaN. + /// If a value equals , the result stored into the corresponding destination location is set to 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Exp(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise result of raising e to the number powers in the specified tensor, minus 1. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .ExpM1([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void ExpM1(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise result of raising 2 to the number powers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Exp2([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Exp2(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise result of raising 2 to the number powers in the specified tensor, minus one. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Exp2M1([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Exp2M1(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise result of raising 10 to the number powers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Exp10([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Exp10(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise result of raising 10 to the number powers in the specified tensor, minus one. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Exp10M1([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Exp10M1(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise floor of numbers in the specified tensor. + /// The first tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Floor([i]). + /// + /// + public static void Floor(ReadOnlySpan x, Span destination) + where T : IFloatingPoint => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hypotensue given values from two tensors representing the lengths of the shorter sides in a right-angled triangle. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Hypot([i], [i]). + /// + /// + public static void Hypot(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IRootFunctions => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise remainder of the numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Ieee754Remainder([i], [i]). + /// + /// + public static void Ieee754Remainder(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise remainder of the numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Ieee754Remainder([i], ). + /// + /// + public static void Ieee754Remainder(ReadOnlySpan x, T y, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise remainder of the numbers in the specified tensors. + /// The first tensor, represented as a scalar. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Ieee754Remainder(, [i]). + /// + /// + public static void Ieee754Remainder(T x, ReadOnlySpan y, Span destination) + where T : IFloatingPointIeee754 => + InvokeScalarSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise integer logarithm of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.ILogB([i]). + /// + /// + public static void ILogB(ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 + { + if (typeof(T) == typeof(double)) + { + // Special-case double as the only vectorizable floating-point type whose size != sizeof(int). + InvokeSpanIntoSpan_2to1(Rename(x), destination); + } + else + { + InvokeSpanIntoSpan>(x, destination); + } + } + + /// Searches for the index of the largest number in the specified tensor. + /// The tensor, represented as a span. + /// The index of the maximum element in , or -1 if is empty. + /// + /// + /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If any value equal to NaN + /// is present, the index of the first is returned. Positive 0 is considered greater than negative 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static int IndexOfMax(ReadOnlySpan x) + where T : INumber => + IndexOfMinMaxCore>(x); + + /// Searches for the index of the number with the largest magnitude in the specified tensor. + /// The tensor, represented as a span. + /// The index of the element in with the largest magnitude (absolute value), or -1 if is empty. + /// + /// + /// The determination of the maximum magnitude matches the IEEE 754:2019 `maximumMagnitude` function. If any value equal to NaN + /// is present, the index of the first is returned. If two values have the same magnitude and one is positive and the other is negative, + /// the positive value is considered to have the larger magnitude. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static int IndexOfMaxMagnitude(ReadOnlySpan x) + where T : INumber => + IndexOfMinMaxCore>(x); + + /// Searches for the index of the smallest number in the specified tensor. + /// The tensor, represented as a span. + /// The index of the minimum element in , or -1 if is empty. + /// + /// + /// The determination of the minimum element matches the IEEE 754:2019 `minimum` function. If any value equal to NaN + /// is present, the index of the first is returned. Negative 0 is considered smaller than positive 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static int IndexOfMin(ReadOnlySpan x) + where T : INumber => + IndexOfMinMaxCore>(x); + + /// Searches for the index of the number with the smallest magnitude in the specified tensor. + /// The tensor, represented as a span. + /// The index of the element in with the smallest magnitude (absolute value), or -1 if is empty. + /// + /// + /// The determination of the minimum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If any value equal to NaN + /// is present, the index of the first is returned. If two values have the same magnitude and one is positive and the other is negative, + /// the negative value is considered to have the smaller magnitude. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static int IndexOfMinMagnitude(ReadOnlySpan x) + where T : INumber => + IndexOfMinMaxCore>(x); + + /// Computes the element-wise leading zero count of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.LeadingZeroCount([i]). + /// + /// + public static void LeadingZeroCount(ReadOnlySpan x, Span destination) + where T : IBinaryInteger => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise linear interpolation between two values based on the given weight in the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of and length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Lerp([i], [i], [i]). + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Lerp(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan amount, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanSpanIntoSpan>(x, y, amount, destination); + + /// Computes the element-wise linear interpolation between two values based on the given weight in the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Lerp([i], [i], ). + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Lerp(ReadOnlySpan x, ReadOnlySpan y, T amount, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanScalarIntoSpan>(x, y, amount, destination); + + /// Computes the element-wise linear interpolation between two values based on the given weight in the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Lerp([i], , [i]). + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Lerp(ReadOnlySpan x, T y, ReadOnlySpan amount, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanScalarSpanIntoSpan>(x, y, amount, destination); + + /// Computes the element-wise natural (base e) logarithm of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log([i]). + /// + /// + /// If a value equals 0, the result stored into the corresponding destination location is set to . + /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. + /// If a value is positive infinity, the result stored into the corresponding destination location is set to . + /// Otherwise, if a value is positive, its natural logarithm is stored into the corresponding destination location. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log(ReadOnlySpan x, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise base 2 logarithm of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log2([i]). + /// + /// + /// If a value equals 0, the result stored into the corresponding destination location is set to . + /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. + /// If a value is positive infinity, the result stored into the corresponding destination location is set to . + /// Otherwise, if a value is positive, its base 2 logarithm is stored into the corresponding destination location. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log2(ReadOnlySpan x, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise base 10 logarithm of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log10([i]). + /// + /// + /// If a value equals 0, the result stored into the corresponding destination location is set to . + /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. + /// If a value is positive infinity, the result stored into the corresponding destination location is set to . + /// Otherwise, if a value is positive, its base 10 logarithm is stored into the corresponding destination location. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log10(ReadOnlySpan x, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise natural (base e) logarithm of numbers in the specified tensor plus 1. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .LogP1([i]). + /// + /// + /// If a value equals 0, the result stored into the corresponding destination location is set to . + /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. + /// If a value is positive infinity, the result stored into the corresponding destination location is set to . + /// Otherwise, if a value is positive, its natural logarithm plus 1 is stored into the corresponding destination location. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void LogP1(ReadOnlySpan x, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise base 2 logarithm of numbers in the specified tensor plus 1. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log2P1([i]). + /// + /// + /// If a value equals 0, the result stored into the corresponding destination location is set to . + /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. + /// If a value is positive infinity, the result stored into the corresponding destination location is set to . + /// Otherwise, if a value is positive, its base 2 logarithm plus 1 is stored into the corresponding destination location. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log2P1(ReadOnlySpan x, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise base 10 logarithm of numbers in the specified tensor plus 1. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log10P1([i]). + /// + /// + /// If a value equals 0, the result stored into the corresponding destination location is set to . + /// If a value is negative or equal to , the result stored into the corresponding destination location is set to NaN. + /// If a value is positive infinity, the result stored into the corresponding destination location is set to . + /// Otherwise, if a value is positive, its base 10 logarithm plus 1 is stored into the corresponding destination location. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log10P1(ReadOnlySpan x, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise logarithm of the numbers in a specified tensor to the specified base in another specified tensor. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log([i], [i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise logarithm of the numbers in a specified tensor to the specified base in another specified tensor. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Log([i], ). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Log(ReadOnlySpan x, T y, Span destination) + where T : ILogarithmicFunctions => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Searches for the largest number in the specified tensor. + /// The tensor, represented as a span. + /// The maximum element in . + /// Length of must be greater than zero. + /// + /// + /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If any value equal to + /// is present, the first is returned. Positive 0 is considered greater than negative 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Max(ReadOnlySpan x) + where T : INumber => + MinMaxCore>(x); + + /// Computes the element-wise maximum of the numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Max([i], [i]). + /// + /// + /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , + /// that value is stored as the result. Positive 0 is considered greater than negative 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Max(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : INumber => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise maximum of the numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Max([i], ). + /// + /// + /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , + /// that value is stored as the result. Positive 0 is considered greater than negative 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Max(ReadOnlySpan x, T y, Span destination) + where T : INumber => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Searches for the number with the largest magnitude in the specified tensor. + /// The tensor, represented as a span. + /// The element in with the largest magnitude (absolute value). + /// Length of must be greater than zero. + /// + /// + /// The determination of the maximum magnitude matches the IEEE 754:2019 `maximumMagnitude` function. If any value equal to + /// is present, the first is returned. If two values have the same magnitude and one is positive and the other is negative, + /// the positive value is considered to have the larger magnitude. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T MaxMagnitude(ReadOnlySpan x) + where T : INumberBase => + MinMaxCore>(x); + + /// Computes the element-wise number with the largest magnitude in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// This method effectively computes [i] = .MaxMagnitude([i], [i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void MaxMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : INumberBase => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise number with the largest magnitude in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// This method effectively computes [i] = .MaxMagnitude([i], ). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void MaxMagnitude(ReadOnlySpan x, T y, Span destination) + where T : INumberBase => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Searches for the smallest number in the specified tensor. + /// The tensor, represented as a span. + /// The minimum element in . + /// Length of must be greater than zero. + /// + /// + /// The determination of the minimum element matches the IEEE 754:2019 `minimum` function. If any value is equal to + /// is present, the first is returned. Negative 0 is considered smaller than positive 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Min(ReadOnlySpan x) + where T : INumber => + MinMaxCore>(x); + + /// Computes the element-wise minimum of the numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Max([i], [i]). + /// + /// + /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , + /// that value is stored as the result. Positive 0 is considered greater than negative 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Min(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : INumber => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise minimum of the numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Max([i], ). + /// + /// + /// The determination of the maximum element matches the IEEE 754:2019 `maximum` function. If either value is equal to , + /// that value is stored as the result. Positive 0 is considered greater than negative 0. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Min(ReadOnlySpan x, T y, Span destination) + where T : INumber => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Searches for the number with the smallest magnitude in the specified tensor. + /// The tensor, represented as a span. + /// The element in with the smallest magnitude (absolute value). + /// Length of must be greater than zero. + /// + /// + /// The determination of the minimum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If any value equal to + /// is present, the first is returned. If two values have the same magnitude and one is positive and the other is negative, + /// the negative value is considered to have the smaller magnitude. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T MinMagnitude(ReadOnlySpan x) + where T : INumberBase => + MinMaxCore>(x); + + /// Computes the element-wise number with the smallest magnitude in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// This method effectively computes [i] = .MinMagnitude([i], [i]). + /// + /// + /// The determination of the maximum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If either value is equal to , + /// that value is stored as the result. If the two values have the same magnitude and one is positive and the other is negative, + /// the negative value is considered to have the smaller magnitude. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void MinMagnitude(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : INumberBase => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise number with the smallest magnitude in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// This method effectively computes [i] = .MinMagnitude([i], ). + /// + /// + /// The determination of the maximum magnitude matches the IEEE 754:2019 `minimumMagnitude` function. If either value is equal to , + /// that value is stored as the result. If the two values have the same magnitude and one is positive and the other is negative, + /// the negative value is considered to have the smaller magnitude. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void MinMagnitude(ReadOnlySpan x, T y, Span destination) + where T : INumberBase => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise product of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] * [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Multiply(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IMultiplyOperators, IMultiplicativeIdentity => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise product of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] * . + /// It corresponds to the scal method defined by BLAS1. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Multiply(ReadOnlySpan x, T y, Span destination) + where T : IMultiplyOperators, IMultiplicativeIdentity => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of and length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * [i]) + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void MultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan addend, Span destination) + where T : IAdditionOperators, IMultiplyOperators => + InvokeSpanSpanSpanIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * [i]) + . + /// It corresponds to the axpy method defined by BLAS1. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void MultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, T addend, Span destination) + where T : IAdditionOperators, IMultiplyOperators => + InvokeSpanSpanScalarIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * ) + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void MultiplyAdd(ReadOnlySpan x, T y, ReadOnlySpan addend, Span destination) + where T : IAdditionOperators, IMultiplyOperators => + InvokeSpanScalarSpanIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of and length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * [i]) + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + /// Behaves the same as either or + /// depending on the current machine's capabilities. + /// + /// + public static void MultiplyAddEstimate(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan addend, Span destination) + where T : INumberBase => + InvokeSpanSpanSpanIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * [i]) + . + /// It corresponds to the axpy method defined by BLAS1. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + /// Behaves the same as either or + /// depending on the current machine's capabilities. + /// + /// + public static void MultiplyAddEstimate(ReadOnlySpan x, ReadOnlySpan y, T addend, Span destination) + where T : INumberBase => + InvokeSpanSpanScalarIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * ) + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + /// Behaves the same as either or + /// depending on the current machine's capabilities. + /// + /// + public static void MultiplyAddEstimate(ReadOnlySpan x, T y, ReadOnlySpan addend, Span destination) + where T : INumberBase => + InvokeSpanScalarSpanIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of and length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * [i]) + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + /// This computes ( * ) as if to infinite precision, adds to that result as if to + /// infinite precision, and finally rounds to the nearest representable value. This differs from the non-fused sequence which would compute + /// ( * ) as if to infinite precision, round the result to the nearest representable value, add to the + /// rounded result as if to infinite precision, and finally round to the nearest representable value. + /// + /// + public static void FusedMultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan addend, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanSpanIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The third tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * [i]) + . + /// It corresponds to the axpy method defined by BLAS1. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + /// This computes ( * ) as if to infinite precision, adds to that result as if to + /// infinite precision, and finally rounds to the nearest representable value. This differs from the non-fused sequence which would compute + /// ( * ) as if to infinite precision, round the result to the nearest representable value, add to the + /// rounded result as if to infinite precision, and finally round to the nearest representable value. + /// + /// + public static void FusedMultiplyAdd(ReadOnlySpan x, ReadOnlySpan y, T addend, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanSpanScalarIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise result of ( * ) * for the specified tensors of numbers. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The third tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ([i] * ) + [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + /// This computes ( * ) as if to infinite precision, adds to that result as if to + /// infinite precision, and finally rounds to the nearest representable value. This differs from the non-fused sequence which would compute + /// ( * ) as if to infinite precision, round the result to the nearest representable value, add to the + /// rounded result as if to infinite precision, and finally round to the nearest representable value. + /// + /// + public static void FusedMultiplyAdd(ReadOnlySpan x, T y, ReadOnlySpan addend, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanScalarSpanIntoSpan>(x, y, addend, destination); + + /// Computes the element-wise negation of each number in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = -[i]. + /// + /// + /// If any of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Negate(ReadOnlySpan x, Span destination) + where T : IUnaryNegationOperators => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the Euclidean norm of the specified tensor of numbers. + /// The first tensor, represented as a span. + /// The norm. + /// + /// + /// This method effectively computes .Sqrt(TensorPrimitives.SumOfSquares(x)). + /// This is often referred to as the Euclidean norm or L2 norm. + /// It corresponds to the nrm2 method defined by BLAS1. + /// + /// + /// If any of the input values is equal to , the result value is also NaN. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Norm(ReadOnlySpan x) + where T : IRootFunctions => + T.Sqrt(SumOfSquares(x)); + + /// Computes the element-wise one's complement of numbers in the specified tensor. + /// The first tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = ~[i]. + /// + /// + public static void OnesComplement(ReadOnlySpan x, Span destination) + where T : IBitwiseOperators => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise population count of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.PopCount([i]). + /// + /// + public static void PopCount(ReadOnlySpan x, Span destination) + where T : IBinaryInteger => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise power of a number in a specified tensor raised to a number in another specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Pow([i], [i]). + /// + /// + public static void Pow(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IPowerFunctions => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise power of a number in a specified tensor raised to a number in another specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Pow([i], ). + /// + /// + public static void Pow(ReadOnlySpan x, T y, Span destination) + where T : IPowerFunctions => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise power of a number in a specified tensor raised to a number in another specified tensors. + /// The first tensor, represented as a scalar. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Pow(, [i]). + /// + /// + public static void Pow(T x, ReadOnlySpan y, Span destination) + where T : IPowerFunctions => + InvokeScalarSpanIntoSpan>(x, y, destination); + + /// Computes the product of all elements in the specified non-empty tensor of numbers. + /// The tensor, represented as a span. + /// The result of multiplying all elements in . + /// Length of must be greater than zero. + /// + /// + /// If any of the input values is equal to , the result value is also NaN. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Product(ReadOnlySpan x) + where T : IMultiplyOperators, IMultiplicativeIdentity + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + return Aggregate, MultiplyOperator>(x); + } + + /// Computes the product of the element-wise differences of the numbers in the specified non-empty tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The result of multiplying the element-wise subtraction of the elements in the second tensor from the first tensor. + /// Length of both input spans must be greater than zero. + /// and must have the same length. + /// + /// + /// This method effectively computes: + /// + /// Span<T> differences = ...; + /// TensorPrimitives.Subtract(x, y, differences); + /// T result = TensorPrimitives.Product(differences); + /// + /// but without requiring additional temporary storage for the intermediate differences. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T ProductOfDifferences(ReadOnlySpan x, ReadOnlySpan y) + where T : ISubtractionOperators, IMultiplyOperators, IMultiplicativeIdentity + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + return Aggregate, MultiplyOperator>(x, y); + } + + /// Computes the product of the element-wise sums of the numbers in the specified non-empty tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The result of multiplying the element-wise additions of the elements in each tensor. + /// Length of both input spans must be greater than zero. + /// and must have the same length. + /// + /// + /// This method effectively computes: + /// + /// Span<T> sums = ...; + /// TensorPrimitives.Add(x, y, sums); + /// T result = TensorPrimitives.Product(sums); + /// + /// but without requiring additional temporary storage for the intermediate sums. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T ProductOfSums(ReadOnlySpan x, ReadOnlySpan y) + where T : IAdditionOperators, IAdditiveIdentity, IMultiplyOperators, IMultiplicativeIdentity + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + return Aggregate, MultiplyOperator>(x, y); + } + + /// Computes the element-wise conversion of each number of radians in the specified tensor to degrees. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .RadiansToDegrees([i]). + /// + /// + public static void RadiansToDegrees(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise reciprocal of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and an element in is equal to zero. + /// + /// + /// This method effectively computes [i] = 1 / [i]. + /// + /// + public static void Reciprocal(ReadOnlySpan x, Span destination) + where T : IFloatingPoint => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise reciprocal of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and an element in is equal to zero. + /// + /// + /// This method effectively computes [i] = 1 / [i]. + /// + /// + public static void ReciprocalEstimate(ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise reciprocal of the square root of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and an element in is equal to zero. + /// + /// + /// This method effectively computes [i] = 1 / [i]. + /// + /// + public static void ReciprocalSqrt(ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise reciprocal of the square root of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is an integer type and an element in is equal to zero. + /// + /// + /// This method effectively computes [i] = 1 / [i]. + /// + /// + public static void ReciprocalSqrtEstimate(ReadOnlySpan x, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise n-th root of the values in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// The degree of the root to be computed, represented as a scalar. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.RootN([i], ). + /// + /// + public static void RootN(ReadOnlySpan x, int n, Span destination) + where T : IRootFunctions => + InvokeSpanIntoSpan(x, new RootNOperator(n), destination); + + /// Computes the element-wise rotation left of numbers in the specified tensor by the specified rotation amount. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// The number of bits to rotate, represented as a scalar. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.RotateLeft([i], ). + /// + /// + public static void RotateLeft(ReadOnlySpan x, int rotateAmount, Span destination) + where T : IBinaryInteger => + InvokeSpanIntoSpan(x, new RotateLeftOperator(rotateAmount), destination); + + /// Computes the element-wise rotation right of numbers in the specified tensor by the specified rotation amount. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// The number of bits to rotate, represented as a scalar. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.RotateRight([i], ). + /// + /// + public static void RotateRight(ReadOnlySpan x, int rotateAmount, Span destination) + where T : IBinaryInteger => + InvokeSpanIntoSpan(x, new RotateRightOperator(rotateAmount), destination); + + /// Computes the element-wise rounding of the numbers in the specified tensor + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Round([i]). + /// + /// + public static void Round(ReadOnlySpan x, Span destination) + where T : IFloatingPoint => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise rounding of the numbers in the specified tensor + /// The tensor, represented as a span. + /// The mode under which should be rounded. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Round([i], ). + /// + /// + public static void Round(ReadOnlySpan x, MidpointRounding mode, Span destination) + where T : IFloatingPoint + { + switch (mode) + { + case MidpointRounding.ToEven: + Round(x, destination); + return; + + case MidpointRounding.AwayFromZero: + InvokeSpanIntoSpan>(x, destination); + break; + + case MidpointRounding.ToZero: + Truncate(x, destination); + return; + + case MidpointRounding.ToNegativeInfinity: + Floor(x, destination); + return; + + case MidpointRounding.ToPositiveInfinity: + Ceiling(x, destination); + return; + + default: + throw new ArgumentException(SR.Format(SR.Argument_InvalidEnumValue, mode, typeof(MidpointRounding)), nameof(mode)); + } + } + + /// Computes the element-wise rounding of the numbers in the specified tensor + /// The tensor, represented as a span. + /// The number of fractional digits to which the numbers in should be rounded. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Round([i], ). + /// + /// + public static void Round(ReadOnlySpan x, int digits, Span destination) where T : IFloatingPoint => + Round(x, digits, MidpointRounding.ToEven, destination); + + /// Computes the element-wise rounding of the numbers in the specified tensor + /// The tensor, represented as a span. + /// The number of fractional digits to which the numbers in should be rounded. + /// The mode under which should be rounded. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// is invalid. + /// is invalid. + /// + /// + /// This method effectively computes [i] = T.Round([i], , ). + /// + /// + public static void Round(ReadOnlySpan x, int digits, MidpointRounding mode, Span destination) + where T : IFloatingPoint + { + if (digits == 0) + { + Round(x, mode, destination); + } + + ReadOnlySpan roundPower10; + if (typeof(T) == typeof(float)) + { + ReadOnlySpan roundPower10Single = [1e0f, 1e1f, 1e2f, 1e3f, 1e4f, 1e5f, 1e6f]; + roundPower10 = MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(roundPower10Single)), roundPower10Single.Length); + } + else if (typeof(T) == typeof(double)) + { + Debug.Assert(typeof(T) == typeof(double)); + ReadOnlySpan roundPower10Double = [1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, 1e12, 1e13, 1e14, 1e15]; + roundPower10 = MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(roundPower10Double)), roundPower10Double.Length); + } + else + { + if ((uint)mode > (uint)MidpointRounding.ToPositiveInfinity) + { + throw new ArgumentException(SR.Format(SR.Argument_InvalidEnumValue, mode, typeof(MidpointRounding)), nameof(mode)); + } + + InvokeSpanIntoSpan(x, new RoundFallbackOperator(digits, mode), destination); + return; + } + + if ((uint)digits >= (uint)roundPower10.Length) + { + throw new ArgumentOutOfRangeException(nameof(digits)); + } + + T power10 = roundPower10[digits]; + switch (mode) + { + case MidpointRounding.ToEven: + InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); + return; + + case MidpointRounding.AwayFromZero: + InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); + break; + + case MidpointRounding.ToZero: + InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); + return; + + case MidpointRounding.ToNegativeInfinity: + InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); + return; + + case MidpointRounding.ToPositiveInfinity: + InvokeSpanIntoSpan(x, new MultiplyRoundDivideOperator>(power10), destination); + return; + + default: + throw new ArgumentException(SR.Format(SR.Argument_InvalidEnumValue, mode, typeof(MidpointRounding)), nameof(mode)); + } + } + + /// Computes the element-wise product of numbers in the specified tensor and their base-radix raised to the specified power. + /// The tensor, represented as a span. + /// The value to which base-radix is raised before multipliying x, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.ILogB([i]). + /// + /// + public static void ScaleB(ReadOnlySpan x, int n, Span destination) + where T : IFloatingPointIeee754 => + InvokeSpanIntoSpan(x, new ScaleBOperator(n), destination); + + /// Computes the element-wise shifting left of numbers in the specified tensor by the specified shift amount. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// The number of bits to shift, represented as a scalar. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] << . + /// + /// + public static void ShiftLeft(ReadOnlySpan x, int shiftAmount, Span destination) + where T : IShiftOperators => + InvokeSpanIntoSpan(x, new ShiftLeftOperator(shiftAmount), destination); + + /// Computes the element-wise arithmetic (signed) shifting right of numbers in the specified tensor by the specified shift amount. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// The number of bits to shift, represented as a scalar. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] >> . + /// + /// + public static void ShiftRightArithmetic(ReadOnlySpan x, int shiftAmount, Span destination) + where T : IShiftOperators => + InvokeSpanIntoSpan(x, new ShiftRightArithmeticOperator(shiftAmount), destination); + + /// Computes the element-wise logical (unsigned) shifting right of numbers in the specified tensor by the specified shift amount. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// The number of bits to shift, represented as a scalar. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] >>> . + /// + /// + public static void ShiftRightLogical(ReadOnlySpan x, int shiftAmount, Span destination) + where T : IShiftOperators => + InvokeSpanIntoSpan(x, new ShiftRightLogicalOperator(shiftAmount), destination); + + /// Computes the element-wise sigmoid function on the specified non-empty tensor of numbers. + /// The tensor, represented as a span. + /// The destination tensor. + /// Destination is too short. + /// must not be empty. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = 1f / (1f + .Exp(-[i])). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Sigmoid(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + InvokeSpanIntoSpan>(x, destination); + } + + /// Computes the element-wise sine of the value in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Sin([i]). + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Sin(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise sine of the value in the specified tensor that has been multiplied by Pi. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .SinPi([i]). + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void SinPi(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hyperbolic sine of each radian angle in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Sinh([i]). + /// + /// + /// If a value is equal to , , or , + /// the corresponding destination location is set to that value. + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi / 180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Sinh(ReadOnlySpan x, Span destination) + where T : IHyperbolicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise sine and cosine of the value in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor for the element-wise sine result, represented as a span. + /// The destination tensor for the element-wise cosine result, represented as a span. + /// Destination is too short. + /// and or reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes ([i], [i]) = .SinCos([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void SinCos(ReadOnlySpan x, Span sinDestination, Span cosDestination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan_TwoOutputs>(x, sinDestination, cosDestination); + + /// Computes the element-wise sine and cosine of the value in the specified tensor that has been multiplied by Pi. + /// The tensor, represented as a span. + /// The destination tensor for the element-wise sine result, represented as a span. + /// The destination tensor for the element-wise cosine result, represented as a span. + /// Destination is too short. + /// and or reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes ([i], [i]) = .SinCos([i]). + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void SinCosPi(ReadOnlySpan x, Span sinPiDestination, Span cosPiDestination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan_TwoOutputs>(x, sinPiDestination, cosPiDestination); + + /// Computes the softmax function over the specified non-empty tensor of numbers. + /// The tensor, represented as a span. + /// The destination tensor. + /// Destination is too short. + /// must not be empty. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes a sum of .Exp(x[i]) for all elements in . + /// It then effectively computes [i] = .Exp([i]) / sum. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void SoftMax(ReadOnlySpan x, Span destination) + where T : IExponentialFunctions + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + + T expSum = Aggregate, AddOperator>(x); + + InvokeSpanScalarIntoSpan, DivideOperator>(x, expSum, destination); + } + + /// Computes the element-wise square root of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Sqrt([i]). + /// + /// + public static void Sqrt(ReadOnlySpan x, Span destination) + where T : IRootFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise difference between numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] - [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Subtract(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : ISubtractionOperators => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise difference between numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] - . + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Subtract(ReadOnlySpan x, T y, Span destination) + where T : ISubtractionOperators => + InvokeSpanScalarIntoSpan>(x, y, destination); + + /// Computes the element-wise difference between numbers in the specified tensors. + /// The first tensor, represented as a scalar. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = - [i]. + /// + /// + /// If either of the element-wise input values is equal to , the resulting element-wise value is also NaN. + /// + /// + public static void Subtract(T x, ReadOnlySpan y, Span destination) + where T : ISubtractionOperators => + InvokeScalarSpanIntoSpan>(x, y, destination); + + /// Computes the sum of all elements in the specified tensor of numbers. + /// The tensor, represented as a span. + /// The result of adding all elements in , or zero if is empty. + /// + /// + /// If any of the values in the input is equal to , the result is also NaN. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T Sum(ReadOnlySpan x) + where T : IAdditionOperators, IAdditiveIdentity => + Aggregate, AddOperator>(x); + + /// Computes the sum of the absolute values of every element in the specified tensor of numbers. + /// The tensor, represented as a span. + /// The result of adding the absolute value of every element in , or zero if is empty. + /// is a signed integer type and contained a value equal to 's minimum value. + /// + /// + /// This method effectively computes: + /// + /// Span<T> absoluteValues = ...; + /// TensorPrimitives.Abs(x, absoluteValues); + /// T result = TensorPrimitives.Sum(absoluteValues); + /// + /// but without requiring intermediate storage for the absolute values. It corresponds to the asum method defined by BLAS1. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T SumOfMagnitudes(ReadOnlySpan x) + where T : INumberBase => + Aggregate, AddOperator>(x); + + /// Computes the sum of the square of every element in the specified tensor of numbers. + /// The tensor, represented as a span. + /// The result of adding the square of every element in , or zero if is empty. + /// + /// + /// This method effectively computes: + /// + /// Span<T> squaredValues = ...; + /// TensorPrimitives.Multiply(x, x, squaredValues); + /// T result = TensorPrimitives.Sum(squaredValues); + /// + /// but without requiring intermediate storage for the squared values. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static T SumOfSquares(ReadOnlySpan x) + where T : IAdditionOperators, IAdditiveIdentity, IMultiplyOperators => + Aggregate, AddOperator>(x); + + /// Computes the element-wise tangent of the value in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Tan([i]). + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Tan(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise tangent of the value in the specified tensor that has been multiplied by Pi. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .TanPi([i]). + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void TanPi(ReadOnlySpan x, Span destination) + where T : ITrigonometricFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise hyperbolic tangent of each radian angle in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = .Tanh([i]). + /// + /// + /// If a value is equal to , the corresponding destination location is set to -1. + /// If a value is equal to , the corresponding destination location is set to 1. + /// If a value is equal to , the corresponding destination location is set to NaN. + /// + /// + /// The angles in x must be in radians. Use or multiply by .Pi / 180 to convert degrees to radians. + /// + /// + /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different + /// operating systems or architectures. + /// + /// + public static void Tanh(ReadOnlySpan x, Span destination) + where T : IHyperbolicFunctions => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise trailing zero count of numbers in the specified tensor. + /// The tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.TrailingZeroCount([i]). + /// + /// + public static void TrailingZeroCount(ReadOnlySpan x, Span destination) + where T : IBinaryInteger => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise truncation of numbers in the specified tensor. + /// The first tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = T.Truncate([i]). + /// + /// + public static void Truncate(ReadOnlySpan x, Span destination) + where T : IFloatingPoint => + InvokeSpanIntoSpan>(x, destination); + + /// Computes the element-wise XOR of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a span. + /// The destination tensor, represented as a span. + /// Length of must be same as length of . + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] ^ [i]. + /// + /// + public static void Xor(ReadOnlySpan x, ReadOnlySpan y, Span destination) + where T : IBitwiseOperators => + InvokeSpanSpanIntoSpan>(x, y, destination); + + /// Computes the element-wise XOR of numbers in the specified tensors. + /// The first tensor, represented as a span. + /// The second tensor, represented as a scalar. + /// The destination tensor, represented as a span. + /// Destination is too short. + /// and reference overlapping memory locations and do not begin at the same location. + /// + /// + /// This method effectively computes [i] = [i] ^ . + /// + /// + public static void Xor(ReadOnlySpan x, T y, Span destination) + where T : IBitwiseOperators => + InvokeSpanScalarIntoSpan>(x, y, destination); + } +} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Tan.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Tan.cs deleted file mode 100644 index 926bca5221a6b7..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Tan.cs +++ /dev/null @@ -1,396 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise tangent of the value in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Tan([i]). - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Tan(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Tan(x) - private readonly struct TanOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - // This code is based on `vrs4_tan` and `vrd2_tan` from amd/aocl-libm-ose - // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // Implementation notes from amd/aocl-libm-ose: - // -------------------------------------------- - // A given x is reduced into the form: - // |x| = (N * π/2) + F - // Where N is an integer obtained using: - // N = round(x * 2/π) - // And F is a fraction part lying in the interval - // [-π/4, +π/4]; - // obtained as F = |x| - (N * π/2) - // Thus tan(x) is given by - // tan(x) = tan((N * π/2) + F) = tan(F) - // when N is even, = -cot(F) = -1/tan(F) - // when N is odd, tan(F) is approximated using a polynomial - // obtained from Remez approximation from Sollya. - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Tan(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - return TanOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return TanOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - return TanOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return TanOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - return TanOperatorSingle.Invoke(x.AsSingle()).As(); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - return TanOperatorDouble.Invoke(x.AsDouble()).As(); - } - } - } - - /// float.Tan(x) - private readonly struct TanOperatorSingle : IUnaryOperator - { - internal const uint SignMask = 0x7FFFFFFFu; - internal const uint MaxVectorizedValue = 0x49800000u; - private const float AlmHuge = 1.2582912e7f; - private const float Pi_Tail2 = 4.371139e-8f; - private const float Pi_Tail3 = 1.7151245e-15f; - private const float C1 = 0.33333358f; - private const float C2 = 0.13332522f; - private const float C3 = 0.05407107f; - private const float C4 = 0.021237267f; - private const float C5 = 0.010932301f; - private const float C6 = -1.5722344e-5f; - private const float C7 = 0.0044221194f; - - public static bool Vectorizable => true; - - public static float Invoke(float x) => float.Tan(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 uxMasked = Vector128.Abs(x); - if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(2 / float.Pi), Vector128.Create(AlmHuge)); - Vector128 odd = dn.AsUInt32() << 31; - dn -= Vector128.Create(AlmHuge); - - Vector128 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-float.Pi / 2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail3), f); - - // POLY_EVAL_ODD_15 - Vector128 f2 = f * f; - Vector128 f4 = f2 * f2; - Vector128 f8 = f4 * f4; - Vector128 f12 = f8 * f4; - Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); - Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C4), f2, Vector128.Create(C3)); - Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C6), f2, Vector128.Create(C5)); - Vector128 b1 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector128 b2 = MultiplyAddEstimateOperator.Invoke(f8, a3, f12 * Vector128.Create(C7)); - Vector128 poly = MultiplyAddEstimateOperator.Invoke(f * f2, b1 + b2, f); - - Vector128 result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector128.Create(~SignMask))).AsSingle(); - return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128.Zero).AsSingle(), - result, - Vector128.Create(-1f) / result); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 uxMasked = Vector256.Abs(x); - if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(2 / float.Pi), Vector256.Create(AlmHuge)); - Vector256 odd = dn.AsUInt32() << 31; - dn -= Vector256.Create(AlmHuge); - - Vector256 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-float.Pi / 2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail3), f); - - // POLY_EVAL_ODD_15 - Vector256 f2 = f * f; - Vector256 f4 = f2 * f2; - Vector256 f8 = f4 * f4; - Vector256 f12 = f8 * f4; - Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); - Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C4), f2, Vector256.Create(C3)); - Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C6), f2, Vector256.Create(C5)); - Vector256 b1 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector256 b2 = MultiplyAddEstimateOperator.Invoke(f8, a3, f12 * Vector256.Create(C7)); - Vector256 poly = MultiplyAddEstimateOperator.Invoke(f * f2, b1 + b2, f); - - Vector256 result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector256.Create(~SignMask))).AsSingle(); - return Vector256.ConditionalSelect(Vector256.Equals(odd, Vector256.Zero).AsSingle(), - result, - Vector256.Create(-1f) / result); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 uxMasked = Vector512.Abs(x); - if (Vector512.GreaterThanAny(uxMasked.AsUInt32(), Vector512.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(2 / float.Pi), Vector512.Create(AlmHuge)); - Vector512 odd = dn.AsUInt32() << 31; - dn -= Vector512.Create(AlmHuge); - - Vector512 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-float.Pi / 2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail3), f); - - // POLY_EVAL_ODD_15 - Vector512 f2 = f * f; - Vector512 f4 = f2 * f2; - Vector512 f8 = f4 * f4; - Vector512 f12 = f8 * f4; - Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); - Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C4), f2, Vector512.Create(C3)); - Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C6), f2, Vector512.Create(C5)); - Vector512 b1 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); - Vector512 b2 = MultiplyAddEstimateOperator.Invoke(f8, a3, f12 * Vector512.Create(C7)); - Vector512 poly = MultiplyAddEstimateOperator.Invoke(f * f2, b1 + b2, f); - - Vector512 result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector512.Create(~SignMask))).AsSingle(); - return Vector512.ConditionalSelect(Vector512.Equals(odd, Vector512.Zero).AsSingle(), - result, - Vector512.Create(-1f) / result); - } - } - - /// double.Tan(x) - private readonly struct TanOperatorDouble : IUnaryOperator - { - internal const ulong SignMask = 0x7FFFFFFFFFFFFFFFul; - internal const ulong MaxVectorizedValue = 0x4160000000000000ul; - private const double AlmHuge = 6.755399441055744e15; - private const double HalfPi2 = 6.123233995736766E-17; - private const double HalfPi3 = -1.4973849048591698E-33; - private const double C1 = 0.33333333333332493; - private const double C3 = 0.133333333334343; - private const double C5 = 0.0539682539203796; - private const double C7 = 0.02186948972198256; - private const double C9 = 0.008863217894198291; - private const double C11 = 0.003592298593761111; - private const double C13 = 0.0014547086183165365; - private const double C15 = 5.952456856028558E-4; - private const double C17 = 2.2190741289936845E-4; - private const double C19 = 1.3739809957985104E-4; - private const double C21 = -2.7500197359895707E-5; - private const double C23 = 9.038741690184683E-5; - private const double C25 = -4.534076545538694E-5; - private const double C27 = 2.0966522562190197E-5; - - public static bool Vectorizable => true; - - public static double Invoke(double x) => double.Tan(x); - - public static Vector128 Invoke(Vector128 x) - { - Vector128 uxMasked = Vector128.Abs(x); - if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = |x| * (2/π) - Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(2 / double.Pi), Vector128.Create(AlmHuge)); - Vector128 odd = dn.AsUInt64() << 63; - dn -= Vector128.Create(AlmHuge); - - // f = |x| - (dn * π/2) - Vector128 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-double.Pi / 2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-HalfPi2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-HalfPi3), f); - - // POLY_EVAL_ODD_29 - Vector128 g = f * f; - Vector128 g2 = g * g; - Vector128 g3 = g * g2; - Vector128 g5 = g3 * g2; - Vector128 g7 = g5 * g2; - Vector128 g9 = g7 * g2; - Vector128 g11 = g9 * g2; - Vector128 g13 = g11 * g2; - Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C3), g, Vector128.Create(C1)); - Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C7), g, Vector128.Create(C5)); - Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C11), g, Vector128.Create(C9)); - Vector128 a4 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C15), g, Vector128.Create(C13)); - Vector128 a5 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C19), g, Vector128.Create(C17)); - Vector128 a6 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C23), g, Vector128.Create(C21)); - Vector128 a7 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C27), g, Vector128.Create(C25)); - Vector128 b1 = MultiplyAddEstimateOperator.Invoke(g, a1, g3 * a2); - Vector128 b2 = MultiplyAddEstimateOperator.Invoke(g5, a3, g7 * a4); - Vector128 b3 = MultiplyAddEstimateOperator.Invoke(g9, a5, g11 * a6); - Vector128 q = MultiplyAddEstimateOperator.Invoke(g13, a7, b1 + b2 + b3); - Vector128 poly = MultiplyAddEstimateOperator.Invoke(f, q, f); - - Vector128 result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector128.Create(~SignMask))).AsDouble(); - return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128.Zero).AsDouble(), - result, - Vector128.Create(-1.0) / result); - } - - public static Vector256 Invoke(Vector256 x) - { - Vector256 uxMasked = Vector256.Abs(x); - if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = |x| * (2/π) - Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(2 / double.Pi), Vector256.Create(AlmHuge)); - Vector256 odd = dn.AsUInt64() << 63; - dn -= Vector256.Create(AlmHuge); - - // f = |x| - (dn * π/2) - Vector256 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-double.Pi / 2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-HalfPi2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-HalfPi3), f); - - // POLY_EVAL_ODD_29 - Vector256 g = f * f; - Vector256 g2 = g * g; - Vector256 g3 = g * g2; - Vector256 g5 = g3 * g2; - Vector256 g7 = g5 * g2; - Vector256 g9 = g7 * g2; - Vector256 g11 = g9 * g2; - Vector256 g13 = g11 * g2; - Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C3), g, Vector256.Create(C1)); - Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C7), g, Vector256.Create(C5)); - Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C11), g, Vector256.Create(C9)); - Vector256 a4 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C15), g, Vector256.Create(C13)); - Vector256 a5 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C19), g, Vector256.Create(C17)); - Vector256 a6 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C23), g, Vector256.Create(C21)); - Vector256 a7 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C27), g, Vector256.Create(C25)); - Vector256 b1 = MultiplyAddEstimateOperator.Invoke(g, a1, g3 * a2); - Vector256 b2 = MultiplyAddEstimateOperator.Invoke(g5, a3, g7 * a4); - Vector256 b3 = MultiplyAddEstimateOperator.Invoke(g9, a5, g11 * a6); - Vector256 q = MultiplyAddEstimateOperator.Invoke(g13, a7, b1 + b2 + b3); - Vector256 poly = MultiplyAddEstimateOperator.Invoke(f, q, f); - - Vector256 result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector256.Create(~SignMask))).AsDouble(); - return Vector256.ConditionalSelect(Vector256.Equals(odd, Vector256.Zero).AsDouble(), - result, - Vector256.Create(-1.0) / result); - } - - public static Vector512 Invoke(Vector512 x) - { - Vector512 uxMasked = Vector512.Abs(x); - if (Vector512.GreaterThanAny(uxMasked.AsUInt64(), Vector512.Create(MaxVectorizedValue))) - { - return ApplyScalar(x); - } - - // dn = |x| * (2/π) - Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(2 / double.Pi), Vector512.Create(AlmHuge)); - Vector512 odd = dn.AsUInt64() << 63; - dn -= Vector512.Create(AlmHuge); - - // f = |x| - (dn * π/2) - Vector512 f = uxMasked; - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-double.Pi / 2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-HalfPi2), f); - f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-HalfPi3), f); - - // POLY_EVAL_ODD_29 - Vector512 g = f * f; - Vector512 g2 = g * g; - Vector512 g3 = g * g2; - Vector512 g5 = g3 * g2; - Vector512 g7 = g5 * g2; - Vector512 g9 = g7 * g2; - Vector512 g11 = g9 * g2; - Vector512 g13 = g11 * g2; - Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C3), g, Vector512.Create(C1)); - Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C7), g, Vector512.Create(C5)); - Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C11), g, Vector512.Create(C9)); - Vector512 a4 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C15), g, Vector512.Create(C13)); - Vector512 a5 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C19), g, Vector512.Create(C17)); - Vector512 a6 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C23), g, Vector512.Create(C21)); - Vector512 a7 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C27), g, Vector512.Create(C25)); - Vector512 b1 = MultiplyAddEstimateOperator.Invoke(g, a1, g3 * a2); - Vector512 b2 = MultiplyAddEstimateOperator.Invoke(g5, a3, g7 * a4); - Vector512 b3 = MultiplyAddEstimateOperator.Invoke(g9, a5, g11 * a6); - Vector512 q = MultiplyAddEstimateOperator.Invoke(g13, a7, b1 + b2 + b3); - Vector512 poly = MultiplyAddEstimateOperator.Invoke(f, q, f); - - Vector512 result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector512.Create(~SignMask))).AsDouble(); - return Vector512.ConditionalSelect(Vector512.Equals(odd, Vector512.Zero).AsDouble(), - result, - Vector512.Create(-1.0) / result); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.TanPi.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.TanPi.cs deleted file mode 100644 index 962dba7c185839..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.TanPi.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise tangent of the value in the specified tensor that has been multiplied by Pi. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .TanPi([i]). - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi/180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void TanPi(ReadOnlySpan x, Span destination) - where T : ITrigonometricFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.TanPi(x) - private readonly struct TanPiOperator : IUnaryOperator - where T : ITrigonometricFunctions - { - public static bool Vectorizable => false; - public static T Invoke(T x) => T.TanPi(x); - public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); - public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); - public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Tanh.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Tanh.cs deleted file mode 100644 index 7ac7d9b37619e9..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Tanh.cs +++ /dev/null @@ -1,130 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise hyperbolic tangent of each radian angle in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = .Tanh([i]). - /// - /// - /// If a value is equal to , the corresponding destination location is set to -1. - /// If a value is equal to , the corresponding destination location is set to 1. - /// If a value is equal to , the corresponding destination location is set to NaN. - /// - /// - /// The angles in x must be in radians. Use or multiply by .Pi / 180 to convert degrees to radians. - /// - /// - /// This method may call into the underlying C runtime or employ instructions specific to the current architecture. Exact results may differ between different - /// operating systems or architectures. - /// - /// - public static void Tanh(ReadOnlySpan x, Span destination) - where T : IHyperbolicFunctions => - InvokeSpanIntoSpan>(x, destination); - - /// T.Tanh(x) - internal readonly struct TanhOperator : IUnaryOperator - where T : IHyperbolicFunctions - { - // This code is based on `vrs4_tanhf` from amd/aocl-libm-ose - // Copyright (C) 2008-2022 Advanced Micro Devices, Inc. All rights reserved. - // - // Licensed under the BSD 3-Clause "New" or "Revised" License - // See THIRD-PARTY-NOTICES.TXT for the full license text - - // To compute vrs4_tanhf(v_f32x4_t x) - // Let y = |x| - // If 0 <= y < 0x1.154246p3 - // Let z = e^(-2.0 * y) - 1 -(1) - // - // Using (1), tanhf(y) can be calculated as, - // tanhf(y) = -z / (z + 2.0) - // - // For other cases, call scalar tanhf() - // - // If x < 0, then we use the identity - // tanhf(-x) = -tanhf(x) - - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Tanh(x); - - public static Vector128 Invoke(Vector128 t) - { - if (typeof(T) == typeof(float)) - { - Vector128 x = t.AsSingle(); - - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2f) * y); - Vector128 sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue); - return (sign ^ (-z / (z + Vector128.Create(2f))).AsUInt32()).As(); - } - else - { - Vector128 x = t.AsDouble(); - - Vector128 y = Vector128.Abs(x); - Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2d) * y); - Vector128 sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue); - return (sign ^ (-z / (z + Vector128.Create(2d))).AsUInt64()).As(); - } - } - - public static Vector256 Invoke(Vector256 t) - { - if (typeof(T) == typeof(float)) - { - Vector256 x = t.AsSingle(); - - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2f) * y); - Vector256 sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue); - return (sign ^ (-z / (z + Vector256.Create(2f))).AsUInt32()).As(); - } - else - { - Vector256 x = t.AsDouble(); - - Vector256 y = Vector256.Abs(x); - Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2d) * y); - Vector256 sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue); - return (sign ^ (-z / (z + Vector256.Create(2d))).AsUInt64()).As(); - } - } - - public static Vector512 Invoke(Vector512 t) - { - if (typeof(T) == typeof(float)) - { - Vector512 x = t.AsSingle(); - - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2f) * y); - Vector512 sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue); - return (sign ^ (-z / (z + Vector512.Create(2f))).AsUInt32()).As(); - } - else - { - Vector512 x = t.AsDouble(); - - Vector512 y = Vector512.Abs(x); - Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2d) * y); - Vector512 sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue); - return (sign ^ (-z / (z + Vector512.Create(2d))).AsUInt64()).As(); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.TrailingZeroCount.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.TrailingZeroCount.cs deleted file mode 100644 index 156bafd3697b96..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.TrailingZeroCount.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; - -#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise trailing zero count of numbers in the specified tensor. - /// The tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.TrailingZeroCount([i]). - /// - /// - public static void TrailingZeroCount(ReadOnlySpan x, Span destination) - where T : IBinaryInteger => - InvokeSpanIntoSpan>(x, destination); - - /// T.TrailingZeroCount(x) - private readonly unsafe struct TrailingZeroCountOperator : IUnaryOperator where T : IBinaryInteger - { - public static bool Vectorizable => - (AdvSimd.Arm64.IsSupported && sizeof(T) == 1) || - PopCountOperator.Vectorizable; // http://0x80.pl/notesen/2023-01-31-avx512-bsf.html#trailing-zeros-simplified - - public static T Invoke(T x) => T.TrailingZeroCount(x); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector128 Invoke(Vector128 x) - { - if (AdvSimd.Arm64.IsSupported && sizeof(T) == 1) - { - return AdvSimd.LeadingZeroCount(AdvSimd.Arm64.ReverseElementBits(x.AsByte())).As(); - } - - Debug.Assert(PopCountOperator.Vectorizable); - return PopCountOperator.Invoke(~x & (x - Vector128.One)); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector256 Invoke(Vector256 x) - { - if (PopCountOperator.Vectorizable) - { - return PopCountOperator.Invoke(~x & (x - Vector256.One)); - } - - return Vector256.Create(Invoke(x.GetLower()), Invoke(x.GetUpper())); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static Vector512 Invoke(Vector512 x) - { - if (PopCountOperator.Vectorizable) - { - return PopCountOperator.Invoke(~x & (x - Vector512.One)); - } - - return Vector512.Create(Invoke(x.GetLower()), Invoke(x.GetUpper())); - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Truncate.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Truncate.cs deleted file mode 100644 index 3a2e3cea290afb..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Truncate.cs +++ /dev/null @@ -1,102 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise truncation of numbers in the specified tensor. - /// The first tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = T.Truncate([i]). - /// - /// - public static void Truncate(ReadOnlySpan x, Span destination) - where T : IFloatingPoint => - InvokeSpanIntoSpan>(x, destination); - - private readonly struct TruncateOperator : IUnaryOperator where T : IFloatingPoint - { - public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); - - public static T Invoke(T x) => T.Truncate(x); - - public static Vector128 Invoke(Vector128 x) - { - if (typeof(T) == typeof(float)) - { - if (Sse41.IsSupported) return Sse41.RoundToZero(x.AsSingle()).As(); - if (AdvSimd.IsSupported) return AdvSimd.RoundToZero(x.AsSingle()).As(); - - return Vector128.ConditionalSelect(Vector128.GreaterThanOrEqual(x, Vector128.Zero), - Vector128.Floor(x.AsSingle()).As(), - Vector128.Ceiling(x.AsSingle()).As()); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - - if (Sse41.IsSupported) return Sse41.RoundToZero(x.AsDouble()).As(); - if (AdvSimd.Arm64.IsSupported) return AdvSimd.Arm64.RoundToZero(x.AsDouble()).As(); - - return Vector128.ConditionalSelect(Vector128.GreaterThanOrEqual(x, Vector128.Zero), - Vector128.Floor(x.AsDouble()).As(), - Vector128.Ceiling(x.AsDouble()).As()); - } - } - - public static Vector256 Invoke(Vector256 x) - { - if (typeof(T) == typeof(float)) - { - if (Avx.IsSupported) return Avx.RoundToZero(x.AsSingle()).As(); - - return Vector256.ConditionalSelect(Vector256.GreaterThanOrEqual(x, Vector256.Zero), - Vector256.Floor(x.AsSingle()).As(), - Vector256.Ceiling(x.AsSingle()).As()); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - - if (Avx.IsSupported) return Avx.RoundToZero(x.AsDouble()).As(); - - return Vector256.ConditionalSelect(Vector256.GreaterThanOrEqual(x, Vector256.Zero), - Vector256.Floor(x.AsDouble()).As(), - Vector256.Ceiling(x.AsDouble()).As()); - } - } - - public static Vector512 Invoke(Vector512 x) - { - if (typeof(T) == typeof(float)) - { - if (Avx512F.IsSupported) return Avx512F.RoundScale(x.AsSingle(), 0b11).As(); - - return Vector512.ConditionalSelect(Vector512.GreaterThanOrEqual(x, Vector512.Zero), - Vector512.Floor(x.AsSingle()).As(), - Vector512.Ceiling(x.AsSingle()).As()); - } - else - { - Debug.Assert(typeof(T) == typeof(double)); - - if (Avx512F.IsSupported) return Avx512F.RoundScale(x.AsDouble(), 0b11).As(); - - return Vector512.ConditionalSelect(Vector512.GreaterThanOrEqual(x, Vector512.Zero), - Vector512.Floor(x.AsDouble()).As(), - Vector512.Ceiling(x.AsDouble()).As()); - } - } - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Xor.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Xor.cs deleted file mode 100644 index 545298192dd7e0..00000000000000 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.Xor.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.Intrinsics; - -namespace System.Numerics.Tensors -{ - public static partial class TensorPrimitives - { - /// Computes the element-wise XOR of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a span. - /// The destination tensor, represented as a span. - /// Length of must be same as length of . - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] ^ [i]. - /// - /// - public static void Xor(ReadOnlySpan x, ReadOnlySpan y, Span destination) - where T : IBitwiseOperators => - InvokeSpanSpanIntoSpan>(x, y, destination); - - /// Computes the element-wise XOR of numbers in the specified tensors. - /// The first tensor, represented as a span. - /// The second tensor, represented as a scalar. - /// The destination tensor, represented as a span. - /// Destination is too short. - /// and reference overlapping memory locations and do not begin at the same location. - /// - /// - /// This method effectively computes [i] = [i] ^ . - /// - /// - public static void Xor(ReadOnlySpan x, T y, Span destination) - where T : IBitwiseOperators => - InvokeSpanScalarIntoSpan>(x, y, destination); - - /// x ^ y - private readonly struct XorOperator : IBinaryOperator where T : IBitwiseOperators - { - public static bool Vectorizable => true; - public static T Invoke(T x, T y) => x ^ y; - public static Vector128 Invoke(Vector128 x, Vector128 y) => x ^ y; - public static Vector256 Invoke(Vector256 x, Vector256 y) => x ^ y; - public static Vector512 Invoke(Vector512 x, Vector512 y) => x ^ y; - } - } -} diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs new file mode 100644 index 00000000000000..dfd0897e701816 --- /dev/null +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/netcore/TensorPrimitives.netcore.cs @@ -0,0 +1,20366 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; + +#pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type + +namespace System.Numerics.Tensors +{ + public static unsafe partial class TensorPrimitives + { + /// Defines the threshold, in bytes, at which non-temporal stores will be used. + /// + /// A non-temporal store is one that allows the CPU to bypass the cache when writing to memory. + /// + /// This can be beneficial when working with large amounts of memory where the writes would otherwise + /// cause large amounts of repeated updates and evictions. The hardware optimization manuals recommend + /// the threshold to be roughly half the size of the last level of on-die cache -- that is, if you have approximately + /// 4MB of L3 cache per core, you'd want this to be approx. 1-2MB, depending on if hyperthreading was enabled. + /// + /// However, actually computing the amount of L3 cache per core can be tricky or error prone. Native memcpy + /// algorithms use a constant threshold that is typically around 256KB and we match that here for simplicity. This + /// threshold accounts for most processors in the last 10-15 years that had approx. 1MB L3 per core and support + /// hyperthreading, giving a per core last level cache of approx. 512KB. + /// + private const nuint NonTemporalByteThreshold = 256 * 1024; + + /// + /// Copies to , converting each + /// value to its nearest representable half-precision floating-point value. + /// + /// The source span from which to copy values. + /// The destination span into which the converted values should be written. + /// Destination is too short. + /// + /// + /// This method effectively computes [i] = (Half)[i]. + /// + /// + /// and must not overlap. If they do, behavior is undefined. + /// + /// + public static void ConvertToHalf(ReadOnlySpan source, Span destination) => + ConvertTruncating(source, destination); + + /// + /// Copies to , converting each half-precision + /// floating-point value to its nearest representable value. + /// + /// The source span from which to copy values. + /// The destination span into which the converted values should be written. + /// Destination is too short. + /// + /// + /// This method effectively computes [i] = (float)[i]. + /// + /// + /// and must not overlap. If they do, behavior is undefined. + /// + /// + public static void ConvertToSingle(ReadOnlySpan source, Span destination) => + ConvertTruncating(source, destination); + + /// Computes the cosine similarity between the two specified non-empty, equal-length tensors of single-precision floating-point numbers. + /// Assumes arguments have already been validated to be non-empty and equal length. + private static T CosineSimilarityCore(ReadOnlySpan x, ReadOnlySpan y) where T : IRootFunctions + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + // Compute the same as: + // TensorPrimitives.Dot(x, y) / (Math.Sqrt(TensorPrimitives.SumOfSquares(x)) * Math.Sqrt(TensorPrimitives.SumOfSquares(y))) + // but only looping over each span once. + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + + Vector512 dotProductVector = Vector512.Zero; + Vector512 xSumOfSquaresVector = Vector512.Zero; + Vector512 ySumOfSquaresVector = Vector512.Zero; + + // Process vectors, summing their dot products and squares, as long as there's a vector's worth remaining. + int oneVectorFromEnd = x.Length - Vector512.Count; + int i = 0; + do + { + Vector512 xVec = Vector512.LoadUnsafe(ref xRef, (uint)i); + Vector512 yVec = Vector512.LoadUnsafe(ref yRef, (uint)i); + + dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); + xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); + ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); + + i += Vector512.Count; + } + while (i <= oneVectorFromEnd); + + // Process the last vector in the span, masking off elements already processed. + if (i != x.Length) + { + Vector512 xVec = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + Vector512 yVec = Vector512.LoadUnsafe(ref yRef, (uint)(x.Length - Vector512.Count)); + + Vector512 remainderMask = CreateRemainderMaskVector512(x.Length - i); + xVec &= remainderMask; + yVec &= remainderMask; + + dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); + xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); + ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); + } + + // Sum(X * Y) / (|X| * |Y|) + return + Vector512.Sum(dotProductVector) / + (T.Sqrt(Vector512.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector512.Sum(ySumOfSquaresVector))); + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + + Vector256 dotProductVector = Vector256.Zero; + Vector256 xSumOfSquaresVector = Vector256.Zero; + Vector256 ySumOfSquaresVector = Vector256.Zero; + + // Process vectors, summing their dot products and squares, as long as there's a vector's worth remaining. + int oneVectorFromEnd = x.Length - Vector256.Count; + int i = 0; + do + { + Vector256 xVec = Vector256.LoadUnsafe(ref xRef, (uint)i); + Vector256 yVec = Vector256.LoadUnsafe(ref yRef, (uint)i); + + dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); + xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); + ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); + + i += Vector256.Count; + } + while (i <= oneVectorFromEnd); + + // Process the last vector in the span, masking off elements already processed. + if (i != x.Length) + { + Vector256 xVec = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + Vector256 yVec = Vector256.LoadUnsafe(ref yRef, (uint)(x.Length - Vector256.Count)); + + Vector256 remainderMask = CreateRemainderMaskVector256(x.Length - i); + xVec &= remainderMask; + yVec &= remainderMask; + + dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); + xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); + ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); + } + + // Sum(X * Y) / (|X| * |Y|) + return + Vector256.Sum(dotProductVector) / + (T.Sqrt(Vector256.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector256.Sum(ySumOfSquaresVector))); + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + { + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + + Vector128 dotProductVector = Vector128.Zero; + Vector128 xSumOfSquaresVector = Vector128.Zero; + Vector128 ySumOfSquaresVector = Vector128.Zero; + + // Process vectors, summing their dot products and squares, as long as there's a vector's worth remaining. + int oneVectorFromEnd = x.Length - Vector128.Count; + int i = 0; + do + { + Vector128 xVec = Vector128.LoadUnsafe(ref xRef, (uint)i); + Vector128 yVec = Vector128.LoadUnsafe(ref yRef, (uint)i); + + dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); + xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); + ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); + + i += Vector128.Count; + } + while (i <= oneVectorFromEnd); + + // Process the last vector in the span, masking off elements already processed. + if (i != x.Length) + { + Vector128 xVec = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + Vector128 yVec = Vector128.LoadUnsafe(ref yRef, (uint)(x.Length - Vector128.Count)); + + Vector128 remainderMask = CreateRemainderMaskVector128(x.Length - i); + xVec &= remainderMask; + yVec &= remainderMask; + + dotProductVector = MultiplyAddEstimateOperator.Invoke(xVec, yVec, dotProductVector); + xSumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(xVec, xVec, xSumOfSquaresVector); + ySumOfSquaresVector = MultiplyAddEstimateOperator.Invoke(yVec, yVec, ySumOfSquaresVector); + } + + // Sum(X * Y) / (|X| * |Y|) + return + Vector128.Sum(dotProductVector) / + (T.Sqrt(Vector128.Sum(xSumOfSquaresVector)) * T.Sqrt(Vector128.Sum(ySumOfSquaresVector))); + } + + // Vectorization isn't supported or there are too few elements to vectorize. + // Use a scalar implementation. + T dotProduct = T.Zero, xSumOfSquares = T.Zero, ySumOfSquares = T.Zero; + for (int i = 0; i < x.Length; i++) + { + dotProduct = MultiplyAddEstimateOperator.Invoke(x[i], y[i], dotProduct); + xSumOfSquares = MultiplyAddEstimateOperator.Invoke(x[i], x[i], xSumOfSquares); + ySumOfSquares = MultiplyAddEstimateOperator.Invoke(y[i], y[i], ySumOfSquares); + } + + // Sum(X * Y) / (|X| * |Y|) + return + dotProduct / + (T.Sqrt(xSumOfSquares) * T.Sqrt(ySumOfSquares)); + } + + /// Performs an aggregation over all elements in to produce a single-precision floating-point value. + /// The element type. + /// Specifies the transform operation that should be applied to each element loaded from . + /// + /// Specifies the aggregation binary operation that should be applied to multiple values to aggregate them into a single value. + /// The aggregation is applied after the transform is applied to each element. + /// + private static T Aggregate( + ReadOnlySpan x) + where TTransformOperator : struct, IUnaryOperator + where TAggregationOperator : struct, IAggregationOperator + { + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TTransformOperator.Vectorizable) + { + T result; + + if (remainder >= (uint)Vector512.Count) + { + result = Vectorized512(ref xRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + result = VectorizedSmall(ref xRef, remainder); + } + + return result; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TTransformOperator.Vectorizable) + { + T result; + + if (remainder >= (uint)Vector256.Count) + { + result = Vectorized256(ref xRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + result = VectorizedSmall(ref xRef, remainder); + } + + return result; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TTransformOperator.Vectorizable) + { + T result; + + if (remainder >= (uint)Vector128.Count) + { + result = Vectorized128(ref xRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + result = VectorizedSmall(ref xRef, remainder); + } + + return result; + } + + // This is the software fallback when no acceleration is available. + // It requires no branches to hit. + + return SoftwareFallback(ref xRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T SoftwareFallback(ref T xRef, nuint length) + { + T result = TAggregationOperator.IdentityValue; + + for (nuint i = 0; i < length; i++) + { + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, i))); + } + + return result; + } + + static T Vectorized128(ref T xRef, nuint remainder) + { + Vector128 vresult = Vector128.Create(TAggregationOperator.IdentityValue); + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + nuint misalignment = 0; + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + { + T* xPtr = px; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + misalignment = ((uint)sizeof(Vector128) - ((nuint)xPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + + Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + // We only need to load, so there isn't a lot of benefit to doing non-temporal operations + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); + vector2 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); + vector3 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); + vector4 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We load, process, and store the next four vectors + + vector1 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); + vector2 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); + vector3 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); + vector4 = TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + } + } + + // Store the first block. Handling this separately simplifies the latter code as we know + // they come after and so we can relegate it to full blocks or the trailing elements + + beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, beg); + + // Process the remaining [0, Count * 7] elements via a jump table + // + // We end up handling any trailing elements in case 0 and in the + // worst case end up just doing the identity operation here if there + // were no trailing elements. + + (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector128.Count); + blocks -= (misalignment == 0) ? 1u : 0u; + remainder -= trailing; + + switch (blocks) + { + case 7: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 6; + } + + case 6: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 5; + } + + case 5: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 4; + } + + case 4: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 3; + } + + case 3: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 2; + } + + case 2: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 1; + } + + case 1: + { + Vector128 vector = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 1))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 0; + } + + case 0: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, end); + break; + } + } + + return TAggregationOperator.Invoke(vresult); + } + + static T Vectorized256(ref T xRef, nuint remainder) + { + Vector256 vresult = Vector256.Create(TAggregationOperator.IdentityValue); + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + nuint misalignment = 0; + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + { + T* xPtr = px; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + misalignment = ((uint)sizeof(Vector256) - ((nuint)xPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + + Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + // We only need to load, so there isn't a lot of benefit to doing non-temporal operations + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); + vector2 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); + vector3 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); + vector4 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We load, process, and store the next four vectors + + vector1 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); + vector2 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); + vector3 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); + vector4 = TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + } + } + + // Store the first block. Handling this separately simplifies the latter code as we know + // they come after and so we can relegate it to full blocks or the trailing elements + + beg = Vector256.ConditionalSelect(CreateAlignmentMaskVector256((int)misalignment), beg, Vector256.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, beg); + + // Process the remaining [0, Count * 7] elements via a jump table + // + // We end up handling any trailing elements in case 0 and in the + // worst case end up just doing the identity operation here if there + // were no trailing elements. + + (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector256.Count); + blocks -= (misalignment == 0) ? 1u : 0u; + remainder -= trailing; + + switch (blocks) + { + case 7: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 6; + } + + case 6: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 5; + } + + case 5: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 4; + } + + case 4: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 3; + } + + case 3: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 2; + } + + case 2: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 1; + } + + case 1: + { + Vector256 vector = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 1))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 0; + } + + case 0: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)trailing), end, Vector256.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, end); + break; + } + } + + return TAggregationOperator.Invoke(vresult); + } + + static T Vectorized512(ref T xRef, nuint remainder) + { + Vector512 vresult = Vector512.Create(TAggregationOperator.IdentityValue); + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 beg = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef)); + Vector512 end = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)); + + nuint misalignment = 0; + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + { + T* xPtr = px; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + misalignment = ((uint)sizeof(Vector512) - ((nuint)xPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + + Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + // We only need to load, so there isn't a lot of benefit to doing non-temporal operations + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); + vector2 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); + vector3 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); + vector4 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We load, process, and store the next four vectors + + vector1 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); + vector2 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); + vector3 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); + vector4 = TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + } + } + + // Store the first block. Handling this separately simplifies the latter code as we know + // they come after and so we can relegate it to full blocks or the trailing elements + + beg = Vector512.ConditionalSelect(CreateAlignmentMaskVector512((int)misalignment), beg, Vector512.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, beg); + + // Process the remaining [0, Count * 7] elements via a jump table + // + // We end up handling any trailing elements in case 0 and in the + // worst case end up just doing the identity operation here if there + // were no trailing elements. + + (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector512.Count); + blocks -= (misalignment == 0) ? 1u : 0u; + remainder -= trailing; + + switch (blocks) + { + case 7: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 6; + } + + case 6: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 5; + } + + case 5: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 4; + } + + case 4: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 3; + } + + case 3: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 2; + } + + case 2: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 1; + } + + case 1: + { + Vector512 vector = TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 1))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 0; + } + + case 0: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end = Vector512.ConditionalSelect(CreateRemainderMaskVector512((int)trailing), end, Vector512.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, end); + break; + } + } + + return TAggregationOperator.Invoke(vresult); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall(ref T xRef, nuint remainder) + { + if (sizeof(T) == 1) + { + return VectorizedSmall1(ref xRef, remainder); + } + else if (sizeof(T) == 2) + { + return VectorizedSmall2(ref xRef, remainder); + } + else if (sizeof(T) == 4) + { + return VectorizedSmall4(ref xRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + return VectorizedSmall8(ref xRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall1(ref T xRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 14))); + goto case 14; + + case 14: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 13))); + goto case 13; + + case 13: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 12))); + goto case 12; + + case 12: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 11))); + goto case 11; + + case 11: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 10))); + goto case 10; + + case 10: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 9))); + goto case 9; + + case 9: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 8))); + goto case 8; + + case 8: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 7))); + goto case 7; + + case 7: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6))); + goto case 6; + + case 6: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5))); + goto case 5; + + case 5: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4))); + goto case 4; + + case 4: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3))); + goto case 3; + + case 3: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2))); + goto case 2; + + case 2: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1))); + goto case 1; + + case 1: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); + goto case 0; + + case 0: + break; + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall2(ref T xRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6))); + goto case 6; + + case 6: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5))); + goto case 5; + + case 5: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4))); + goto case 4; + + case 4: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3))); + goto case 3; + + case 3: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2))); + goto case 2; + + case 2: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1))); + goto case 1; + + case 1: + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); + goto case 0; + + case 0: + break; + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall4(ref T xRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 3: + { + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2))); + goto case 2; + } + + case 2: + { + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1))); + goto case 1; + } + + case 1: + { + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); + goto case 0; + } + + case 0: + { + break; + } + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall8(ref T xRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 1: + { + result = TAggregationOperator.Invoke(result, TTransformOperator.Invoke(xRef)); + goto case 0; + } + + case 0: + { + break; + } + } + + return result; + } + } + + /// Performs an aggregation over all pair-wise elements in and to produce a single-precision floating-point value. + /// The element type. + /// Specifies the binary operation that should be applied to the pair-wise elements loaded from and . + /// + /// Specifies the aggregation binary operation that should be applied to multiple values to aggregate them into a single value. + /// The aggregation is applied to the results of the binary operations on the pair-wise values. + /// + private static T Aggregate( + ReadOnlySpan x, ReadOnlySpan y) + where TBinaryOperator : struct, IBinaryOperator + where TAggregationOperator : struct, IAggregationOperator + { + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TBinaryOperator.Vectorizable) + { + T result; + + if (remainder >= (uint)Vector512.Count) + { + result = Vectorized512(ref xRef, ref yRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + result = VectorizedSmall(ref xRef, ref yRef, remainder); + } + + return result; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TBinaryOperator.Vectorizable) + { + T result; + + if (remainder >= (uint)Vector256.Count) + { + result = Vectorized256(ref xRef, ref yRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + result = VectorizedSmall(ref xRef, ref yRef, remainder); + } + + return result; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TBinaryOperator.Vectorizable) + { + T result; + + if (remainder >= (uint)Vector128.Count) + { + result = Vectorized128(ref xRef, ref yRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + result = VectorizedSmall(ref xRef, ref yRef, remainder); + } + + return result; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + return SoftwareFallback(ref xRef, ref yRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T SoftwareFallback(ref T xRef, ref T yRef, nuint length) + { + T result = TAggregationOperator.IdentityValue; + + for (nuint i = 0; i < length; i++) + { + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, i), + Unsafe.Add(ref yRef, i))); + } + + return result; + } + + static T Vectorized128(ref T xRef, ref T yRef, nuint remainder) + { + Vector128 vresult = Vector128.Create(TAggregationOperator.IdentityValue); + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + nuint misalignment = 0; + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + { + T* xPtr = px; + T* yPtr = py; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + misalignment = ((uint)sizeof(Vector128) - ((nuint)xPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + + Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + // We only need to load, so there isn't a lot of benefit to doing non-temporal operations + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + } + } + + // Store the first block. Handling this separately simplifies the latter code as we know + // they come after and so we can relegate it to full blocks or the trailing elements + + beg = Vector128.ConditionalSelect(CreateAlignmentMaskVector128((int)misalignment), beg, Vector128.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, beg); + + // Process the remaining [0, Count * 7] elements via a jump table + // + // We end up handling any trailing elements in case 0 and in the + // worst case end up just doing the identity operation here if there + // were no trailing elements. + + (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector128.Count); + blocks -= (misalignment == 0) ? 1u : 0u; + remainder -= trailing; + + switch (blocks) + { + case 7: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 6; + } + + case 6: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 5; + } + + case 5: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 4; + } + + case 4: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 3; + } + + case 3: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 2; + } + + case 2: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 1; + } + + case 1: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 1)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 1))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 0; + } + + case 0: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)trailing), end, Vector128.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, end); + break; + } + } + + return TAggregationOperator.Invoke(vresult); + } + + static T Vectorized256(ref T xRef, ref T yRef, nuint remainder) + { + Vector256 vresult = Vector256.Create(TAggregationOperator.IdentityValue); + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + nuint misalignment = 0; + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + { + T* xPtr = px; + T* yPtr = py; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + misalignment = ((uint)sizeof(Vector256) - ((nuint)xPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + + Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + // We only need to load, so there isn't a lot of benefit to doing non-temporal operations + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + } + } + + // Store the first block. Handling this separately simplifies the latter code as we know + // they come after and so we can relegate it to full blocks or the trailing elements + + beg = Vector256.ConditionalSelect(CreateAlignmentMaskVector256((int)misalignment), beg, Vector256.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, beg); + + // Process the remaining [0, Count * 7] elements via a jump table + // + // We end up handling any trailing elements in case 0 and in the + // worst case end up just doing the identity operation here if there + // were no trailing elements. + + (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector256.Count); + blocks -= (misalignment == 0) ? 1u : 0u; + remainder -= trailing; + + switch (blocks) + { + case 7: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 6; + } + + case 6: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 5; + } + + case 5: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 4; + } + + case 4: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 3; + } + + case 3: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 2; + } + + case 2: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 1; + } + + case 1: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 1)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 1))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 0; + } + + case 0: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)trailing), end, Vector256.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, end); + break; + } + } + + return TAggregationOperator.Invoke(vresult); + } + + static T Vectorized512(ref T xRef, ref T yRef, nuint remainder) + { + Vector512 vresult = Vector512.Create(TAggregationOperator.IdentityValue); + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 beg = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), + Vector512.LoadUnsafe(ref yRef)); + Vector512 end = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count)); + + nuint misalignment = 0; + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + { + T* xPtr = px; + T* yPtr = py; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + misalignment = ((uint)sizeof(Vector512) - ((nuint)xPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + + Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + // We only need to load, so there isn't a lot of benefit to doing non-temporal operations + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7))); + + vresult = TAggregationOperator.Invoke(vresult, vector1); + vresult = TAggregationOperator.Invoke(vresult, vector2); + vresult = TAggregationOperator.Invoke(vresult, vector3); + vresult = TAggregationOperator.Invoke(vresult, vector4); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + } + } + + // Store the first block. Handling this separately simplifies the latter code as we know + // they come after and so we can relegate it to full blocks or the trailing elements + + beg = Vector512.ConditionalSelect(CreateAlignmentMaskVector512((int)misalignment), beg, Vector512.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, beg); + + // Process the remaining [0, Count * 7] elements via a jump table + // + // We end up handling any trailing elements in case 0 and in the + // worst case end up just doing the identity operation here if there + // were no trailing elements. + + (nuint blocks, nuint trailing) = Math.DivRem(remainder, (nuint)Vector512.Count); + blocks -= (misalignment == 0) ? 1u : 0u; + remainder -= trailing; + + switch (blocks) + { + case 7: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 6; + } + + case 6: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 5; + } + + case 5: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 4; + } + + case 4: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 3; + } + + case 3: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 2; + } + + case 2: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 1; + } + + case 1: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 1)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 1))); + vresult = TAggregationOperator.Invoke(vresult, vector); + goto case 0; + } + + case 0: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end = Vector512.ConditionalSelect(CreateRemainderMaskVector512((int)trailing), end, Vector512.Create(TAggregationOperator.IdentityValue)); + vresult = TAggregationOperator.Invoke(vresult, end); + break; + } + } + + return TAggregationOperator.Invoke(vresult); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall(ref T xRef, ref T yRef, nuint remainder) + { + if (sizeof(T) == 1) + { + return VectorizedSmall1(ref xRef, ref yRef, remainder); + } + else if (sizeof(T) == 2) + { + return VectorizedSmall2(ref xRef, ref yRef, remainder); + } + else if (sizeof(T) == 4) + { + return VectorizedSmall4(ref xRef, ref yRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + return VectorizedSmall8(ref xRef, ref yRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall1(ref T xRef, ref T yRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 14), Unsafe.Add(ref yRef, 14))); + goto case 14; + + case 14: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 13), Unsafe.Add(ref yRef, 13))); + goto case 13; + + case 13: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 12), Unsafe.Add(ref yRef, 12))); + goto case 12; + + case 12: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 11), Unsafe.Add(ref yRef, 11))); + goto case 11; + + case 11: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 10), Unsafe.Add(ref yRef, 10))); + goto case 10; + + case 10: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 9), Unsafe.Add(ref yRef, 9))); + goto case 9; + + case 9: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 8), Unsafe.Add(ref yRef, 8))); + goto case 8; + + case 8: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 7), Unsafe.Add(ref yRef, 7))); + goto case 7; + + case 7: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), Unsafe.Add(ref yRef, 6))); + goto case 6; + + case 6: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), Unsafe.Add(ref yRef, 5))); + goto case 5; + + case 5: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), Unsafe.Add(ref yRef, 4))); + goto case 4; + + case 4: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), Unsafe.Add(ref yRef, 3))); + goto case 3; + + case 3: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), Unsafe.Add(ref yRef, 2))); + goto case 2; + + case 2: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), Unsafe.Add(ref yRef, 1))); + goto case 1; + + case 1: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); + goto case 0; + + case 0: + break; + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall2(ref T xRef, ref T yRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), Unsafe.Add(ref yRef, 6))); + goto case 6; + + case 6: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), Unsafe.Add(ref yRef, 5))); + goto case 5; + + case 5: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), Unsafe.Add(ref yRef, 4))); + goto case 4; + + case 4: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), Unsafe.Add(ref yRef, 3))); + goto case 3; + + case 3: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), Unsafe.Add(ref yRef, 2))); + goto case 2; + + case 2: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), Unsafe.Add(ref yRef, 1))); + goto case 1; + + case 1: + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); + goto case 0; + + case 0: + break; + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall4(ref T xRef, ref T yRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 3: + { + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2))); + goto case 2; + } + + case 2: + { + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1))); + goto case 1; + } + + case 1: + { + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); + goto case 0; + } + + case 0: + { + break; + } + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static T VectorizedSmall8(ref T xRef, ref T yRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + T result = TAggregationOperator.IdentityValue; + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + end = Vector256.ConditionalSelect(CreateRemainderMaskVector256((int)(remainder % (uint)Vector256.Count)), end, Vector256.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + end = Vector128.ConditionalSelect(CreateRemainderMaskVector128((int)(remainder % (uint)Vector128.Count)), end, Vector128.Create(TAggregationOperator.IdentityValue)); + + result = TAggregationOperator.Invoke(TAggregationOperator.Invoke(beg, end)); + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + + result = TAggregationOperator.Invoke(beg); + break; + } + + case 1: + { + result = TAggregationOperator.Invoke(result, TBinaryOperator.Invoke(xRef, yRef)); + goto case 0; + } + + case 0: + { + break; + } + } + + return result; + } + } + + /// + /// This is the same as + /// with an identity transform, except it early exits on NaN. + /// + private static T MinMaxCore(ReadOnlySpan x) + where T : INumberBase + where TMinMaxOperator : struct, IAggregationOperator + { + if (x.IsEmpty) + { + ThrowHelper.ThrowArgument_SpansMustBeNonEmpty(); + } + + // This matches the IEEE 754:2019 `maximum`/`minimum` functions. + // It propagates NaN inputs back to the caller and + // otherwise returns the greater of the inputs. + // It treats +0 as greater than -0 as per the specification. + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) + { + ref T xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector512 result = Vector512.LoadUnsafe(ref xRef, 0); + Vector512 current; + + Vector512 nanMask; + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector512.Equals(result, result); + if (nanMask != Vector512.Zero) + { + return result.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + int oneVectorFromEnd = x.Length - Vector512.Count; + int i = Vector512.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector512.LoadUnsafe(ref xRef, (uint)i); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector512.Equals(current, current); + if (nanMask != Vector512.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, current); + i += Vector512.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector512.Equals(current, current); + if (nanMask != Vector512.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TMinMaxOperator.Invoke(result); + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + { + ref T xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector256 result = Vector256.LoadUnsafe(ref xRef, 0); + Vector256 current; + + Vector256 nanMask; + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector256.Equals(result, result); + if (nanMask != Vector256.Zero) + { + return result.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + int oneVectorFromEnd = x.Length - Vector256.Count; + int i = Vector256.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector256.LoadUnsafe(ref xRef, (uint)i); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector256.Equals(current, current); + if (nanMask != Vector256.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, current); + i += Vector256.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector256.Equals(current, current); + if (nanMask != Vector256.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TMinMaxOperator.Invoke(result); + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + { + ref T xRef = ref MemoryMarshal.GetReference(x); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector128 result = Vector128.LoadUnsafe(ref xRef, 0); + Vector128 current; + + Vector128 nanMask; + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector128.Equals(result, result); + if (nanMask != Vector128.Zero) + { + return result.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + int oneVectorFromEnd = x.Length - Vector128.Count; + int i = Vector128.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector128.LoadUnsafe(ref xRef, (uint)i); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector128.Equals(current, current); + if (nanMask != Vector128.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, current); + i += Vector128.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // Check for NaNs + nanMask = ~Vector128.Equals(current, current); + if (nanMask != Vector128.Zero) + { + return current.GetElement(IndexOfFirstMatch(nanMask)); + } + } + + result = TMinMaxOperator.Invoke(result, current); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TMinMaxOperator.Invoke(result); + } + + // Scalar path used when either vectorization is not supported or the input is too small to vectorize. + T curResult = x[0]; + if (T.IsNaN(curResult)) + { + return curResult; + } + + for (int i = 1; i < x.Length; i++) + { + T current = x[i]; + if (T.IsNaN(current)) + { + return current; + } + + curResult = TMinMaxOperator.Invoke(curResult, current); + } + + return curResult; + } + + private static int IndexOfMinMaxCore(ReadOnlySpan x) + where T : INumber + where TIndexOfMinMax : struct, IIndexOfOperator + { + if (x.IsEmpty) + { + return -1; + } + + // This matches the IEEE 754:2019 `maximum`/`minimum` functions. + // It propagates NaN inputs back to the caller and + // otherwise returns the index of the greater of the inputs. + // It treats +0 as greater than -0 as per the specification. + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && x.Length >= Vector512.Count) + { + Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static Vector512 CreateVector512T(int i) => + sizeof(T) == sizeof(long) ? Vector512.Create((long)i).As() : + sizeof(T) == sizeof(int) ? Vector512.Create(i).As() : + sizeof(T) == sizeof(short) ? Vector512.Create((short)i).As() : + Vector512.Create((byte)i).As(); + + ref T xRef = ref MemoryMarshal.GetReference(x); + Vector512 resultIndex = +#if NET9_0_OR_GREATER + sizeof(T) == sizeof(long) ? Vector512.Indices.As() : + sizeof(T) == sizeof(int) ? Vector512.Indices.As() : + sizeof(T) == sizeof(short) ? Vector512.Indices.As() : + Vector512.Indices.As(); +#else + sizeof(T) == sizeof(long) ? Vector512.Create(0L, 1, 2, 3, 4, 5, 6, 7).As() : + sizeof(T) == sizeof(int) ? Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).As() : + sizeof(T) == sizeof(short) ? Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31).As() : + Vector512.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63).As(); +#endif + Vector512 currentIndex = resultIndex; + Vector512 increment = CreateVector512T(Vector512.Count); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector512 result = Vector512.LoadUnsafe(ref xRef); + Vector512 current; + + Vector512 nanMask; + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector512.Equals(result, result); + if (nanMask != Vector512.Zero) + { + return IndexOfFirstMatch(nanMask); + } + } + + int oneVectorFromEnd = x.Length - Vector512.Count; + int i = Vector512.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector512.LoadUnsafe(ref xRef, (uint)i); + currentIndex += increment; + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector512.Equals(current, current); + if (nanMask != Vector512.Zero) + { + return i + IndexOfFirstMatch(nanMask); + } + } + + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + + i += Vector512.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + currentIndex += CreateVector512T(x.Length - i); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector512.Equals(current, current); + if (nanMask != Vector512.Zero) + { + int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + return typeof(T) == typeof(double) ? + (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : + (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; + } + } + + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return IndexOfFinalAggregate(result, resultIndex); + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && x.Length >= Vector256.Count) + { + Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static Vector256 CreateVector256T(int i) => + sizeof(T) == sizeof(long) ? Vector256.Create((long)i).As() : + sizeof(T) == sizeof(int) ? Vector256.Create(i).As() : + sizeof(T) == sizeof(short) ? Vector256.Create((short)i).As() : + Vector256.Create((byte)i).As(); + + ref T xRef = ref MemoryMarshal.GetReference(x); + Vector256 resultIndex = +#if NET9_0_OR_GREATER + sizeof(T) == sizeof(long) ? Vector256.Indices.As() : + sizeof(T) == sizeof(int) ? Vector256.Indices.As() : + sizeof(T) == sizeof(short) ? Vector256.Indices.As() : + Vector256.Indices.As(); +#else + sizeof(T) == sizeof(long) ? Vector256.Create(0L, 1, 2, 3).As() : + sizeof(T) == sizeof(int) ? Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7).As() : + sizeof(T) == sizeof(short) ? Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).As() : + Vector256.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31).As(); +#endif + Vector256 currentIndex = resultIndex; + Vector256 increment = CreateVector256T(Vector256.Count); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector256 result = Vector256.LoadUnsafe(ref xRef); + Vector256 current; + + Vector256 nanMask; + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector256.Equals(result, result); + if (nanMask != Vector256.Zero) + { + return IndexOfFirstMatch(nanMask); + } + } + + int oneVectorFromEnd = x.Length - Vector256.Count; + int i = Vector256.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector256.LoadUnsafe(ref xRef, (uint)i); + currentIndex += increment; + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector256.Equals(current, current); + if (nanMask != Vector256.Zero) + { + return i + IndexOfFirstMatch(nanMask); + } + } + + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + + i += Vector256.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + currentIndex += CreateVector256T(x.Length - i); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector256.Equals(current, current); + if (nanMask != Vector256.Zero) + { + int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + return typeof(T) == typeof(double) ? + (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : + (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; + } + } + + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return IndexOfFinalAggregate(result, resultIndex); + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && x.Length >= Vector128.Count) + { + Debug.Assert(sizeof(T) is 1 or 2 or 4 or 8); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static Vector128 CreateVector128T(int i) => + sizeof(T) == sizeof(long) ? Vector128.Create((long)i).As() : + sizeof(T) == sizeof(int) ? Vector128.Create(i).As() : + sizeof(T) == sizeof(short) ? Vector128.Create((short)i).As() : + Vector128.Create((byte)i).As(); + + ref T xRef = ref MemoryMarshal.GetReference(x); + Vector128 resultIndex = +#if NET9_0_OR_GREATER + sizeof(T) == sizeof(long) ? Vector128.Indices.As() : + sizeof(T) == sizeof(int) ? Vector128.Indices.As() : + sizeof(T) == sizeof(short) ? Vector128.Indices.As() : + Vector128.Indices.As(); +#else + sizeof(T) == sizeof(long) ? Vector128.Create(0L, 1).As() : + sizeof(T) == sizeof(int) ? Vector128.Create(0, 1, 2, 3).As() : + sizeof(T) == sizeof(short) ? Vector128.Create(0, 1, 2, 3, 4, 5, 6, 7).As() : + Vector128.Create((byte)0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15).As(); +#endif + Vector128 currentIndex = resultIndex; + Vector128 increment = CreateVector128T(Vector128.Count); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector128 result = Vector128.LoadUnsafe(ref xRef); + Vector128 current; + + Vector128 nanMask; + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector128.Equals(result, result); + if (nanMask != Vector128.Zero) + { + return IndexOfFirstMatch(nanMask); + } + } + + int oneVectorFromEnd = x.Length - Vector128.Count; + int i = Vector128.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector128.LoadUnsafe(ref xRef, (uint)i); + currentIndex += increment; + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector128.Equals(current, current); + if (nanMask != Vector128.Zero) + { + return i + IndexOfFirstMatch(nanMask); + } + } + + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + + i += Vector128.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + currentIndex += CreateVector128T(x.Length - i); + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + nanMask = ~Vector128.Equals(current, current); + if (nanMask != Vector128.Zero) + { + int indexInVectorOfFirstMatch = IndexOfFirstMatch(nanMask); + return typeof(T) == typeof(double) ? + (int)(long)(object)currentIndex.As()[indexInVectorOfFirstMatch] : + (int)(object)currentIndex.As()[indexInVectorOfFirstMatch]; + } + } + + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, currentIndex); + } + + // Aggregate the lanes in the vector to create the final scalar result. + return IndexOfFinalAggregate(result, resultIndex); + } + + // Scalar path used when either vectorization is not supported or the input is too small to vectorize. + T curResult = x[0]; + int curIn = 0; + if (T.IsNaN(curResult)) + { + return curIn; + } + + for (int i = 1; i < x.Length; i++) + { + T current = x[i]; + if (T.IsNaN(current)) + { + return i; + } + + curIn = TIndexOfMinMax.Invoke(ref curResult, current, curIn, i); + } + + return curIn; + } + + private static int IndexOfFirstMatch(Vector128 mask) => + BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + + private static int IndexOfFirstMatch(Vector256 mask) => + BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + + private static int IndexOfFirstMatch(Vector512 mask) => + BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + + /// Performs an element-wise operation on and writes the results to . + /// The element input type. + /// Specifies the operation to perform on each element loaded from . + private static void InvokeSpanIntoSpan( + ReadOnlySpan x, Span destination) + where TUnaryOperator : struct, IUnaryOperator => + InvokeSpanIntoSpan(x, destination); + + /// Performs an element-wise operation on and writes the results to . + /// The element input type. + /// The element output type. Must be the same size as TInput if TInput and TOutput both support vectorization. + /// Specifies the operation to perform on each element loaded from . + /// + /// This supports vectorizing the operation if and are the same size. + /// Otherwise, it'll fall back to scalar operations. + /// + private static void InvokeSpanIntoSpan( + ReadOnlySpan x, Span destination) + where TUnaryOperator : struct, IUnaryOperator + { + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + if (typeof(TInput) == typeof(TOutput)) + { + // This ignores the unsafe case where a developer passes in overlapping spans for distinct types. + ValidateInputOutputSpanNonOverlapping(x, Rename(destination)); + } + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref TInput xRef = ref MemoryMarshal.GetReference(x); + ref TOutput dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && Vector512.IsSupported && TUnaryOperator.Vectorizable && Unsafe.SizeOf() == Unsafe.SizeOf()) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref dRef, remainder); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && Vector256.IsSupported && TUnaryOperator.Vectorizable && Unsafe.SizeOf() == Unsafe.SizeOf()) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref dRef, remainder); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && Vector128.IsSupported && TUnaryOperator.Vectorizable && Unsafe.SizeOf() == Unsafe.SizeOf()) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref dRef, remainder); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, ref dRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref TInput xRef, ref TOutput dRef, nuint length) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, i)); + } + } + + static void Vectorized128(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + ref TOutput dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (TInput* px = &xRef) + fixed (TOutput* pd = &dRef) + { + TInput* xPtr = px; + TOutput* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(TInput)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(TInput); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(TInput))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); + vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); + vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); + vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); + vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); + vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); + vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); + vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); + vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); + vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); + vector2 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); + vector3 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); + vector4 = TUnaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + ref TOutput dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (TInput* px = &xRef) + fixed (TOutput* pd = &dRef) + { + TInput* xPtr = px; + TOutput* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(TInput)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(TInput); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(TInput))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); + vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); + vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); + vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); + vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); + vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); + vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); + vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); + vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); + vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); + vector2 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); + vector3 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); + vector4 = TUnaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + ref TOutput dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 beg = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef)); + Vector512 end = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (TInput* px = &xRef) + fixed (TOutput* pd = &dRef) + { + TInput* xPtr = px; + TOutput* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(TInput)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(TInput); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(TInput))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); + vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); + vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); + vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); + vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); + vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); + vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); + vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); + vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); + vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); + vector2 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); + vector3 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); + vector4 = TUnaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + if (sizeof(TInput) == 1) + { + VectorizedSmall1(ref xRef, ref dRef, remainder); + } + else if (sizeof(TInput) == 2) + { + VectorizedSmall2(ref xRef, ref dRef, remainder); + } + else if (sizeof(TInput) == 4) + { + VectorizedSmall4(ref xRef, ref dRef, remainder); + } + else + { + Debug.Assert(sizeof(TInput) == 8); + VectorizedSmall8(ref xRef, ref dRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + Debug.Assert(sizeof(TInput) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 14)); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 13)); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 12)); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 11)); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 10)); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 9)); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 8)); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 7)); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 1)); + goto case 1; + + case 1: + dRef = TUnaryOperator.Invoke(xRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + Debug.Assert(sizeof(TInput) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 1)); + goto case 1; + + case 1: + dRef = TUnaryOperator.Invoke(xRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + Debug.Assert(sizeof(TInput) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 2)); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, 1)); + goto case 1; + } + + case 1: + { + dRef = TUnaryOperator.Invoke(xRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref TInput xRef, ref TOutput dRef, nuint remainder) + { + Debug.Assert(sizeof(TInput) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = TUnaryOperator.Invoke(xRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// Performs an element-wise operation on and writes the results to . + /// The element input type. + /// The element output type. Must be the same size as TInput if TInput and TOutput both support vectorization. + /// Specifies the operation to perform on each element loaded from . + /// This should only be used when it's known that TInput/TOutput are vectorizable and the size of TInput is twice that of TOutput. + private static void InvokeSpanIntoSpan_2to1( + ReadOnlySpan x, Span destination) + where TUnaryOperator : struct, IUnaryTwoToOneOperator + { + Debug.Assert(sizeof(TInput) == sizeof(TOutput) * 2); + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ref TInput xRef = ref MemoryMarshal.GetReference(x); + ref TOutput destinationRef = ref MemoryMarshal.GetReference(destination); + int i = 0, twoVectorsFromEnd; + + if (Vector512.IsHardwareAccelerated && TUnaryOperator.Vectorizable) + { + Debug.Assert(Vector512.IsSupported); + Debug.Assert(Vector512.IsSupported); + + twoVectorsFromEnd = x.Length - (Vector512.Count * 2); + if (i <= twoVectorsFromEnd) + { + // Loop handling two input vectors / one output vector at a time. + do + { + TUnaryOperator.Invoke( + Vector512.LoadUnsafe(ref xRef, (uint)i), + Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count))).StoreUnsafe(ref destinationRef, (uint)i); + + i += Vector512.Count * 2; + } + while (i <= twoVectorsFromEnd); + + // Handle any remaining elements with final vectors. + if (i != x.Length) + { + i = x.Length - (Vector512.Count * 2); + + TUnaryOperator.Invoke( + Vector512.LoadUnsafe(ref xRef, (uint)i), + Vector512.LoadUnsafe(ref xRef, (uint)(i + Vector512.Count))).StoreUnsafe(ref destinationRef, (uint)i); + } + + return; + } + } + + if (Vector256.IsHardwareAccelerated && TUnaryOperator.Vectorizable) + { + Debug.Assert(Vector256.IsSupported); + Debug.Assert(Vector256.IsSupported); + + twoVectorsFromEnd = x.Length - (Vector256.Count * 2); + if (i <= twoVectorsFromEnd) + { + // Loop handling two input vectors / one output vector at a time. + do + { + TUnaryOperator.Invoke( + Vector256.LoadUnsafe(ref xRef, (uint)i), + Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count))).StoreUnsafe(ref destinationRef, (uint)i); + + i += Vector256.Count * 2; + } + while (i <= twoVectorsFromEnd); + + // Handle any remaining elements with final vectors. + if (i != x.Length) + { + i = x.Length - (Vector256.Count * 2); + + TUnaryOperator.Invoke( + Vector256.LoadUnsafe(ref xRef, (uint)i), + Vector256.LoadUnsafe(ref xRef, (uint)(i + Vector256.Count))).StoreUnsafe(ref destinationRef, (uint)i); + } + + return; + } + } + + if (Vector128.IsHardwareAccelerated && TUnaryOperator.Vectorizable) + { + Debug.Assert(Vector128.IsSupported); + Debug.Assert(Vector128.IsSupported); + + twoVectorsFromEnd = x.Length - (Vector128.Count * 2); + if (i <= twoVectorsFromEnd) + { + // Loop handling two input vectors / one output vector at a time. + do + { + TUnaryOperator.Invoke( + Vector128.LoadUnsafe(ref xRef, (uint)i), + Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count))).StoreUnsafe(ref destinationRef, (uint)i); + + i += Vector128.Count * 2; + } + while (i <= twoVectorsFromEnd); + + // Handle any remaining elements with final vectors. + if (i != x.Length) + { + i = x.Length - (Vector128.Count * 2); + + TUnaryOperator.Invoke( + Vector128.LoadUnsafe(ref xRef, (uint)i), + Vector128.LoadUnsafe(ref xRef, (uint)(i + Vector128.Count))).StoreUnsafe(ref destinationRef, (uint)i); + } + + return; + } + } + + while (i < x.Length) + { + Unsafe.Add(ref destinationRef, i) = TUnaryOperator.Invoke(Unsafe.Add(ref xRef, i)); + i++; + } + } + + /// Performs an element-wise operation on and writes the results to . + /// The element type. + /// Specifies the operation to perform on each element loaded from . + private static void InvokeSpanIntoSpan( + ReadOnlySpan x, TStatefulUnaryOperator op, Span destination) + where TStatefulUnaryOperator : struct, IStatefulUnaryOperator + { + // NOTE: This implementation is an exact copy of InvokeSpanIntoSpan, + // except it accepts an operator that carries state with it, using instance rather than + // static invocation methods. + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TStatefulUnaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, ref dRef, remainder, op); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref dRef, remainder, op); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TStatefulUnaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, ref dRef, remainder, op); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref dRef, remainder, op); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TStatefulUnaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, ref dRef, remainder, op); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref dRef, remainder, op); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, ref dRef, remainder, op); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref T xRef, ref T dRef, nuint length, TStatefulUnaryOperator op) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = op.Invoke(Unsafe.Add(ref xRef, i)); + } + } + + static void Vectorized128(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); + vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); + vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); + vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); + vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); + vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); + vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))); + vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))); + vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))); + vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))); + vector2 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))); + vector3 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))); + vector4 = op.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); + vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); + vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); + vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); + vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); + vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); + vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))); + vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))); + vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))); + vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))); + vector2 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))); + vector3 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))); + vector4 = op.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 beg = op.Invoke(Vector512.LoadUnsafe(ref xRef)); + Vector512 end = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); + vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); + vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); + vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); + vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); + vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); + vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))); + vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))); + vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))); + vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))); + vector2 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))); + vector3 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))); + vector4 = op.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = op.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + if (sizeof(T) == 1) + { + VectorizedSmall1(ref xRef, ref dRef, remainder, op); + } + else if (sizeof(T) == 2) + { + VectorizedSmall2(ref xRef, ref dRef, remainder, op); + } + else if (sizeof(T) == 4) + { + VectorizedSmall4(ref xRef, ref dRef, remainder, op); + } + else + { + Debug.Assert(sizeof(T) == 8); + VectorizedSmall8(ref xRef, ref dRef, remainder, op); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + Debug.Assert(sizeof(T) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = op.Invoke(Unsafe.Add(ref xRef, 14)); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = op.Invoke(Unsafe.Add(ref xRef, 13)); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = op.Invoke(Unsafe.Add(ref xRef, 12)); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = op.Invoke(Unsafe.Add(ref xRef, 11)); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = op.Invoke(Unsafe.Add(ref xRef, 10)); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = op.Invoke(Unsafe.Add(ref xRef, 9)); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = op.Invoke(Unsafe.Add(ref xRef, 8)); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = op.Invoke(Unsafe.Add(ref xRef, 7)); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = op.Invoke(Unsafe.Add(ref xRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = op.Invoke(Unsafe.Add(ref xRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = op.Invoke(Unsafe.Add(ref xRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = op.Invoke(Unsafe.Add(ref xRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = op.Invoke(Unsafe.Add(ref xRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = op.Invoke(Unsafe.Add(ref xRef, 1)); + goto case 1; + + case 1: + dRef = op.Invoke(xRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + Debug.Assert(sizeof(T) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = op.Invoke(Unsafe.Add(ref xRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = op.Invoke(Unsafe.Add(ref xRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = op.Invoke(Unsafe.Add(ref xRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = op.Invoke(Unsafe.Add(ref xRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = op.Invoke(Unsafe.Add(ref xRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = op.Invoke(Unsafe.Add(ref xRef, 1)); + goto case 1; + + case 1: + dRef = op.Invoke(xRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + Debug.Assert(sizeof(T) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = op.Invoke(Unsafe.Add(ref xRef, 2)); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = op.Invoke(Unsafe.Add(ref xRef, 1)); + goto case 1; + } + + case 1: + { + dRef = op.Invoke(xRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref T xRef, ref T dRef, nuint remainder, TStatefulUnaryOperator op) + { + Debug.Assert(sizeof(T) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + Vector256 end = op.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = op.Invoke(Vector256.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + Vector128 end = op.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = op.Invoke(Vector128.LoadUnsafe(ref xRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = op.Invoke(xRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// Performs an element-wise operation on and writes the results to . + /// The element input type. + /// The element output type. Must be the same size as TInput if TInput and TOutput both support vectorization. + /// Specifies the operation to perform on each element loaded from . + /// This should only be used when it's known that TInput/TOutput are vectorizable and the size of TInput is half that of TOutput. + private static void InvokeSpanIntoSpan_1to2( + ReadOnlySpan x, Span destination) + where TUnaryOperator : struct, IUnaryOneToTwoOperator + { + Debug.Assert(sizeof(TInput) * 2 == sizeof(TOutput)); + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ref TInput sourceRef = ref MemoryMarshal.GetReference(x); + ref TOutput destinationRef = ref MemoryMarshal.GetReference(destination); + int i = 0, oneVectorFromEnd; + + if (Vector512.IsHardwareAccelerated && TUnaryOperator.Vectorizable) + { + Debug.Assert(Vector512.IsSupported); + Debug.Assert(Vector512.IsSupported); + + oneVectorFromEnd = x.Length - Vector512.Count; + if (i <= oneVectorFromEnd) + { + // Loop handling one input vector / two output vectors at a time. + do + { + (Vector512 lower, Vector512 upper) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); + lower.StoreUnsafe(ref destinationRef, (uint)i); + upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector512.Count)); + + i += Vector512.Count; + } + while (i <= oneVectorFromEnd); + + // Handle any remaining elements with a final input vector. + if (i != x.Length) + { + i = x.Length - Vector512.Count; + + (Vector512 lower, Vector512 upper) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); + lower.StoreUnsafe(ref destinationRef, (uint)i); + upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector512.Count)); + } + + return; + } + } + + if (Vector256.IsHardwareAccelerated && TUnaryOperator.Vectorizable) + { + Debug.Assert(Vector256.IsSupported); + Debug.Assert(Vector256.IsSupported); + + oneVectorFromEnd = x.Length - Vector256.Count; + if (i <= oneVectorFromEnd) + { + // Loop handling one input vector / two output vectors at a time. + do + { + (Vector256 lower, Vector256 upper) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); + lower.StoreUnsafe(ref destinationRef, (uint)i); + upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector256.Count)); + + i += Vector256.Count; + } + while (i <= oneVectorFromEnd); + + // Handle any remaining elements with a final input vector. + if (i != x.Length) + { + i = x.Length - Vector256.Count; + + (Vector256 lower, Vector256 upper) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); + lower.StoreUnsafe(ref destinationRef, (uint)i); + upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector256.Count)); + } + + return; + } + } + + if (Vector128.IsHardwareAccelerated && TUnaryOperator.Vectorizable) + { + Debug.Assert(Vector128.IsSupported); + Debug.Assert(Vector128.IsSupported); + + oneVectorFromEnd = x.Length - Vector128.Count; + if (i <= oneVectorFromEnd) + { + // Loop handling one input vector / two output vectors at a time. + do + { + (Vector128 lower, Vector128 upper) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); + lower.StoreUnsafe(ref destinationRef, (uint)i); + upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector128.Count)); + + i += Vector128.Count; + } + while (i <= oneVectorFromEnd); + + // Handle any remaining elements with a final input vector. + if (i != x.Length) + { + i = x.Length - Vector128.Count; + + (Vector128 lower, Vector128 upper) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); + lower.StoreUnsafe(ref destinationRef, (uint)i); + upper.StoreUnsafe(ref destinationRef, (uint)(i + Vector128.Count)); + } + + return; + } + } + + while (i < x.Length) + { + Unsafe.Add(ref destinationRef, i) = TUnaryOperator.Invoke(Unsafe.Add(ref sourceRef, i)); + i++; + } + } + + /// Performs an element-wise operation on and writes the results to and . + /// The element type. + /// Specifies the operation to perform on each element loaded from . + private static void InvokeSpanIntoSpan_TwoOutputs( + ReadOnlySpan x, Span destination1, Span destination2) + where TUnaryOperator : struct, IUnaryInputBinaryOutput + { + if (x.Length > destination1.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(nameof(destination1)); + } + + if (x.Length > destination2.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(nameof(destination2)); + } + + ValidateInputOutputSpanNonOverlapping(x, destination1); + ValidateInputOutputSpanNonOverlapping(x, destination2); + + ref T sourceRef = ref MemoryMarshal.GetReference(x); + ref T destination1Ref = ref MemoryMarshal.GetReference(destination1); + ref T destination2Ref = ref MemoryMarshal.GetReference(destination2); + int i = 0, oneVectorFromEnd; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TUnaryOperator.Vectorizable) + { + oneVectorFromEnd = x.Length - Vector512.Count; + if (i <= oneVectorFromEnd) + { + // Loop handling one input vector / two destination vectors at a time. + do + { + (Vector512 first, Vector512 second) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); + first.StoreUnsafe(ref destination1Ref, (uint)i); + second.StoreUnsafe(ref destination2Ref, (uint)i); + + i += Vector512.Count; + } + while (i <= oneVectorFromEnd); + + // Handle any remaining elements with a final input vector. + if (i != x.Length) + { + i = x.Length - Vector512.Count; + + (Vector512 first, Vector512 second) = TUnaryOperator.Invoke(Vector512.LoadUnsafe(ref sourceRef, (uint)i)); + first.StoreUnsafe(ref destination1Ref, (uint)i); + second.StoreUnsafe(ref destination2Ref, (uint)i); + } + + return; + } + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TUnaryOperator.Vectorizable) + { + oneVectorFromEnd = x.Length - Vector256.Count; + if (i <= oneVectorFromEnd) + { + // Loop handling one input vector / two destination vectors at a time. + do + { + (Vector256 first, Vector256 second) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); + first.StoreUnsafe(ref destination1Ref, (uint)i); + second.StoreUnsafe(ref destination2Ref, (uint)i); + + i += Vector256.Count; + } + while (i <= oneVectorFromEnd); + + // Handle any remaining elements with a final input vector. + if (i != x.Length) + { + i = x.Length - Vector256.Count; + + (Vector256 first, Vector256 second) = TUnaryOperator.Invoke(Vector256.LoadUnsafe(ref sourceRef, (uint)i)); + first.StoreUnsafe(ref destination1Ref, (uint)i); + second.StoreUnsafe(ref destination2Ref, (uint)i); + } + + return; + } + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TUnaryOperator.Vectorizable) + { + oneVectorFromEnd = x.Length - Vector128.Count; + if (i <= oneVectorFromEnd) + { + // Loop handling one input vector / two destination vectors at a time. + do + { + (Vector128 first, Vector128 second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); + first.StoreUnsafe(ref destination1Ref, (uint)i); + second.StoreUnsafe(ref destination2Ref, (uint)i); + + i += Vector128.Count; + } + while (i <= oneVectorFromEnd); + + // Handle any remaining elements with a final input vector. + if (i != x.Length) + { + i = x.Length - Vector128.Count; + + (Vector128 first, Vector128 second) = TUnaryOperator.Invoke(Vector128.LoadUnsafe(ref sourceRef, (uint)i)); + first.StoreUnsafe(ref destination1Ref, (uint)i); + second.StoreUnsafe(ref destination2Ref, (uint)i); + } + + return; + } + } + + while (i < x.Length) + { + (T first, T second) = TUnaryOperator.Invoke(Unsafe.Add(ref sourceRef, i)); + Unsafe.Add(ref destination1Ref, i) = first; + Unsafe.Add(ref destination2Ref, i) = second; + i++; + } + } + + /// + /// Performs an element-wise operation on and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on the pair-wise elements loaded from and . + /// + private static void InvokeSpanSpanIntoSpan( + ReadOnlySpan x, ReadOnlySpan y, Span destination) + where TBinaryOperator : struct, IBinaryOperator + { + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + ValidateInputOutputSpanNonOverlapping(y, destination); + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + ref T dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TBinaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, ref yRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, ref dRef, remainder); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TBinaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, ref yRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, ref dRef, remainder); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TBinaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, ref yRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, ref dRef, remainder); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, ref yRef, ref dRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref T xRef, ref T yRef, ref T dRef, nuint length) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, i), + Unsafe.Add(ref yRef, i)); + } + } + + static void Vectorized128(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 beg = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), + Vector512.LoadUnsafe(ref yRef)); + Vector512 end = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count)); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0))); + vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1))); + vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2))); + vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4))); + vector2 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5))); + vector3 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6))); + vector4 = TBinaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = TBinaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + if (sizeof(T) == 1) + { + VectorizedSmall1(ref xRef, ref yRef, ref dRef, remainder); + } + else if (sizeof(T) == 2) + { + VectorizedSmall2(ref xRef, ref yRef, ref dRef, remainder); + } + else if (sizeof(T) == 4) + { + VectorizedSmall4(ref xRef, ref yRef, ref dRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + VectorizedSmall8(ref xRef, ref yRef, ref dRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 14), + Unsafe.Add(ref yRef, 14)); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 13), + Unsafe.Add(ref yRef, 13)); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 12), + Unsafe.Add(ref yRef, 12)); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 11), + Unsafe.Add(ref yRef, 11)); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 10), + Unsafe.Add(ref yRef, 10)); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 9), + Unsafe.Add(ref yRef, 9)); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 8), + Unsafe.Add(ref yRef, 8)); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 7), + Unsafe.Add(ref yRef, 7)); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + Unsafe.Add(ref yRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + Unsafe.Add(ref yRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + Unsafe.Add(ref yRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + Unsafe.Add(ref yRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1)); + goto case 1; + + case 1: + dRef = TBinaryOperator.Invoke(xRef, yRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + Unsafe.Add(ref yRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + Unsafe.Add(ref yRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + Unsafe.Add(ref yRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + Unsafe.Add(ref yRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1)); + goto case 1; + + case 1: + dRef = TBinaryOperator.Invoke(xRef, yRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2)); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1)); + goto case 1; + } + + case 1: + { + dRef = TBinaryOperator.Invoke(xRef, yRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref T xRef, ref T yRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + Vector256 end = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + Vector128 end = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = TBinaryOperator.Invoke(xRef, yRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// + /// Performs an element-wise operation on and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on each element loaded from with . + /// + private static void InvokeScalarSpanIntoSpan( + T x, ReadOnlySpan y, Span destination) + where TBinaryOperator : struct, IBinaryOperator => + InvokeSpanScalarIntoSpan, InvertedBinaryOperator>(y, x, destination); + + /// + /// Performs an element-wise operation on and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on each element loaded from with . + /// + private static void InvokeSpanScalarIntoSpan( + ReadOnlySpan x, T y, Span destination) + where TBinaryOperator : struct, IBinaryOperator => + InvokeSpanScalarIntoSpan, TBinaryOperator>(x, y, destination); + + /// + /// Performs an element-wise operation on and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on each element loaded from . + /// It is not used with . + /// + /// + /// Specifies the operation to perform on the transformed value from with . + /// + private static void InvokeSpanScalarIntoSpan( + ReadOnlySpan x, T y, Span destination) + where TTransformOperator : struct, IUnaryOperator + where TBinaryOperator : struct, IBinaryOperator + { + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, y, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, y, ref dRef, remainder); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, y, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, y, ref dRef, remainder); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported && TTransformOperator.Vectorizable && TBinaryOperator.Vectorizable) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, y, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, y, ref dRef, remainder); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, y, ref dRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref T xRef, T y, ref T dRef, nuint length) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, i)), + y); + } + } + + static void Vectorized128(ref T xRef, T y, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + yVec); + Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), + yVec); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))), + yVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))), + yVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3))), + yVec); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7))), + yVec); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref T xRef, T y, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + yVec); + Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), + yVec); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))), + yVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))), + yVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3))), + yVec); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7))), + yVec); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref T xRef, T y, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 yVec = Vector512.Create(y); + + Vector512 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef)), + yVec); + Vector512 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count)), + yVec); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))), + yVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))), + yVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3))), + yVec); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4))), + yVec); + vector2 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5))), + yVec); + vector3 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6))), + yVec); + vector4 = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7))), + yVec); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2))), + yVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref T xRef, T y, ref T dRef, nuint remainder) + { + if (sizeof(T) == 1) + { + VectorizedSmall1(ref xRef, y, ref dRef, remainder); + } + else if (sizeof(T) == 2) + { + VectorizedSmall2(ref xRef, y, ref dRef, remainder); + } + else if (sizeof(T) == 4) + { + VectorizedSmall4(ref xRef, y, ref dRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + VectorizedSmall8(ref xRef, y, ref dRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref T xRef, T y, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + yVec); + Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + Vector256.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + yVec); + Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + Vector128.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 14)), + y); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 13)), + y); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 12)), + y); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 11)), + y); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 10)), + y); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 9)), + y); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 8)), + y); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 7)), + y); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6)), + y); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5)), + y); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4)), + y); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3)), + y); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2)), + y); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1)), + y); + goto case 1; + + case 1: + dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref T xRef, T y, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + yVec); + Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + Vector256.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + yVec); + Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + Vector128.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 6)), + y); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 5)), + y); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 4)), + y); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 3)), + y); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2)), + y); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1)), + y); + goto case 1; + + case 1: + dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref T xRef, T y, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + yVec); + Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + Vector256.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + yVec); + Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + Vector128.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 2)), + y); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = TBinaryOperator.Invoke(TTransformOperator.Invoke(Unsafe.Add(ref xRef, 1)), + y); + goto case 1; + } + + case 1: + { + dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref T xRef, T y, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + yVec); + Vector256 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector256.LoadUnsafe(ref xRef)), + Vector256.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + yVec); + Vector128 end = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count)), + yVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TBinaryOperator.Invoke(TTransformOperator.Invoke(Vector128.LoadUnsafe(ref xRef)), + Vector128.Create(y)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = TBinaryOperator.Invoke(TTransformOperator.Invoke(xRef), y); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// + /// Performs an element-wise operation on , , and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on the pair-wise elements loaded from , , + /// and . + /// + private static void InvokeSpanSpanSpanIntoSpan( + ReadOnlySpan x, ReadOnlySpan y, ReadOnlySpan z, Span destination) + where TTernaryOperator : struct, ITernaryOperator + { + if (x.Length != y.Length || x.Length != z.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + ValidateInputOutputSpanNonOverlapping(y, destination); + ValidateInputOutputSpanNonOverlapping(z, destination); + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + ref T zRef = ref MemoryMarshal.GetReference(z); + ref T dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint length) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, i), + Unsafe.Add(ref yRef, i), + Unsafe.Add(ref zRef, i)); + } + } + + static void Vectorized128(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pz = &zRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* zPtr = pz; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + zPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + zPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + zPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + zRef = ref *zPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 8)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2)), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pz = &zRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* zPtr = pz; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (nuint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + zPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + zPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + zPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + zRef = ref *zPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 8)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2)), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 beg = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), + Vector512.LoadUnsafe(ref yRef), + Vector512.LoadUnsafe(ref zRef)); + Vector512 end = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)Vector512.Count)); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pz = &zRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* zPtr = pz; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + zPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + zPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + zPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + zRef = ref *zPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 8)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2)), + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + if (sizeof(T) == 1) + { + VectorizedSmall1(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + else if (sizeof(T) == 2) + { + VectorizedSmall2(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + else if (sizeof(T) == 4) + { + VectorizedSmall4(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + VectorizedSmall8(ref xRef, ref yRef, ref zRef, ref dRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 14), + Unsafe.Add(ref yRef, 14), + Unsafe.Add(ref zRef, 14)); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 13), + Unsafe.Add(ref yRef, 13), + Unsafe.Add(ref zRef, 13)); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 12), + Unsafe.Add(ref yRef, 12), + Unsafe.Add(ref zRef, 12)); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 11), + Unsafe.Add(ref yRef, 11), + Unsafe.Add(ref zRef, 11)); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 10), + Unsafe.Add(ref yRef, 10), + Unsafe.Add(ref zRef, 10)); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 9), + Unsafe.Add(ref yRef, 9), + Unsafe.Add(ref zRef, 9)); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 8), + Unsafe.Add(ref yRef, 8), + Unsafe.Add(ref zRef, 8)); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 7), + Unsafe.Add(ref yRef, 7), + Unsafe.Add(ref zRef, 7)); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + Unsafe.Add(ref yRef, 6), + Unsafe.Add(ref zRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + Unsafe.Add(ref yRef, 5), + Unsafe.Add(ref zRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + Unsafe.Add(ref yRef, 4), + Unsafe.Add(ref zRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + Unsafe.Add(ref yRef, 3), + Unsafe.Add(ref zRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2), + Unsafe.Add(ref zRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1), + Unsafe.Add(ref zRef, 1)); + goto case 1; + + case 1: + dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + Unsafe.Add(ref yRef, 6), + Unsafe.Add(ref zRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + Unsafe.Add(ref yRef, 5), + Unsafe.Add(ref zRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + Unsafe.Add(ref yRef, 4), + Unsafe.Add(ref zRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + Unsafe.Add(ref yRef, 3), + Unsafe.Add(ref zRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2), + Unsafe.Add(ref zRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1), + Unsafe.Add(ref zRef, 1)); + goto case 1; + + case 1: + dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2), + Unsafe.Add(ref zRef, 2)); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1), + Unsafe.Add(ref zRef, 1)); + goto case 1; + } + + case 1: + { + dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref T xRef, ref T yRef, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = TTernaryOperator.Invoke(xRef, yRef, zRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// + /// Performs an element-wise operation on , , and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on the pair-wise elements loaded from and + /// with . + /// + private static void InvokeSpanSpanScalarIntoSpan( + ReadOnlySpan x, ReadOnlySpan y, T z, Span destination) + where TTernaryOperator : struct, ITernaryOperator + { + if (x.Length != y.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + ValidateInputOutputSpanNonOverlapping(y, destination); + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T yRef = ref MemoryMarshal.GetReference(y); + ref T dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, ref yRef, z, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, z, ref dRef, remainder); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, ref yRef, z, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, z, ref dRef, remainder); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, ref yRef, z, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, ref yRef, z, ref dRef, remainder); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, ref yRef, z, ref dRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref T xRef, ref T yRef, T z, ref T dRef, nuint length) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, i), + Unsafe.Add(ref yRef, i), + z); + } + } + + static void Vectorized128(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 zVec = Vector128.Create(z); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + zVec); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + zVec); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), + zVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), + zVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 0)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 1)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 2)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 3)), + zVec); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 4)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 5)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 6)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + Vector128.Load(yPtr + (uint)(Vector128.Count * 7)), + zVec); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + yPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 8)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 7)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 6)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 5)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 4)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 3)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)(Vector128.Count * 2)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 zVec = Vector256.Create(z); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + zVec); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + zVec); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), + zVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), + zVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 0)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 1)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 2)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 3)), + zVec); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 4)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 5)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 6)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + Vector256.Load(yPtr + (uint)(Vector256.Count * 7)), + zVec); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + yPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 8)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 7)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 6)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 5)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 4)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 3)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)(Vector256.Count * 2)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 zVec = Vector512.Create(z); + + Vector512 beg = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), + Vector512.LoadUnsafe(ref yRef), + zVec); + Vector512 end = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)Vector512.Count), + zVec); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* py = &yRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* yPtr = py; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + yPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), + zVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), + zVec); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 0)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 1)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 2)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 3)), + zVec); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 4)), + zVec); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 5)), + zVec); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 6)), + zVec); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + Vector512.Load(yPtr + (uint)(Vector512.Count * 7)), + zVec); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + yPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + yRef = ref *yPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 8)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 7)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 6)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 5)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 4)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 3)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), + Vector512.LoadUnsafe(ref yRef, remainder - (uint)(Vector512.Count * 2)), + zVec); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + if (sizeof(T) == 1) + { + VectorizedSmall1(ref xRef, ref yRef, z, ref dRef, remainder); + } + else if (sizeof(T) == 2) + { + VectorizedSmall2(ref xRef, ref yRef, z, ref dRef, remainder); + } + else if (sizeof(T) == 4) + { + VectorizedSmall4(ref xRef, ref yRef, z, ref dRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + VectorizedSmall8(ref xRef, ref yRef, z, ref dRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 zVec = Vector256.Create(z); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + zVec); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 zVec = Vector128.Create(z); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + zVec); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 14), + Unsafe.Add(ref yRef, 14), + z); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 13), + Unsafe.Add(ref yRef, 13), + z); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 12), + Unsafe.Add(ref yRef, 12), + z); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 11), + Unsafe.Add(ref yRef, 11), + z); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 10), + Unsafe.Add(ref yRef, 10), + z); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 9), + Unsafe.Add(ref yRef, 9), + z); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 8), + Unsafe.Add(ref yRef, 8), + z); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 7), + Unsafe.Add(ref yRef, 7), + z); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + Unsafe.Add(ref yRef, 6), + z); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + Unsafe.Add(ref yRef, 5), + z); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + Unsafe.Add(ref yRef, 4), + z); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + Unsafe.Add(ref yRef, 3), + z); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2), + z); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1), + z); + goto case 1; + + case 1: + dRef = TTernaryOperator.Invoke(xRef, yRef, z); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 zVec = Vector256.Create(z); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + zVec); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 zVec = Vector128.Create(z); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + zVec); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + Unsafe.Add(ref yRef, 6), + z); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + Unsafe.Add(ref yRef, 5), + z); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + Unsafe.Add(ref yRef, 4), + z); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + Unsafe.Add(ref yRef, 3), + z); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2), + z); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1), + z); + goto case 1; + + case 1: + dRef = TTernaryOperator.Invoke(xRef, yRef, z); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 zVec = Vector256.Create(z); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + zVec); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 zVec = Vector128.Create(z); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + zVec); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + Unsafe.Add(ref yRef, 2), + z); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + Unsafe.Add(ref yRef, 1), + z); + goto case 1; + } + + case 1: + { + dRef = TTernaryOperator.Invoke(xRef, yRef, z); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref T xRef, ref T yRef, T z, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 zVec = Vector256.Create(z); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + zVec); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + Vector256.LoadUnsafe(ref yRef, remainder - (uint)Vector256.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.LoadUnsafe(ref yRef), + Vector256.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 zVec = Vector128.Create(z); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + zVec); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + Vector128.LoadUnsafe(ref yRef, remainder - (uint)Vector128.Count), + zVec); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.LoadUnsafe(ref yRef), + Vector128.Create(z)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = TTernaryOperator.Invoke(xRef, yRef, z); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// + /// Performs an element-wise operation on , , and , + /// and writes the results to . + /// + /// The element type. + /// + /// Specifies the operation to perform on the pair-wise element loaded from , with , + /// and the element loaded from . + /// + private static void InvokeSpanScalarSpanIntoSpan( + ReadOnlySpan x, T y, ReadOnlySpan z, Span destination) + where TTernaryOperator : struct, ITernaryOperator + { + if (x.Length != z.Length) + { + ThrowHelper.ThrowArgument_SpansMustHaveSameLength(); + } + + if (x.Length > destination.Length) + { + ThrowHelper.ThrowArgument_DestinationTooShort(); + } + + ValidateInputOutputSpanNonOverlapping(x, destination); + ValidateInputOutputSpanNonOverlapping(z, destination); + + // Since every branch has a cost and since that cost is + // essentially lost for larger inputs, we do branches + // in a way that allows us to have the minimum possible + // for small sizes + + ref T xRef = ref MemoryMarshal.GetReference(x); + ref T zRef = ref MemoryMarshal.GetReference(z); + ref T dRef = ref MemoryMarshal.GetReference(destination); + + nuint remainder = (uint)x.Length; + + if (Vector512.IsHardwareAccelerated && Vector512.IsSupported) + { + if (remainder >= (uint)Vector512.Count) + { + Vectorized512(ref xRef, y, ref zRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, y, ref zRef, ref dRef, remainder); + } + + return; + } + + if (Vector256.IsHardwareAccelerated && Vector256.IsSupported) + { + if (remainder >= (uint)Vector256.Count) + { + Vectorized256(ref xRef, y, ref zRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, y, ref zRef, ref dRef, remainder); + } + + return; + } + + if (Vector128.IsHardwareAccelerated && Vector128.IsSupported) + { + if (remainder >= (uint)Vector128.Count) + { + Vectorized128(ref xRef, y, ref zRef, ref dRef, remainder); + } + else + { + // We have less than a vector and so we can only handle this as scalar. To do this + // efficiently, we simply have a small jump table and fallthrough. So we get a simple + // length check, single jump, and then linear execution. + + VectorizedSmall(ref xRef, y, ref zRef, ref dRef, remainder); + } + + return; + } + + // This is the software fallback when no acceleration is available + // It requires no branches to hit + + SoftwareFallback(ref xRef, y, ref zRef, ref dRef, remainder); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void SoftwareFallback(ref T xRef, T y, ref T zRef, ref T dRef, nuint length) + { + for (nuint i = 0; i < length; i++) + { + Unsafe.Add(ref dRef, i) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, i), + y, + Unsafe.Add(ref zRef, i)); + } + } + + static void Vectorized128(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + yVec, + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + if (remainder > (uint)(Vector128.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pz = &zRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* zPtr = pz; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector128) - ((nuint)dPtr % (uint)sizeof(Vector128))) / (uint)sizeof(T); + + xPtr += misalignment; + zPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector128)) == 0); + + remainder -= misalignment; + } + + Vector128 vector1; + Vector128 vector2; + Vector128 vector3; + Vector128 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + zPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector128.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 0)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 1)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 2)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 3)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 0)); + vector2.Store(dPtr + (uint)(Vector128.Count * 1)); + vector3.Store(dPtr + (uint)(Vector128.Count * 2)); + vector4.Store(dPtr + (uint)(Vector128.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 4)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 5)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 6)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector128.Load(xPtr + (uint)(Vector128.Count * 7)), + yVec, + Vector128.Load(zPtr + (uint)(Vector128.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector128.Count * 4)); + vector2.Store(dPtr + (uint)(Vector128.Count * 5)); + vector3.Store(dPtr + (uint)(Vector128.Count * 6)); + vector4.Store(dPtr + (uint)(Vector128.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector128.Count * 8); + zPtr += (uint)(Vector128.Count * 8); + dPtr += (uint)(Vector128.Count * 8); + + remainder -= (uint)(Vector128.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + zRef = ref *zPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector128.Count - 1)) & (nuint)(-Vector128.Count); + + switch (remainder / (uint)Vector128.Count) + { + case 8: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 8)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 8)); + goto case 7; + } + + case 7: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 7)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 7)); + goto case 6; + } + + case 6: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 6)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 6)); + goto case 5; + } + + case 5: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 5)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 5)); + goto case 4; + } + + case 4: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 4)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 4)); + goto case 3; + } + + case 3: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 3)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 3)); + goto case 2; + } + + case 2: + { + Vector128 vector = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)(Vector128.Count * 2)), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)(Vector128.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector128.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector128.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized256(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + yVec, + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + if (remainder > (uint)(Vector256.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pz = &zRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* zPtr = pz; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector256) - ((nuint)dPtr % (uint)sizeof(Vector256))) / (uint)sizeof(T); + + xPtr += misalignment; + zPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector256)) == 0); + + remainder -= misalignment; + } + + Vector256 vector1; + Vector256 vector2; + Vector256 vector3; + Vector256 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + zPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector256.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 0)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 1)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 2)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 3)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 0)); + vector2.Store(dPtr + (uint)(Vector256.Count * 1)); + vector3.Store(dPtr + (uint)(Vector256.Count * 2)); + vector4.Store(dPtr + (uint)(Vector256.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 4)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 5)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 6)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector256.Load(xPtr + (uint)(Vector256.Count * 7)), + yVec, + Vector256.Load(zPtr + (uint)(Vector256.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector256.Count * 4)); + vector2.Store(dPtr + (uint)(Vector256.Count * 5)); + vector3.Store(dPtr + (uint)(Vector256.Count * 6)); + vector4.Store(dPtr + (uint)(Vector256.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector256.Count * 8); + zPtr += (uint)(Vector256.Count * 8); + dPtr += (uint)(Vector256.Count * 8); + + remainder -= (uint)(Vector256.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + zRef = ref *zPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector256.Count - 1)) & (nuint)(-Vector256.Count); + + switch (remainder / (uint)Vector256.Count) + { + case 8: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 8)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 8)); + goto case 7; + } + + case 7: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 7)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 7)); + goto case 6; + } + + case 6: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 6)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 6)); + goto case 5; + } + + case 5: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 5)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 5)); + goto case 4; + } + + case 4: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 4)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 4)); + goto case 3; + } + + case 3: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 3)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 3)); + goto case 2; + } + + case 2: + { + Vector256 vector = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)(Vector256.Count * 2)), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)(Vector256.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector256.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector256.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + static void Vectorized512(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + ref T dRefBeg = ref dRef; + + // Preload the beginning and end so that overlapping accesses don't negatively impact the data + + Vector512 yVec = Vector512.Create(y); + + Vector512 beg = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef), + yVec, + Vector512.LoadUnsafe(ref zRef)); + Vector512 end = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)Vector512.Count), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)Vector512.Count)); + + if (remainder > (uint)(Vector512.Count * 8)) + { + // Pinning is cheap and will be short lived for small inputs and unlikely to be impactful + // for large inputs (> 85KB) which are on the LOH and unlikely to be compacted. + + fixed (T* px = &xRef) + fixed (T* pz = &zRef) + fixed (T* pd = &dRef) + { + T* xPtr = px; + T* zPtr = pz; + T* dPtr = pd; + + // We need to the ensure the underlying data can be aligned and only align + // it if it can. It is possible we have an unaligned ref, in which case we + // can never achieve the required SIMD alignment. + + bool canAlign = ((nuint)dPtr % (nuint)sizeof(T)) == 0; + + if (canAlign) + { + // Compute by how many elements we're misaligned and adjust the pointers accordingly + // + // Noting that we are only actually aligning dPtr. This is because unaligned stores + // are more expensive than unaligned loads and aligning both is significantly more + // complex. + + nuint misalignment = ((uint)sizeof(Vector512) - ((nuint)dPtr % (uint)sizeof(Vector512))) / (uint)sizeof(T); + + xPtr += misalignment; + zPtr += misalignment; + dPtr += misalignment; + + Debug.Assert(((nuint)dPtr % (uint)sizeof(Vector512)) == 0); + + remainder -= misalignment; + } + + Vector512 vector1; + Vector512 vector2; + Vector512 vector3; + Vector512 vector4; + + if ((remainder > (NonTemporalByteThreshold / (nuint)sizeof(T))) && canAlign) + { + // This loop stores the data non-temporally, which benefits us when there + // is a large amount of data involved as it avoids polluting the cache. + + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 0)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 1)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 2)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); + + vector1.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 4)); + vector2.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 5)); + vector3.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 6)); + vector4.StoreAlignedNonTemporal(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + zPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + else + { + while (remainder >= (uint)(Vector512.Count * 8)) + { + // We load, process, and store the first four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 0)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 0))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 1)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 1))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 2)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 2))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 3)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 3))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 0)); + vector2.Store(dPtr + (uint)(Vector512.Count * 1)); + vector3.Store(dPtr + (uint)(Vector512.Count * 2)); + vector4.Store(dPtr + (uint)(Vector512.Count * 3)); + + // We load, process, and store the next four vectors + + vector1 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 4)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 4))); + vector2 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 5)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 5))); + vector3 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 6)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 6))); + vector4 = TTernaryOperator.Invoke(Vector512.Load(xPtr + (uint)(Vector512.Count * 7)), + yVec, + Vector512.Load(zPtr + (uint)(Vector512.Count * 7))); + + vector1.Store(dPtr + (uint)(Vector512.Count * 4)); + vector2.Store(dPtr + (uint)(Vector512.Count * 5)); + vector3.Store(dPtr + (uint)(Vector512.Count * 6)); + vector4.Store(dPtr + (uint)(Vector512.Count * 7)); + + // We adjust the source and destination references, then update + // the count of remaining elements to process. + + xPtr += (uint)(Vector512.Count * 8); + zPtr += (uint)(Vector512.Count * 8); + dPtr += (uint)(Vector512.Count * 8); + + remainder -= (uint)(Vector512.Count * 8); + } + } + + // Adjusting the refs here allows us to avoid pinning for very small inputs + + xRef = ref *xPtr; + zRef = ref *zPtr; + dRef = ref *dPtr; + } + } + + // Process the remaining [Count, Count * 8] elements via a jump table + // + // Unless the original length was an exact multiple of Count, then we'll + // end up reprocessing a couple elements in case 1 for end. We'll also + // potentially reprocess a few elements in case 0 for beg, to handle any + // data before the first aligned address. + + nuint endIndex = remainder; + remainder = (remainder + (uint)(Vector512.Count - 1)) & (nuint)(-Vector512.Count); + + switch (remainder / (uint)Vector512.Count) + { + case 8: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 8)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 8))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 8)); + goto case 7; + } + + case 7: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 7)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 7))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 7)); + goto case 6; + } + + case 6: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 6)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 6))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 6)); + goto case 5; + } + + case 5: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 5)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 5))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 5)); + goto case 4; + } + + case 4: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 4)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 4))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 4)); + goto case 3; + } + + case 3: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 3)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 3))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 3)); + goto case 2; + } + + case 2: + { + Vector512 vector = TTernaryOperator.Invoke(Vector512.LoadUnsafe(ref xRef, remainder - (uint)(Vector512.Count * 2)), + yVec, + Vector512.LoadUnsafe(ref zRef, remainder - (uint)(Vector512.Count * 2))); + vector.StoreUnsafe(ref dRef, remainder - (uint)(Vector512.Count * 2)); + goto case 1; + } + + case 1: + { + // Store the last block, which includes any elements that wouldn't fill a full vector + end.StoreUnsafe(ref dRef, endIndex - (uint)Vector512.Count); + goto case 0; + } + + case 0: + { + // Store the first block, which includes any elements preceding the first aligned block + beg.StoreUnsafe(ref dRefBeg); + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + if (sizeof(T) == 1) + { + VectorizedSmall1(ref xRef, y, ref zRef, ref dRef, remainder); + } + else if (sizeof(T) == 2) + { + VectorizedSmall2(ref xRef, y, ref zRef, ref dRef, remainder); + } + else if (sizeof(T) == 4) + { + VectorizedSmall4(ref xRef, y, ref zRef, ref dRef, remainder); + } + else + { + Debug.Assert(sizeof(T) == 8); + VectorizedSmall8(ref xRef, y, ref zRef, ref dRef, remainder); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall1(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 1); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 63: + case 62: + case 61: + case 60: + case 59: + case 58: + case 57: + case 56: + case 55: + case 54: + case 53: + case 52: + case 51: + case 50: + case 49: + case 48: + case 47: + case 46: + case 45: + case 44: + case 43: + case 42: + case 41: + case 40: + case 39: + case 38: + case 37: + case 36: + case 35: + case 34: + case 33: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + yVec, + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 32: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.Create(y), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + yVec, + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 16: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.Create(y), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 15: + Unsafe.Add(ref dRef, 14) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 14), + y, + Unsafe.Add(ref zRef, 14)); + goto case 14; + + case 14: + Unsafe.Add(ref dRef, 13) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 13), + y, + Unsafe.Add(ref zRef, 13)); + goto case 13; + + case 13: + Unsafe.Add(ref dRef, 12) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 12), + y, + Unsafe.Add(ref zRef, 12)); + goto case 12; + + case 12: + Unsafe.Add(ref dRef, 11) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 11), + y, + Unsafe.Add(ref zRef, 11)); + goto case 11; + + case 11: + Unsafe.Add(ref dRef, 10) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 10), + y, + Unsafe.Add(ref zRef, 10)); + goto case 10; + + case 10: + Unsafe.Add(ref dRef, 9) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 9), + y, + Unsafe.Add(ref zRef, 9)); + goto case 9; + + case 9: + Unsafe.Add(ref dRef, 8) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 8), + y, + Unsafe.Add(ref zRef, 8)); + goto case 8; + + case 8: + Unsafe.Add(ref dRef, 7) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 7), + y, + Unsafe.Add(ref zRef, 7)); + goto case 7; + + case 7: + Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + y, + Unsafe.Add(ref zRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + y, + Unsafe.Add(ref zRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + y, + Unsafe.Add(ref zRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + y, + Unsafe.Add(ref zRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + y, + Unsafe.Add(ref zRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + y, + Unsafe.Add(ref zRef, 1)); + goto case 1; + + case 1: + dRef = TTernaryOperator.Invoke(xRef, y, zRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall2(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 2); + + switch (remainder) + { + // Two Vector256's worth of data, with at least one element overlapping. + case 31: + case 30: + case 29: + case 28: + case 27: + case 26: + case 25: + case 24: + case 23: + case 22: + case 21: + case 20: + case 19: + case 18: + case 17: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + yVec, + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + // One Vector256's worth of data. + case 16: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.Create(y), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Two Vector128's worth of data, with at least one element overlapping. + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + yVec, + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + // One Vector128's worth of data. + case 8: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.Create(y), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + // Cases that are smaller than a single vector. No SIMD; just jump to the length and fall through each + // case to unroll the whole processing. + case 7: + Unsafe.Add(ref dRef, 6) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 6), + y, + Unsafe.Add(ref zRef, 6)); + goto case 6; + + case 6: + Unsafe.Add(ref dRef, 5) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 5), + y, + Unsafe.Add(ref zRef, 5)); + goto case 5; + + case 5: + Unsafe.Add(ref dRef, 4) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 4), + y, + Unsafe.Add(ref zRef, 4)); + goto case 4; + + case 4: + Unsafe.Add(ref dRef, 3) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 3), + y, + Unsafe.Add(ref zRef, 3)); + goto case 3; + + case 3: + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + y, + Unsafe.Add(ref zRef, 2)); + goto case 2; + + case 2: + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + y, + Unsafe.Add(ref zRef, 1)); + goto case 1; + + case 1: + dRef = TTernaryOperator.Invoke(xRef, y, zRef); + goto case 0; + + case 0: + break; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall4(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 4); + + switch (remainder) + { + case 15: + case 14: + case 13: + case 12: + case 11: + case 10: + case 9: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + yVec, + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 8: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.Create(y), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 7: + case 6: + case 5: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + yVec, + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.Create(y), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Unsafe.Add(ref dRef, 2) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 2), + y, + Unsafe.Add(ref zRef, 2)); + goto case 2; + } + + case 2: + { + Unsafe.Add(ref dRef, 1) = TTernaryOperator.Invoke(Unsafe.Add(ref xRef, 1), + y, + Unsafe.Add(ref zRef, 1)); + goto case 1; + } + + case 1: + { + dRef = TTernaryOperator.Invoke(xRef, y, zRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + static void VectorizedSmall8(ref T xRef, T y, ref T zRef, ref T dRef, nuint remainder) + { + Debug.Assert(sizeof(T) == 8); + + switch (remainder) + { + case 7: + case 6: + case 5: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 yVec = Vector256.Create(y); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + yVec, + Vector256.LoadUnsafe(ref zRef)); + Vector256 end = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef, remainder - (uint)Vector256.Count), + yVec, + Vector256.LoadUnsafe(ref zRef, remainder - (uint)Vector256.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector256.Count); + + break; + } + + case 4: + { + Debug.Assert(Vector256.IsHardwareAccelerated); + + Vector256 beg = TTernaryOperator.Invoke(Vector256.LoadUnsafe(ref xRef), + Vector256.Create(y), + Vector256.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 3: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 yVec = Vector128.Create(y); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + yVec, + Vector128.LoadUnsafe(ref zRef)); + Vector128 end = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef, remainder - (uint)Vector128.Count), + yVec, + Vector128.LoadUnsafe(ref zRef, remainder - (uint)Vector128.Count)); + + beg.StoreUnsafe(ref dRef); + end.StoreUnsafe(ref dRef, remainder - (uint)Vector128.Count); + + break; + } + + case 2: + { + Debug.Assert(Vector128.IsHardwareAccelerated); + + Vector128 beg = TTernaryOperator.Invoke(Vector128.LoadUnsafe(ref xRef), + Vector128.Create(y), + Vector128.LoadUnsafe(ref zRef)); + beg.StoreUnsafe(ref dRef); + + break; + } + + case 1: + { + dRef = TTernaryOperator.Invoke(xRef, y, zRef); + goto case 0; + } + + case 0: + { + break; + } + } + } + } + + /// Aggregates all of the elements in the into a single value. + /// The element type. + /// Specifies the operation to be performed on each pair of values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T HorizontalAggregate(Vector128 x) where TAggregate : struct, IBinaryOperator + { + // We need to do log2(count) operations to compute the total sum + + if (Unsafe.SizeOf() == 1) + { + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As()); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As()); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As()); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As()); + } + else if (Unsafe.SizeOf() == 2) + { + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As()); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As()); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As()); + } + else if (Unsafe.SizeOf() == 4) + { + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(2, 3, 0, 1)).As()); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt32(), Vector128.Create(1, 0, 3, 2)).As()); + } + else + { + Debug.Assert(Unsafe.SizeOf() == 8); + x = TAggregate.Invoke(x, Vector128.Shuffle(x.AsInt64(), Vector128.Create(1, 0)).As()); + } + + return x.ToScalar(); + } + + /// Aggregates all of the elements in the into a single value. + /// The element type. + /// Specifies the operation to be performed on each pair of values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T HorizontalAggregate(Vector256 x) where TAggregate : struct, IBinaryOperator => + HorizontalAggregate(TAggregate.Invoke(x.GetLower(), x.GetUpper())); + + /// Aggregates all of the elements in the into a single value. + /// The element type. + /// Specifies the operation to be performed on each pair of values. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static T HorizontalAggregate(Vector512 x) where TAggregate : struct, IBinaryOperator => + HorizontalAggregate(TAggregate.Invoke(x.GetLower(), x.GetUpper())); + + /// Gets whether the specified is negative. + private static bool IsNegative(T f) where T : INumberBase => T.IsNegative(f); + + /// Gets whether each specified is negative. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 IsNegative(Vector128 vector) + { + if (typeof(T) == typeof(float)) + { + return Vector128.LessThan(vector.AsInt32(), Vector128.Zero).As(); + } + + if (typeof(T) == typeof(double)) + { + return Vector128.LessThan(vector.AsInt64(), Vector128.Zero).As(); + } + + return Vector128.LessThan(vector, Vector128.Zero); + } + + /// Gets whether each specified is negative. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 IsNegative(Vector256 vector) + { + if (typeof(T) == typeof(float)) + { + return Vector256.LessThan(vector.AsInt32(), Vector256.Zero).As(); + } + + if (typeof(T) == typeof(double)) + { + return Vector256.LessThan(vector.AsInt64(), Vector256.Zero).As(); + } + + return Vector256.LessThan(vector, Vector256.Zero); + } + + /// Gets whether each specified is negative. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 IsNegative(Vector512 vector) + { + if (typeof(T) == typeof(float)) + { + return Vector512.LessThan(vector.AsInt32(), Vector512.Zero).As(); + } + + if (typeof(T) == typeof(double)) + { + return Vector512.LessThan(vector.AsInt64(), Vector512.Zero).As(); + } + + return Vector512.LessThan(vector, Vector512.Zero); + } + + /// + /// Gets a vector mask that will be all-ones-set for the last elements + /// and zero for all other elements. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateAlignmentMaskVector128(int count) + { + if (Unsafe.SizeOf() == 1) + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentByteMask_64x65)), + (uint)(count * 64)); + } + + if (Unsafe.SizeOf() == 2) + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt16Mask_32x33)), + (uint)(count * 32)); + } + + if (Unsafe.SizeOf() == 4) + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt32Mask_16x17)), + (uint)(count * 16)); + } + + Debug.Assert(Unsafe.SizeOf() == 8); + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt64Mask_8x9)), + (uint)(count * 8)); + } + } + + /// + /// Gets a vector mask that will be all-ones-set for the last elements + /// and zero for all other elements. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 CreateAlignmentMaskVector256(int count) + { + if (Unsafe.SizeOf() == 1) + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentByteMask_64x65)), + (uint)(count * 64)); + } + + if (Unsafe.SizeOf() == 2) + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt16Mask_32x33)), + (uint)(count * 32)); + } + + if (Unsafe.SizeOf() == 4) + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt32Mask_16x17)), + (uint)(count * 16)); + } + + Debug.Assert(Unsafe.SizeOf() == 8); + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt64Mask_8x9)), + (uint)(count * 8)); + } + } + + /// + /// Gets a vector mask that will be all-ones-set for the last elements + /// and zero for all other elements. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateAlignmentMaskVector512(int count) + { + if (Unsafe.SizeOf() == 1) + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentByteMask_64x65)), + (uint)(count * 64)); + } + + if (Unsafe.SizeOf() == 2) + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt16Mask_32x33)), + (uint)(count * 32)); + } + + if (Unsafe.SizeOf() == 4) + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt32Mask_16x17)), + (uint)(count * 16)); + } + + Debug.Assert(Unsafe.SizeOf() == 8); + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(AlignmentUInt64Mask_8x9)), + (uint)(count * 8)); + } + } + + /// + /// Gets a vector mask that will be all-ones-set for the last elements + /// and zero for all other elements. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 CreateRemainderMaskVector128(int count) + { + if (Unsafe.SizeOf() == 1) + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderByteMask_64x65)), + (uint)(count * 64) + 48); // last 16 bytes in the row + } + + if (Unsafe.SizeOf() == 2) + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt16Mask_32x33)), + (uint)(count * 32) + 24); // last 8 shorts in the row + } + + if (Unsafe.SizeOf() == 4) + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt32Mask_16x17)), + (uint)(count * 16) + 12); // last 4 ints in the row + } + + Debug.Assert(Unsafe.SizeOf() == 8); + { + return Vector128.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt64Mask_8x9)), + (uint)(count * 8) + 6); // last 2 longs in the row + } + } + + /// + /// Gets a vector mask that will be all-ones-set for the last elements + /// and zero for all other elements. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 CreateRemainderMaskVector256(int count) + { + if (Unsafe.SizeOf() == 1) + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderByteMask_64x65)), + (uint)(count * 64) + 32); // last 32 bytes in the row + } + + if (Unsafe.SizeOf() == 2) + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt16Mask_32x33)), + (uint)(count * 32) + 16); // last 16 shorts in the row + } + + if (Unsafe.SizeOf() == 4) + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt32Mask_16x17)), + (uint)(count * 16) + 8); // last 8 ints in the row + } + + Debug.Assert(Unsafe.SizeOf() == 8); + { + return Vector256.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt64Mask_8x9)), + (uint)(count * 8) + 4); // last 4 longs in the row + } + } + + /// + /// Gets a vector mask that will be all-ones-set for the last elements + /// and zero for all other elements. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 CreateRemainderMaskVector512(int count) + { + if (Unsafe.SizeOf() == 1) + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderByteMask_64x65)), + (uint)(count * 64)); + } + + if (Unsafe.SizeOf() == 2) + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt16Mask_32x33)), + (uint)(count * 32)); + } + + if (Unsafe.SizeOf() == 4) + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt32Mask_16x17)), + (uint)(count * 16)); + } + + Debug.Assert(Unsafe.SizeOf() == 8); + { + return Vector512.LoadUnsafe( + ref Unsafe.As(ref MemoryMarshal.GetReference(RemainderUInt64Mask_8x9)), + (uint)(count * 8)); + } + } + + // TODO: The uses of these ApplyScalar methods are all as part of operators when handling edge cases (NaN, Infinity, really large inputs, etc.) + // Currently, these edge cases are not handled in a vectorized way and instead fall back to scalar processing. We can look into + // handling those in a vectorized manner as well. + + private static Vector128 ApplyScalar(Vector128 floats) where TOperator : IUnaryOperator => + Vector128.Create(TOperator.Invoke(floats[0]), TOperator.Invoke(floats[1]), TOperator.Invoke(floats[2]), TOperator.Invoke(floats[3])); + + private static Vector256 ApplyScalar(Vector256 floats) where TOperator : IUnaryOperator => + Vector256.Create(ApplyScalar(floats.GetLower()), ApplyScalar(floats.GetUpper())); + + private static Vector512 ApplyScalar(Vector512 floats) where TOperator : IUnaryOperator => + Vector512.Create(ApplyScalar(floats.GetLower()), ApplyScalar(floats.GetUpper())); + + private static Vector128 ApplyScalar(Vector128 doubles) where TOperator : IUnaryOperator => + Vector128.Create(TOperator.Invoke(doubles[0]), TOperator.Invoke(doubles[1])); + + private static Vector256 ApplyScalar(Vector256 doubles) where TOperator : IUnaryOperator => + Vector256.Create(ApplyScalar(doubles.GetLower()), ApplyScalar(doubles.GetUpper())); + + private static Vector512 ApplyScalar(Vector512 doubles) where TOperator : IUnaryOperator => + Vector512.Create(ApplyScalar(doubles.GetLower()), ApplyScalar(doubles.GetUpper())); + + /// Creates a span of from a when they're the same type. + private static unsafe ReadOnlySpan Rename(ReadOnlySpan span) + { + Debug.Assert(sizeof(TFrom) == sizeof(TTo)); + return MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); + } + + /// Creates a span of from a when they're the same type. + private static unsafe Span Rename(Span span) + { + Debug.Assert(sizeof(TFrom) == sizeof(TTo)); + return MemoryMarshal.CreateSpan(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length); + } + + /// Gets whether is or if in a 32-bit process. + private static bool IsUInt32Like() => typeof(T) == typeof(uint) || (IntPtr.Size == 4 && typeof(T) == typeof(nuint)); + + /// Gets whether is or if in a 32-bit process. + private static bool IsInt32Like() => typeof(T) == typeof(int) || (IntPtr.Size == 4 && typeof(T) == typeof(nint)); + + /// Gets whether is or if in a 64-bit process. + private static bool IsUInt64Like() => typeof(T) == typeof(ulong) || (IntPtr.Size == 8 && typeof(T) == typeof(nuint)); + + /// Gets whether is or if in a 64-bit process. + private static bool IsInt64Like() => typeof(T) == typeof(long) || (IntPtr.Size == 8 && typeof(T) == typeof(nint)); + + /// x + y + internal readonly struct AddOperator : IAggregationOperator where T : IAdditionOperators, IAdditiveIdentity + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => x + y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x + y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x + y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x + y; + + public static T Invoke(Vector128 x) => Vector128.Sum(x); + public static T Invoke(Vector256 x) => Vector256.Sum(x); + public static T Invoke(Vector512 x) => Vector512.Sum(x); + + public static T IdentityValue => T.AdditiveIdentity; + } + + private readonly struct InvertedBinaryOperator : IBinaryOperator + where TOperator : IBinaryOperator + { + public static bool Vectorizable => TOperator.Vectorizable; + public static T Invoke(T x, T y) => TOperator.Invoke(y, x); + public static Vector128 Invoke(Vector128 x, Vector128 y) => TOperator.Invoke(y, x); + public static Vector256 Invoke(Vector256 x, Vector256 y) => TOperator.Invoke(y, x); + public static Vector512 Invoke(Vector512 x, Vector512 y) => TOperator.Invoke(y, x); + } + + /// x - y + internal readonly struct SubtractOperator : IBinaryOperator where T : ISubtractionOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x, T y) => x - y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x - y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x - y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x - y; + } + + /// (x - y) * (x - y) + internal readonly struct SubtractSquaredOperator : IBinaryOperator where T : ISubtractionOperators, IMultiplyOperators + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) + { + T tmp = x - y; + return tmp * tmp; + } + + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 tmp = x - y; + return tmp * tmp; + } + + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 tmp = x - y; + return tmp * tmp; + } + + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 tmp = x - y; + return tmp * tmp; + } + } + + /// x * y + internal readonly struct MultiplyOperator : IAggregationOperator where T : IMultiplyOperators, IMultiplicativeIdentity + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => x * y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x * y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x * y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x * y; + + public static T Invoke(Vector128 x) => HorizontalAggregate>(x); + public static T Invoke(Vector256 x) => HorizontalAggregate>(x); + public static T Invoke(Vector512 x) => HorizontalAggregate>(x); + + public static T IdentityValue => T.MultiplicativeIdentity; + } + + /// x / y + internal readonly struct DivideOperator : IBinaryOperator where T : IDivisionOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x, T y) => x / y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x / y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x / y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x / y; + } + + /// T.Ieee754Remainder(x, y) + internal readonly struct Ieee754RemainderOperator : IBinaryOperator where T : IFloatingPointIeee754 + { + public static bool Vectorizable => false; + public static T Invoke(T x, T y) => T.Ieee754Remainder(x, y); + public static Vector128 Invoke(Vector128 x, Vector128 y) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x, Vector256 y) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x, Vector512 y) => throw new NotSupportedException(); + } + + // Ieee754Remainder + + internal readonly struct ReciprocalOperator : IUnaryOperator where T : IFloatingPoint + { + public static bool Vectorizable => true; + public static T Invoke(T x) => T.One / x; + public static Vector128 Invoke(Vector128 x) => Vector128.One / x; + public static Vector256 Invoke(Vector256 x) => Vector256.One / x; + public static Vector512 Invoke(Vector512 x) => Vector512.One / x; + } + + private readonly struct ReciprocalSqrtOperator : IUnaryOperator where T : IFloatingPointIeee754 + { + public static bool Vectorizable => true; + public static T Invoke(T x) => T.One / T.Sqrt(x); + public static Vector128 Invoke(Vector128 x) => Vector128.One / Vector128.Sqrt(x); + public static Vector256 Invoke(Vector256 x) => Vector256.One / Vector256.Sqrt(x); + public static Vector512 Invoke(Vector512 x) => Vector512.One / Vector512.Sqrt(x); + } + + private readonly struct ReciprocalEstimateOperator : IUnaryOperator where T : IFloatingPointIeee754 + { + public static bool Vectorizable => true; + + public static T Invoke(T x) => T.ReciprocalEstimate(x); + + public static Vector128 Invoke(Vector128 x) + { + if (Sse.IsSupported) + { + if (typeof(T) == typeof(float)) return Sse.Reciprocal(x.AsSingle()).As(); + } + + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(float)) return AdvSimd.ReciprocalEstimate(x.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.ReciprocalEstimate(x.AsDouble()).As(); + } + + return Vector128.One / x; + } + + public static Vector256 Invoke(Vector256 x) + { + if (Avx.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx.Reciprocal(x.AsSingle()).As(); + } + + return Vector256.One / x; + } + + public static Vector512 Invoke(Vector512 x) + { + if (Avx512F.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx512F.Reciprocal14(x.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Avx512F.Reciprocal14(x.AsDouble()).As(); + } + + return Vector512.One / x; + } + } + + private readonly struct ReciprocalSqrtEstimateOperator : IUnaryOperator where T : IFloatingPointIeee754 + { + public static bool Vectorizable => true; + + public static T Invoke(T x) => T.ReciprocalSqrtEstimate(x); + + public static Vector128 Invoke(Vector128 x) + { + if (Sse.IsSupported) + { + if (typeof(T) == typeof(float)) return Sse.ReciprocalSqrt(x.AsSingle()).As(); + } + + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(float)) return AdvSimd.ReciprocalSquareRootEstimate(x.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.ReciprocalSquareRootEstimate(x.AsDouble()).As(); + } + + return Vector128.One / Vector128.Sqrt(x); + } + + public static Vector256 Invoke(Vector256 x) + { + if (Avx.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx.ReciprocalSqrt(x.AsSingle()).As(); + } + + return Vector256.One / Vector256.Sqrt(x); + } + + public static Vector512 Invoke(Vector512 x) + { + if (Avx512F.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx512F.ReciprocalSqrt14(x.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Avx512F.ReciprocalSqrt14(x.AsDouble()).As(); + } + + return Vector512.One / Vector512.Sqrt(x); + } + } + + /// x & y + internal readonly struct BitwiseAndOperator : IBinaryOperator where T : IBitwiseOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x, T y) => x & y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x & y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x & y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x & y; + } + + /// x | y + internal readonly struct BitwiseOrOperator : IBinaryOperator where T : IBitwiseOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x, T y) => x | y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x | y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x | y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x | y; + } + + /// x ^ y + internal readonly struct XorOperator : IBinaryOperator where T : IBitwiseOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x, T y) => x ^ y; + public static Vector128 Invoke(Vector128 x, Vector128 y) => x ^ y; + public static Vector256 Invoke(Vector256 x, Vector256 y) => x ^ y; + public static Vector512 Invoke(Vector512 x, Vector512 y) => x ^ y; + } + + /// ~x + internal readonly struct OnesComplementOperator : IUnaryOperator where T : IBitwiseOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x) => ~x; + public static Vector128 Invoke(Vector128 x) => ~x; + public static Vector256 Invoke(Vector256 x) => ~x; + public static Vector512 Invoke(Vector512 x) => ~x; + } + + /// T.Max(x, y) (but NaNs may not be propagated) + internal readonly struct MaxOperator : IAggregationOperator where T : INumber + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) + { + if (typeof(T) == typeof(Half) || typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return x == y ? + (IsNegative(x) ? y : x) : + (y > x ? y : x); + } + + return T.Max(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(byte)) return AdvSimd.Max(x.AsByte(), y.AsByte()).As(); + if (typeof(T) == typeof(sbyte)) return AdvSimd.Max(x.AsSByte(), y.AsSByte()).As(); + if (typeof(T) == typeof(short)) return AdvSimd.Max(x.AsInt16(), y.AsInt16()).As(); + if (typeof(T) == typeof(ushort)) return AdvSimd.Max(x.AsUInt16(), y.AsUInt16()).As(); + if (typeof(T) == typeof(int)) return AdvSimd.Max(x.AsInt32(), y.AsInt32()).As(); + if (typeof(T) == typeof(uint)) return AdvSimd.Max(x.AsUInt32(), y.AsUInt32()).As(); + if (typeof(T) == typeof(float)) return AdvSimd.Max(x.AsSingle(), y.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Max(x.AsDouble(), y.AsDouble()).As(); + } + + if (typeof(T) == typeof(float)) + { + return + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x.AsSingle()).As(), y, x), + Vector128.Max(x, y)); + } + + if (typeof(T) == typeof(double)) + { + return + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x.AsDouble()).As(), y, x), + Vector128.Max(x, y)); + } + + return Vector128.Max(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float)) + { + return + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x.AsSingle()).As(), y, x), + Vector256.Max(x, y)); + } + + if (typeof(T) == typeof(double)) + { + return + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x.AsDouble()).As(), y, x), + Vector256.Max(x, y)); + } + + return Vector256.Max(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float)) + { + return + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x.AsSingle()).As(), y, x), + Vector512.Max(x, y)); + } + + if (typeof(T) == typeof(double)) + { + return + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x.AsDouble()).As(), y, x), + Vector512.Max(x, y)); + } + + return Vector512.Max(x, y); + } + + public static T Invoke(Vector128 x) => HorizontalAggregate>(x); + public static T Invoke(Vector256 x) => HorizontalAggregate>(x); + public static T Invoke(Vector512 x) => HorizontalAggregate>(x); + } + + private interface IIndexOfOperator where T : INumber + { + static abstract int Invoke(ref T result, T current, int resultIndex, int currentIndex); + static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex); + static abstract void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex); + static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int IndexOfFinalAggregate(Vector128 result, Vector128 resultIndex) + where T : INumber + where TIndexOfOperator : struct, IIndexOfOperator + { + Vector128 tmpResult; + Vector128 tmpIndex; + + if (sizeof(T) == 8) + { + // Compare 0 with 1 + tmpResult = Vector128.Shuffle(result.AsInt64(), Vector128.Create(1, 0)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsInt64(), Vector128.Create(1, 0)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Return 0 + return (int)resultIndex.As().ToScalar(); + } + + if (sizeof(T) == 4) + { + // Compare 0,1 with 2,3 + tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(2, 3, 0, 1)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(2, 3, 0, 1)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Compare 0 with 1 + tmpResult = Vector128.Shuffle(result.AsInt32(), Vector128.Create(1, 0, 3, 2)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsInt32(), Vector128.Create(1, 0, 3, 2)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Return 0 + return resultIndex.As().ToScalar(); + } + + if (sizeof(T) == 2) + { + // Compare 0,1,2,3 with 4,5,6,7 + tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(4, 5, 6, 7, 0, 1, 2, 3)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Compare 0,1 with 2,3 + tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(2, 3, 0, 1, 4, 5, 6, 7)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Compare 0 with 1 + tmpResult = Vector128.Shuffle(result.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsInt16(), Vector128.Create(1, 0, 2, 3, 4, 5, 6, 7)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Return 0 + return resultIndex.As().ToScalar(); + } + + Debug.Assert(sizeof(T) == 1); + { + // Compare 0,1,2,3,4,5,6,7 with 8,9,10,11,12,13,14,15 + tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)8, 9, 10, 11, 12, 13, 14, 15, 0, 1, 2, 3, 4, 5, 6, 7)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Compare 0,1,2,3 with 4,5,6,7 + tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)4, 5, 6, 7, 0, 1, 2, 3, 8, 9, 10, 11, 12, 13, 14, 15)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Compare 0,1 with 2,3 + tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Compare 0 with 1 + tmpResult = Vector128.Shuffle(result.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); + tmpIndex = Vector128.Shuffle(resultIndex.AsByte(), Vector128.Create((byte)1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)).As(); + TIndexOfOperator.Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); + + // Return 0 + return resultIndex.As().ToScalar(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int IndexOfFinalAggregate(Vector256 result, Vector256 resultIndex) + where T : INumber + where TIndexOfOperator : struct, IIndexOfOperator + { + // Min the upper/lower halves of the Vector256 + Vector128 resultLower = result.GetLower(); + Vector128 indexLower = resultIndex.GetLower(); + + TIndexOfOperator.Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return IndexOfFinalAggregate(resultLower, indexLower); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int IndexOfFinalAggregate(Vector512 result, Vector512 resultIndex) + where T : INumber + where TIndexOfOperator : struct, IIndexOfOperator + { + Vector256 resultLower = result.GetLower(); + Vector256 indexLower = resultIndex.GetLower(); + + TIndexOfOperator.Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return IndexOfFinalAggregate(resultLower, indexLower); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 IndexLessThan(Vector128 indices1, Vector128 indices2) => + sizeof(T) == sizeof(long) ? Vector128.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : + sizeof(T) == sizeof(int) ? Vector128.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : + sizeof(T) == sizeof(short) ? Vector128.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : + Vector128.LessThan(indices1.AsByte(), indices2.AsByte()).As(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 IndexLessThan(Vector256 indices1, Vector256 indices2) => + sizeof(T) == sizeof(long) ? Vector256.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : + sizeof(T) == sizeof(int) ? Vector256.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : + sizeof(T) == sizeof(short) ? Vector256.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : + Vector256.LessThan(indices1.AsByte(), indices2.AsByte()).As(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 IndexLessThan(Vector512 indices1, Vector512 indices2) => + sizeof(T) == sizeof(long) ? Vector512.LessThan(indices1.AsInt64(), indices2.AsInt64()).As() : + sizeof(T) == sizeof(int) ? Vector512.LessThan(indices1.AsInt32(), indices2.AsInt32()).As() : + sizeof(T) == sizeof(short) ? Vector512.LessThan(indices1.AsInt16(), indices2.AsInt16()).As() : + Vector512.LessThan(indices1.AsByte(), indices2.AsByte()).As(); + + /// Returns the index of MathF.Max(x, y) + internal readonly struct IndexOfMaxOperator : IIndexOfOperator where T : INumber + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) + { + Vector128 useResult = Vector128.GreaterThan(result, current); + Vector128 equalMask = Vector128.Equals(result, current); + + if (equalMask != Vector128.Zero) + { + Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); + Vector128 currentNegative = IsNegative(current); + Vector128 sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) + { + Vector256 useResult = Vector256.GreaterThan(result, current); + Vector256 equalMask = Vector256.Equals(result, current); + + if (equalMask != Vector256.Zero) + { + Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); + Vector256 currentNegative = IsNegative(current); + Vector256 sameSign = Vector256.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) + { + Vector512 useResult = Vector512.GreaterThan(result, current); + Vector512 equalMask = Vector512.Equals(result, current); + + if (equalMask != Vector512.Zero) + { + Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); + Vector512 currentNegative = IsNegative(current); + Vector512 sameSign = Vector512.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) + { + if (result == current) + { + bool resultNegative = IsNegative(result); + if ((resultNegative == IsNegative(current)) ? (currentIndex < resultIndex) : resultNegative) + { + result = current; + return currentIndex; + } + } + else if (current > result) + { + result = current; + return currentIndex; + } + + return resultIndex; + } + } + + internal readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator where T : INumber + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) + { + Vector128 resultMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); + Vector128 useResult = Vector128.GreaterThan(resultMag, currentMag); + Vector128 equalMask = Vector128.Equals(resultMag, currentMag); + + if (equalMask != Vector128.Zero) + { + Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); + Vector128 currentNegative = IsNegative(current); + Vector128 sameSign = Vector128.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) + { + Vector256 resultMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); + Vector256 useResult = Vector256.GreaterThan(resultMag, currentMag); + Vector256 equalMask = Vector256.Equals(resultMag, currentMag); + + if (equalMask != Vector256.Zero) + { + Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); + Vector256 currentNegative = IsNegative(current); + Vector256 sameSign = Vector256.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) + { + Vector512 resultMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); + Vector512 useResult = Vector512.GreaterThan(resultMag, currentMag); + Vector512 equalMask = Vector512.Equals(resultMag, currentMag); + + if (equalMask != Vector512.Zero) + { + Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(current)); + Vector512 currentNegative = IsNegative(current); + Vector512 sameSign = Vector512.Equals(IsNegative(result).AsInt32(), currentNegative.AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, currentNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) + { + T resultMag = T.Abs(result); + T currentMag = T.Abs(current); + + if (resultMag == currentMag) + { + bool resultNegative = IsNegative(result); + if ((resultNegative == IsNegative(current)) ? (currentIndex < resultIndex) : resultNegative) + { + result = current; + return currentIndex; + } + } + else if (currentMag > resultMag) + { + result = current; + return currentIndex; + } + + return resultIndex; + } + } + + /// Returns the index of MathF.Min(x, y) + internal readonly struct IndexOfMinOperator : IIndexOfOperator where T : INumber + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) + { + Vector128 useResult = Vector128.LessThan(result, current); + Vector128 equalMask = Vector128.Equals(result, current); + + if (equalMask != Vector128.Zero) + { + Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); + Vector128 resultNegative = IsNegative(result); + Vector128 sameSign = Vector128.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) + { + Vector256 useResult = Vector256.LessThan(result, current); + Vector256 equalMask = Vector256.Equals(result, current); + + if (equalMask != Vector256.Zero) + { + Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); + Vector256 resultNegative = IsNegative(result); + Vector256 sameSign = Vector256.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) + { + Vector512 useResult = Vector512.LessThan(result, current); + Vector512 equalMask = Vector512.Equals(result, current); + + if (equalMask != Vector512.Zero) + { + Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); + Vector512 resultNegative = IsNegative(result); + Vector512 sameSign = Vector512.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) + { + if (result == current) + { + bool currentNegative = IsNegative(current); + if ((IsNegative(result) == currentNegative) ? (currentIndex < resultIndex) : currentNegative) + { + result = current; + return currentIndex; + } + } + else if (current < result) + { + result = current; + return currentIndex; + } + + return resultIndex; + } + } + + internal readonly struct IndexOfMinMagnitudeOperator : IIndexOfOperator where T : INumber + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 currentIndex) + { + Vector128 resultMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); + Vector128 useResult = Vector128.LessThan(resultMag, currentMag); + Vector128 equalMask = Vector128.Equals(resultMag, currentMag); + + if (equalMask != Vector128.Zero) + { + Vector128 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); + Vector128 resultNegative = IsNegative(result); + Vector128 sameSign = Vector128.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 currentIndex) + { + Vector256 resultMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); + Vector256 useResult = Vector256.LessThan(resultMag, currentMag); + Vector256 equalMask = Vector256.Equals(resultMag, currentMag); + + if (equalMask != Vector256.Zero) + { + Vector256 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); + Vector256 resultNegative = IsNegative(result); + Vector256 sameSign = Vector256.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 currentIndex) + { + Vector512 resultMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); + Vector512 useResult = Vector512.LessThan(resultMag, currentMag); + Vector512 equalMask = Vector512.Equals(resultMag, currentMag); + + if (equalMask != Vector512.Zero) + { + Vector512 lessThanIndexMask = IndexLessThan(resultIndex, currentIndex); + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + // bool useResult = equal && ((IsNegative(result) == IsNegative(current)) ? (resultIndex < currentIndex) : IsNegative(result)); + Vector512 resultNegative = IsNegative(result); + Vector512 sameSign = Vector512.Equals(resultNegative.AsInt32(), IsNegative(current).AsInt32()).As(); + useResult |= equalMask & ElementWiseSelect(sameSign, lessThanIndexMask, resultNegative); + } + else + { + useResult |= equalMask & lessThanIndexMask; + } + } + + result = ElementWiseSelect(useResult, result, current); + resultIndex = ElementWiseSelect(useResult, resultIndex, currentIndex); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ref T result, T current, int resultIndex, int currentIndex) + { + T resultMag = T.Abs(result); + T currentMag = T.Abs(current); + + if (resultMag == currentMag) + { + bool currentNegative = IsNegative(current); + if ((IsNegative(result) == currentNegative) ? (currentIndex < resultIndex) : currentNegative) + { + result = current; + return currentIndex; + } + } + else if (currentMag < resultMag) + { + result = current; + return currentIndex; + } + + return resultIndex; + } + } + + /// Max(x, y) + internal readonly struct MaxPropagateNaNOperator : IBinaryOperator + where T : INumber + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.Max(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(byte)) return AdvSimd.Max(x.AsByte(), y.AsByte()).As(); + if (typeof(T) == typeof(sbyte)) return AdvSimd.Max(x.AsSByte(), y.AsSByte()).As(); + if (typeof(T) == typeof(ushort)) return AdvSimd.Max(x.AsUInt16(), y.AsUInt16()).As(); + if (typeof(T) == typeof(short)) return AdvSimd.Max(x.AsInt16(), y.AsInt16()).As(); + if (typeof(T) == typeof(uint)) return AdvSimd.Max(x.AsUInt32(), y.AsUInt32()).As(); + if (typeof(T) == typeof(int)) return AdvSimd.Max(x.AsInt32(), y.AsInt32()).As(); + if (typeof(T) == typeof(float)) return AdvSimd.Max(x.AsSingle(), y.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Max(x.AsDouble(), y.AsDouble()).As(); + } + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.Max(x, y)), + y), + x); + } + + return Vector128.Max(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.Max(x, y)), + y), + x); + } + + return Vector256.Max(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.Max(x, y)), + y), + x); + } + + return Vector512.Max(x, y); + } + } + + /// Operator to get x or y based on which has the larger MathF.Abs (but NaNs may not be propagated) + internal readonly struct MaxMagnitudeOperator : IAggregationOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.MaxMagnitude(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + + Vector128 result = + Vector128.ConditionalSelect(Vector128.Equals(xMag, yMag), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.ConditionalSelect(Vector128.GreaterThan(xMag, yMag), x, y)); + + // Handle minimum signed value that should have the largest magnitude + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector128 negativeMagnitudeX = Vector128.LessThan(xMag, Vector128.Zero); + Vector128 negativeMagnitudeY = Vector128.LessThan(yMag, Vector128.Zero); + result = Vector128.ConditionalSelect(negativeMagnitudeX, + x, + Vector128.ConditionalSelect(negativeMagnitudeY, + y, + result)); + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + + Vector256 result = + Vector256.ConditionalSelect(Vector256.Equals(xMag, yMag), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.ConditionalSelect(Vector256.GreaterThan(xMag, yMag), x, y)); + + // Handle minimum signed value that should have the largest magnitude + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector256 negativeMagnitudeX = Vector256.LessThan(xMag, Vector256.Zero); + Vector256 negativeMagnitudeY = Vector256.LessThan(yMag, Vector256.Zero); + result = Vector256.ConditionalSelect(negativeMagnitudeX, + x, + Vector256.ConditionalSelect(negativeMagnitudeY, + y, + result)); + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + + Vector512 result = + Vector512.ConditionalSelect(Vector512.Equals(xMag, yMag), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.ConditionalSelect(Vector512.GreaterThan(xMag, yMag), x, y)); + + // Handle minimum signed value that should have the largest magnitude + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector512 negativeMagnitudeX = Vector512.LessThan(xMag, Vector512.Zero); + Vector512 negativeMagnitudeY = Vector512.LessThan(yMag, Vector512.Zero); + result = Vector512.ConditionalSelect(negativeMagnitudeX, + x, + Vector512.ConditionalSelect(negativeMagnitudeY, + y, + result)); + } + + return result; + } + + public static T Invoke(Vector128 x) => HorizontalAggregate>(x); + public static T Invoke(Vector256 x) => HorizontalAggregate>(x); + public static T Invoke(Vector512 x) => HorizontalAggregate>(x); + } + + /// Operator to get x or y based on which has the larger MathF.Abs + internal readonly struct MaxMagnitudePropagateNaNOperator : IBinaryOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.MaxMagnitude(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + // Handle NaNs + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), + Vector128.ConditionalSelect(IsNegative(x), y, x), + Vector128.ConditionalSelect(Vector128.GreaterThan(yMag, xMag), y, x)), + y), + x); + } + + return MaxMagnitudeOperator.Invoke(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + // Handle NaNs + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + return + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(xMag, yMag), + Vector256.ConditionalSelect(IsNegative(x), y, x), + Vector256.ConditionalSelect(Vector256.GreaterThan(xMag, yMag), x, y)), + y), + x); + } + + return MaxMagnitudeOperator.Invoke(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + // Handle NaNs + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + return + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(xMag, yMag), + Vector512.ConditionalSelect(IsNegative(x), y, x), + Vector512.ConditionalSelect(Vector512.GreaterThan(xMag, yMag), x, y)), + y), + x); + } + + return MaxMagnitudeOperator.Invoke(x, y); + } + } + + /// T.Min(x, y) (but NaNs may not be propagated) + internal readonly struct MinOperator : IAggregationOperator + where T : INumber + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) + { + if (typeof(T) == typeof(Half) || typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return x == y ? + (IsNegative(y) ? y : x) : + (y < x ? y : x); + } + + return T.Min(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(byte)) return AdvSimd.Min(x.AsByte(), y.AsByte()).As(); + if (typeof(T) == typeof(sbyte)) return AdvSimd.Min(x.AsSByte(), y.AsSByte()).As(); + if (typeof(T) == typeof(short)) return AdvSimd.Min(x.AsInt16(), y.AsInt16()).As(); + if (typeof(T) == typeof(ushort)) return AdvSimd.Min(x.AsUInt16(), y.AsUInt16()).As(); + if (typeof(T) == typeof(int)) return AdvSimd.Min(x.AsInt32(), y.AsInt32()).As(); + if (typeof(T) == typeof(uint)) return AdvSimd.Min(x.AsUInt32(), y.AsUInt32()).As(); + if (typeof(T) == typeof(float)) return AdvSimd.Min(x.AsSingle(), y.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Min(x.AsDouble(), y.AsDouble()).As(); + } + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(y), y, x), + Vector128.Min(x, y)); + } + + return Vector128.Min(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(y), y, x), + Vector256.Min(x, y)); + } + + return Vector256.Min(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(y), y, x), + Vector512.Min(x, y)); + } + + return Vector512.Min(x, y); + } + + public static T Invoke(Vector128 x) => HorizontalAggregate>(x); + public static T Invoke(Vector256 x) => HorizontalAggregate>(x); + public static T Invoke(Vector512 x) => HorizontalAggregate>(x); + } + + /// T.Min(x, y) + internal readonly struct MinPropagateNaNOperator : IBinaryOperator + where T : INumber + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.Min(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(byte)) return AdvSimd.Min(x.AsByte(), y.AsByte()).As(); + if (typeof(T) == typeof(sbyte)) return AdvSimd.Min(x.AsSByte(), y.AsSByte()).As(); + if (typeof(T) == typeof(short)) return AdvSimd.Min(x.AsInt16(), y.AsInt16()).As(); + if (typeof(T) == typeof(ushort)) return AdvSimd.Min(x.AsUInt16(), y.AsUInt16()).As(); + if (typeof(T) == typeof(int)) return AdvSimd.Min(x.AsInt32(), y.AsInt32()).As(); + if (typeof(T) == typeof(uint)) return AdvSimd.Min(x.AsUInt32(), y.AsUInt32()).As(); + if (typeof(T) == typeof(float)) return AdvSimd.Min(x.AsSingle(), y.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.Min(x.AsDouble(), y.AsDouble()).As(); + } + + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(x, y), + Vector128.ConditionalSelect(IsNegative(x), x, y), + Vector128.Min(x, y)), + y), + x); + } + + return Vector128.Min(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(x, y), + Vector256.ConditionalSelect(IsNegative(x), x, y), + Vector256.Min(x, y)), + y), + x); + } + + return Vector256.Min(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + return + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(x, y), + Vector512.ConditionalSelect(IsNegative(x), x, y), + Vector512.Min(x, y)), + y), + x); + } + + return Vector512.Min(x, y); + } + } + + /// Operator to get x or y based on which has the smaller MathF.Abs (but NaNs may not be propagated) + internal readonly struct MinMagnitudeOperator : IAggregationOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.MinMagnitude(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + + Vector128 result = + Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), + Vector128.ConditionalSelect(IsNegative(y), y, x), + Vector128.ConditionalSelect(Vector128.LessThan(yMag, xMag), y, x)); + + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector128 negativeMagnitudeX = Vector128.LessThan(xMag, Vector128.Zero); + Vector128 negativeMagnitudeY = Vector128.LessThan(yMag, Vector128.Zero); + result = Vector128.ConditionalSelect(negativeMagnitudeX, + y, + Vector128.ConditionalSelect(negativeMagnitudeY, + x, + result)); + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + + Vector256 result = + Vector256.ConditionalSelect(Vector256.Equals(yMag, xMag), + Vector256.ConditionalSelect(IsNegative(y), y, x), + Vector256.ConditionalSelect(Vector256.LessThan(yMag, xMag), y, x)); + + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector256 negativeMagnitudeX = Vector256.LessThan(xMag, Vector256.Zero); + Vector256 negativeMagnitudeY = Vector256.LessThan(yMag, Vector256.Zero); + result = Vector256.ConditionalSelect(negativeMagnitudeX, + y, + Vector256.ConditionalSelect(negativeMagnitudeY, + x, + result)); + } + + return result; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + + Vector512 result = + Vector512.ConditionalSelect(Vector512.Equals(yMag, xMag), + Vector512.ConditionalSelect(IsNegative(y), y, x), + Vector512.ConditionalSelect(Vector512.LessThan(yMag, xMag), y, x)); + + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector512 negativeMagnitudeX = Vector512.LessThan(xMag, Vector512.Zero); + Vector512 negativeMagnitudeY = Vector512.LessThan(yMag, Vector512.Zero); + result = Vector512.ConditionalSelect(negativeMagnitudeX, + y, + Vector512.ConditionalSelect(negativeMagnitudeY, + x, + result)); + } + + return result; + } + + public static T Invoke(Vector128 x) => HorizontalAggregate>(x); + public static T Invoke(Vector256 x) => HorizontalAggregate>(x); + public static T Invoke(Vector512 x) => HorizontalAggregate>(x); + } + + /// Operator to get x or y based on which has the smaller MathF.Abs + internal readonly struct MinMagnitudePropagateNaNOperator : IBinaryOperator + where T : INumberBase + { + public static bool Vectorizable => true; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y) => T.MinMagnitude(x, y); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + // Handle NaNs + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + Vector128 xMag = Vector128.Abs(x), yMag = Vector128.Abs(y); + return + Vector128.ConditionalSelect(Vector128.Equals(x, x), + Vector128.ConditionalSelect(Vector128.Equals(y, y), + Vector128.ConditionalSelect(Vector128.Equals(yMag, xMag), + Vector128.ConditionalSelect(IsNegative(x), x, y), + Vector128.ConditionalSelect(Vector128.LessThan(xMag, yMag), x, y)), + y), + x); + } + + return MinMagnitudeOperator.Invoke(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + // Handle NaNs + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + Vector256 xMag = Vector256.Abs(x), yMag = Vector256.Abs(y); + return + Vector256.ConditionalSelect(Vector256.Equals(x, x), + Vector256.ConditionalSelect(Vector256.Equals(y, y), + Vector256.ConditionalSelect(Vector256.Equals(yMag, xMag), + Vector256.ConditionalSelect(IsNegative(x), x, y), + Vector256.ConditionalSelect(Vector256.LessThan(xMag, yMag), x, y)), + y), + x); + } + + return MinMagnitudeOperator.Invoke(x, y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + // Handle NaNs + if (typeof(T) == typeof(float) || typeof(T) == typeof(double)) + { + Vector512 xMag = Vector512.Abs(x), yMag = Vector512.Abs(y); + return + Vector512.ConditionalSelect(Vector512.Equals(x, x), + Vector512.ConditionalSelect(Vector512.Equals(y, y), + Vector512.ConditionalSelect(Vector512.Equals(yMag, xMag), + Vector512.ConditionalSelect(IsNegative(x), x, y), + Vector512.ConditionalSelect(Vector512.LessThan(xMag, yMag), x, y)), + y), + x); + } + + return MinMagnitudeOperator.Invoke(x, y); + } + } + + /// -x + internal readonly struct NegateOperator : IUnaryOperator where T : IUnaryNegationOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x) => -x; + public static Vector128 Invoke(Vector128 x) => -x; + public static Vector256 Invoke(Vector256 x) => -x; + public static Vector512 Invoke(Vector512 x) => -x; + } + + /// (x + y) * z + internal readonly struct AddMultiplyOperator : ITernaryOperator where T : IAdditionOperators, IMultiplyOperators + { + public static T Invoke(T x, T y, T z) => (x + y) * z; + public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) => (x + y) * z; + public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) => (x + y) * z; + public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) => (x + y) * z; + } + + /// (x * y) + z + internal readonly struct MultiplyAddOperator : ITernaryOperator where T : IAdditionOperators, IMultiplyOperators + { + public static T Invoke(T x, T y, T z) => (x * y) + z; + public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) => (x * y) + z; + public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) => (x * y) + z; + public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) => (x * y) + z; + } + + /// (x * y) + z + internal readonly struct MultiplyAddEstimateOperator : ITernaryOperator where T : INumberBase + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static T Invoke(T x, T y, T z) + { + // TODO https://github.com/dotnet/runtime/issues/98053: Use T.MultiplyAddEstimate when it's available. + + if (Fma.IsSupported || AdvSimd.IsSupported) + { + if (typeof(T) == typeof(Half)) + { + Half result = Half.FusedMultiplyAdd(Unsafe.As(ref x), Unsafe.As(ref y), Unsafe.As(ref z)); + return Unsafe.As(ref result); + } + + if (typeof(T) == typeof(float)) + { + float result = float.FusedMultiplyAdd(Unsafe.As(ref x), Unsafe.As(ref y), Unsafe.As(ref z)); + return Unsafe.As(ref result); + } + + if (typeof(T) == typeof(double)) + { + double result = double.FusedMultiplyAdd(Unsafe.As(ref x), Unsafe.As(ref y), Unsafe.As(ref z)); + return Unsafe.As(ref result); + } + } + + return (x * y) + z; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) + { + if (Fma.IsSupported) + { + if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); + } + + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(float)) return AdvSimd.FusedMultiplyAdd(z.AsSingle(), x.AsSingle(), y.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.FusedMultiplyAdd(z.AsDouble(), x.AsDouble(), y.AsDouble()).As(); + } + + return (x * y) + z; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) + { + if (Fma.IsSupported) + { + if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); + } + + return (x * y) + z; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) + { + if (Avx512F.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx512F.FusedMultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Avx512F.FusedMultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); + } + + return (x * y) + z; + } + } + + /// (x * y) + z + internal readonly struct FusedMultiplyAddOperator : ITernaryOperator where T : IFloatingPointIeee754 + { + public static T Invoke(T x, T y, T z) => T.FusedMultiplyAdd(x, y, z); + + public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z) + { + if (Fma.IsSupported) + { + if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); + } + + if (AdvSimd.IsSupported) + { + if (typeof(T) == typeof(float)) return AdvSimd.FusedMultiplyAdd(z.AsSingle(), x.AsSingle(), y.AsSingle()).As(); + } + + if (AdvSimd.Arm64.IsSupported) + { + if (typeof(T) == typeof(double)) return AdvSimd.Arm64.FusedMultiplyAdd(z.AsDouble(), x.AsDouble(), y.AsDouble()).As(); + } + + if (typeof(T) == typeof(float)) + { + Vector128 xFloats = x.AsSingle(); + Vector128 yFloats = y.AsSingle(); + Vector128 zFloats = z.AsSingle(); + return Vector128.Create( + float.FusedMultiplyAdd(xFloats[0], yFloats[0], zFloats[0]), + float.FusedMultiplyAdd(xFloats[1], yFloats[1], zFloats[1]), + float.FusedMultiplyAdd(xFloats[2], yFloats[2], zFloats[2]), + float.FusedMultiplyAdd(xFloats[3], yFloats[3], zFloats[3])).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector128 xDoubles = x.AsDouble(); + Vector128 yDoubles = y.AsDouble(); + Vector128 zDoubles = z.AsDouble(); + return Vector128.Create( + double.FusedMultiplyAdd(xDoubles[0], yDoubles[0], zDoubles[0]), + double.FusedMultiplyAdd(xDoubles[1], yDoubles[1], zDoubles[1])).As(); + } + } + + public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z) + { + if (Fma.IsSupported) + { + if (typeof(T) == typeof(float)) return Fma.MultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Fma.MultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); + } + + return Vector256.Create( + Invoke(x.GetLower(), y.GetLower(), z.GetLower()), + Invoke(x.GetUpper(), y.GetUpper(), z.GetUpper())); + } + + public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z) + { + if (Avx512F.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx512F.FusedMultiplyAdd(x.AsSingle(), y.AsSingle(), z.AsSingle()).As(); + if (typeof(T) == typeof(double)) return Avx512F.FusedMultiplyAdd(x.AsDouble(), y.AsDouble(), z.AsDouble()).As(); + } + + return Vector512.Create( + Invoke(x.GetLower(), y.GetLower(), z.GetLower()), + Invoke(x.GetUpper(), y.GetUpper(), z.GetUpper())); + } + } + + /// (x * (1 - z)) + (y * z) + internal readonly struct LerpOperator : ITernaryOperator where T : IFloatingPointIeee754 + { + public static T Invoke(T x, T y, T amount) => T.Lerp(x, y, amount); + public static Vector128 Invoke(Vector128 x, Vector128 y, Vector128 amount) => (x * (Vector128.One - amount)) + (y * amount); + public static Vector256 Invoke(Vector256 x, Vector256 y, Vector256 amount) => (x * (Vector256.One - amount)) + (y * amount); + public static Vector512 Invoke(Vector512 x, Vector512 y, Vector512 amount) => (x * (Vector512.One - amount)) + (y * amount); + } + + /// x + internal readonly struct IdentityOperator : IUnaryOperator + { + public static bool Vectorizable => true; + public static T Invoke(T x) => x; + public static Vector128 Invoke(Vector128 x) => x; + public static Vector256 Invoke(Vector256 x) => x; + public static Vector512 Invoke(Vector512 x) => x; + } + + /// x * x + internal readonly struct SquaredOperator : IUnaryOperator where T : IMultiplyOperators + { + public static bool Vectorizable => true; + public static T Invoke(T x) => x * x; + public static Vector128 Invoke(Vector128 x) => x * x; + public static Vector256 Invoke(Vector256 x) => x * x; + public static Vector512 Invoke(Vector512 x) => x * x; + } + + /// T.Abs(x) + internal readonly struct AbsoluteOperator : IUnaryOperator where T : INumberBase + { + public static bool Vectorizable => true; + + public static T Invoke(T x) => T.Abs(x); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(sbyte) || + typeof(T) == typeof(short) || + typeof(T) == typeof(int) || + typeof(T) == typeof(long) || + typeof(T) == typeof(nint)) + { + // Handle signed integers specially, in order to throw if any attempt is made to + // take the absolute value of the minimum value of the type, which doesn't have + // a positive absolute value representation. + Vector128 abs = Vector128.ConditionalSelect(Vector128.LessThan(x, Vector128.Zero), -x, x); + if (Vector128.LessThan(abs, Vector128.Zero) != Vector128.Zero) + { + ThrowNegateTwosCompOverflow(); + } + } + + return Vector128.Abs(x); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(sbyte) || + typeof(T) == typeof(short) || + typeof(T) == typeof(int) || + typeof(T) == typeof(long) || + typeof(T) == typeof(nint)) + { + // Handle signed integers specially, in order to throw if any attempt is made to + // take the absolute value of the minimum value of the type, which doesn't have + // a positive absolute value representation. + Vector256 abs = Vector256.ConditionalSelect(Vector256.LessThan(x, Vector256.Zero), -x, x); + if (Vector256.LessThan(abs, Vector256.Zero) != Vector256.Zero) + { + ThrowNegateTwosCompOverflow(); + } + } + + return Vector256.Abs(x); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(sbyte) || + typeof(T) == typeof(short) || + typeof(T) == typeof(int) || + typeof(T) == typeof(long) || + typeof(T) == typeof(nint)) + { + // Handle signed integers specially, in order to throw if any attempt is made to + // take the absolute value of the minimum value of the type, which doesn't have + // a positive absolute value representation. + Vector512 abs = Vector512.ConditionalSelect(Vector512.LessThan(x, Vector512.Zero), -x, x); + if (Vector512.LessThan(abs, Vector512.Zero) != Vector512.Zero) + { + ThrowNegateTwosCompOverflow(); + } + } + + return Vector512.Abs(x); + } + } + + /// T.Exp(x) + internal readonly struct ExpOperator : IUnaryOperator + where T : IExponentialFunctions + { + public static bool Vectorizable => (typeof(T) == typeof(double)) + || (typeof(T) == typeof(float)); + + public static T Invoke(T x) => T.Exp(x); + + public static Vector128 Invoke(Vector128 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector128.Exp(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector128.Exp(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return ExpOperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return ExpOperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + + public static Vector256 Invoke(Vector256 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector256.Exp(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector256.Exp(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return ExpOperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return ExpOperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + + public static Vector512 Invoke(Vector512 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector512.Exp(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector512.Exp(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return ExpOperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return ExpOperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + } + +#if !NET9_0_OR_GREATER + /// double.Exp(x) + internal readonly struct ExpOperatorDouble : IUnaryOperator + { + // This code is based on `vrd2_exp` from amd/aocl-libm-ose + // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Implementation Notes + // ---------------------- + // 1. Argument Reduction: + // e^x = 2^(x/ln2) = 2^(x*(64/ln(2))/64) --- (1) + // + // Choose 'n' and 'f', such that + // x * 64/ln2 = n + f --- (2) | n is integer + // | |f| <= 0.5 + // Choose 'm' and 'j' such that, + // n = (64 * m) + j --- (3) + // + // From (1), (2) and (3), + // e^x = 2^((64*m + j + f)/64) + // = (2^m) * (2^(j/64)) * 2^(f/64) + // = (2^m) * (2^(j/64)) * e^(f*(ln(2)/64)) + // + // 2. Table Lookup + // Values of (2^(j/64)) are precomputed, j = 0, 1, 2, 3 ... 63 + // + // 3. Polynomial Evaluation + // From (2), + // f = x*(64/ln(2)) - n + // Let, + // r = f*(ln(2)/64) = x - n*(ln(2)/64) + // + // 4. Reconstruction + // Thus, + // e^x = (2^m) * (2^(j/64)) * e^r + + private const ulong V_ARG_MAX = 0x40862000_00000000; + private const ulong V_DP64_BIAS = 1023; + + private const double V_EXPF_MIN = -709.782712893384; + private const double V_EXPF_MAX = +709.782712893384; + + private const double V_EXPF_HUGE = 6755399441055744; + private const double V_TBL_LN2 = 1.4426950408889634; + + private const double V_LN2_HEAD = +0.693359375; + private const double V_LN2_TAIL = -0.00021219444005469057; + + private const double C3 = 0.5000000000000018; + private const double C4 = 0.1666666666666617; + private const double C5 = 0.04166666666649277; + private const double C6 = 0.008333333333559272; + private const double C7 = 0.001388888895122404; + private const double C8 = 0.00019841269432677495; + private const double C9 = 2.4801486521374483E-05; + private const double C10 = 2.7557622532543023E-06; + private const double C11 = 2.7632293298250954E-07; + private const double C12 = 2.499430431958571E-08; + + public static bool Vectorizable => true; + + public static double Invoke(double x) => double.Exp(x); + + public static Vector128 Invoke(Vector128 x) + { + // x * (64.0 / ln(2)) + Vector128 z = x * Vector128.Create(V_TBL_LN2); + + Vector128 dn = z + Vector128.Create(V_EXPF_HUGE); + + // n = (int)z + Vector128 n = dn.AsUInt64(); + + // dn = (double)n + dn -= Vector128.Create(V_EXPF_HUGE); + + // r = x - (dn * (ln(2) / 64)) + // where ln(2) / 64 is split into Head and Tail values + Vector128 r = x - (dn * Vector128.Create(V_LN2_HEAD)) - (dn * Vector128.Create(V_LN2_TAIL)); + + Vector128 r2 = r * r; + Vector128 r4 = r2 * r2; + Vector128 r8 = r4 * r4; + + // Compute polynomial + Vector128 poly = ((Vector128.Create(C12) * r + Vector128.Create(C11)) * r2 + + Vector128.Create(C10) * r + Vector128.Create(C9)) * r8 + + ((Vector128.Create(C8) * r + Vector128.Create(C7)) * r2 + + (Vector128.Create(C6) * r + Vector128.Create(C5))) * r4 + + ((Vector128.Create(C4) * r + Vector128.Create(C3)) * r2 + (r + Vector128.One)); + + // m = (n - j) / 64 + // result = polynomial * 2^m + Vector128 ret = poly * ((n + Vector128.Create(V_DP64_BIAS)) << 52).AsDouble(); + + // Check if -709 < vx < 709 + if (Vector128.GreaterThanAny(Vector128.Abs(x).AsUInt64(), Vector128.Create(V_ARG_MAX))) + { + // (x > V_EXPF_MAX) ? double.PositiveInfinity : x + Vector128 infinityMask = Vector128.GreaterThan(x, Vector128.Create(V_EXPF_MAX)); + + ret = Vector128.ConditionalSelect( + infinityMask, + Vector128.Create(double.PositiveInfinity), + ret + ); + + // (x < V_EXPF_MIN) ? 0 : x + ret = Vector128.AndNot(ret, Vector128.LessThan(x, Vector128.Create(V_EXPF_MIN))); + } + + return ret; + } + + public static Vector256 Invoke(Vector256 x) + { + // x * (64.0 / ln(2)) + Vector256 z = x * Vector256.Create(V_TBL_LN2); + + Vector256 dn = z + Vector256.Create(V_EXPF_HUGE); + + // n = (int)z + Vector256 n = dn.AsUInt64(); + + // dn = (double)n + dn -= Vector256.Create(V_EXPF_HUGE); + + // r = x - (dn * (ln(2) / 64)) + // where ln(2) / 64 is split into Head and Tail values + Vector256 r = x - (dn * Vector256.Create(V_LN2_HEAD)) - (dn * Vector256.Create(V_LN2_TAIL)); + + Vector256 r2 = r * r; + Vector256 r4 = r2 * r2; + Vector256 r8 = r4 * r4; + + // Compute polynomial + Vector256 poly = ((Vector256.Create(C12) * r + Vector256.Create(C11)) * r2 + + Vector256.Create(C10) * r + Vector256.Create(C9)) * r8 + + ((Vector256.Create(C8) * r + Vector256.Create(C7)) * r2 + + (Vector256.Create(C6) * r + Vector256.Create(C5))) * r4 + + ((Vector256.Create(C4) * r + Vector256.Create(C3)) * r2 + (r + Vector256.One)); + + // m = (n - j) / 64 + // result = polynomial * 2^m + Vector256 ret = poly * ((n + Vector256.Create(V_DP64_BIAS)) << 52).AsDouble(); + + // Check if -709 < vx < 709 + if (Vector256.GreaterThanAny(Vector256.Abs(x).AsUInt64(), Vector256.Create(V_ARG_MAX))) + { + // (x > V_EXPF_MAX) ? double.PositiveInfinity : x + Vector256 infinityMask = Vector256.GreaterThan(x, Vector256.Create(V_EXPF_MAX)); + + ret = Vector256.ConditionalSelect( + infinityMask, + Vector256.Create(double.PositiveInfinity), + ret + ); + + // (x < V_EXPF_MIN) ? 0 : x + ret = Vector256.AndNot(ret, Vector256.LessThan(x, Vector256.Create(V_EXPF_MIN))); + } + + return ret; + } + + public static Vector512 Invoke(Vector512 x) + { + // x * (64.0 / ln(2)) + Vector512 z = x * Vector512.Create(V_TBL_LN2); + + Vector512 dn = z + Vector512.Create(V_EXPF_HUGE); + + // n = (int)z + Vector512 n = dn.AsUInt64(); + + // dn = (double)n + dn -= Vector512.Create(V_EXPF_HUGE); + + // r = x - (dn * (ln(2) / 64)) + // where ln(2) / 64 is split into Head and Tail values + Vector512 r = x - (dn * Vector512.Create(V_LN2_HEAD)) - (dn * Vector512.Create(V_LN2_TAIL)); + + Vector512 r2 = r * r; + Vector512 r4 = r2 * r2; + Vector512 r8 = r4 * r4; + + // Compute polynomial + Vector512 poly = ((Vector512.Create(C12) * r + Vector512.Create(C11)) * r2 + + Vector512.Create(C10) * r + Vector512.Create(C9)) * r8 + + ((Vector512.Create(C8) * r + Vector512.Create(C7)) * r2 + + (Vector512.Create(C6) * r + Vector512.Create(C5))) * r4 + + ((Vector512.Create(C4) * r + Vector512.Create(C3)) * r2 + (r + Vector512.One)); + + // m = (n - j) / 64 + // result = polynomial * 2^m + Vector512 ret = poly * ((n + Vector512.Create(V_DP64_BIAS)) << 52).AsDouble(); + + // Check if -709 < vx < 709 + if (Vector512.GreaterThanAny(Vector512.Abs(x).AsUInt64(), Vector512.Create(V_ARG_MAX))) + { + // (x > V_EXPF_MAX) ? double.PositiveInfinity : x + Vector512 infinityMask = Vector512.GreaterThan(x, Vector512.Create(V_EXPF_MAX)); + + ret = Vector512.ConditionalSelect( + infinityMask, + Vector512.Create(double.PositiveInfinity), + ret + ); + + // (x < V_EXPF_MIN) ? 0 : x + ret = Vector512.AndNot(ret, Vector512.LessThan(x, Vector512.Create(V_EXPF_MIN))); + } + + return ret; + } + } + + /// float.Exp(x) + internal readonly struct ExpOperatorSingle : IUnaryOperator + { + // This code is based on `vrs4_expf` from amd/aocl-libm-ose + // Copyright (C) 2019-2022 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Implementation Notes: + // 1. Argument Reduction: + // e^x = 2^(x/ln2) --- (1) + // + // Let x/ln(2) = z --- (2) + // + // Let z = n + r , where n is an integer --- (3) + // |r| <= 1/2 + // + // From (1), (2) and (3), + // e^x = 2^z + // = 2^(N+r) + // = (2^N)*(2^r) --- (4) + // + // 2. Polynomial Evaluation + // From (4), + // r = z - N + // 2^r = C1 + C2*r + C3*r^2 + C4*r^3 + C5 *r^4 + C6*r^5 + // + // 4. Reconstruction + // Thus, + // e^x = (2^N) * (2^r) + + private const uint V_ARG_MAX = 0x42AE0000; + + private const float V_EXPF_MIN = -103.97208f; + private const float V_EXPF_MAX = +88.72284f; + + private const double V_EXPF_HUGE = 6755399441055744; + private const double V_TBL_LN2 = 1.4426950408889634; + + private const double C1 = 1.0000000754895704; + private const double C2 = 0.6931472254087585; + private const double C3 = 0.2402210737432219; + private const double C4 = 0.05550297297702539; + private const double C5 = 0.009676036358193323; + private const double C6 = 0.001341000536524434; + + public static bool Vectorizable => true; + + public static float Invoke(float x) => float.Exp(x); + + public static Vector128 Invoke(Vector128 x) + { + // Convert x to double precision + (Vector128 xl, Vector128 xu) = Vector128.Widen(x); + + // x * (64.0 / ln(2)) + Vector128 v_tbl_ln2 = Vector128.Create(V_TBL_LN2); + + Vector128 zl = xl * v_tbl_ln2; + Vector128 zu = xu * v_tbl_ln2; + + Vector128 v_expf_huge = Vector128.Create(V_EXPF_HUGE); + + Vector128 dnl = zl + v_expf_huge; + Vector128 dnu = zu + v_expf_huge; + + // n = (int)z + Vector128 nl = dnl.AsUInt64(); + Vector128 nu = dnu.AsUInt64(); + + // dn = (double)n + dnl -= v_expf_huge; + dnu -= v_expf_huge; + + // r = z - dn + Vector128 c1 = Vector128.Create(C1); + Vector128 c2 = Vector128.Create(C2); + Vector128 c3 = Vector128.Create(C3); + Vector128 c4 = Vector128.Create(C4); + Vector128 c5 = Vector128.Create(C5); + Vector128 c6 = Vector128.Create(C6); + + Vector128 rl = zl - dnl; + + Vector128 rl2 = rl * rl; + Vector128 rl4 = rl2 * rl2; + + Vector128 polyl = (c4 * rl + c3) * rl2 + + ((c6 * rl + c5) * rl4 + + (c2 * rl + c1)); + + + Vector128 ru = zu - dnu; + + Vector128 ru2 = ru * ru; + Vector128 ru4 = ru2 * ru2; + + Vector128 polyu = (c4 * ru + c3) * ru2 + + ((c6 * ru + c5) * ru4 + + (c2 * ru + c1)); + + // result = (float)(poly + (n << 52)) + Vector128 ret = Vector128.Narrow( + (polyl.AsUInt64() + (nl << 52)).AsDouble(), + (polyu.AsUInt64() + (nu << 52)).AsDouble() + ); + + // Check if -103 < |x| < 88 + if (Vector128.GreaterThanAny(Vector128.Abs(x).AsUInt32(), Vector128.Create(V_ARG_MAX))) + { + // (x > V_EXPF_MAX) ? float.PositiveInfinity : x + Vector128 infinityMask = Vector128.GreaterThan(x, Vector128.Create(V_EXPF_MAX)); + + ret = Vector128.ConditionalSelect( + infinityMask, + Vector128.Create(float.PositiveInfinity), + ret + ); + + // (x < V_EXPF_MIN) ? 0 : x + ret = Vector128.AndNot(ret, Vector128.LessThan(x, Vector128.Create(V_EXPF_MIN))); + } + + return ret; + } + + public static Vector256 Invoke(Vector256 x) + { + // Convert x to double precision + (Vector256 xl, Vector256 xu) = Vector256.Widen(x); + + // x * (64.0 / ln(2)) + Vector256 v_tbl_ln2 = Vector256.Create(V_TBL_LN2); + + Vector256 zl = xl * v_tbl_ln2; + Vector256 zu = xu * v_tbl_ln2; + + Vector256 v_expf_huge = Vector256.Create(V_EXPF_HUGE); + + Vector256 dnl = zl + v_expf_huge; + Vector256 dnu = zu + v_expf_huge; + + // n = (int)z + Vector256 nl = dnl.AsUInt64(); + Vector256 nu = dnu.AsUInt64(); + + // dn = (double)n + dnl -= v_expf_huge; + dnu -= v_expf_huge; + + // r = z - dn + Vector256 c1 = Vector256.Create(C1); + Vector256 c2 = Vector256.Create(C2); + Vector256 c3 = Vector256.Create(C3); + Vector256 c4 = Vector256.Create(C4); + Vector256 c5 = Vector256.Create(C5); + Vector256 c6 = Vector256.Create(C6); + + Vector256 rl = zl - dnl; + + Vector256 rl2 = rl * rl; + Vector256 rl4 = rl2 * rl2; + + Vector256 polyl = (c4 * rl + c3) * rl2 + + ((c6 * rl + c5) * rl4 + + (c2 * rl + c1)); + + + Vector256 ru = zu - dnu; + + Vector256 ru2 = ru * ru; + Vector256 ru4 = ru2 * ru2; + + Vector256 polyu = (c4 * ru + c3) * ru2 + + ((c6 * ru + c5) * ru4 + + (c2 * ru + c1)); + + // result = (float)(poly + (n << 52)) + Vector256 ret = Vector256.Narrow( + (polyl.AsUInt64() + (nl << 52)).AsDouble(), + (polyu.AsUInt64() + (nu << 52)).AsDouble() + ); + + // Check if -103 < |x| < 88 + if (Vector256.GreaterThanAny(Vector256.Abs(x).AsUInt32(), Vector256.Create(V_ARG_MAX))) + { + // (x > V_EXPF_MAX) ? float.PositiveInfinity : x + Vector256 infinityMask = Vector256.GreaterThan(x, Vector256.Create(V_EXPF_MAX)); + + ret = Vector256.ConditionalSelect( + infinityMask, + Vector256.Create(float.PositiveInfinity), + ret + ); + + // (x < V_EXPF_MIN) ? 0 : x + ret = Vector256.AndNot(ret, Vector256.LessThan(x, Vector256.Create(V_EXPF_MIN))); + } + + return ret; + } + + public static Vector512 Invoke(Vector512 x) + { + // Convert x to double precision + (Vector512 xl, Vector512 xu) = Vector512.Widen(x); + + // x * (64.0 / ln(2)) + Vector512 v_tbl_ln2 = Vector512.Create(V_TBL_LN2); + + Vector512 zl = xl * v_tbl_ln2; + Vector512 zu = xu * v_tbl_ln2; + + Vector512 v_expf_huge = Vector512.Create(V_EXPF_HUGE); + + Vector512 dnl = zl + v_expf_huge; + Vector512 dnu = zu + v_expf_huge; + + // n = (int)z + Vector512 nl = dnl.AsUInt64(); + Vector512 nu = dnu.AsUInt64(); + + // dn = (double)n + dnl -= v_expf_huge; + dnu -= v_expf_huge; + + // r = z - dn + Vector512 c1 = Vector512.Create(C1); + Vector512 c2 = Vector512.Create(C2); + Vector512 c3 = Vector512.Create(C3); + Vector512 c4 = Vector512.Create(C4); + Vector512 c5 = Vector512.Create(C5); + Vector512 c6 = Vector512.Create(C6); + + Vector512 rl = zl - dnl; + + Vector512 rl2 = rl * rl; + Vector512 rl4 = rl2 * rl2; + + Vector512 polyl = (c4 * rl + c3) * rl2 + + ((c6 * rl + c5) * rl4 + + (c2 * rl + c1)); + + + Vector512 ru = zu - dnu; + + Vector512 ru2 = ru * ru; + Vector512 ru4 = ru2 * ru2; + + Vector512 polyu = (c4 * ru + c3) * ru2 + + ((c6 * ru + c5) * ru4 + + (c2 * ru + c1)); + + // result = (float)(poly + (n << 52)) + Vector512 ret = Vector512.Narrow( + (polyl.AsUInt64() + (nl << 52)).AsDouble(), + (polyu.AsUInt64() + (nu << 52)).AsDouble() + ); + + // Check if -103 < |x| < 88 + if (Vector512.GreaterThanAny(Vector512.Abs(x).AsUInt32(), Vector512.Create(V_ARG_MAX))) + { + // (x > V_EXPF_MAX) ? float.PositiveInfinity : x + Vector512 infinityMask = Vector512.GreaterThan(x, Vector512.Create(V_EXPF_MAX)); + + ret = Vector512.ConditionalSelect( + infinityMask, + Vector512.Create(float.PositiveInfinity), + ret + ); + + // (x < V_EXPF_MIN) ? 0 : x + ret = Vector512.AndNot(ret, Vector512.LessThan(x, Vector512.Create(V_EXPF_MIN))); + } + + return ret; + } + } +#endif + + /// T.ExpM1(x) + internal readonly struct ExpM1Operator : IUnaryOperator + where T : IExponentialFunctions + { + public static bool Vectorizable => ExpOperator.Vectorizable; + + public static T Invoke(T x) => T.ExpM1(x); + public static Vector128 Invoke(Vector128 x) => ExpOperator.Invoke(x) - Vector128.One; + public static Vector256 Invoke(Vector256 x) => ExpOperator.Invoke(x) - Vector256.One; + public static Vector512 Invoke(Vector512 x) => ExpOperator.Invoke(x) - Vector512.One; + } + + /// T.Exp2(x) + internal readonly struct Exp2Operator : IUnaryOperator + where T : IExponentialFunctions + { + private const double NaturalLog2 = 0.6931471805599453; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Exp2(x); + public static Vector128 Invoke(Vector128 x) => ExpOperator.Invoke(x * Vector128.Create(T.CreateTruncating(NaturalLog2))); + public static Vector256 Invoke(Vector256 x) => ExpOperator.Invoke(x * Vector256.Create(T.CreateTruncating(NaturalLog2))); + public static Vector512 Invoke(Vector512 x) => ExpOperator.Invoke(x * Vector512.Create(T.CreateTruncating(NaturalLog2))); + } + + /// T.Exp2M1(x) + internal readonly struct Exp2M1Operator : IUnaryOperator + where T : IExponentialFunctions + { + public static bool Vectorizable => Exp2Operator.Vectorizable; + + public static T Invoke(T x) => T.Exp2M1(x); + public static Vector128 Invoke(Vector128 x) => Exp2Operator.Invoke(x) - Vector128.One; + public static Vector256 Invoke(Vector256 x) => Exp2Operator.Invoke(x) - Vector256.One; + public static Vector512 Invoke(Vector512 x) => Exp2Operator.Invoke(x) - Vector512.One; + } + + /// T.Exp10(x) + internal readonly struct Exp10Operator : IUnaryOperator + where T : IExponentialFunctions + { + private const double NaturalLog10 = 2.302585092994046; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Exp10(x); + public static Vector128 Invoke(Vector128 x) => ExpOperator.Invoke(x * Vector128.Create(T.CreateTruncating(NaturalLog10))); + public static Vector256 Invoke(Vector256 x) => ExpOperator.Invoke(x * Vector256.Create(T.CreateTruncating(NaturalLog10))); + public static Vector512 Invoke(Vector512 x) => ExpOperator.Invoke(x * Vector512.Create(T.CreateTruncating(NaturalLog10))); + } + + /// T.Exp10M1(x) + internal readonly struct Exp10M1Operator : IUnaryOperator + where T : IExponentialFunctions + { + public static bool Vectorizable => Exp2Operator.Vectorizable; + + public static T Invoke(T x) => T.Exp10M1(x); + public static Vector128 Invoke(Vector128 x) => Exp10Operator.Invoke(x) - Vector128.One; + public static Vector256 Invoke(Vector256 x) => Exp10Operator.Invoke(x) - Vector256.One; + public static Vector512 Invoke(Vector512 x) => Exp10Operator.Invoke(x) - Vector512.One; + } + + /// T.Pow(x, y) + internal readonly struct PowOperator : IBinaryOperator + where T : IPowerFunctions + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x, T y) => T.Pow(x, y); + + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(y.AsSingle() * LogOperator.Invoke(x.AsSingle())).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(y.AsDouble() * LogOperator.Invoke(x.AsDouble())).As(); + } + } + + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(y.AsSingle() * LogOperator.Invoke(x.AsSingle())).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(y.AsDouble() * LogOperator.Invoke(x.AsDouble())).As(); + } + } + + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(y.AsSingle() * LogOperator.Invoke(x.AsSingle())).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(y.AsDouble() * LogOperator.Invoke(x.AsDouble())).As(); + } + } + } + + /// T.Sqrt(x) + internal readonly struct SqrtOperator : IUnaryOperator + where T : IRootFunctions + { + public static bool Vectorizable => true; + public static T Invoke(T x) => T.Sqrt(x); + public static Vector128 Invoke(Vector128 x) => Vector128.Sqrt(x); + public static Vector256 Invoke(Vector256 x) => Vector256.Sqrt(x); + public static Vector512 Invoke(Vector512 x) => Vector512.Sqrt(x); + } + + /// T.Cbrt(x) + internal readonly struct CbrtOperator : IUnaryOperator + where T : IRootFunctions + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Cbrt(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector128.Create(3f)).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector128.Create(3d)).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector256.Create(3f)).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector256.Create(3d)).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector512.Create(3f)).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector512.Create(3d)).As(); + } + } + } + + /// T.Hypot(x, y) + internal readonly struct HypotOperator : IBinaryOperator + where T : IRootFunctions + { + public static bool Vectorizable => true; + public static T Invoke(T x, T y) => T.Hypot(x, y); + public static Vector128 Invoke(Vector128 x, Vector128 y) => Vector128.Sqrt((x * x) + (y * y)); + public static Vector256 Invoke(Vector256 x, Vector256 y) => Vector256.Sqrt((x * x) + (y * y)); + public static Vector512 Invoke(Vector512 x, Vector512 y) => Vector512.Sqrt((x * x) + (y * y)); + } + + /// T.Acos(x) + internal readonly struct AcosOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.Acos(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.Acosh(x) + internal readonly struct AcoshOperator : IUnaryOperator + where T : IHyperbolicFunctions + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.Acosh(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.AcosPi(x) + internal readonly struct AcosPiOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => AcosOperator.Vectorizable; + public static T Invoke(T x) => T.AcosPi(x); + public static Vector128 Invoke(Vector128 x) => AcosOperator.Invoke(x) / Vector128.Create(T.Pi); + public static Vector256 Invoke(Vector256 x) => AcosOperator.Invoke(x) / Vector256.Create(T.Pi); + public static Vector512 Invoke(Vector512 x) => AcosOperator.Invoke(x) / Vector512.Create(T.Pi); + } + + /// T.Asin(x) + internal readonly struct AsinOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.Asin(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.Asinh(x) + internal readonly struct AsinhOperator : IUnaryOperator + where T : IHyperbolicFunctions + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.Asinh(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.AsinPi(x) + internal readonly struct AsinPiOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => AsinOperator.Vectorizable; + public static T Invoke(T x) => T.AsinPi(x); + public static Vector128 Invoke(Vector128 x) => AsinOperator.Invoke(x) / Vector128.Create(T.Pi); + public static Vector256 Invoke(Vector256 x) => AsinOperator.Invoke(x) / Vector256.Create(T.Pi); + public static Vector512 Invoke(Vector512 x) => AsinOperator.Invoke(x) / Vector512.Create(T.Pi); + } + + /// T.Atan(x) + internal readonly struct AtanOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.Atan(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.Atanh(x) + internal readonly struct AtanhOperator : IUnaryOperator + where T : IHyperbolicFunctions + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.Atanh(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.AtanPi(x) + internal readonly struct AtanPiOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => AtanOperator.Vectorizable; + public static T Invoke(T x) => T.AtanPi(x); + public static Vector128 Invoke(Vector128 x) => AtanOperator.Invoke(x) / Vector128.Create(T.Pi); + public static Vector256 Invoke(Vector256 x) => AtanOperator.Invoke(x) / Vector256.Create(T.Pi); + public static Vector512 Invoke(Vector512 x) => AtanOperator.Invoke(x) / Vector512.Create(T.Pi); + } + + /// T.Atan2(y, x) + internal readonly struct Atan2Operator : IBinaryOperator + where T : IFloatingPointIeee754 + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T y, T x) => T.Atan2(y, x); + public static Vector128 Invoke(Vector128 y, Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 y, Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 y, Vector512 x) => throw new NotSupportedException(); + } + + /// T.Atan2Pi(y, x) + internal readonly struct Atan2PiOperator : IBinaryOperator + where T : IFloatingPointIeee754 + { + public static bool Vectorizable => Atan2Operator.Vectorizable; + public static T Invoke(T y, T x) => T.Atan2Pi(y, x); + public static Vector128 Invoke(Vector128 y, Vector128 x) => Atan2Operator.Invoke(y, x) / Vector128.Create(T.Pi); + public static Vector256 Invoke(Vector256 y, Vector256 x) => Atan2Operator.Invoke(y, x) / Vector256.Create(T.Pi); + public static Vector512 Invoke(Vector512 y, Vector512 x) => Atan2Operator.Invoke(y, x) / Vector512.Create(T.Pi); + } + + /// T.Cos(x) + internal readonly struct CosOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + // This code is based on `vrs4_cos` and `vrd2_cos` from amd/aocl-libm-ose + // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Implementation notes from amd/aocl-libm-ose: + // -------------------------------------------- + // To compute cosf(float x) + // Using the identity, + // cos(x) = sin(x + pi/2) (1) + // + // 1. Argument Reduction + // Now, let x be represented as, + // |x| = N * pi + f (2) | N is an integer, + // -pi/2 <= f <= pi/2 + // + // From (2), N = int( (x + pi/2) / pi) - 0.5 + // f = |x| - (N * pi) + // + // 2. Polynomial Evaluation + // From (1) and (2),sin(f) can be calculated using a polynomial + // sin(f) = f*(1 + C1*f^2 + C2*f^4 + C3*f^6 + c4*f^8) + // + // 3. Reconstruction + // Hence, cos(x) = sin(x + pi/2) = (-1)^N * sin(f) + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Cos(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return CosOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return CosOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return CosOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return CosOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return CosOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return CosOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + } + + /// float.Cos(x) + private readonly struct CosOperatorSingle : IUnaryOperator + { + internal const uint MaxVectorizedValue = 0x4A989680u; + internal const uint SignMask = 0x7FFFFFFFu; + private const float AlmHuge = 1.2582912e7f; + private const float Pi_Tail1 = 8.742278e-8f; + private const float Pi_Tail2 = 3.430249e-15f; + private const float C1 = -0.16666657f; + private const float C2 = 0.008332962f; + private const float C3 = -1.9801206e-4f; + private const float C4 = 2.5867037e-6f; + + public static bool Vectorizable => true; + + public static float Invoke(float x) => float.Cos(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 uxMasked = Vector128.Abs(x); + if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector128 almHuge = Vector128.Create(AlmHuge); + Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked + Vector128.Create(float.Pi / 2), Vector128.Create(1 / float.Pi), almHuge); + Vector128 odd = dn.AsUInt32() << 31; + dn = dn - almHuge - Vector128.Create(0.5f); + + Vector128 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-float.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail1), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); + + // POLY_EVAL_ODD_9 + Vector128 f2 = f * f; + Vector128 f4 = f2 * f2; + Vector128 a0 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); + Vector128 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector128.One); + Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C3), f2, Vector128.Create(C4) * f4); + Vector128 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector128 poly = f * a3; + + return (poly.AsUInt32() ^ odd).AsSingle(); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 uxMasked = Vector256.Abs(x); + if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector256 almHuge = Vector256.Create(AlmHuge); + Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked + Vector256.Create(float.Pi / 2), Vector256.Create(1 / float.Pi), almHuge); + Vector256 odd = dn.AsUInt32() << 31; + dn = dn - almHuge - Vector256.Create(0.5f); + + Vector256 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-float.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail1), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); + + // POLY_EVAL_ODD_9 + Vector256 f2 = f * f; + Vector256 f4 = f2 * f2; + Vector256 a0 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); + Vector256 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector256.One); + Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C3), f2, Vector256.Create(C4) * f4); + Vector256 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector256 poly = f * a3; + + return (poly.AsUInt32() ^ odd).AsSingle(); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 uxMasked = Vector512.Abs(x); + if (Vector512.GreaterThanAny(uxMasked.AsUInt32(), Vector512.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector512 almHuge = Vector512.Create(AlmHuge); + Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked + Vector512.Create(float.Pi / 2), Vector512.Create(1 / float.Pi), almHuge); + Vector512 odd = dn.AsUInt32() << 31; + dn = dn - almHuge - Vector512.Create(0.5f); + + Vector512 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-float.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail1), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); + + // POLY_EVAL_ODD_9 + Vector512 f2 = f * f; + Vector512 f4 = f2 * f2; + Vector512 a0 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); + Vector512 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector512.One); + Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C3), f2, Vector512.Create(C4) * f4); + Vector512 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector512 poly = f * a3; + + return (poly.AsUInt32() ^ odd).AsSingle(); + } + } + + /// double.Cos(x) + internal readonly struct CosOperatorDouble : IUnaryOperator + { + internal const ulong SignMask = 0x7FFFFFFFFFFFFFFFul; + internal const ulong MaxVectorizedValue = 0x4160000000000000ul; + private const double AlmHuge = 6.755399441055744E15; + private const double Pi_Tail2 = -1.2246467991473532E-16; + private const double Pi_Tail3 = 2.9947698097183397E-33; + private const double C1 = -0.16666666666666666; + private const double C2 = 0.008333333333333165; + private const double C3 = -1.984126984120184E-4; + private const double C4 = 2.7557319210152756E-6; + private const double C5 = -2.5052106798274616E-8; + private const double C6 = 1.6058936490373254E-10; + private const double C7 = -7.642917806937501E-13; + private const double C8 = 2.7204790963151784E-15; + + public static bool Vectorizable => true; + + public static double Invoke(double x) => double.Cos(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 uxMasked = Vector128.Abs(x); + if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector128 almHuge = Vector128.Create(AlmHuge); + Vector128 dn = (uxMasked * Vector128.Create(1 / double.Pi)) + Vector128.Create(double.Pi / 2) + almHuge; + Vector128 odd = dn.AsUInt64() << 63; + dn = dn - almHuge - Vector128.Create(0.5); + + Vector128 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-double.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail3), f); + + // POLY_EVAL_ODD_17 + Vector128 f2 = f * f; + Vector128 f4 = f2 * f2; + Vector128 f6 = f4 * f2; + Vector128 f10 = f6 * f4; + Vector128 f14 = f10 * f4; + Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); + Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C4), f2, Vector128.Create(C3)); + Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C6), f2, Vector128.Create(C5)); + Vector128 a4 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C8), f2, Vector128.Create(C7)); + Vector128 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); + Vector128 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); + Vector128 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); + + return (poly.AsUInt64() ^ odd).AsDouble(); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 uxMasked = Vector256.Abs(x); + if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector256 almHuge = Vector256.Create(AlmHuge); + Vector256 dn = (uxMasked * Vector256.Create(1 / double.Pi)) + Vector256.Create(double.Pi / 2) + almHuge; + Vector256 odd = dn.AsUInt64() << 63; + dn = dn - almHuge - Vector256.Create(0.5); + + Vector256 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-double.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail3), f); + + // POLY_EVAL_ODD_17 + Vector256 f2 = f * f; + Vector256 f4 = f2 * f2; + Vector256 f6 = f4 * f2; + Vector256 f10 = f6 * f4; + Vector256 f14 = f10 * f4; + Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); + Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C4), f2, Vector256.Create(C3)); + Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C6), f2, Vector256.Create(C5)); + Vector256 a4 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C8), f2, Vector256.Create(C7)); + Vector256 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); + Vector256 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); + Vector256 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); + + return (poly.AsUInt64() ^ odd).AsDouble(); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 uxMasked = Vector512.Abs(x); + if (Vector512.GreaterThanAny(uxMasked.AsUInt64(), Vector512.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector512 almHuge = Vector512.Create(AlmHuge); + Vector512 dn = (uxMasked * Vector512.Create(1 / double.Pi)) + Vector512.Create(double.Pi / 2) + almHuge; + Vector512 odd = dn.AsUInt64() << 63; + dn = dn - almHuge - Vector512.Create(0.5); + + Vector512 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-double.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail3), f); + + // POLY_EVAL_ODD_17 + Vector512 f2 = f * f; + Vector512 f4 = f2 * f2; + Vector512 f6 = f4 * f2; + Vector512 f10 = f6 * f4; + Vector512 f14 = f10 * f4; + Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); + Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C4), f2, Vector512.Create(C3)); + Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C6), f2, Vector512.Create(C5)); + Vector512 a4 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C8), f2, Vector512.Create(C7)); + Vector512 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); + Vector512 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); + Vector512 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); + + return (poly.AsUInt64() ^ odd).AsDouble(); + } + } + + /// T.CosPi(x) + internal readonly struct CosPiOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.CosPi(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 xpi = x * Vector128.Create(T.Pi); + if (typeof(T) == typeof(float)) + { + if (Vector128.GreaterThanAny(xpi.AsUInt32() & Vector128.Create(CosOperatorSingle.SignMask), Vector128.Create(CosOperatorSingle.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsSingle()).As(); + } + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + if (Vector128.GreaterThanAny(xpi.AsUInt64() & Vector128.Create(CosOperatorDouble.SignMask), Vector128.Create(CosOperatorDouble.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsDouble()).As(); + } + } + + return CosOperator.Invoke(xpi); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 xpi = x * Vector256.Create(T.Pi); + if (typeof(T) == typeof(float)) + { + if (Vector256.GreaterThanAny(xpi.AsUInt32() & Vector256.Create(CosOperatorSingle.SignMask), Vector256.Create(CosOperatorSingle.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsSingle()).As(); + } + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + if (Vector256.GreaterThanAny(xpi.AsUInt64() & Vector256.Create(CosOperatorDouble.SignMask), Vector256.Create(CosOperatorDouble.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsDouble()).As(); + } + } + + return CosOperator.Invoke(xpi); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 xpi = x * Vector512.Create(T.Pi); + if (typeof(T) == typeof(float)) + { + if (Vector512.GreaterThanAny(xpi.AsUInt32() & Vector512.Create(CosOperatorSingle.SignMask), Vector512.Create(CosOperatorSingle.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsSingle()).As(); + } + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + if (Vector512.GreaterThanAny(xpi.AsUInt64() & Vector512.Create(CosOperatorDouble.SignMask), Vector512.Create(CosOperatorDouble.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsDouble()).As(); + } + } + + return CosOperator.Invoke(xpi); + } + } + + /// T.Cosh(x) + internal readonly struct CoshOperator : IUnaryOperator + where T : IHyperbolicFunctions + { + // This code is based on `vrs4_coshf` from amd/aocl-libm-ose + // Copyright (C) 2008-2022 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Spec: + // coshf(|x| > 89.415985107421875) = Infinity + // coshf(Infinity) = infinity + // coshf(-Infinity) = infinity + // + // cosh(x) = (exp(x) + exp(-x))/2 + // cosh(-x) = +cosh(x) + // + // checks for special cases + // if ( asint(x) > infinity) return x with overflow exception and + // return x. + // if x is NaN then raise invalid FP operation exception and return x. + // + // coshf = v/2 * exp(x - log(v)) where v = 0x1.0000e8p-1 + + private const float Single_LOGV = 0.693161f; + private const float Single_HALFV = 1.0000138f; + private const float Single_INVV2 = 0.24999309f; + + private const double Double_LOGV = 0.6931471805599453; + private const double Double_HALFV = 1.0; + private const double Double_INVV2 = 0.25; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Cosh(x); + + public static Vector128 Invoke(Vector128 t) + { + if (typeof(T) == typeof(float)) + { + Vector128 x = t.AsSingle(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create((float)Single_LOGV)); + return (Vector128.Create((float)Single_HALFV) * (z + (Vector128.Create((float)Single_INVV2) / z))).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector128 x = t.AsDouble(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create(Double_LOGV)); + return (Vector128.Create(Double_HALFV) * (z + (Vector128.Create(Double_INVV2) / z))).As(); + } + } + + public static Vector256 Invoke(Vector256 t) + { + if (typeof(T) == typeof(float)) + { + Vector256 x = t.AsSingle(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create((float)Single_LOGV)); + return (Vector256.Create((float)Single_HALFV) * (z + (Vector256.Create((float)Single_INVV2) / z))).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector256 x = t.AsDouble(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create(Double_LOGV)); + return (Vector256.Create(Double_HALFV) * (z + (Vector256.Create(Double_INVV2) / z))).As(); + } + } + + public static Vector512 Invoke(Vector512 t) + { + if (typeof(T) == typeof(float)) + { + Vector512 x = t.AsSingle(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create((float)Single_LOGV)); + return (Vector512.Create((float)Single_HALFV) * (z + (Vector512.Create((float)Single_INVV2) / z))).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector512 x = t.AsDouble(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create(Double_LOGV)); + return (Vector512.Create(Double_HALFV) * (z + (Vector512.Create(Double_INVV2) / z))).As(); + } + } + } + + /// T.Sin(x) + internal readonly struct SinOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + // This code is based on `vrs4_sin` and `vrd2_sin` from amd/aocl-libm-ose + // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Implementation notes from amd/aocl-libm-ose: + // ----------------------------------------------------------------- + // Convert given x into the form + // |x| = N * pi + f where N is an integer and f lies in [-pi/2,pi/2] + // N is obtained by : N = round(x/pi) + // f is obtained by : f = abs(x)-N*pi + // sin(x) = sin(N * pi + f) = sin(N * pi)*cos(f) + cos(N*pi)*sin(f) + // sin(x) = sign(x)*sin(f)*(-1)**N + // + // The term sin(f) can be approximated by using a polynomial + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Sin(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return SinOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return SinOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return SinOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return SinOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return SinOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return SinOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + } + + /// float.Sin(x) + private readonly struct SinOperatorSingle : IUnaryOperator + { + internal const uint SignMask = 0x7FFFFFFFu; + internal const uint MaxVectorizedValue = 0x49800000u; + private const float AlmHuge = 1.2582912e7f; + private const float Pi_Tail1 = 8.742278e-8f; + private const float Pi_Tail2 = 3.430249e-15f; + private const float C1 = -0.16666657f; + private const float C2 = 0.0083330255f; + private const float C3 = -1.980742e-4f; + private const float C4 = 2.6019031e-6f; + + public static bool Vectorizable => true; + + public static float Invoke(float x) => float.Sin(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 uxMasked = Vector128.Abs(x); + if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector128 almHuge = Vector128.Create(AlmHuge); + Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(1 / float.Pi), almHuge); + Vector128 odd = dn.AsUInt32() << 31; + dn -= almHuge; + + Vector128 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-float.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail1), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); + + // POLY_EVAL_ODD_9 + Vector128 f2 = f * f; + Vector128 f4 = f2 * f2; + Vector128 a0 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); + Vector128 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector128.One); + Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C3), f2, Vector128.Create(C4) * f4); + Vector128 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector128 poly = f * a3; + + return (poly.AsUInt32() ^ (x.AsUInt32() & Vector128.Create(~SignMask)) ^ odd).AsSingle(); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 uxMasked = Vector256.Abs(x); + if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector256 almHuge = Vector256.Create(AlmHuge); + Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(1 / float.Pi), almHuge); + Vector256 odd = dn.AsUInt32() << 31; + dn -= almHuge; + + Vector256 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-float.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail1), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); + + // POLY_EVAL_ODD_9 + Vector256 f2 = f * f; + Vector256 f4 = f2 * f2; + Vector256 a0 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); + Vector256 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector256.One); + Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C3), f2, Vector256.Create(C4) * f4); + Vector256 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector256 poly = f * a3; + + return (poly.AsUInt32() ^ (x.AsUInt32() & Vector256.Create(~SignMask)) ^ odd).AsSingle(); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 uxMasked = Vector512.Abs(x); + if (Vector512.GreaterThanAny(uxMasked.AsUInt32(), Vector512.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector512 almHuge = Vector512.Create(AlmHuge); + Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(1 / float.Pi), almHuge); + Vector512 odd = dn.AsUInt32() << 31; + dn -= almHuge; + + Vector512 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-float.Pi), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail1), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); + + // POLY_EVAL_ODD_9 + Vector512 f2 = f * f; + Vector512 f4 = f2 * f2; + Vector512 a0 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); + Vector512 a1 = MultiplyAddEstimateOperator.Invoke(a0, f2, Vector512.One); + Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C3), f2, Vector512.Create(C4) * f4); + Vector512 a3 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector512 poly = f * a3; + + return (poly.AsUInt32() ^ (x.AsUInt32() & Vector512.Create(~SignMask)) ^ odd).AsSingle(); + } + } + + /// double.Sin(x) + private readonly struct SinOperatorDouble : IUnaryOperator + { + internal const ulong SignMask = 0x7FFFFFFFFFFFFFFFul; + internal const ulong MaxVectorizedValue = 0x4160000000000000ul; + private const double AlmHuge = 6.755399441055744e15; + private const double Pi_Tail1 = 1.224646799147353e-16; + private const double Pi_Tail2 = 2.165713347843828e-32; + private const double C0 = -0.16666666666666666; + private const double C2 = 0.008333333333333165; + private const double C4 = -1.984126984120184e-4; + private const double C6 = 2.7557319210152756e-6; + private const double C8 = -2.5052106798274583e-8; + private const double C10 = 1.605893649037159e-10; + private const double C12 = -7.642917806891047e-13; + private const double C14 = 2.7204790957888847e-15; + + public static bool Vectorizable => true; + + public static double Invoke(double x) => double.Sin(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 uxMasked = Vector128.Abs(x); + if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector128 almHuge = Vector128.Create(AlmHuge); + Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(1 / double.Pi), almHuge); + Vector128 odd = dn.AsUInt64() << 63; + dn -= almHuge; + Vector128 f = uxMasked - (dn * Vector128.Create(double.Pi)) - (dn * Vector128.Create(Pi_Tail1)) - (dn * Vector128.Create(Pi_Tail2)); + + // POLY_EVAL_ODD_17 + Vector128 f2 = f * f; + Vector128 f4 = f2 * f2; + Vector128 f6 = f4 * f2; + Vector128 f10 = f6 * f4; + Vector128 f14 = f10 * f4; + Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C0)); + Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C6), f2, Vector128.Create(C4)); + Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C10), f2, Vector128.Create(C8)); + Vector128 a4 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C14), f2, Vector128.Create(C12)); + Vector128 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); + Vector128 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); + Vector128 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); + + return (poly.AsUInt64() ^ (x.AsUInt64() & Vector128.Create(~SignMask)) ^ odd).AsDouble(); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 uxMasked = Vector256.Abs(x); + if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector256 almHuge = Vector256.Create(AlmHuge); + Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(1 / double.Pi), almHuge); + Vector256 odd = dn.AsUInt64() << 63; + dn -= almHuge; + Vector256 f = uxMasked - (dn * Vector256.Create(double.Pi)) - (dn * Vector256.Create(Pi_Tail1)) - (dn * Vector256.Create(Pi_Tail2)); + + // POLY_EVAL_ODD_17 + Vector256 f2 = f * f; + Vector256 f4 = f2 * f2; + Vector256 f6 = f4 * f2; + Vector256 f10 = f6 * f4; + Vector256 f14 = f10 * f4; + Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C0)); + Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C6), f2, Vector256.Create(C4)); + Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C10), f2, Vector256.Create(C8)); + Vector256 a4 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C14), f2, Vector256.Create(C12)); + Vector256 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); + Vector256 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); + Vector256 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); + + return (poly.AsUInt64() ^ (x.AsUInt64() & Vector256.Create(~SignMask)) ^ odd).AsDouble(); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 uxMasked = Vector512.Abs(x); + if (Vector512.GreaterThanAny(uxMasked.AsUInt64(), Vector512.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector512 almHuge = Vector512.Create(AlmHuge); + Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(1 / double.Pi), almHuge); + Vector512 odd = dn.AsUInt64() << 63; + dn -= almHuge; + Vector512 f = uxMasked - (dn * Vector512.Create(double.Pi)) - (dn * Vector512.Create(Pi_Tail1)) - (dn * Vector512.Create(Pi_Tail2)); + + // POLY_EVAL_ODD_17 + Vector512 f2 = f * f; + Vector512 f4 = f2 * f2; + Vector512 f6 = f4 * f2; + Vector512 f10 = f6 * f4; + Vector512 f14 = f10 * f4; + Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C0)); + Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C6), f2, Vector512.Create(C4)); + Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C10), f2, Vector512.Create(C8)); + Vector512 a4 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C14), f2, Vector512.Create(C12)); + Vector512 b1 = MultiplyAddEstimateOperator.Invoke(a1, f2, a2 * f6); + Vector512 b2 = MultiplyAddEstimateOperator.Invoke(f10, a3, f14 * a4); + Vector512 poly = MultiplyAddEstimateOperator.Invoke(f, b1 + b2, f); + + return (poly.AsUInt64() ^ (x.AsUInt64() & Vector512.Create(~SignMask)) ^ odd).AsDouble(); + } + } + + /// T.SinPi(x) + internal readonly struct SinPiOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.SinPi(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 xpi = x * Vector128.Create(T.Pi); + if (typeof(T) == typeof(float)) + { + if (Vector128.GreaterThanAny(xpi.AsUInt32() & Vector128.Create(SinOperatorSingle.SignMask), Vector128.Create(SinOperatorSingle.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsSingle()).As(); + } + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + if (Vector128.GreaterThanAny(xpi.AsUInt64() & Vector128.Create(SinOperatorDouble.SignMask), Vector128.Create(SinOperatorDouble.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsDouble()).As(); + } + } + + return SinOperator.Invoke(xpi); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 xpi = x * Vector256.Create(T.Pi); + if (typeof(T) == typeof(float)) + { + if (Vector256.GreaterThanAny(xpi.AsUInt32() & Vector256.Create(SinOperatorSingle.SignMask), Vector256.Create(SinOperatorSingle.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsSingle()).As(); + } + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + if (Vector256.GreaterThanAny(xpi.AsUInt64() & Vector256.Create(SinOperatorDouble.SignMask), Vector256.Create(SinOperatorDouble.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsDouble()).As(); + } + } + + return SinOperator.Invoke(xpi); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 xpi = x * Vector512.Create(T.Pi); + if (typeof(T) == typeof(float)) + { + if (Vector512.GreaterThanAny(xpi.AsUInt32() & Vector512.Create(SinOperatorSingle.SignMask), Vector512.Create(SinOperatorSingle.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsSingle()).As(); + } + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + if (Vector512.GreaterThanAny(xpi.AsUInt64() & Vector512.Create(SinOperatorDouble.SignMask), Vector512.Create(SinOperatorDouble.MaxVectorizedValue))) + { + return ApplyScalar>(x.AsDouble()).As(); + } + } + + return SinOperator.Invoke(xpi); + } + } + + /// T.Sinh(x) + internal readonly struct SinhOperator : IUnaryOperator + where T : IHyperbolicFunctions + { + // Same as cosh, but with `z -` rather than `z +`, and with the sign + // flipped on the result based on the sign of the input. + + private const float Single_LOGV = 0.693161f; + private const float Single_HALFV = 1.0000138f; + private const float Single_INVV2 = 0.24999309f; + + private const double Double_LOGV = 0.6931471805599453; + private const double Double_HALFV = 1.0; + private const double Double_INVV2 = 0.25; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Sinh(x); + + public static Vector128 Invoke(Vector128 t) + { + if (typeof(T) == typeof(float)) + { + Vector128 x = t.AsSingle(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create((float)Single_LOGV)); + Vector128 result = Vector128.Create((float)Single_HALFV) * (z - (Vector128.Create((float)Single_INVV2) / z)); + Vector128 sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue); + return (sign ^ result.AsUInt32()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector128 x = t.AsDouble(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpOperator.Invoke(y - Vector128.Create(Double_LOGV)); + Vector128 result = Vector128.Create(Double_HALFV) * (z - (Vector128.Create(Double_INVV2) / z)); + Vector128 sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue); + return (sign ^ result.AsUInt64()).As(); + } + } + + public static Vector256 Invoke(Vector256 t) + { + if (typeof(T) == typeof(float)) + { + Vector256 x = t.AsSingle(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create((float)Single_LOGV)); + Vector256 result = Vector256.Create((float)Single_HALFV) * (z - (Vector256.Create((float)Single_INVV2) / z)); + Vector256 sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue); + return (sign ^ result.AsUInt32()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector256 x = t.AsDouble(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpOperator.Invoke(y - Vector256.Create(Double_LOGV)); + Vector256 result = Vector256.Create(Double_HALFV) * (z - (Vector256.Create(Double_INVV2) / z)); + Vector256 sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue); + return (sign ^ result.AsUInt64()).As(); + } + } + + public static Vector512 Invoke(Vector512 t) + { + if (typeof(T) == typeof(float)) + { + Vector512 x = t.AsSingle(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create((float)Single_LOGV)); + Vector512 result = Vector512.Create((float)Single_HALFV) * (z - (Vector512.Create((float)Single_INVV2) / z)); + Vector512 sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue); + return (sign ^ result.AsUInt32()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + Vector512 x = t.AsDouble(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpOperator.Invoke(y - Vector512.Create(Double_LOGV)); + Vector512 result = Vector512.Create(Double_HALFV) * (z - (Vector512.Create(Double_INVV2) / z)); + Vector512 sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue); + return (sign ^ result.AsUInt64()).As(); + } + } + } + + /// T.Tan(x) + internal readonly struct TanOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + // This code is based on `vrs4_tan` and `vrd2_tan` from amd/aocl-libm-ose + // Copyright (C) 2019-2020 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Implementation notes from amd/aocl-libm-ose: + // -------------------------------------------- + // A given x is reduced into the form: + // |x| = (N * π/2) + F + // Where N is an integer obtained using: + // N = round(x * 2/π) + // And F is a fraction part lying in the interval + // [-π/4, +π/4]; + // obtained as F = |x| - (N * π/2) + // Thus tan(x) is given by + // tan(x) = tan((N * π/2) + F) = tan(F) + // when N is even, = -cot(F) = -1/tan(F) + // when N is odd, tan(F) is approximated using a polynomial + // obtained from Remez approximation from Sollya. + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Tan(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return TanOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return TanOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return TanOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return TanOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return TanOperatorSingle.Invoke(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return TanOperatorDouble.Invoke(x.AsDouble()).As(); + } + } + } + + /// float.Tan(x) + internal readonly struct TanOperatorSingle : IUnaryOperator + { + internal const uint SignMask = 0x7FFFFFFFu; + internal const uint MaxVectorizedValue = 0x49800000u; + private const float AlmHuge = 1.2582912e7f; + private const float Pi_Tail2 = 4.371139e-8f; + private const float Pi_Tail3 = 1.7151245e-15f; + private const float C1 = 0.33333358f; + private const float C2 = 0.13332522f; + private const float C3 = 0.05407107f; + private const float C4 = 0.021237267f; + private const float C5 = 0.010932301f; + private const float C6 = -1.5722344e-5f; + private const float C7 = 0.0044221194f; + + public static bool Vectorizable => true; + + public static float Invoke(float x) => float.Tan(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 uxMasked = Vector128.Abs(x); + if (Vector128.GreaterThanAny(uxMasked.AsUInt32(), Vector128.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(2 / float.Pi), Vector128.Create(AlmHuge)); + Vector128 odd = dn.AsUInt32() << 31; + dn -= Vector128.Create(AlmHuge); + + Vector128 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(-float.Pi / 2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector128.Create(Pi_Tail3), f); + + // POLY_EVAL_ODD_15 + Vector128 f2 = f * f; + Vector128 f4 = f2 * f2; + Vector128 f8 = f4 * f4; + Vector128 f12 = f8 * f4; + Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C2), f2, Vector128.Create(C1)); + Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C4), f2, Vector128.Create(C3)); + Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C6), f2, Vector128.Create(C5)); + Vector128 b1 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector128 b2 = MultiplyAddEstimateOperator.Invoke(f8, a3, f12 * Vector128.Create(C7)); + Vector128 poly = MultiplyAddEstimateOperator.Invoke(f * f2, b1 + b2, f); + + Vector128 result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector128.Create(~SignMask))).AsSingle(); + return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128.Zero).AsSingle(), + result, + Vector128.Create(-1f) / result); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 uxMasked = Vector256.Abs(x); + if (Vector256.GreaterThanAny(uxMasked.AsUInt32(), Vector256.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(2 / float.Pi), Vector256.Create(AlmHuge)); + Vector256 odd = dn.AsUInt32() << 31; + dn -= Vector256.Create(AlmHuge); + + Vector256 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(-float.Pi / 2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector256.Create(Pi_Tail3), f); + + // POLY_EVAL_ODD_15 + Vector256 f2 = f * f; + Vector256 f4 = f2 * f2; + Vector256 f8 = f4 * f4; + Vector256 f12 = f8 * f4; + Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C2), f2, Vector256.Create(C1)); + Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C4), f2, Vector256.Create(C3)); + Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C6), f2, Vector256.Create(C5)); + Vector256 b1 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector256 b2 = MultiplyAddEstimateOperator.Invoke(f8, a3, f12 * Vector256.Create(C7)); + Vector256 poly = MultiplyAddEstimateOperator.Invoke(f * f2, b1 + b2, f); + + Vector256 result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector256.Create(~SignMask))).AsSingle(); + return Vector256.ConditionalSelect(Vector256.Equals(odd, Vector256.Zero).AsSingle(), + result, + Vector256.Create(-1f) / result); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 uxMasked = Vector512.Abs(x); + if (Vector512.GreaterThanAny(uxMasked.AsUInt32(), Vector512.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(2 / float.Pi), Vector512.Create(AlmHuge)); + Vector512 odd = dn.AsUInt32() << 31; + dn -= Vector512.Create(AlmHuge); + + Vector512 f = uxMasked; + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(-float.Pi / 2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail2), f); + f = MultiplyAddEstimateOperator.Invoke(dn, Vector512.Create(Pi_Tail3), f); + + // POLY_EVAL_ODD_15 + Vector512 f2 = f * f; + Vector512 f4 = f2 * f2; + Vector512 f8 = f4 * f4; + Vector512 f12 = f8 * f4; + Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C2), f2, Vector512.Create(C1)); + Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C4), f2, Vector512.Create(C3)); + Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C6), f2, Vector512.Create(C5)); + Vector512 b1 = MultiplyAddEstimateOperator.Invoke(a2, f4, a1); + Vector512 b2 = MultiplyAddEstimateOperator.Invoke(f8, a3, f12 * Vector512.Create(C7)); + Vector512 poly = MultiplyAddEstimateOperator.Invoke(f * f2, b1 + b2, f); + + Vector512 result = (poly.AsUInt32() ^ (x.AsUInt32() & Vector512.Create(~SignMask))).AsSingle(); + return Vector512.ConditionalSelect(Vector512.Equals(odd, Vector512.Zero).AsSingle(), + result, + Vector512.Create(-1f) / result); + } + } + + /// double.Tan(x) + internal readonly struct TanOperatorDouble : IUnaryOperator + { + internal const ulong SignMask = 0x7FFFFFFFFFFFFFFFul; + internal const ulong MaxVectorizedValue = 0x4160000000000000ul; + private const double AlmHuge = 6.755399441055744e15; + private const double HalfPi2 = 6.123233995736766E-17; + private const double HalfPi3 = -1.4973849048591698E-33; + private const double C1 = 0.33333333333332493; + private const double C3 = 0.133333333334343; + private const double C5 = 0.0539682539203796; + private const double C7 = 0.02186948972198256; + private const double C9 = 0.008863217894198291; + private const double C11 = 0.003592298593761111; + private const double C13 = 0.0014547086183165365; + private const double C15 = 5.952456856028558E-4; + private const double C17 = 2.2190741289936845E-4; + private const double C19 = 1.3739809957985104E-4; + private const double C21 = -2.7500197359895707E-5; + private const double C23 = 9.038741690184683E-5; + private const double C25 = -4.534076545538694E-5; + private const double C27 = 2.0966522562190197E-5; + + public static bool Vectorizable => true; + + public static double Invoke(double x) => double.Tan(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 uxMasked = Vector128.Abs(x); + if (Vector128.GreaterThanAny(uxMasked.AsUInt64(), Vector128.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector128 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector128.Create(2 / double.Pi), Vector128.Create(AlmHuge)); + Vector128 odd = dn.AsUInt64() << 63; + dn -= Vector128.Create(AlmHuge); + Vector128 f = uxMasked.AsDouble() - (dn * (double.Pi / 2)) - (dn * HalfPi2) - (dn * HalfPi3); + + // POLY_EVAL_ODD_29 + Vector128 g = f * f; + Vector128 g2 = g * g; + Vector128 g3 = g * g2; + Vector128 g5 = g3 * g2; + Vector128 g7 = g5 * g2; + Vector128 g9 = g7 * g2; + Vector128 g11 = g9 * g2; + Vector128 g13 = g11 * g2; + Vector128 a1 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C3), g, Vector128.Create(C1)); + Vector128 a2 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C7), g, Vector128.Create(C5)); + Vector128 a3 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C11), g, Vector128.Create(C9)); + Vector128 a4 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C15), g, Vector128.Create(C13)); + Vector128 a5 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C19), g, Vector128.Create(C17)); + Vector128 a6 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C23), g, Vector128.Create(C21)); + Vector128 a7 = MultiplyAddEstimateOperator.Invoke(Vector128.Create(C27), g, Vector128.Create(C25)); + Vector128 b1 = MultiplyAddEstimateOperator.Invoke(g, a1, g3 * a2); + Vector128 b2 = MultiplyAddEstimateOperator.Invoke(g5, a3, g7 * a4); + Vector128 b3 = MultiplyAddEstimateOperator.Invoke(g9, a5, g11 * a6); + Vector128 q = MultiplyAddEstimateOperator.Invoke(g13, a7, b1 + b2 + b3); + Vector128 poly = MultiplyAddEstimateOperator.Invoke(f, q, f); + + Vector128 result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector128.Create(~SignMask))).AsDouble(); + return Vector128.ConditionalSelect(Vector128.Equals(odd, Vector128.Zero).AsDouble(), + result, + Vector128.Create(-1.0) / result); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 uxMasked = Vector256.Abs(x); + if (Vector256.GreaterThanAny(uxMasked.AsUInt64(), Vector256.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector256 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector256.Create(2 / double.Pi), Vector256.Create(AlmHuge)); + Vector256 odd = dn.AsUInt64() << 63; + dn -= Vector256.Create(AlmHuge); + Vector256 f = uxMasked.AsDouble() - (dn * (double.Pi / 2)) - (dn * HalfPi2) - (dn * HalfPi3); + + // POLY_EVAL_ODD_29 + Vector256 g = f * f; + Vector256 g2 = g * g; + Vector256 g3 = g * g2; + Vector256 g5 = g3 * g2; + Vector256 g7 = g5 * g2; + Vector256 g9 = g7 * g2; + Vector256 g11 = g9 * g2; + Vector256 g13 = g11 * g2; + Vector256 a1 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C3), g, Vector256.Create(C1)); + Vector256 a2 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C7), g, Vector256.Create(C5)); + Vector256 a3 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C11), g, Vector256.Create(C9)); + Vector256 a4 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C15), g, Vector256.Create(C13)); + Vector256 a5 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C19), g, Vector256.Create(C17)); + Vector256 a6 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C23), g, Vector256.Create(C21)); + Vector256 a7 = MultiplyAddEstimateOperator.Invoke(Vector256.Create(C27), g, Vector256.Create(C25)); + Vector256 b1 = MultiplyAddEstimateOperator.Invoke(g, a1, g3 * a2); + Vector256 b2 = MultiplyAddEstimateOperator.Invoke(g5, a3, g7 * a4); + Vector256 b3 = MultiplyAddEstimateOperator.Invoke(g9, a5, g11 * a6); + Vector256 q = MultiplyAddEstimateOperator.Invoke(g13, a7, b1 + b2 + b3); + Vector256 poly = MultiplyAddEstimateOperator.Invoke(f, q, f); + + Vector256 result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector256.Create(~SignMask))).AsDouble(); + return Vector256.ConditionalSelect(Vector256.Equals(odd, Vector256.Zero).AsDouble(), + result, + Vector256.Create(-1.0) / result); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 uxMasked = Vector512.Abs(x); + if (Vector512.GreaterThanAny(uxMasked.AsUInt64(), Vector512.Create(MaxVectorizedValue))) + { + return ApplyScalar(x); + } + + Vector512 dn = MultiplyAddEstimateOperator.Invoke(uxMasked, Vector512.Create(2 / double.Pi), Vector512.Create(AlmHuge)); + Vector512 odd = dn.AsUInt64() << 63; + dn -= Vector512.Create(AlmHuge); + Vector512 f = uxMasked.AsDouble() - (dn * (double.Pi / 2)) - (dn * HalfPi2) - (dn * HalfPi3); + + // POLY_EVAL_ODD_29 + Vector512 g = f * f; + Vector512 g2 = g * g; + Vector512 g3 = g * g2; + Vector512 g5 = g3 * g2; + Vector512 g7 = g5 * g2; + Vector512 g9 = g7 * g2; + Vector512 g11 = g9 * g2; + Vector512 g13 = g11 * g2; + Vector512 a1 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C3), g, Vector512.Create(C1)); + Vector512 a2 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C7), g, Vector512.Create(C5)); + Vector512 a3 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C11), g, Vector512.Create(C9)); + Vector512 a4 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C15), g, Vector512.Create(C13)); + Vector512 a5 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C19), g, Vector512.Create(C17)); + Vector512 a6 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C23), g, Vector512.Create(C21)); + Vector512 a7 = MultiplyAddEstimateOperator.Invoke(Vector512.Create(C27), g, Vector512.Create(C25)); + Vector512 b1 = MultiplyAddEstimateOperator.Invoke(g, a1, g3 * a2); + Vector512 b2 = MultiplyAddEstimateOperator.Invoke(g5, a3, g7 * a4); + Vector512 b3 = MultiplyAddEstimateOperator.Invoke(g9, a5, g11 * a6); + Vector512 q = MultiplyAddEstimateOperator.Invoke(g13, a7, b1 + b2 + b3); + Vector512 poly = MultiplyAddEstimateOperator.Invoke(f, q, f); + + Vector512 result = (poly.AsUInt64() ^ (x.AsUInt64() & Vector512.Create(~SignMask))).AsDouble(); + return Vector512.ConditionalSelect(Vector512.Equals(odd, Vector512.Zero).AsDouble(), + result, + Vector512.Create(-1.0) / result); + } + } + + /// T.TanPi(x) + internal readonly struct TanPiOperator : IUnaryOperator + where T : ITrigonometricFunctions + { + public static bool Vectorizable => false; + public static T Invoke(T x) => T.TanPi(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.Tanh(x) + internal readonly struct TanhOperator : IUnaryOperator + where T : IHyperbolicFunctions + { + // This code is based on `vrs4_tanhf` from amd/aocl-libm-ose + // Copyright (C) 2008-2022 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // To compute vrs4_tanhf(v_f32x4_t x) + // Let y = |x| + // If 0 <= y < 0x1.154246p3 + // Let z = e^(-2.0 * y) - 1 -(1) + // + // Using (1), tanhf(y) can be calculated as, + // tanhf(y) = -z / (z + 2.0) + // + // For other cases, call scalar tanhf() + // + // If x < 0, then we use the identity + // tanhf(-x) = -tanhf(x) + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Tanh(x); + + public static Vector128 Invoke(Vector128 t) + { + if (typeof(T) == typeof(float)) + { + Vector128 x = t.AsSingle(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2f) * y); + Vector128 sign = x.AsUInt32() & Vector128.Create(~(uint)int.MaxValue); + return (sign ^ (-z / (z + Vector128.Create(2f))).AsUInt32()).As(); + } + else + { + Vector128 x = t.AsDouble(); + + Vector128 y = Vector128.Abs(x); + Vector128 z = ExpM1Operator.Invoke(Vector128.Create(-2d) * y); + Vector128 sign = x.AsUInt64() & Vector128.Create(~(ulong)long.MaxValue); + return (sign ^ (-z / (z + Vector128.Create(2d))).AsUInt64()).As(); + } + } + + public static Vector256 Invoke(Vector256 t) + { + if (typeof(T) == typeof(float)) + { + Vector256 x = t.AsSingle(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2f) * y); + Vector256 sign = x.AsUInt32() & Vector256.Create(~(uint)int.MaxValue); + return (sign ^ (-z / (z + Vector256.Create(2f))).AsUInt32()).As(); + } + else + { + Vector256 x = t.AsDouble(); + + Vector256 y = Vector256.Abs(x); + Vector256 z = ExpM1Operator.Invoke(Vector256.Create(-2d) * y); + Vector256 sign = x.AsUInt64() & Vector256.Create(~(ulong)long.MaxValue); + return (sign ^ (-z / (z + Vector256.Create(2d))).AsUInt64()).As(); + } + } + + public static Vector512 Invoke(Vector512 t) + { + if (typeof(T) == typeof(float)) + { + Vector512 x = t.AsSingle(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2f) * y); + Vector512 sign = x.AsUInt32() & Vector512.Create(~(uint)int.MaxValue); + return (sign ^ (-z / (z + Vector512.Create(2f))).AsUInt32()).As(); + } + else + { + Vector512 x = t.AsDouble(); + + Vector512 y = Vector512.Abs(x); + Vector512 z = ExpM1Operator.Invoke(Vector512.Create(-2d) * y); + Vector512 sign = x.AsUInt64() & Vector512.Create(~(ulong)long.MaxValue); + return (sign ^ (-z / (z + Vector512.Create(2d))).AsUInt64()).As(); + } + } + } + + /// T.Log(x) + internal readonly struct LogOperator : IUnaryOperator + where T : ILogarithmicFunctions + { + public static bool Vectorizable => (typeof(T) == typeof(double)) + || (typeof(T) == typeof(float)); + + public static T Invoke(T x) => T.Log(x); + + public static Vector128 Invoke(Vector128 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector128.Log(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector128.Log(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return LogOperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return LogOperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + + public static Vector256 Invoke(Vector256 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector256.Log(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector256.Log(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return LogOperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return LogOperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + + public static Vector512 Invoke(Vector512 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector512.Log(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector512.Log(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return LogOperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return LogOperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + } + +#if !NET9_0_OR_GREATER + /// double.Log(x) + internal readonly struct LogOperatorDouble : IUnaryOperator + { + // This code is based on `vrd2_log` from amd/aocl-libm-ose + // Copyright (C) 2018-2020 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Reduce x into the form: + // x = (-1)^s*2^n*m + // s will be always zero, as log is defined for positive numbers + // n is an integer known as the exponent + // m is mantissa + // + // x is reduced such that the mantissa, m lies in [2/3,4/3] + // x = 2^n*m where m is in [2/3,4/3] + // log(x) = log(2^n*m) We have log(a*b) = log(a)+log(b) + // = log(2^n) + log(m) We have log(a^n) = n*log(a) + // = n*log(2) + log(m) + // = n*log(2) + log(1+(m-1)) + // = n*log(2) + log(1+f) Where f = m-1 + // = n*log(2) + log1p(f) f lies in [-1/3,+1/3] + // + // Thus we have : + // log(x) = n*log(2) + log1p(f) + // In the above, the first term n*log(2), n can be calculated by using right shift operator and the value of log(2) + // is known and is stored as a constant + // The second term log1p(F) is approximated by using a polynomial + + private const ulong V_MIN = 0x00100000_00000000; // SmallestNormal + private const ulong V_MAX = 0x7FF00000_00000000; // +Infinity + private const ulong V_MSK = 0x000FFFFF_FFFFFFFF; // (1 << 52) - 1 + private const ulong V_OFF = 0x3FE55555_55555555; // 2.0 / 3.0 + + private const double LN2_HEAD = 0.693359375; + private const double LN2_TAIL = -0.00021219444005469057; + + private const double C02 = -0.499999999999999560; + private const double C03 = +0.333333333333414750; + private const double C04 = -0.250000000000297430; + private const double C05 = +0.199999999975985220; + private const double C06 = -0.166666666608919500; + private const double C07 = +0.142857145600277100; + private const double C08 = -0.125000005127831270; + private const double C09 = +0.111110952357159440; + private const double C10 = -0.099999750495501240; + private const double C11 = +0.090914349823462390; + private const double C12 = -0.083340600527551860; + private const double C13 = +0.076817603328311300; + private const double C14 = -0.071296718946287310; + private const double C15 = +0.067963465211535730; + private const double C16 = -0.063995035098960040; + private const double C17 = +0.049370587082412105; + private const double C18 = -0.045370170994891980; + private const double C19 = +0.088970636003577750; + private const double C20 = -0.086906174116908760; + + public static bool Vectorizable => true; + + public static double Invoke(double x) => double.Log(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 specialResult = x; + + // x is zero, subnormal, infinity, or NaN + Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt64() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); + + if (specialMask != Vector128.Zero) + { + Vector128 xBits = x.AsInt64(); + + // (x < 0) ? float.NaN : x + Vector128 lessThanZeroMask = Vector128.LessThan(xBits, Vector128.Zero).AsDouble(); + + specialResult = Vector128.ConditionalSelect( + lessThanZeroMask, + Vector128.Create(double.NaN), + specialResult + ); + + // double.IsZero(x) ? double.NegativeInfinity : x + Vector128 zeroMask = Vector128.Equals(xBits << 1, Vector128.Zero).AsDouble(); + + specialResult = Vector128.ConditionalSelect( + zeroMask, + Vector128.Create(double.NegativeInfinity), + specialResult + ); + + // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) + Vector128 temp = zeroMask + | lessThanZeroMask + | Vector128.GreaterThanOrEqual(xBits, Vector128.Create(double.PositiveInfinity).AsInt64()).AsDouble(); + + // subnormal + Vector128 subnormalMask = Vector128.AndNot(specialMask.AsDouble(), temp); + + // multiply by 2^52, then normalize + x = Vector128.ConditionalSelect( + subnormalMask, + ((x * 4503599627370496.0).AsUInt64() - Vector128.Create(52ul << 52)).AsDouble(), + x + ); + + specialMask = temp.AsUInt64(); + } + + // Reduce the mantissa to [+2/3, +4/3] + Vector128 vx = x.AsUInt64() - Vector128.Create(V_OFF); + Vector128 n = Vector128.ConvertToDouble(vx.AsInt64() >> 52); + vx = (vx & Vector128.Create(V_MSK)) + Vector128.Create(V_OFF); + + // Adjust the mantissa to [-1/3, +1/3] + Vector128 r = vx.AsDouble() - Vector128.One; + + Vector128 r02 = r * r; + Vector128 r04 = r02 * r02; + Vector128 r08 = r04 * r04; + Vector128 r16 = r08 * r08; + + // Compute log(x + 1) using Polynomial approximation + // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) + + Vector128 poly = (((r04 * C20) + + ((((r * C19) + Vector128.Create(C18)) * r02) + + ((r * C17) + Vector128.Create(C16)))) * r16) + + (((((((r * C15) + Vector128.Create(C14)) * r02) + + ((r * C13) + Vector128.Create(C12))) * r04) + + ((((r * C11) + Vector128.Create(C10)) * r02) + + ((r * C09) + Vector128.Create(C08)))) * r08) + + (((((r * C07) + Vector128.Create(C06)) * r02) + + ((r * C05) + Vector128.Create(C04))) * r04) + + ((((r * C03) + Vector128.Create(C02)) * r02) + r); + + return Vector128.ConditionalSelect( + specialMask.AsDouble(), + specialResult, + (n * LN2_HEAD) + ((n * LN2_TAIL) + poly) + ); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 specialResult = x; + + // x is zero, subnormal, infinity, or NaN + Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt64() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); + + if (specialMask != Vector256.Zero) + { + Vector256 xBits = x.AsInt64(); + + // (x < 0) ? float.NaN : x + Vector256 lessThanZeroMask = Vector256.LessThan(xBits, Vector256.Zero).AsDouble(); + + specialResult = Vector256.ConditionalSelect( + lessThanZeroMask, + Vector256.Create(double.NaN), + specialResult + ); + + // double.IsZero(x) ? double.NegativeInfinity : x + Vector256 zeroMask = Vector256.Equals(xBits << 1, Vector256.Zero).AsDouble(); + + specialResult = Vector256.ConditionalSelect( + zeroMask, + Vector256.Create(double.NegativeInfinity), + specialResult + ); + + // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) + Vector256 temp = zeroMask + | lessThanZeroMask + | Vector256.GreaterThanOrEqual(xBits, Vector256.Create(double.PositiveInfinity).AsInt64()).AsDouble(); + + // subnormal + Vector256 subnormalMask = Vector256.AndNot(specialMask.AsDouble(), temp); + + // multiply by 2^52, then normalize + x = Vector256.ConditionalSelect( + subnormalMask, + ((x * 4503599627370496.0).AsUInt64() - Vector256.Create(52ul << 52)).AsDouble(), + x + ); + + specialMask = temp.AsUInt64(); + } + + // Reduce the mantissa to [+2/3, +4/3] + Vector256 vx = x.AsUInt64() - Vector256.Create(V_OFF); + Vector256 n = Vector256.ConvertToDouble(vx.AsInt64() >> 52); + vx = (vx & Vector256.Create(V_MSK)) + Vector256.Create(V_OFF); + + // Adjust the mantissa to [-1/3, +1/3] + Vector256 r = vx.AsDouble() - Vector256.One; + + Vector256 r02 = r * r; + Vector256 r04 = r02 * r02; + Vector256 r08 = r04 * r04; + Vector256 r16 = r08 * r08; + + // Compute log(x + 1) using Polynomial approximation + // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) + + Vector256 poly = (((r04 * C20) + + ((((r * C19) + Vector256.Create(C18)) * r02) + + ((r * C17) + Vector256.Create(C16)))) * r16) + + (((((((r * C15) + Vector256.Create(C14)) * r02) + + ((r * C13) + Vector256.Create(C12))) * r04) + + ((((r * C11) + Vector256.Create(C10)) * r02) + + ((r * C09) + Vector256.Create(C08)))) * r08) + + (((((r * C07) + Vector256.Create(C06)) * r02) + + ((r * C05) + Vector256.Create(C04))) * r04) + + ((((r * C03) + Vector256.Create(C02)) * r02) + r); + + return Vector256.ConditionalSelect( + specialMask.AsDouble(), + specialResult, + (n * LN2_HEAD) + ((n * LN2_TAIL) + poly) + ); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 specialResult = x; + + // x is zero, subnormal, infinity, or NaN + Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt64() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); + + if (specialMask != Vector512.Zero) + { + Vector512 xBits = x.AsInt64(); + + // (x < 0) ? float.NaN : x + Vector512 lessThanZeroMask = Vector512.LessThan(xBits, Vector512.Zero).AsDouble(); + + specialResult = Vector512.ConditionalSelect( + lessThanZeroMask, + Vector512.Create(double.NaN), + specialResult + ); + + // double.IsZero(x) ? double.NegativeInfinity : x + Vector512 zeroMask = Vector512.Equals(xBits << 1, Vector512.Zero).AsDouble(); + + specialResult = Vector512.ConditionalSelect( + zeroMask, + Vector512.Create(double.NegativeInfinity), + specialResult + ); + + // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) + Vector512 temp = zeroMask + | lessThanZeroMask + | Vector512.GreaterThanOrEqual(xBits, Vector512.Create(double.PositiveInfinity).AsInt64()).AsDouble(); + + // subnormal + Vector512 subnormalMask = Vector512.AndNot(specialMask.AsDouble(), temp); + + // multiply by 2^52, then normalize + x = Vector512.ConditionalSelect( + subnormalMask, + ((x * 4503599627370496.0).AsUInt64() - Vector512.Create(52ul << 52)).AsDouble(), + x + ); + + specialMask = temp.AsUInt64(); + } + + // Reduce the mantissa to [+2/3, +4/3] + Vector512 vx = x.AsUInt64() - Vector512.Create(V_OFF); + Vector512 n = Vector512.ConvertToDouble(vx.AsInt64() >> 52); + vx = (vx & Vector512.Create(V_MSK)) + Vector512.Create(V_OFF); + + // Adjust the mantissa to [-1/3, +1/3] + Vector512 r = vx.AsDouble() - Vector512.One; + + Vector512 r02 = r * r; + Vector512 r04 = r02 * r02; + Vector512 r08 = r04 * r04; + Vector512 r16 = r08 * r08; + + // Compute log(x + 1) using Polynomial approximation + // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) + + Vector512 poly = (((r04 * C20) + + ((((r * C19) + Vector512.Create(C18)) * r02) + + ((r * C17) + Vector512.Create(C16)))) * r16) + + (((((((r * C15) + Vector512.Create(C14)) * r02) + + ((r * C13) + Vector512.Create(C12))) * r04) + + ((((r * C11) + Vector512.Create(C10)) * r02) + + ((r * C09) + Vector512.Create(C08)))) * r08) + + (((((r * C07) + Vector512.Create(C06)) * r02) + + ((r * C05) + Vector512.Create(C04))) * r04) + + ((((r * C03) + Vector512.Create(C02)) * r02) + r); + + return Vector512.ConditionalSelect( + specialMask.AsDouble(), + specialResult, + (n * LN2_HEAD) + ((n * LN2_TAIL) + poly) + ); + } + } + + /// float.Log(x) + internal readonly struct LogOperatorSingle : IUnaryOperator + { + // This code is based on `vrs4_logf` from amd/aocl-libm-ose + // Copyright (C) 2018-2019 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Spec: + // logf(x) + // = logf(x) if x ∈ F and x > 0 + // = x if x = qNaN + // = 0 if x = 1 + // = -inf if x = (-0, 0} + // = NaN otherwise + // + // Assumptions/Expectations + // - ULP is derived to be << 4 (always) + // - Some FPU Exceptions may not be available + // - Performance is at least 3x + // + // Implementation Notes: + // 1. Range Reduction: + // x = 2^n*(1+f) .... (1) + // where n is exponent and is an integer + // (1+f) is mantissa ∈ [1,2). i.e., 1 ≤ 1+f < 2 .... (2) + // + // From (1), taking log on both sides + // log(x) = log(2^n * (1+f)) + // = log(2^n) + log(1+f) + // = n*log(2) + log(1+f) .... (3) + // + // let z = 1 + f + // log(z) = log(k) + log(z) - log(k) + // log(z) = log(kz) - log(k) + // + // From (2), range of z is [1, 2) + // by simply dividing range by 'k', z is in [1/k, 2/k) .... (4) + // Best choice of k is the one which gives equal and opposite values + // at extrema +- -+ + // 1 | 2 | + // --- - 1 = - |--- - 1 | + // k | k | .... (5) + // +- -+ + // + // Solving for k, k = 3/2, + // From (4), using 'k' value, range is therefore [-0.3333, 0.3333] + // + // 2. Polynomial Approximation: + // More information refer to tools/sollya/vrs4_logf.sollya + // + // 7th Deg - Error abs: 0x1.04c4ac98p-22 rel: 0x1.2216e6f8p-19 + // 6th Deg - Error abs: 0x1.179e97d8p-19 rel: 0x1.db676c1p-17 + + private const uint V_MIN = 0x00800000; + private const uint V_MAX = 0x7F800000; + private const uint V_MASK = 0x007FFFFF; + private const uint V_OFF = 0x3F2AAAAB; + + private const float V_LN2 = 0.6931472f; + + private const float C0 = 0.0f; + private const float C1 = 1.0f; + private const float C2 = -0.5000001f; + private const float C3 = 0.33332965f; + private const float C4 = -0.24999046f; + private const float C5 = 0.20018855f; + private const float C6 = -0.16700386f; + private const float C7 = 0.13902695f; + private const float C8 = -0.1197452f; + private const float C9 = 0.14401625f; + private const float C10 = -0.13657966f; + + public static bool Vectorizable => true; + + public static float Invoke(float x) => float.Log(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 specialResult = x; + + // x is subnormal or infinity or NaN + Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt32() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); + + if (specialMask != Vector128.Zero) + { + // float.IsZero(x) ? float.NegativeInfinity : x + Vector128 zeroMask = Vector128.Equals(x, Vector128.Zero); + + specialResult = Vector128.ConditionalSelect( + zeroMask, + Vector128.Create(float.NegativeInfinity), + specialResult + ); + + // (x < 0) ? float.NaN : x + Vector128 lessThanZeroMask = Vector128.LessThan(x, Vector128.Zero); + + specialResult = Vector128.ConditionalSelect( + lessThanZeroMask, + Vector128.Create(float.NaN), + specialResult + ); + + // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) + Vector128 temp = zeroMask + | lessThanZeroMask + | ~Vector128.Equals(x, x) + | Vector128.Equals(x, Vector128.Create(float.PositiveInfinity)); + + // subnormal + Vector128 subnormalMask = Vector128.AndNot(specialMask.AsSingle(), temp); + + x = Vector128.ConditionalSelect( + subnormalMask, + ((x * 8388608.0f).AsUInt32() - Vector128.Create(23u << 23)).AsSingle(), + x + ); + + specialMask = temp.AsUInt32(); + } + + Vector128 vx = x.AsUInt32() - Vector128.Create(V_OFF); + Vector128 n = Vector128.ConvertToSingle(Vector128.ShiftRightArithmetic(vx.AsInt32(), 23)); + + vx = (vx & Vector128.Create(V_MASK)) + Vector128.Create(V_OFF); + + Vector128 r = vx.AsSingle() - Vector128.One; + + Vector128 r2 = r * r; + Vector128 r4 = r2 * r2; + Vector128 r8 = r4 * r4; + + Vector128 q = (Vector128.Create(C10) * r2 + (Vector128.Create(C9) * r + Vector128.Create(C8))) + * r8 + (((Vector128.Create(C7) * r + Vector128.Create(C6)) + * r2 + (Vector128.Create(C5) * r + Vector128.Create(C4))) + * r4 + ((Vector128.Create(C3) * r + Vector128.Create(C2)) + * r2 + (Vector128.Create(C1) * r + Vector128.Create(C0)))); + + return Vector128.ConditionalSelect( + specialMask.AsSingle(), + specialResult, + n * Vector128.Create(V_LN2) + q + ); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 specialResult = x; + + // x is subnormal or infinity or NaN + Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt32() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); + + if (specialMask != Vector256.Zero) + { + // float.IsZero(x) ? float.NegativeInfinity : x + Vector256 zeroMask = Vector256.Equals(x, Vector256.Zero); + + specialResult = Vector256.ConditionalSelect( + zeroMask, + Vector256.Create(float.NegativeInfinity), + specialResult + ); + + // (x < 0) ? float.NaN : x + Vector256 lessThanZeroMask = Vector256.LessThan(x, Vector256.Zero); + + specialResult = Vector256.ConditionalSelect( + lessThanZeroMask, + Vector256.Create(float.NaN), + specialResult + ); + + // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) + Vector256 temp = zeroMask + | lessThanZeroMask + | ~Vector256.Equals(x, x) + | Vector256.Equals(x, Vector256.Create(float.PositiveInfinity)); + + // subnormal + Vector256 subnormalMask = Vector256.AndNot(specialMask.AsSingle(), temp); + + x = Vector256.ConditionalSelect( + subnormalMask, + ((x * 8388608.0f).AsUInt32() - Vector256.Create(23u << 23)).AsSingle(), + x + ); + + specialMask = temp.AsUInt32(); + } + + Vector256 vx = x.AsUInt32() - Vector256.Create(V_OFF); + Vector256 n = Vector256.ConvertToSingle(Vector256.ShiftRightArithmetic(vx.AsInt32(), 23)); + + vx = (vx & Vector256.Create(V_MASK)) + Vector256.Create(V_OFF); + + Vector256 r = vx.AsSingle() - Vector256.One; + + Vector256 r2 = r * r; + Vector256 r4 = r2 * r2; + Vector256 r8 = r4 * r4; + + Vector256 q = (Vector256.Create(C10) * r2 + (Vector256.Create(C9) * r + Vector256.Create(C8))) + * r8 + (((Vector256.Create(C7) * r + Vector256.Create(C6)) + * r2 + (Vector256.Create(C5) * r + Vector256.Create(C4))) + * r4 + ((Vector256.Create(C3) * r + Vector256.Create(C2)) + * r2 + (Vector256.Create(C1) * r + Vector256.Create(C0)))); + + return Vector256.ConditionalSelect( + specialMask.AsSingle(), + specialResult, + n * Vector256.Create(V_LN2) + q + ); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 specialResult = x; + + // x is subnormal or infinity or NaN + Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt32() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); + + if (specialMask != Vector512.Zero) + { + // float.IsZero(x) ? float.NegativeInfinity : x + Vector512 zeroMask = Vector512.Equals(x, Vector512.Zero); + + specialResult = Vector512.ConditionalSelect( + zeroMask, + Vector512.Create(float.NegativeInfinity), + specialResult + ); + + // (x < 0) ? float.NaN : x + Vector512 lessThanZeroMask = Vector512.LessThan(x, Vector512.Zero); + + specialResult = Vector512.ConditionalSelect( + lessThanZeroMask, + Vector512.Create(float.NaN), + specialResult + ); + + // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) + Vector512 temp = zeroMask + | lessThanZeroMask + | ~Vector512.Equals(x, x) + | Vector512.Equals(x, Vector512.Create(float.PositiveInfinity)); + + // subnormal + Vector512 subnormalMask = Vector512.AndNot(specialMask.AsSingle(), temp); + + x = Vector512.ConditionalSelect( + subnormalMask, + ((x * 8388608.0f).AsUInt32() - Vector512.Create(23u << 23)).AsSingle(), + x + ); + + specialMask = temp.AsUInt32(); + } + + Vector512 vx = x.AsUInt32() - Vector512.Create(V_OFF); + Vector512 n = Vector512.ConvertToSingle(Vector512.ShiftRightArithmetic(vx.AsInt32(), 23)); + + vx = (vx & Vector512.Create(V_MASK)) + Vector512.Create(V_OFF); + + Vector512 r = vx.AsSingle() - Vector512.One; + + Vector512 r2 = r * r; + Vector512 r4 = r2 * r2; + Vector512 r8 = r4 * r4; + + Vector512 q = (Vector512.Create(C10) * r2 + (Vector512.Create(C9) * r + Vector512.Create(C8))) + * r8 + (((Vector512.Create(C7) * r + Vector512.Create(C6)) + * r2 + (Vector512.Create(C5) * r + Vector512.Create(C4))) + * r4 + ((Vector512.Create(C3) * r + Vector512.Create(C2)) + * r2 + (Vector512.Create(C1) * r + Vector512.Create(C0)))); + + return Vector512.ConditionalSelect( + specialMask.AsSingle(), + specialResult, + n * Vector512.Create(V_LN2) + q + ); + } + } +#endif + + /// T.Log2(x) + internal readonly struct Log2Operator : IUnaryOperator + where T : ILogarithmicFunctions + { + public static bool Vectorizable => (typeof(T) == typeof(double)) + || (typeof(T) == typeof(float)); + + public static T Invoke(T x) => T.Log2(x); + + public static Vector128 Invoke(Vector128 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector128.Log2(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector128.Log2(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return Log2OperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Log2OperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + + public static Vector256 Invoke(Vector256 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector256.Log2(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector256.Log2(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return Log2OperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Log2OperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + + public static Vector512 Invoke(Vector512 x) + { +#if NET9_0_OR_GREATER + if (typeof(T) == typeof(double)) + { + return Vector512.Log2(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Vector512.Log2(x.AsSingle()).As(); + } +#else + if (typeof(T) == typeof(double)) + { + return Log2OperatorDouble.Invoke(x.AsDouble()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(float)); + return Log2OperatorSingle.Invoke(x.AsSingle()).As(); + } +#endif + } + } + +#if !NET9_0_OR_GREATER + /// double.Log2(x) + internal readonly struct Log2OperatorDouble : IUnaryOperator + { + // This code is based on `vrd2_log2` from amd/aocl-libm-ose + // Copyright (C) 2021-2022 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Reduce x into the form: + // x = (-1)^s*2^n*m + // s will be always zero, as log is defined for positive numbers + // n is an integer known as the exponent + // m is mantissa + // + // x is reduced such that the mantissa, m lies in [2/3,4/3] + // x = 2^n*m where m is in [2/3,4/3] + // log2(x) = log2(2^n*m) We have log(a*b) = log(a)+log(b) + // = log2(2^n) + log2(m) We have log(a^n) = n*log(a) + // = n + log2(m) + // = n + log2(1+(m-1)) + // = n + ln(1+f) * log2(e) Where f = m-1 + // = n + log1p(f) * log2(e) f lies in [-1/3,+1/3] + // + // Thus we have : + // log(x) = n + log1p(f) * log2(e) + // The second term log1p(F) is approximated by using a polynomial + + private const ulong V_MIN = 0x00100000_00000000; // SmallestNormal + private const ulong V_MAX = 0x7FF00000_00000000; // +Infinity + private const ulong V_MSK = 0x000FFFFF_FFFFFFFF; // (1 << 52) - 1 + private const ulong V_OFF = 0x3FE55555_55555555; // 2.0 / 3.0 + + private const double LN2_HEAD = 1.44269180297851562500E+00; + private const double LN2_TAIL = 3.23791044778235969970E-06; + + private const double C02 = -0.499999999999999560; + private const double C03 = +0.333333333333414750; + private const double C04 = -0.250000000000297430; + private const double C05 = +0.199999999975985220; + private const double C06 = -0.166666666608919500; + private const double C07 = +0.142857145600277100; + private const double C08 = -0.125000005127831270; + private const double C09 = +0.111110952357159440; + private const double C10 = -0.099999750495501240; + private const double C11 = +0.090914349823462390; + private const double C12 = -0.083340600527551860; + private const double C13 = +0.076817603328311300; + private const double C14 = -0.071296718946287310; + private const double C15 = +0.067963465211535730; + private const double C16 = -0.063995035098960040; + private const double C17 = +0.049370587082412105; + private const double C18 = -0.045370170994891980; + private const double C19 = +0.088970636003577750; + private const double C20 = -0.086906174116908760; + + public static bool Vectorizable => true; + + public static double Invoke(double x) => double.Log2(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 specialResult = x; + + // x is zero, subnormal, infinity, or NaN + Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt64() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); + + if (specialMask != Vector128.Zero) + { + Vector128 xBits = x.AsInt64(); + + // (x < 0) ? float.NaN : x + Vector128 lessThanZeroMask = Vector128.LessThan(xBits, Vector128.Zero).AsDouble(); + + specialResult = Vector128.ConditionalSelect( + lessThanZeroMask, + Vector128.Create(double.NaN), + specialResult + ); + + // double.IsZero(x) ? double.NegativeInfinity : x + Vector128 zeroMask = Vector128.Equals(xBits << 1, Vector128.Zero).AsDouble(); + + specialResult = Vector128.ConditionalSelect( + zeroMask, + Vector128.Create(double.NegativeInfinity), + specialResult + ); + + // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) + Vector128 temp = zeroMask + | lessThanZeroMask + | Vector128.GreaterThanOrEqual(xBits, Vector128.Create(double.PositiveInfinity).AsInt64()).AsDouble(); + + // subnormal + Vector128 subnormalMask = Vector128.AndNot(specialMask.AsDouble(), temp); + + // multiply by 2^52, then normalize + x = Vector128.ConditionalSelect( + subnormalMask, + ((x * 4503599627370496.0).AsUInt64() - Vector128.Create(52ul << 52)).AsDouble(), + x + ); + + specialMask = temp.AsUInt64(); + } + + // Reduce the mantissa to [+2/3, +4/3] + Vector128 vx = x.AsUInt64() - Vector128.Create(V_OFF); + Vector128 n = Vector128.ConvertToDouble(vx.AsInt64() >> 52); + vx = (vx & Vector128.Create(V_MSK)) + Vector128.Create(V_OFF); + + // Adjust the mantissa to [-1/3, +1/3] + Vector128 r = vx.AsDouble() - Vector128.One; + + Vector128 r02 = r * r; + Vector128 r04 = r02 * r02; + Vector128 r08 = r04 * r04; + Vector128 r16 = r08 * r08; + + // Compute log(x + 1) using polynomial approximation + // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) + + Vector128 poly = (((r04 * C20) + + ((((r * C19) + Vector128.Create(C18)) * r02) + + ((r * C17) + Vector128.Create(C16)))) * r16) + + (((((((r * C15) + Vector128.Create(C14)) * r02) + + ((r * C13) + Vector128.Create(C12))) * r04) + + ((((r * C11) + Vector128.Create(C10)) * r02) + + ((r * C09) + Vector128.Create(C08)))) * r08) + + (((((r * C07) + Vector128.Create(C06)) * r02) + + ((r * C05) + Vector128.Create(C04))) * r04) + + ((((r * C03) + Vector128.Create(C02)) * r02) + r); + + return Vector128.ConditionalSelect( + specialMask.AsDouble(), + specialResult, + (poly * LN2_HEAD) + ((poly * LN2_TAIL) + n) + ); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 specialResult = x; + + // x is zero, subnormal, infinity, or NaN + Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt64() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); + + if (specialMask != Vector256.Zero) + { + Vector256 xBits = x.AsInt64(); + + // (x < 0) ? float.NaN : x + Vector256 lessThanZeroMask = Vector256.LessThan(xBits, Vector256.Zero).AsDouble(); + + specialResult = Vector256.ConditionalSelect( + lessThanZeroMask, + Vector256.Create(double.NaN), + specialResult + ); + + // double.IsZero(x) ? double.NegativeInfinity : x + Vector256 zeroMask = Vector256.Equals(xBits << 1, Vector256.Zero).AsDouble(); + + specialResult = Vector256.ConditionalSelect( + zeroMask, + Vector256.Create(double.NegativeInfinity), + specialResult + ); + + // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) + Vector256 temp = zeroMask + | lessThanZeroMask + | Vector256.GreaterThanOrEqual(xBits, Vector256.Create(double.PositiveInfinity).AsInt64()).AsDouble(); + + // subnormal + Vector256 subnormalMask = Vector256.AndNot(specialMask.AsDouble(), temp); + + // multiply by 2^52, then normalize + x = Vector256.ConditionalSelect( + subnormalMask, + ((x * 4503599627370496.0).AsUInt64() - Vector256.Create(52ul << 52)).AsDouble(), + x + ); + + specialMask = temp.AsUInt64(); + } + + // Reduce the mantissa to [+2/3, +4/3] + Vector256 vx = x.AsUInt64() - Vector256.Create(V_OFF); + Vector256 n = Vector256.ConvertToDouble(vx.AsInt64() >> 52); + vx = (vx & Vector256.Create(V_MSK)) + Vector256.Create(V_OFF); + + // Adjust the mantissa to [-1/3, +1/3] + Vector256 r = vx.AsDouble() - Vector256.One; + + Vector256 r02 = r * r; + Vector256 r04 = r02 * r02; + Vector256 r08 = r04 * r04; + Vector256 r16 = r08 * r08; + + // Compute log(x + 1) using polynomial approximation + // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) + + Vector256 poly = (((r04 * C20) + + ((((r * C19) + Vector256.Create(C18)) * r02) + + ((r * C17) + Vector256.Create(C16)))) * r16) + + (((((((r * C15) + Vector256.Create(C14)) * r02) + + ((r * C13) + Vector256.Create(C12))) * r04) + + ((((r * C11) + Vector256.Create(C10)) * r02) + + ((r * C09) + Vector256.Create(C08)))) * r08) + + (((((r * C07) + Vector256.Create(C06)) * r02) + + ((r * C05) + Vector256.Create(C04))) * r04) + + ((((r * C03) + Vector256.Create(C02)) * r02) + r); + + return Vector256.ConditionalSelect( + specialMask.AsDouble(), + specialResult, + (poly * LN2_HEAD) + ((poly * LN2_TAIL) + n) + ); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 specialResult = x; + + // x is zero, subnormal, infinity, or NaN + Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt64() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); + + if (specialMask != Vector512.Zero) + { + Vector512 xBits = x.AsInt64(); + + // (x < 0) ? float.NaN : x + Vector512 lessThanZeroMask = Vector512.LessThan(xBits, Vector512.Zero).AsDouble(); + + specialResult = Vector512.ConditionalSelect( + lessThanZeroMask, + Vector512.Create(double.NaN), + specialResult + ); + + // double.IsZero(x) ? double.NegativeInfinity : x + Vector512 zeroMask = Vector512.Equals(xBits << 1, Vector512.Zero).AsDouble(); + + specialResult = Vector512.ConditionalSelect( + zeroMask, + Vector512.Create(double.NegativeInfinity), + specialResult + ); + + // double.IsZero(x) | (x < 0) | double.IsNaN(x) | double.IsPositiveInfinity(x) + Vector512 temp = zeroMask + | lessThanZeroMask + | Vector512.GreaterThanOrEqual(xBits, Vector512.Create(double.PositiveInfinity).AsInt64()).AsDouble(); + + // subnormal + Vector512 subnormalMask = Vector512.AndNot(specialMask.AsDouble(), temp); + + // multiply by 2^52, then normalize + x = Vector512.ConditionalSelect( + subnormalMask, + ((x * 4503599627370496.0).AsUInt64() - Vector512.Create(52ul << 52)).AsDouble(), + x + ); + + specialMask = temp.AsUInt64(); + } + + // Reduce the mantissa to [+2/3, +4/3] + Vector512 vx = x.AsUInt64() - Vector512.Create(V_OFF); + Vector512 n = Vector512.ConvertToDouble(vx.AsInt64() >> 52); + vx = (vx & Vector512.Create(V_MSK)) + Vector512.Create(V_OFF); + + // Adjust the mantissa to [-1/3, +1/3] + Vector512 r = vx.AsDouble() - Vector512.One; + + Vector512 r02 = r * r; + Vector512 r04 = r02 * r02; + Vector512 r08 = r04 * r04; + Vector512 r16 = r08 * r08; + + // Compute log(x + 1) using polynomial approximation + // C0 + (r * C1) + (r^2 * C2) + ... + (r^20 * C20) + + Vector512 poly = (((r04 * C20) + + ((((r * C19) + Vector512.Create(C18)) * r02) + + ((r * C17) + Vector512.Create(C16)))) * r16) + + (((((((r * C15) + Vector512.Create(C14)) * r02) + + ((r * C13) + Vector512.Create(C12))) * r04) + + ((((r * C11) + Vector512.Create(C10)) * r02) + + ((r * C09) + Vector512.Create(C08)))) * r08) + + (((((r * C07) + Vector512.Create(C06)) * r02) + + ((r * C05) + Vector512.Create(C04))) * r04) + + ((((r * C03) + Vector512.Create(C02)) * r02) + r); + + return Vector512.ConditionalSelect( + specialMask.AsDouble(), + specialResult, + (poly * LN2_HEAD) + ((poly * LN2_TAIL) + n) + ); + } + } + + /// float.Log2(x) + internal readonly struct Log2OperatorSingle : IUnaryOperator + { + // This code is based on `vrs4_log2f` from amd/aocl-libm-ose + // Copyright (C) 2021-2022 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + // Spec: + // log2f(x) + // = log2f(x) if x ∈ F and x > 0 + // = x if x = qNaN + // = 0 if x = 1 + // = -inf if x = (-0, 0} + // = NaN otherwise + // + // Assumptions/Expectations + // - Maximum ULP is observed to be at 4 + // - Some FPU Exceptions may not be available + // - Performance is at least 3x + // + // Implementation Notes: + // 1. Range Reduction: + // x = 2^n*(1+f) .... (1) + // where n is exponent and is an integer + // (1+f) is mantissa ∈ [1,2). i.e., 1 ≤ 1+f < 2 .... (2) + // + // From (1), taking log on both sides + // log2(x) = log2(2^n * (1+f)) + // = n + log2(1+f) .... (3) + // + // let z = 1 + f + // log2(z) = log2(k) + log2(z) - log2(k) + // log2(z) = log2(kz) - log2(k) + // + // From (2), range of z is [1, 2) + // by simply dividing range by 'k', z is in [1/k, 2/k) .... (4) + // Best choice of k is the one which gives equal and opposite values + // at extrema +- -+ + // 1 | 2 | + // --- - 1 = - |--- - 1 | + // k | k | .... (5) + // +- -+ + // + // Solving for k, k = 3/2, + // From (4), using 'k' value, range is therefore [-0.3333, 0.3333] + // + // 2. Polynomial Approximation: + // More information refer to tools/sollya/vrs4_logf.sollya + // + // 7th Deg - Error abs: 0x1.04c4ac98p-22 rel: 0x1.2216e6f8p-19 + + private const uint V_MIN = 0x00800000; + private const uint V_MAX = 0x7F800000; + private const uint V_MASK = 0x007FFFFF; + private const uint V_OFF = 0x3F2AAAAB; + + private const float C0 = 0.0f; + private const float C1 = 1.4426951f; + private const float C2 = -0.72134554f; + private const float C3 = 0.48089063f; + private const float C4 = -0.36084408f; + private const float C5 = 0.2888971f; + private const float C6 = -0.23594281f; + private const float C7 = 0.19948183f; + private const float C8 = -0.22616665f; + private const float C9 = 0.21228963f; + + public static bool Vectorizable => true; + + public static float Invoke(float x) => float.Log2(x); + + public static Vector128 Invoke(Vector128 x) + { + Vector128 specialResult = x; + + // x is subnormal or infinity or NaN + Vector128 specialMask = Vector128.GreaterThanOrEqual(x.AsUInt32() - Vector128.Create(V_MIN), Vector128.Create(V_MAX - V_MIN)); + + if (specialMask != Vector128.Zero) + { + // float.IsZero(x) ? float.NegativeInfinity : x + Vector128 zeroMask = Vector128.Equals(x, Vector128.Zero); + + specialResult = Vector128.ConditionalSelect( + zeroMask, + Vector128.Create(float.NegativeInfinity), + specialResult + ); + + // (x < 0) ? float.NaN : x + Vector128 lessThanZeroMask = Vector128.LessThan(x, Vector128.Zero); + + specialResult = Vector128.ConditionalSelect( + lessThanZeroMask, + Vector128.Create(float.NaN), + specialResult + ); + + // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) + Vector128 temp = zeroMask + | lessThanZeroMask + | ~Vector128.Equals(x, x) + | Vector128.Equals(x, Vector128.Create(float.PositiveInfinity)); + + // subnormal + Vector128 subnormalMask = Vector128.AndNot(specialMask.AsSingle(), temp); + + x = Vector128.ConditionalSelect( + subnormalMask, + ((x * 8388608.0f).AsUInt32() - Vector128.Create(23u << 23)).AsSingle(), + x + ); + + specialMask = temp.AsUInt32(); + } + + Vector128 vx = x.AsUInt32() - Vector128.Create(V_OFF); + Vector128 n = Vector128.ConvertToSingle(Vector128.ShiftRightArithmetic(vx.AsInt32(), 23)); + + vx = (vx & Vector128.Create(V_MASK)) + Vector128.Create(V_OFF); + + Vector128 r = vx.AsSingle() - Vector128.One; + + Vector128 r2 = r * r; + Vector128 r4 = r2 * r2; + Vector128 r8 = r4 * r4; + + Vector128 poly = (Vector128.Create(C9) * r + Vector128.Create(C8)) * r8 + + (((Vector128.Create(C7) * r + Vector128.Create(C6)) * r2 + + (Vector128.Create(C5) * r + Vector128.Create(C4))) * r4 + + ((Vector128.Create(C3) * r + Vector128.Create(C2)) * r2 + + (Vector128.Create(C1) * r + Vector128.Create(C0)))); + + return Vector128.ConditionalSelect( + specialMask.AsSingle(), + specialResult, + n + poly + ); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 specialResult = x; + + // x is subnormal or infinity or NaN + Vector256 specialMask = Vector256.GreaterThanOrEqual(x.AsUInt32() - Vector256.Create(V_MIN), Vector256.Create(V_MAX - V_MIN)); + + if (specialMask != Vector256.Zero) + { + // float.IsZero(x) ? float.NegativeInfinity : x + Vector256 zeroMask = Vector256.Equals(x, Vector256.Zero); + + specialResult = Vector256.ConditionalSelect( + zeroMask, + Vector256.Create(float.NegativeInfinity), + specialResult + ); + + // (x < 0) ? float.NaN : x + Vector256 lessThanZeroMask = Vector256.LessThan(x, Vector256.Zero); + + specialResult = Vector256.ConditionalSelect( + lessThanZeroMask, + Vector256.Create(float.NaN), + specialResult + ); + + // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) + Vector256 temp = zeroMask + | lessThanZeroMask + | ~Vector256.Equals(x, x) + | Vector256.Equals(x, Vector256.Create(float.PositiveInfinity)); + + // subnormal + Vector256 subnormalMask = Vector256.AndNot(specialMask.AsSingle(), temp); + + x = Vector256.ConditionalSelect( + subnormalMask, + ((x * 8388608.0f).AsUInt32() - Vector256.Create(23u << 23)).AsSingle(), + x + ); + + specialMask = temp.AsUInt32(); + } + + Vector256 vx = x.AsUInt32() - Vector256.Create(V_OFF); + Vector256 n = Vector256.ConvertToSingle(Vector256.ShiftRightArithmetic(vx.AsInt32(), 23)); + + vx = (vx & Vector256.Create(V_MASK)) + Vector256.Create(V_OFF); + + Vector256 r = vx.AsSingle() - Vector256.One; + + Vector256 r2 = r * r; + Vector256 r4 = r2 * r2; + Vector256 r8 = r4 * r4; + + Vector256 poly = (Vector256.Create(C9) * r + Vector256.Create(C8)) * r8 + + (((Vector256.Create(C7) * r + Vector256.Create(C6)) * r2 + + (Vector256.Create(C5) * r + Vector256.Create(C4))) * r4 + + ((Vector256.Create(C3) * r + Vector256.Create(C2)) * r2 + + (Vector256.Create(C1) * r + Vector256.Create(C0)))); + + return Vector256.ConditionalSelect( + specialMask.AsSingle(), + specialResult, + n + poly + ); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 specialResult = x; + + // x is subnormal or infinity or NaN + Vector512 specialMask = Vector512.GreaterThanOrEqual(x.AsUInt32() - Vector512.Create(V_MIN), Vector512.Create(V_MAX - V_MIN)); + + if (specialMask != Vector512.Zero) + { + // float.IsZero(x) ? float.NegativeInfinity : x + Vector512 zeroMask = Vector512.Equals(x, Vector512.Zero); + + specialResult = Vector512.ConditionalSelect( + zeroMask, + Vector512.Create(float.NegativeInfinity), + specialResult + ); + + // (x < 0) ? float.NaN : x + Vector512 lessThanZeroMask = Vector512.LessThan(x, Vector512.Zero); + + specialResult = Vector512.ConditionalSelect( + lessThanZeroMask, + Vector512.Create(float.NaN), + specialResult + ); + + // float.IsZero(x) | (x < 0) | float.IsNaN(x) | float.IsPositiveInfinity(x) + Vector512 temp = zeroMask + | lessThanZeroMask + | ~Vector512.Equals(x, x) + | Vector512.Equals(x, Vector512.Create(float.PositiveInfinity)); + + // subnormal + Vector512 subnormalMask = Vector512.AndNot(specialMask.AsSingle(), temp); + + x = Vector512.ConditionalSelect( + subnormalMask, + ((x * 8388608.0f).AsUInt32() - Vector512.Create(23u << 23)).AsSingle(), + x + ); + + specialMask = temp.AsUInt32(); + } + + Vector512 vx = x.AsUInt32() - Vector512.Create(V_OFF); + Vector512 n = Vector512.ConvertToSingle(Vector512.ShiftRightArithmetic(vx.AsInt32(), 23)); + + vx = (vx & Vector512.Create(V_MASK)) + Vector512.Create(V_OFF); + + Vector512 r = vx.AsSingle() - Vector512.One; + + Vector512 r2 = r * r; + Vector512 r4 = r2 * r2; + Vector512 r8 = r4 * r4; + + Vector512 poly = (Vector512.Create(C9) * r + Vector512.Create(C8)) * r8 + + (((Vector512.Create(C7) * r + Vector512.Create(C6)) * r2 + + (Vector512.Create(C5) * r + Vector512.Create(C4))) * r4 + + ((Vector512.Create(C3) * r + Vector512.Create(C2)) * r2 + + (Vector512.Create(C1) * r + Vector512.Create(C0)))); + + return Vector512.ConditionalSelect( + specialMask.AsSingle(), + specialResult, + n + poly + ); + } + } +#endif + + /// T.Log10(x) + internal readonly struct Log10Operator : IUnaryOperator + where T : ILogarithmicFunctions + { + private const double NaturalLog10 = 2.302585092994046; + public static bool Vectorizable => LogOperator.Vectorizable; + public static T Invoke(T x) => T.Log10(x); + public static Vector128 Invoke(Vector128 x) => LogOperator.Invoke(x) / Vector128.Create(T.CreateTruncating(NaturalLog10)); + public static Vector256 Invoke(Vector256 x) => LogOperator.Invoke(x) / Vector256.Create(T.CreateTruncating(NaturalLog10)); + public static Vector512 Invoke(Vector512 x) => LogOperator.Invoke(x) / Vector512.Create(T.CreateTruncating(NaturalLog10)); + } + + /// T.LogP1(x) + internal readonly struct LogP1Operator : IUnaryOperator + where T : ILogarithmicFunctions + { + public static bool Vectorizable => LogOperator.Vectorizable; + public static T Invoke(T x) => T.LogP1(x); + public static Vector128 Invoke(Vector128 x) => LogOperator.Invoke(x + Vector128.One); + public static Vector256 Invoke(Vector256 x) => LogOperator.Invoke(x + Vector256.One); + public static Vector512 Invoke(Vector512 x) => LogOperator.Invoke(x + Vector512.One); + } + + /// T.Log2P1(x) + internal readonly struct Log2P1Operator : IUnaryOperator + where T : ILogarithmicFunctions + { + public static bool Vectorizable => Log2Operator.Vectorizable; + public static T Invoke(T x) => T.Log2P1(x); + public static Vector128 Invoke(Vector128 x) => Log2Operator.Invoke(x + Vector128.One); + public static Vector256 Invoke(Vector256 x) => Log2Operator.Invoke(x + Vector256.One); + public static Vector512 Invoke(Vector512 x) => Log2Operator.Invoke(x + Vector512.One); + } + + /// T.Log10P1(x) + internal readonly struct Log10P1Operator : IUnaryOperator + where T : ILogarithmicFunctions + { + public static bool Vectorizable => Log10Operator.Vectorizable; + public static T Invoke(T x) => T.Log10P1(x); + public static Vector128 Invoke(Vector128 x) => Log10Operator.Invoke(x + Vector128.One); + public static Vector256 Invoke(Vector256 x) => Log10Operator.Invoke(x + Vector256.One); + public static Vector512 Invoke(Vector512 x) => Log10Operator.Invoke(x + Vector512.One); + } + + /// T.Log(x, y) + internal readonly struct LogBaseOperator : IBinaryOperator + where T : ILogarithmicFunctions + { + public static bool Vectorizable => LogOperator.Vectorizable; + public static T Invoke(T x, T y) => T.Log(x, y); + public static Vector128 Invoke(Vector128 x, Vector128 y) => LogOperator.Invoke(x) / LogOperator.Invoke(y); + public static Vector256 Invoke(Vector256 x, Vector256 y) => LogOperator.Invoke(x) / LogOperator.Invoke(y); + public static Vector512 Invoke(Vector512 x, Vector512 y) => LogOperator.Invoke(x) / LogOperator.Invoke(y); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) + { + if (Sse41.IsSupported) + { + if (typeof(T) == typeof(float)) return Sse41.BlendVariable(left.AsSingle(), right.AsSingle(), (~mask).AsSingle()).As(); + if (typeof(T) == typeof(double)) return Sse41.BlendVariable(left.AsDouble(), right.AsDouble(), (~mask).AsDouble()).As(); + + if (sizeof(T) == 1) return Sse41.BlendVariable(left.AsByte(), right.AsByte(), (~mask).AsByte()).As(); + if (sizeof(T) == 2) return Sse41.BlendVariable(left.AsUInt16(), right.AsUInt16(), (~mask).AsUInt16()).As(); + if (sizeof(T) == 4) return Sse41.BlendVariable(left.AsUInt32(), right.AsUInt32(), (~mask).AsUInt32()).As(); + if (sizeof(T) == 8) return Sse41.BlendVariable(left.AsUInt64(), right.AsUInt64(), (~mask).AsUInt64()).As(); + } + + return Vector128.ConditionalSelect(mask, left, right); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) + { + if (Avx2.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx2.BlendVariable(left.AsSingle(), right.AsSingle(), (~mask).AsSingle()).As(); + if (typeof(T) == typeof(double)) return Avx2.BlendVariable(left.AsDouble(), right.AsDouble(), (~mask).AsDouble()).As(); + + if (sizeof(T) == 1) return Avx2.BlendVariable(left.AsByte(), right.AsByte(), (~mask).AsByte()).As(); + if (sizeof(T) == 2) return Avx2.BlendVariable(left.AsUInt16(), right.AsUInt16(), (~mask).AsUInt16()).As(); + if (sizeof(T) == 4) return Avx2.BlendVariable(left.AsUInt32(), right.AsUInt32(), (~mask).AsUInt32()).As(); + if (sizeof(T) == 8) return Avx2.BlendVariable(left.AsUInt64(), right.AsUInt64(), (~mask).AsUInt64()).As(); + } + + return Vector256.ConditionalSelect(mask, left, right); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported) + { + if (typeof(T) == typeof(float)) return Avx512F.BlendVariable(left.AsSingle(), right.AsSingle(), (~mask).AsSingle()).As(); + if (typeof(T) == typeof(double)) return Avx512F.BlendVariable(left.AsDouble(), right.AsDouble(), (~mask).AsDouble()).As(); + + if (sizeof(T) == 4) return Avx512F.BlendVariable(left.AsUInt32(), right.AsUInt32(), (~mask).AsUInt32()).As(); + if (sizeof(T) == 8) return Avx512F.BlendVariable(left.AsUInt64(), right.AsUInt64(), (~mask).AsUInt64()).As(); + } + + return Vector512.ConditionalSelect(mask, left, right); + } + + /// 1 / (1 + T.Exp(-x)) + internal readonly struct SigmoidOperator : IUnaryOperator where T : IExponentialFunctions + { + public static bool Vectorizable => ExpOperator.Vectorizable; + public static T Invoke(T x) => T.One / (T.One + T.Exp(-x)); + public static Vector128 Invoke(Vector128 x) => Vector128.Create(T.One) / (Vector128.Create(T.One) + ExpOperator.Invoke(-x)); + public static Vector256 Invoke(Vector256 x) => Vector256.Create(T.One) / (Vector256.Create(T.One) + ExpOperator.Invoke(-x)); + public static Vector512 Invoke(Vector512 x) => Vector512.Create(T.One) / (Vector512.Create(T.One) + ExpOperator.Invoke(-x)); + } + + internal readonly struct CeilingOperator : IUnaryOperator where T : IFloatingPoint + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Ceiling(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return Vector128.Ceiling(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return Vector128.Ceiling(x.AsDouble()).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return Vector256.Ceiling(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return Vector256.Ceiling(x.AsDouble()).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return Vector512.Ceiling(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return Vector512.Ceiling(x.AsDouble()).As(); + } + } + } + + internal readonly struct FloorOperator : IUnaryOperator where T : IFloatingPoint + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Floor(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return Vector128.Floor(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return Vector128.Floor(x.AsDouble()).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return Vector256.Floor(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return Vector256.Floor(x.AsDouble()).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return Vector512.Floor(x.AsSingle()).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return Vector512.Floor(x.AsDouble()).As(); + } + } + } + + private readonly struct TruncateOperator : IUnaryOperator where T : IFloatingPoint + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Truncate(x); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + if (Sse41.IsSupported) return Sse41.RoundToZero(x.AsSingle()).As(); + if (AdvSimd.IsSupported) return AdvSimd.RoundToZero(x.AsSingle()).As(); + + return Vector128.ConditionalSelect(Vector128.GreaterThanOrEqual(x, Vector128.Zero), + Vector128.Floor(x.AsSingle()).As(), + Vector128.Ceiling(x.AsSingle()).As()); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + + if (Sse41.IsSupported) return Sse41.RoundToZero(x.AsDouble()).As(); + if (AdvSimd.Arm64.IsSupported) return AdvSimd.Arm64.RoundToZero(x.AsDouble()).As(); + + return Vector128.ConditionalSelect(Vector128.GreaterThanOrEqual(x, Vector128.Zero), + Vector128.Floor(x.AsDouble()).As(), + Vector128.Ceiling(x.AsDouble()).As()); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + if (Avx.IsSupported) return Avx.RoundToZero(x.AsSingle()).As(); + + return Vector256.ConditionalSelect(Vector256.GreaterThanOrEqual(x, Vector256.Zero), + Vector256.Floor(x.AsSingle()).As(), + Vector256.Ceiling(x.AsSingle()).As()); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + + if (Avx.IsSupported) return Avx.RoundToZero(x.AsDouble()).As(); + + return Vector256.ConditionalSelect(Vector256.GreaterThanOrEqual(x, Vector256.Zero), + Vector256.Floor(x.AsDouble()).As(), + Vector256.Ceiling(x.AsDouble()).As()); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + if (Avx512F.IsSupported) return Avx512F.RoundScale(x.AsSingle(), 0b11).As(); + + return Vector512.ConditionalSelect(Vector512.GreaterThanOrEqual(x, Vector512.Zero), + Vector512.Floor(x.AsSingle()).As(), + Vector512.Ceiling(x.AsSingle()).As()); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + + if (Avx512F.IsSupported) return Avx512F.RoundScale(x.AsDouble(), 0b11).As(); + + return Vector512.ConditionalSelect(Vector512.GreaterThanOrEqual(x, Vector512.Zero), + Vector512.Floor(x.AsDouble()).As(), + Vector512.Ceiling(x.AsDouble()).As()); + } + } + } + + /// T.PopCount(x) + internal readonly struct PopCountOperator : IUnaryOperator where T : IBinaryInteger + { + // TODO https://github.com/dotnet/runtime/issues/96162: Use AVX512 popcount operations when available + + public static bool Vectorizable => + // The implementation uses a vectorized version of the BitOperations.PopCount software fallback: + // https://github.com/dotnet/runtime/blob/aff061bab1b6d9ccd5731bd16fa8e89ad82ab75a/src/libraries/System.Private.CoreLib/src/System/Numerics/BitOperations.cs#L496-L508 + // This relies on 64-bit shifts for sizeof(T) == 8, and such shifts aren't accelerated on today's hardware. + // Alternative approaches, such as doing two 32-bit operations and combining them were observed to not + // provide any meaningfuls speedup over scalar. So for now, we don't vectorize when sizeof(T) == 8. + sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4; + + public static T Invoke(T x) => T.PopCount(x); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector128 Invoke(Vector128 x) + { + if (sizeof(T) == 1) + { + if (AdvSimd.IsSupported) + { + return AdvSimd.PopCount(x.AsByte()).As(); + } + + if (PackedSimd.IsSupported) + { + return PackedSimd.PopCount(x.AsByte()).As(); + } + + Vector128 c1 = Vector128.Create((byte)0x55); + Vector128 c2 = Vector128.Create((byte)0x33); + Vector128 c3 = Vector128.Create((byte)0x0F); + + // We don't have a per element shuffle for byte on some platforms. + // However, we do currently always have a 16-bit shift available and + // due to how the algorithm works, we don't need to worry about + // any bits that shift into the lower 8-bits from the upper 8-bits. + Vector128 tmp = x.AsByte(); + tmp -= (x.AsUInt16() >> 1).AsByte() & c1; + tmp = (tmp & c2) + ((tmp.AsUInt16() >> 2).AsByte() & c2); + return ((tmp + (tmp.AsUInt16() >> 4).AsByte()) & c3).As(); + } + + if (sizeof(T) == 2) + { + Vector128 c1 = Vector128.Create((ushort)0x5555); + Vector128 c2 = Vector128.Create((ushort)0x3333); + Vector128 c3 = Vector128.Create((ushort)0x0F0F); + Vector128 c4 = Vector128.Create((ushort)0x0101); + + Vector128 tmp = x.AsUInt16(); + tmp -= (tmp >> 1) & c1; + tmp = (tmp & c2) + ((tmp >> 2) & c2); + tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 8; + return tmp.As(); + } + + Debug.Assert(sizeof(T) == 4); + { + Vector128 c1 = Vector128.Create(0x55555555u); + Vector128 c2 = Vector128.Create(0x33333333u); + Vector128 c3 = Vector128.Create(0x0F0F0F0Fu); + Vector128 c4 = Vector128.Create(0x01010101u); + + Vector128 tmp = x.AsUInt32(); + tmp -= (tmp >> 1) & c1; + tmp = (tmp & c2) + ((tmp >> 2) & c2); + tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 24; + return tmp.As(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector256 Invoke(Vector256 x) + { + if (sizeof(T) == 1) + { + Vector256 c1 = Vector256.Create((byte)0x55); + Vector256 c2 = Vector256.Create((byte)0x33); + Vector256 c3 = Vector256.Create((byte)0x0F); + + // We don't have a per element shuffle for byte on some platforms. + // However, we do currently always have a 16-bit shift available and + // due to how the algorithm works, we don't need to worry about + // any bits that shift into the lower 8-bits from the upper 8-bits. + Vector256 tmp = x.AsByte(); + tmp -= (x.AsUInt16() >> 1).AsByte() & c1; + tmp = (tmp & c2) + ((tmp.AsUInt16() >> 2).AsByte() & c2); + return ((tmp + (tmp.AsUInt16() >> 4).AsByte()) & c3).As(); + } + + if (sizeof(T) == 2) + { + Vector256 c1 = Vector256.Create((ushort)0x5555); + Vector256 c2 = Vector256.Create((ushort)0x3333); + Vector256 c3 = Vector256.Create((ushort)0x0F0F); + Vector256 c4 = Vector256.Create((ushort)0x0101); + + Vector256 tmp = x.AsUInt16(); + tmp -= (tmp >> 1) & c1; + tmp = (tmp & c2) + ((tmp >> 2) & c2); + tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 8; + return tmp.As(); + } + + Debug.Assert(sizeof(T) == 4); + { + Vector256 c1 = Vector256.Create(0x55555555u); + Vector256 c2 = Vector256.Create(0x33333333u); + Vector256 c3 = Vector256.Create(0x0F0F0F0Fu); + Vector256 c4 = Vector256.Create(0x01010101u); + + Vector256 tmp = x.AsUInt32(); + tmp -= (tmp >> 1) & c1; + tmp = (tmp & c2) + ((tmp >> 2) & c2); + tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 24; + return tmp.As(); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Vector512 Invoke(Vector512 x) + { + if (sizeof(T) == 1) + { + Vector512 c1 = Vector512.Create((byte)0x55); + Vector512 c2 = Vector512.Create((byte)0x33); + Vector512 c3 = Vector512.Create((byte)0x0F); + + // We don't have a per element shuffle for byte on some platforms. + // However, we do currently always have a 16-bit shift available and + // due to how the algorithm works, we don't need to worry about + // any bits that shift into the lower 8-bits from the upper 8-bits. + Vector512 tmp = x.AsByte(); + tmp -= (x.AsUInt16() >> 1).AsByte() & c1; + tmp = (tmp & c2) + ((tmp.AsUInt16() >> 2).AsByte() & c2); + return ((tmp + (tmp.AsUInt16() >> 4).AsByte()) & c3).As(); + } + + if (sizeof(T) == 2) + { + Vector512 c1 = Vector512.Create((ushort)0x5555); + Vector512 c2 = Vector512.Create((ushort)0x3333); + Vector512 c3 = Vector512.Create((ushort)0x0F0F); + Vector512 c4 = Vector512.Create((ushort)0x0101); + + Vector512 tmp = x.AsUInt16(); + tmp -= (tmp >> 1) & c1; + tmp = (tmp & c2) + ((tmp >> 2) & c2); + tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 8; + return tmp.As(); + } + + Debug.Assert(sizeof(T) == 4); + { + Vector512 c1 = Vector512.Create(0x55555555u); + Vector512 c2 = Vector512.Create(0x33333333u); + Vector512 c3 = Vector512.Create(0x0F0F0F0Fu); + Vector512 c4 = Vector512.Create(0x01010101u); + + Vector512 tmp = x.AsUInt32(); + tmp -= (tmp >> 1) & c1; + tmp = (tmp & c2) + ((tmp >> 2) & c2); + tmp = (((tmp + (tmp >> 4)) & c3) * c4) >> 24; + return tmp.As(); + } + } + } + + /// T.LeadingZeroCount(x) + internal readonly struct LeadingZeroCountOperator : IUnaryOperator where T : IBinaryInteger + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.LeadingZeroCount(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.TrailingZeroCount(x) + internal readonly struct TrailingZeroCountOperator : IUnaryOperator where T : IBinaryInteger + { + public static bool Vectorizable => false; // TODO: Vectorize + public static T Invoke(T x) => T.TrailingZeroCount(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + private readonly struct CopySignOperator : IBinaryOperator where T : INumber + { + public static bool Vectorizable => true; + + public static T Invoke(T x, T y) => T.CopySign(x, y); + + public static Vector128 Invoke(Vector128 x, Vector128 y) + { + if (typeof(T) == typeof(float)) + { + return Vector128.ConditionalSelect(Vector128.Create(-0.0f).As(), y, x); + } + + if (typeof(T) == typeof(double)) + { + return Vector128.ConditionalSelect(Vector128.Create(-0.0d).As(), y, x); + } + + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector128 absValue = Vector128.Abs(x); + Vector128 sign = Vector128.GreaterThanOrEqual(y, Vector128.Zero); + Vector128 error = sign & Vector128.LessThan(absValue, Vector128.Zero); + if (error != Vector128.Zero) + { + Math.Abs(int.MinValue); // throw OverflowException + } + + return Vector128.ConditionalSelect(sign, absValue, -absValue); + } + + return x; + } + + public static Vector256 Invoke(Vector256 x, Vector256 y) + { + if (typeof(T) == typeof(float)) + { + return Vector256.ConditionalSelect(Vector256.Create(-0.0f).As(), y, x); + } + + if (typeof(T) == typeof(double)) + { + return Vector256.ConditionalSelect(Vector256.Create(-0.0d).As(), y, x); + } + + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector256 absValue = Vector256.Abs(x); + Vector256 sign = Vector256.GreaterThanOrEqual(y, Vector256.Zero); + Vector256 error = sign & Vector256.LessThan(absValue, Vector256.Zero); + if (error != Vector256.Zero) + { + Math.Abs(int.MinValue); // throw OverflowException + } + + return Vector256.ConditionalSelect(sign, absValue, -absValue); + } + + return x; + } + + public static Vector512 Invoke(Vector512 x, Vector512 y) + { + if (typeof(T) == typeof(float)) + { + return Vector512.ConditionalSelect(Vector512.Create(-0.0f).As(), y, x); + } + + if (typeof(T) == typeof(double)) + { + return Vector512.ConditionalSelect(Vector512.Create(-0.0d).As(), y, x); + } + + if (typeof(T) == typeof(sbyte) || typeof(T) == typeof(short) || typeof(T) == typeof(int) || typeof(T) == typeof(long) || typeof(T) == typeof(nint)) + { + Vector512 absValue = Vector512.Abs(x); + Vector512 sign = Vector512.GreaterThanOrEqual(y, Vector512.Zero); + Vector512 error = sign & Vector512.LessThan(absValue, Vector512.Zero); + if (error != Vector512.Zero) + { + Math.Abs(int.MinValue); // throw OverflowException + } + + return Vector512.ConditionalSelect(sign, absValue, -absValue); + } + + return x; + } + } + + /// T.DegreesToRadians(x) + internal readonly struct DegreesToRadiansOperator : IUnaryOperator where T : ITrigonometricFunctions + { + public static bool Vectorizable => true; + public static T Invoke(T x) => T.DegreesToRadians(x); + public static Vector128 Invoke(Vector128 x) => (x * T.Pi) / T.CreateChecked(180); + public static Vector256 Invoke(Vector256 x) => (x * T.Pi) / T.CreateChecked(180); + public static Vector512 Invoke(Vector512 x) => (x * T.Pi) / T.CreateChecked(180); + } + + /// T.RadiansToDegrees(x) + internal readonly struct RadiansToDegreesOperator : IUnaryOperator where T : ITrigonometricFunctions + { + public static bool Vectorizable => true; + public static T Invoke(T x) => T.RadiansToDegrees(x); + public static Vector128 Invoke(Vector128 x) => (x * T.CreateChecked(180)) / T.Pi; + public static Vector256 Invoke(Vector256 x) => (x * T.CreateChecked(180)) / T.Pi; + public static Vector512 Invoke(Vector512 x) => (x * T.CreateChecked(180)) / T.Pi; + } + + /// T << amount + internal readonly struct ShiftLeftOperator(int amount) : IStatefulUnaryOperator where T : IShiftOperators + { + private readonly int _amount = amount; + + public static bool Vectorizable => true; + + public T Invoke(T x) => x << _amount; + public Vector128 Invoke(Vector128 x) => x << _amount; + public Vector256 Invoke(Vector256 x) => x << _amount; + public Vector512 Invoke(Vector512 x) => x << _amount; + } + + /// T >> amount + internal readonly struct ShiftRightArithmeticOperator(int amount) : IStatefulUnaryOperator where T : IShiftOperators + { + private readonly int _amount = amount; + + public static bool Vectorizable => true; + + public T Invoke(T x) => x >> _amount; + public Vector128 Invoke(Vector128 x) => x >> _amount; + public Vector256 Invoke(Vector256 x) => x >> _amount; + public Vector512 Invoke(Vector512 x) => x >> _amount; + } + + /// T >>> amount + internal readonly struct ShiftRightLogicalOperator(int amount) : IStatefulUnaryOperator where T : IShiftOperators + { + private readonly int _amount = amount; + + public static bool Vectorizable => true; + + public T Invoke(T x) => x >>> _amount; + public Vector128 Invoke(Vector128 x) => x >>> _amount; + public Vector256 Invoke(Vector256 x) => x >>> _amount; + public Vector512 Invoke(Vector512 x) => x >>> _amount; + } + + /// T.RotateLeft(amount) + internal readonly struct RotateLeftOperator(int amount) : IStatefulUnaryOperator where T : IBinaryInteger + { + private readonly int _amount = amount; + + public static bool Vectorizable => true; + + public T Invoke(T x) => T.RotateLeft(x, _amount); + public Vector128 Invoke(Vector128 x) => (x << _amount) | (x >>> ((sizeof(T) * 8) - _amount)); + public Vector256 Invoke(Vector256 x) => (x << _amount) | (x >>> ((sizeof(T) * 8) - _amount)); + public Vector512 Invoke(Vector512 x) => (x << _amount) | (x >>> ((sizeof(T) * 8) - _amount)); + } + + /// T.RotateRight(amount) + internal readonly struct RotateRightOperator(int amount) : IStatefulUnaryOperator where T : IBinaryInteger + { + private readonly int _amount = amount; + + public static bool Vectorizable => true; + + public T Invoke(T x) => T.RotateRight(x, _amount); + public Vector128 Invoke(Vector128 x) => (x >>> _amount) | (x << ((sizeof(T) * 8) - _amount)); + public Vector256 Invoke(Vector256 x) => (x >>> _amount) | (x << ((sizeof(T) * 8) - _amount)); + public Vector512 Invoke(Vector512 x) => (x >>> _amount) | (x << ((sizeof(T) * 8) - _amount)); + } + + /// T.ScaleB(x, n) + internal readonly struct ScaleBOperator(int n) : IStatefulUnaryOperator where T : IFloatingPointIeee754 + { + private readonly int _n = n; + private readonly T _pow2n = typeof(T) == typeof(float) || typeof(T) == typeof(double) ? T.Pow(T.CreateTruncating(2), T.CreateTruncating(n)) : default!; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public T Invoke(T x) => T.ScaleB(x, _n); + public Vector128 Invoke(Vector128 x) => x * Vector128.Create(_pow2n); + public Vector256 Invoke(Vector256 x) => x * Vector256.Create(_pow2n); + public Vector512 Invoke(Vector512 x) => x * Vector512.Create(_pow2n); + } + + /// T.RootN(x, n) + internal readonly struct RootNOperator(int n) : IStatefulUnaryOperator where T : IRootFunctions + { + private readonly int _n = n; + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public T Invoke(T x) => T.RootN(x, _n); + + public Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector128.Create((float)_n)).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector128.Create((double)_n)).As(); + } + } + + public Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector256.Create((float)_n)).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector256.Create((double)_n)).As(); + } + } + + public Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return ExpOperator.Invoke(LogOperator.Invoke(x.AsSingle()) / Vector512.Create((float)_n)).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return ExpOperator.Invoke(LogOperator.Invoke(x.AsDouble()) / Vector512.Create((double)_n)).As(); + } + } + } + + /// T.Round(x) + internal readonly struct RoundToEvenOperator : IUnaryOperator where T : IFloatingPoint + { + // This code is based on `nearbyint` from amd/aocl-libm-ose + // Copyright (C) 2008-2022 Advanced Micro Devices, Inc. All rights reserved. + // + // Licensed under the BSD 3-Clause "New" or "Revised" License + // See THIRD-PARTY-NOTICES.TXT for the full license text + + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Round(x); + + private const float SingleBoundary = 8388608.0f; // 2^23 + private const double DoubleBoundary = 4503599627370496.0; // 2^52 + + public static Vector128 Invoke(Vector128 x) + { + Vector128 boundary = Vector128.Create(typeof(T) == typeof(float) ? T.CreateTruncating(SingleBoundary) : T.CreateTruncating(DoubleBoundary)); + Vector128 temp = CopySignOperator.Invoke(boundary, x); + return Vector128.ConditionalSelect(Vector128.GreaterThan(Vector128.Abs(x), boundary), x, CopySignOperator.Invoke((x + temp) - temp, x)); + } + + public static Vector256 Invoke(Vector256 x) + { + Vector256 boundary = Vector256.Create(typeof(T) == typeof(float) ? T.CreateTruncating(SingleBoundary) : T.CreateTruncating(DoubleBoundary)); + Vector256 temp = CopySignOperator.Invoke(boundary, x); + return Vector256.ConditionalSelect(Vector256.GreaterThan(Vector256.Abs(x), boundary), x, CopySignOperator.Invoke((x + temp) - temp, x)); + } + + public static Vector512 Invoke(Vector512 x) + { + Vector512 boundary = Vector512.Create(typeof(T) == typeof(float) ? T.CreateTruncating(SingleBoundary) : T.CreateTruncating(DoubleBoundary)); + Vector512 temp = CopySignOperator.Invoke(boundary, x); + return Vector512.ConditionalSelect(Vector512.GreaterThan(Vector512.Abs(x), boundary), x, CopySignOperator.Invoke((x + temp) - temp, x)); + } + } + + /// T.Round(x, MidpointRounding.AwayFromZero) + internal readonly struct RoundAwayFromZeroOperator : IUnaryOperator where T : IFloatingPoint + { + public static bool Vectorizable => typeof(T) == typeof(float) || typeof(T) == typeof(double); + + public static T Invoke(T x) => T.Round(x, MidpointRounding.AwayFromZero); + + public static Vector128 Invoke(Vector128 x) + { + if (typeof(T) == typeof(float)) + { + if (AdvSimd.IsSupported) + { + return AdvSimd.RoundAwayFromZero(x.AsSingle()).As(); + } + + return TruncateOperator.Invoke(x.AsSingle() + CopySignOperator.Invoke(Vector128.Create(0.49999997f), x.AsSingle())).As(); + } + else + { + if (AdvSimd.Arm64.IsSupported) + { + return AdvSimd.Arm64.RoundAwayFromZero(x.AsDouble()).As(); + } + + Debug.Assert(typeof(T) == typeof(double)); + return TruncateOperator.Invoke(x.AsDouble() + CopySignOperator.Invoke(Vector128.Create(0.49999999999999994), x.AsDouble())).As(); + } + } + + public static Vector256 Invoke(Vector256 x) + { + if (typeof(T) == typeof(float)) + { + return TruncateOperator.Invoke(x.AsSingle() + CopySignOperator.Invoke(Vector256.Create(0.49999997f), x.AsSingle())).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return TruncateOperator.Invoke(x.AsDouble() + CopySignOperator.Invoke(Vector256.Create(0.49999999999999994), x.AsDouble())).As(); + } + } + + public static Vector512 Invoke(Vector512 x) + { + if (typeof(T) == typeof(float)) + { + return TruncateOperator.Invoke(x.AsSingle() + CopySignOperator.Invoke(Vector512.Create(0.49999997f), x.AsSingle())).As(); + } + else + { + Debug.Assert(typeof(T) == typeof(double)); + return TruncateOperator.Invoke(x.AsDouble() + CopySignOperator.Invoke(Vector512.Create(0.49999999999999994), x.AsDouble())).As(); + } + } + } + + /// (T.Round(x * power10, digits, mode)) / power10 + internal readonly struct MultiplyRoundDivideOperator : IStatefulUnaryOperator + where T : IFloatingPoint + where TDelegatedRound : IUnaryOperator + { + private readonly T _factor; + + public MultiplyRoundDivideOperator(T factor) + { + Debug.Assert(typeof(T) == typeof(float) || typeof(T) == typeof(double)); + _factor = factor; + } + + public static bool Vectorizable => true; + + private const float Single_RoundLimit = 1e8f; + private const double Double_RoundLimit = 1e16d; + + public T Invoke(T x) + { + T limit = typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit); + return T.Abs(x) < limit ? + TDelegatedRound.Invoke(x * _factor) / _factor : + x; + } + + public Vector128 Invoke(Vector128 x) + { + Vector128 limit = Vector128.Create(typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit)); + return Vector128.ConditionalSelect(Vector128.LessThan(Vector128.Abs(x), limit), + TDelegatedRound.Invoke(x * _factor) / _factor, + x); + } + + public Vector256 Invoke(Vector256 x) + { + Vector256 limit = Vector256.Create(typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit)); + return Vector256.ConditionalSelect(Vector256.LessThan(Vector256.Abs(x), limit), + TDelegatedRound.Invoke(x * _factor) / _factor, + x); + } + + public Vector512 Invoke(Vector512 x) + { + Vector512 limit = Vector512.Create(typeof(T) == typeof(float) ? T.CreateTruncating(Single_RoundLimit) : T.CreateTruncating(Double_RoundLimit)); + return Vector512.ConditionalSelect(Vector512.LessThan(Vector512.Abs(x), limit), + TDelegatedRound.Invoke(x * _factor) / _factor, + x); + } + } + + /// T.Round(x, digits, mode) + internal readonly struct RoundFallbackOperator(int digits, MidpointRounding mode) : IStatefulUnaryOperator + where T : IFloatingPoint + { + private readonly int _digits = digits; + private readonly MidpointRounding _mode = mode; + + public static bool Vectorizable => false; + + public T Invoke(T x) => T.Round(x, _digits, _mode); + + public Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.ILogB(x) + internal readonly struct ILogBOperator : IUnaryOperator where T : IFloatingPointIeee754 + { + public static bool Vectorizable => false; // TODO: vectorize for float + + public static int Invoke(T x) => T.ILogB(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// double.ILogB(x) + internal readonly struct ILogBDoubleOperator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => false; // TODO: vectorize + + public static int Invoke(double x) => double.ILogB(x); + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => throw new NotSupportedException(); + } + + /// T.CreateChecked(x) + internal readonly struct ConvertCheckedFallbackOperator : IUnaryOperator where TFrom : INumberBase where TTo : INumberBase + { + public static bool Vectorizable => false; + + public static TTo Invoke(TFrom x) => TTo.CreateChecked(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.CreateSaturating(x) + internal readonly struct ConvertSaturatingFallbackOperator : IUnaryOperator where TFrom : INumberBase where TTo : INumberBase + { + public static bool Vectorizable => false; + + public static TTo Invoke(TFrom x) => TTo.CreateSaturating(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.CreateTruncating(x) + internal readonly struct ConvertTruncatingFallbackOperator : IUnaryOperator where TFrom : INumberBase where TTo : INumberBase + { + public static bool Vectorizable => false; + + public static TTo Invoke(TFrom x) => TTo.CreateTruncating(x); + public static Vector128 Invoke(Vector128 x) => throw new NotSupportedException(); + public static Vector256 Invoke(Vector256 x) => throw new NotSupportedException(); + public static Vector512 Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// (uint)float + internal readonly struct ConvertUInt32ToSingle : IUnaryOperator + { + public static bool Vectorizable => true; + + public static float Invoke(uint x) => x; + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToSingle(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToSingle(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToSingle(x); + } + + /// (int)float + internal readonly struct ConvertInt32ToSingle : IUnaryOperator + { + public static bool Vectorizable => true; + + public static float Invoke(int x) => x; + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToSingle(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToSingle(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToSingle(x); + } + + /// (float)uint + internal readonly struct ConvertSingleToUInt32 : IUnaryOperator + { + public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar + + public static uint Invoke(float x) => uint.CreateTruncating(x); + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToUInt32(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToUInt32(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToUInt32(x); + } + + /// (float)int + internal readonly struct ConvertSingleToInt32 : IUnaryOperator + { + public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar + + public static int Invoke(float x) => int.CreateTruncating(x); + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToInt32(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToInt32(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToInt32(x); + } + + /// (double)ulong + internal readonly struct ConvertUInt64ToDouble : IUnaryOperator + { + public static bool Vectorizable => true; + + public static double Invoke(ulong x) => x; + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToDouble(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToDouble(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToDouble(x); + } + + /// (double)long + internal readonly struct ConvertInt64ToDouble : IUnaryOperator + { + public static bool Vectorizable => true; + + public static double Invoke(long x) => x; + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToDouble(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToDouble(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToDouble(x); + } + + /// (ulong)double + internal readonly struct ConvertDoubleToUInt64 : IUnaryOperator + { + public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar + + public static ulong Invoke(double x) => ulong.CreateTruncating(x); + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToUInt64(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToUInt64(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToUInt64(x); + } + + /// (long)double + internal readonly struct ConvertDoubleToInt64 : IUnaryOperator + { + public static bool Vectorizable => false; // TODO https://github.com/dotnet/runtime/pull/97529: make this true once vectorized behavior matches scalar + + public static long Invoke(double x) => long.CreateTruncating(x); + public static Vector128 Invoke(Vector128 x) => Vector128.ConvertToInt64(x); + public static Vector256 Invoke(Vector256 x) => Vector256.ConvertToInt64(x); + public static Vector512 Invoke(Vector512 x) => Vector512.ConvertToInt64(x); + } + + /// (double)float + internal readonly struct WidenSingleToDoubleOperator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static double Invoke(float x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (float)double + internal readonly struct NarrowDoubleToSingleOperator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static float Invoke(double x) => (float)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + /// (ushort)byte + internal readonly struct WidenByteToUInt16Operator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static ushort Invoke(byte x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (byte)ushort + internal readonly struct NarrowUInt16ToByteOperator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static byte Invoke(ushort x) => (byte)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + /// (short)sbyte + internal readonly struct WidenSByteToInt16Operator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static short Invoke(sbyte x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (sbyte)short + internal readonly struct NarrowInt16ToSByteOperator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static sbyte Invoke(short x) => (sbyte)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + /// (uint)ushort + internal readonly struct WidenUInt16ToUInt32Operator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static uint Invoke(ushort x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (ushort)uint + internal readonly struct NarrowUInt32ToUInt16Operator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static ushort Invoke(uint x) => (ushort)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + /// (int)short + internal readonly struct WidenInt16ToInt32Operator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static int Invoke(short x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (short)int + internal readonly struct NarrowInt32ToInt16Operator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static short Invoke(int x) => (short)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + /// (ulong)uint + internal readonly struct WidenUInt32ToUInt64Operator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static ulong Invoke(uint x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (uint)ulong + internal readonly struct NarrowUInt64ToUInt32Operator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static uint Invoke(ulong x) => (uint)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + /// (long)int + internal readonly struct WidenInt32ToInt64Operator : IUnaryOneToTwoOperator + { + public static bool Vectorizable => true; + + public static long Invoke(int x) => x; + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) => Vector128.Widen(x); + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) => Vector256.Widen(x); + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) => Vector512.Widen(x); + } + + /// (int)long + internal readonly struct NarrowInt64ToInt32Operator : IUnaryTwoToOneOperator + { + public static bool Vectorizable => true; + + public static int Invoke(long x) => (int)x; + public static Vector128 Invoke(Vector128 lower, Vector128 upper) => Vector128.Narrow(lower, upper); + public static Vector256 Invoke(Vector256 lower, Vector256 upper) => Vector256.Narrow(lower, upper); + public static Vector512 Invoke(Vector512 lower, Vector512 upper) => Vector512.Narrow(lower, upper); + } + + internal readonly struct WidenHalfAsInt16ToSingleOperator : IUnaryOneToTwoOperator + { + // This implements a vectorized version of the `explicit operator float(Half value) operator`. + // See detailed description of the algorithm used here: + // https://github.com/dotnet/runtime/blob/3bf40a378f00cb5bf18ff62796bc7097719b974c/src/libraries/System.Private.CoreLib/src/System/Half.cs#L1010-L1040 + // The cast operator converts a Half represented as uint to a float. This does the same, with an input VectorXx and an output VectorXx. + // The VectorXx is created by reading a vector of Halfs as a VectorXx then widened to two VectorXxs and cast to VectorXxs. + // We loop handling one input vector at a time, producing two output float vectors. + + private const uint ExponentLowerBound = 0x3880_0000u; // The smallest positive normal number in Half, converted to Single + private const uint ExponentOffset = 0x3800_0000u; // BitConverter.SingleToUInt32Bits(1.0f) - ((uint)BitConverter.HalfToUInt16Bits((Half)1.0f) << 13) + private const uint SingleSignMask = 0x8000_0000; // float.SignMask; // Mask for sign bit in Single + private const uint HalfExponentMask = 0x7C00; // Mask for exponent bits in Half + private const uint HalfToSingleBitsMask = 0x0FFF_E000; // Mask for bits in Single converted from Half + + public static bool Vectorizable => true; + + public static float Invoke(short x) => (float)Unsafe.BitCast(x); + + public static (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x) + { + (Vector128 lowerInt32, Vector128 upperInt32) = Vector128.Widen(x); + return + (HalfAsWidenedUInt32ToSingle(lowerInt32.AsUInt32()), + HalfAsWidenedUInt32ToSingle(upperInt32.AsUInt32())); + + static Vector128 HalfAsWidenedUInt32ToSingle(Vector128 value) + { + // Extract sign bit of value + Vector128 sign = value & Vector128.Create(SingleSignMask); + + // Copy sign bit to upper bits + Vector128 bitValueInProcess = value; + + // Extract exponent bits of value (BiasedExponent is not for here as it performs unnecessary shift) + Vector128 offsetExponent = bitValueInProcess & Vector128.Create(HalfExponentMask); + + // ~0u when value is subnormal, 0 otherwise + Vector128 subnormalMask = Vector128.Equals(offsetExponent, Vector128.Zero); + + // ~0u when value is either Infinity or NaN, 0 otherwise + Vector128 infinityOrNaNMask = Vector128.Equals(offsetExponent, Vector128.Create(HalfExponentMask)); + + // 0x3880_0000u if value is subnormal, 0 otherwise + Vector128 maskedExponentLowerBound = subnormalMask & Vector128.Create(ExponentLowerBound); + + // 0x3880_0000u if value is subnormal, 0x3800_0000u otherwise + Vector128 offsetMaskedExponentLowerBound = Vector128.Create(ExponentOffset) | maskedExponentLowerBound; + + // Match the position of the boundary of exponent bits and fraction bits with IEEE 754 Binary32(Single) + bitValueInProcess = Vector128.ShiftLeft(bitValueInProcess, 13); + + // Double the offsetMaskedExponentLowerBound if value is either Infinity or NaN + offsetMaskedExponentLowerBound = Vector128.ConditionalSelect(Vector128.Equals(infinityOrNaNMask, Vector128.Zero), + offsetMaskedExponentLowerBound, + Vector128.ShiftLeft(offsetMaskedExponentLowerBound, 1)); + + // Extract exponent bits and fraction bits of value + bitValueInProcess &= Vector128.Create(HalfToSingleBitsMask); + + // Adjust exponent to match the range of exponent + bitValueInProcess += offsetMaskedExponentLowerBound; + + // If value is subnormal, remove unnecessary 1 on top of fraction bits. + Vector128 absoluteValue = (bitValueInProcess.AsSingle() - maskedExponentLowerBound.AsSingle()).AsUInt32(); + + // Merge sign bit with rest + return (absoluteValue | sign).AsSingle(); + } + } + + public static (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x) + { + (Vector256 lowerInt32, Vector256 upperInt32) = Vector256.Widen(x); + return + (HalfAsWidenedUInt32ToSingle(lowerInt32.AsUInt32()), + HalfAsWidenedUInt32ToSingle(upperInt32.AsUInt32())); + + static Vector256 HalfAsWidenedUInt32ToSingle(Vector256 value) + { + // Extract sign bit of value + Vector256 sign = value & Vector256.Create(SingleSignMask); + + // Copy sign bit to upper bits + Vector256 bitValueInProcess = value; + + // Extract exponent bits of value (BiasedExponent is not for here as it performs unnecessary shift) + Vector256 offsetExponent = bitValueInProcess & Vector256.Create(HalfExponentMask); + + // ~0u when value is subnormal, 0 otherwise + Vector256 subnormalMask = Vector256.Equals(offsetExponent, Vector256.Zero); + + // ~0u when value is either Infinity or NaN, 0 otherwise + Vector256 infinityOrNaNMask = Vector256.Equals(offsetExponent, Vector256.Create(HalfExponentMask)); + + // 0x3880_0000u if value is subnormal, 0 otherwise + Vector256 maskedExponentLowerBound = subnormalMask & Vector256.Create(ExponentLowerBound); + + // 0x3880_0000u if value is subnormal, 0x3800_0000u otherwise + Vector256 offsetMaskedExponentLowerBound = Vector256.Create(ExponentOffset) | maskedExponentLowerBound; + + // Match the position of the boundary of exponent bits and fraction bits with IEEE 754 Binary32(Single) + bitValueInProcess = Vector256.ShiftLeft(bitValueInProcess, 13); + + // Double the offsetMaskedExponentLowerBound if value is either Infinity or NaN + offsetMaskedExponentLowerBound = Vector256.ConditionalSelect(Vector256.Equals(infinityOrNaNMask, Vector256.Zero), + offsetMaskedExponentLowerBound, + Vector256.ShiftLeft(offsetMaskedExponentLowerBound, 1)); + + // Extract exponent bits and fraction bits of value + bitValueInProcess &= Vector256.Create(HalfToSingleBitsMask); + + // Adjust exponent to match the range of exponent + bitValueInProcess += offsetMaskedExponentLowerBound; + + // If value is subnormal, remove unnecessary 1 on top of fraction bits. + Vector256 absoluteValue = (bitValueInProcess.AsSingle() - maskedExponentLowerBound.AsSingle()).AsUInt32(); + + // Merge sign bit with rest + return (absoluteValue | sign).AsSingle(); + } + } + + public static (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x) + { + (Vector512 lowerInt32, Vector512 upperInt32) = Vector512.Widen(x); + return + (HalfAsWidenedUInt32ToSingle(lowerInt32.AsUInt32()), + HalfAsWidenedUInt32ToSingle(upperInt32.AsUInt32())); + + static Vector512 HalfAsWidenedUInt32ToSingle(Vector512 value) + { + // Extract sign bit of value + Vector512 sign = value & Vector512.Create(SingleSignMask); + + // Copy sign bit to upper bits + Vector512 bitValueInProcess = value; + + // Extract exponent bits of value (BiasedExponent is not for here as it performs unnecessary shift) + Vector512 offsetExponent = bitValueInProcess & Vector512.Create(HalfExponentMask); + + // ~0u when value is subnormal, 0 otherwise + Vector512 subnormalMask = Vector512.Equals(offsetExponent, Vector512.Zero); + + // ~0u when value is either Infinity or NaN, 0 otherwise + Vector512 infinityOrNaNMask = Vector512.Equals(offsetExponent, Vector512.Create(HalfExponentMask)); + + // 0x3880_0000u if value is subnormal, 0 otherwise + Vector512 maskedExponentLowerBound = subnormalMask & Vector512.Create(ExponentLowerBound); + + // 0x3880_0000u if value is subnormal, 0x3800_0000u otherwise + Vector512 offsetMaskedExponentLowerBound = Vector512.Create(ExponentOffset) | maskedExponentLowerBound; + + // Match the position of the boundary of exponent bits and fraction bits with IEEE 754 Binary32(Single) + bitValueInProcess = Vector512.ShiftLeft(bitValueInProcess, 13); + + // Double the offsetMaskedExponentLowerBound if value is either Infinity or NaN + offsetMaskedExponentLowerBound = Vector512.ConditionalSelect(Vector512.Equals(infinityOrNaNMask, Vector512.Zero), + offsetMaskedExponentLowerBound, + Vector512.ShiftLeft(offsetMaskedExponentLowerBound, 1)); + + // Extract exponent bits and fraction bits of value + bitValueInProcess &= Vector512.Create(HalfToSingleBitsMask); + + // Adjust exponent to match the range of exponent + bitValueInProcess += offsetMaskedExponentLowerBound; + + // If value is subnormal, remove unnecessary 1 on top of fraction bits. + Vector512 absoluteValue = (bitValueInProcess.AsSingle() - maskedExponentLowerBound.AsSingle()).AsUInt32(); + + // Merge sign bit with rest + return (absoluteValue | sign).AsSingle(); + } + } + } + + internal readonly struct NarrowSingleToHalfAsUInt16Operator : IUnaryTwoToOneOperator + { + // This implements a vectorized version of the `explicit operator Half(float value) operator`. + // See detailed description of the algorithm used here: + // https://github.com/dotnet/runtime/blob/ca8d6f0420096831766ec11c7d400e4f7ccc7a34/src/libraries/System.Private.CoreLib/src/System/Half.cs#L606-L714 + // The cast operator converts a float to a Half represented as a UInt32, then narrows to a UInt16, and reinterpret casts to Half. + // This does the same, with an input VectorXx and an output VectorXx. + // Loop handling two input vectors at a time; each input float is double the size of each output Half, + // so we need two vectors of floats to produce one vector of Halfs. Half isn't supported in VectorXx, + // so we convert the VectorXx to a VectorXx, and the caller then uses this twice, narrows the combination + // into a VectorXx, and then saves that out to the destination `ref Half` reinterpreted as `ref ushort`. + + private const uint MinExp = 0x3880_0000u; // Minimum exponent for rounding + private const uint Exponent126 = 0x3f00_0000u; // Exponent displacement #1 + private const uint SingleBiasedExponentMask = 0x7F80_0000; // float.BiasedExponentMask; // Exponent mask + private const uint Exponent13 = 0x0680_0000u; // Exponent displacement #2 + private const float MaxHalfValueBelowInfinity = 65520.0f; // Maximum value that is not Infinity in Half + private const uint ExponentMask = 0x7C00; // Mask for exponent bits in Half + private const uint SingleSignMask = 0x8000_0000u; // float.SignMask; // Mask for sign bit in float + + public static bool Vectorizable => true; + + public static ushort Invoke(float x) => Unsafe.BitCast((Half)x); + + public static Vector128 Invoke(Vector128 lower, Vector128 upper) + { + return Vector128.Narrow( + SingleToHalfAsWidenedUInt32(lower), + SingleToHalfAsWidenedUInt32(upper)); + + static Vector128 SingleToHalfAsWidenedUInt32(Vector128 value) + { + Vector128 bitValue = value.AsUInt32(); + + // Extract sign bit + Vector128 sign = Vector128.ShiftRightLogical(bitValue & Vector128.Create(SingleSignMask), 16); + + // Detecting NaN (0u if value is NaN; otherwise, ~0u) + Vector128 realMask = Vector128.Equals(value, value).AsUInt32(); + + // Clear sign bit + value = Vector128.Abs(value); + + // Rectify values that are Infinity in Half. + value = Vector128.Min(Vector128.Create(MaxHalfValueBelowInfinity), value); + + // Rectify lower exponent + Vector128 exponentOffset0 = Vector128.Max(value, Vector128.Create(MinExp).AsSingle()).AsUInt32(); + + // Extract exponent + exponentOffset0 &= Vector128.Create(SingleBiasedExponentMask); + + // Add exponent by 13 + exponentOffset0 += Vector128.Create(Exponent13); + + // Round Single into Half's precision (NaN also gets modified here, just setting the MSB of fraction) + value += exponentOffset0.AsSingle(); + bitValue = value.AsUInt32(); + + // Only exponent bits will be modified if NaN + Vector128 maskedHalfExponentForNaN = ~realMask & Vector128.Create(ExponentMask); + + // Subtract exponent by 126 + bitValue -= Vector128.Create(Exponent126); + + // Shift bitValue right by 13 bits to match the boundary of exponent part and fraction part. + Vector128 newExponent = Vector128.ShiftRightLogical(bitValue, 13); + + // Clear the fraction parts if the value was NaN. + bitValue &= realMask; + + // Merge the exponent part with fraction part, and add the exponent part and fraction part's overflow. + bitValue += newExponent; + + // Clear exponents if value is NaN + bitValue &= ~maskedHalfExponentForNaN; + + // Merge sign bit with possible NaN exponent + Vector128 signAndMaskedExponent = maskedHalfExponentForNaN | sign; + + // Merge sign bit and possible NaN exponent + bitValue |= signAndMaskedExponent; + + // The final result + return bitValue; + } + } + + public static Vector256 Invoke(Vector256 lower, Vector256 upper) + { + return Vector256.Narrow( + SingleToHalfAsWidenedUInt32(lower), + SingleToHalfAsWidenedUInt32(upper)); + + static Vector256 SingleToHalfAsWidenedUInt32(Vector256 value) + { + Vector256 bitValue = value.AsUInt32(); + + // Extract sign bit + Vector256 sign = Vector256.ShiftRightLogical(bitValue & Vector256.Create(SingleSignMask), 16); + + // Detecting NaN (0u if value is NaN; otherwise, ~0u) + Vector256 realMask = Vector256.Equals(value, value).AsUInt32(); + + // Clear sign bit + value = Vector256.Abs(value); + + // Rectify values that are Infinity in Half. + value = Vector256.Min(Vector256.Create(MaxHalfValueBelowInfinity), value); + + // Rectify lower exponent + Vector256 exponentOffset0 = Vector256.Max(value, Vector256.Create(MinExp).AsSingle()).AsUInt32(); + + // Extract exponent + exponentOffset0 &= Vector256.Create(SingleBiasedExponentMask); + + // Add exponent by 13 + exponentOffset0 += Vector256.Create(Exponent13); + + // Round Single into Half's precision (NaN also gets modified here, just setting the MSB of fraction) + value += exponentOffset0.AsSingle(); + bitValue = value.AsUInt32(); + + // Only exponent bits will be modified if NaN + Vector256 maskedHalfExponentForNaN = ~realMask & Vector256.Create(ExponentMask); + + // Subtract exponent by 126 + bitValue -= Vector256.Create(Exponent126); + + // Shift bitValue right by 13 bits to match the boundary of exponent part and fraction part. + Vector256 newExponent = Vector256.ShiftRightLogical(bitValue, 13); + + // Clear the fraction parts if the value was NaN. + bitValue &= realMask; + + // Merge the exponent part with fraction part, and add the exponent part and fraction part's overflow. + bitValue += newExponent; + + // Clear exponents if value is NaN + bitValue &= ~maskedHalfExponentForNaN; + + // Merge sign bit with possible NaN exponent + Vector256 signAndMaskedExponent = maskedHalfExponentForNaN | sign; + + // Merge sign bit and possible NaN exponent + bitValue |= signAndMaskedExponent; + + // The final result + return bitValue; + } + } + + public static Vector512 Invoke(Vector512 lower, Vector512 upper) + { + return Vector512.Narrow( + SingleToHalfAsWidenedUInt32(lower), + SingleToHalfAsWidenedUInt32(upper)); + + static Vector512 SingleToHalfAsWidenedUInt32(Vector512 value) + { + Vector512 bitValue = value.AsUInt32(); + + // Extract sign bit + Vector512 sign = Vector512.ShiftRightLogical(bitValue & Vector512.Create(SingleSignMask), 16); + + // Detecting NaN (0u if value is NaN; otherwise, ~0u) + Vector512 realMask = Vector512.Equals(value, value).AsUInt32(); + + // Clear sign bit + value = Vector512.Abs(value); + + // Rectify values that are Infinity in Half. + value = Vector512.Min(Vector512.Create(MaxHalfValueBelowInfinity), value); + + // Rectify lower exponent + Vector512 exponentOffset0 = Vector512.Max(value, Vector512.Create(MinExp).AsSingle()).AsUInt32(); + + // Extract exponent + exponentOffset0 &= Vector512.Create(SingleBiasedExponentMask); + + // Add exponent by 13 + exponentOffset0 += Vector512.Create(Exponent13); + + // Round Single into Half's precision (NaN also gets modified here, just setting the MSB of fraction) + value += exponentOffset0.AsSingle(); + bitValue = value.AsUInt32(); + + // Only exponent bits will be modified if NaN + Vector512 maskedHalfExponentForNaN = ~realMask & Vector512.Create(ExponentMask); + + // Subtract exponent by 126 + bitValue -= Vector512.Create(Exponent126); + + // Shift bitValue right by 13 bits to match the boundary of exponent part and fraction part. + Vector512 newExponent = Vector512.ShiftRightLogical(bitValue, 13); + + // Clear the fraction parts if the value was NaN. + bitValue &= realMask; + + // Merge the exponent part with fraction part, and add the exponent part and fraction part's overflow. + bitValue += newExponent; + + // Clear exponents if value is NaN + bitValue &= ~maskedHalfExponentForNaN; + + // Merge sign bit with possible NaN exponent + Vector512 signAndMaskedExponent = maskedHalfExponentForNaN | sign; + + // Merge sign bit and possible NaN exponent + bitValue |= signAndMaskedExponent; + + // The final result + return bitValue; + } + } + } + + /// T.SinCos(x) + internal readonly struct SinCosOperator : IUnaryInputBinaryOutput where T : ITrigonometricFunctions + { + public static bool Vectorizable => false; // TODO: vectorize + + public static (T, T) Invoke(T x) => T.SinCos(x); + public static (Vector128 First, Vector128 Second) Invoke(Vector128 x) => throw new NotSupportedException(); + public static (Vector256 First, Vector256 Second) Invoke(Vector256 x) => throw new NotSupportedException(); + public static (Vector512 First, Vector512 Second) Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// T.SinCosPi(x) + internal readonly struct SinCosPiOperator : IUnaryInputBinaryOutput where T : ITrigonometricFunctions + { + public static bool Vectorizable => false; // TODO: vectorize + + public static (T, T) Invoke(T x) => T.SinCosPi(x); + public static (Vector128 First, Vector128 Second) Invoke(Vector128 x) => throw new NotSupportedException(); + public static (Vector256 First, Vector256 Second) Invoke(Vector256 x) => throw new NotSupportedException(); + public static (Vector512 First, Vector512 Second) Invoke(Vector512 x) => throw new NotSupportedException(); + } + + /// Operator that takes one input value and returns a single value. + /// The input and output type must be of the same size if vectorization is desired. + internal interface IUnaryOperator + { + static abstract bool Vectorizable { get; } + static abstract TOutput Invoke(TInput x); + static abstract Vector128 Invoke(Vector128 x); + static abstract Vector256 Invoke(Vector256 x); + static abstract Vector512 Invoke(Vector512 x); + } + + /// Operator that takes one input value and returns a single value. + /// The input type must be half the size of the output type. + private interface IUnaryOneToTwoOperator + { + static abstract bool Vectorizable { get; } + static abstract TOutput Invoke(TInput x); + static abstract (Vector128 Lower, Vector128 Upper) Invoke(Vector128 x); + static abstract (Vector256 Lower, Vector256 Upper) Invoke(Vector256 x); + static abstract (Vector512 Lower, Vector512 Upper) Invoke(Vector512 x); + } + + /// Operator that takes one input value and returns a single value. + /// The input type must be twice the size of the output type. + private interface IUnaryTwoToOneOperator + { + static abstract bool Vectorizable { get; } + static abstract TOutput Invoke(TInput x); + static abstract Vector128 Invoke(Vector128 lower, Vector128 upper); + static abstract Vector256 Invoke(Vector256 lower, Vector256 upper); + static abstract Vector512 Invoke(Vector512 lower, Vector512 upper); + } + + /// Operator that takes one input value and returns two output values. + private interface IUnaryInputBinaryOutput + { + static abstract bool Vectorizable { get; } + static abstract (T, T) Invoke(T x); + static abstract (Vector128 First, Vector128 Second) Invoke(Vector128 x); + static abstract (Vector256 First, Vector256 Second) Invoke(Vector256 x); + static abstract (Vector512 First, Vector512 Second) Invoke(Vector512 x); + } + + /// Operator that takes one input value and returns a single value. + private interface IStatefulUnaryOperator + { + static abstract bool Vectorizable { get; } + T Invoke(T x); + Vector128 Invoke(Vector128 x); + Vector256 Invoke(Vector256 x); + Vector512 Invoke(Vector512 x); + } + + /// Operator that takes two input values and returns a single value. + private interface IBinaryOperator + { + static abstract bool Vectorizable { get; } + static abstract T Invoke(T x, T y); + static abstract Vector128 Invoke(Vector128 x, Vector128 y); + static abstract Vector256 Invoke(Vector256 x, Vector256 y); + static abstract Vector512 Invoke(Vector512 x, Vector512 y); + } + + /// that specializes horizontal aggregation of all elements in a vector. + private interface IAggregationOperator : IBinaryOperator + { + static abstract T Invoke(Vector128 x); + static abstract T Invoke(Vector256 x); + static abstract T Invoke(Vector512 x); + + static virtual T IdentityValue => throw new NotSupportedException(); + } + + /// Operator that takes three input values and returns a single value. + private interface ITernaryOperator + { + static abstract T Invoke(T x, T y, T z); + static abstract Vector128 Invoke(Vector128 x, Vector128 y, Vector128 z); + static abstract Vector256 Invoke(Vector256 x, Vector256 y, Vector256 z); + static abstract Vector512 Invoke(Vector512 x, Vector512 y, Vector512 z); + } + } +} diff --git a/src/libraries/System.Numerics.Tensors/tests/Helpers.cs b/src/libraries/System.Numerics.Tensors/tests/Helpers.cs index 729cacda351695..d6b5eef63d9dae 100644 --- a/src/libraries/System.Numerics.Tensors/tests/Helpers.cs +++ b/src/libraries/System.Numerics.Tensors/tests/Helpers.cs @@ -11,60 +11,5 @@ internal static class Helpers public static IEnumerable TensorLengthsIncluding0 => Enumerable.Range(0, 257); public static IEnumerable TensorLengths => Enumerable.Range(1, 256); - - // Tolerances taken from testing in the scalar math routines: - // cf. https://github.com/dotnet/runtime/blob/89f7ad3b276fb0b48f20cb4e8408bdce85c2b415/src/libraries/System.Runtime/tests/System.Runtime.Extensions.Tests/System/Math.cs - // and https://github.com/dotnet/runtime/blob/fd48b6f5d1ff81a81d09e9d72982cc9e8d139852/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs - public const double DefaultDoubleTolerance = 8.8817841970012523e-16; - public const float DefaultFloatTolerance = 4.76837158e-07f; - public const float DefaultHalfTolerance = 3.90625e-03f; - public const double DefaultToleranceForEstimates = 1.171875e-02; - -#if NETCOREAPP - private static class DefaultTolerance where T : unmanaged, INumber - { - public static readonly T Value = DetermineTolerance(DefaultDoubleTolerance, DefaultFloatTolerance, Half.CreateTruncating(DefaultHalfTolerance)) ?? T.CreateTruncating(0); - } - - public static bool IsEqualWithTolerance(T expected, T actual, T? tolerance = null) where T : unmanaged, INumber - { - tolerance = tolerance ?? DefaultTolerance.Value; - T diff = T.Abs(expected - actual); - return !(diff > tolerance && diff > T.Max(T.Abs(expected), T.Abs(actual)) * tolerance); - } -#else - public static bool IsEqualWithTolerance(float expected, float actual, float? tolerance = null) - { - tolerance ??= DefaultFloatTolerance; - float diff = MathF.Abs(expected - actual); - return !(diff > tolerance && diff > MathF.Max(MathF.Abs(expected), MathF.Abs(actual)) * tolerance); - } -#endif - - public static T? DetermineTolerance( - double? doubleTolerance = null, - float? floatTolerance = null -#if NETCOREAPP - , Half? halfTolerance = null -#endif - ) where T : struct - { - if (typeof(T) == typeof(double) && doubleTolerance != null) - { - return (T?)(object)doubleTolerance; - } - else if (typeof(T) == typeof(float) && floatTolerance != null) - { - return (T?)(object)floatTolerance; - } -#if NETCOREAPP - else if (typeof(T) == typeof(Half) && halfTolerance != null) - { - return (T?)(object)halfTolerance; - } -#endif - - return null; - } } } diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs index cba1f4f1b95ea2..cd6ae2455491c8 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.Generic.cs @@ -6,8 +6,6 @@ using System.Linq; using System.Reflection; using System.Runtime.InteropServices; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; using Xunit; using Xunit.Sdk; @@ -115,7 +113,7 @@ private static void ConvertTruncatingImpl() for (int i = 0; i < tensorLength; i++) { - if (!Helpers.IsEqualWithTolerance(TTo.CreateTruncating(source.Span[i]), destination.Span[i])) + if (!IsEqualWithTolerance(TTo.CreateTruncating(source.Span[i]), destination.Span[i])) { throw new XunitException($"{typeof(TFrom).Name} => {typeof(TTo).Name}. Input: {source.Span[i]}. Actual: {destination.Span[i]}. Expected: {TTo.CreateTruncating(source.Span[i])}."); } @@ -147,7 +145,7 @@ private static void ConvertSaturatingImpl() for (int i = 0; i < tensorLength; i++) { - if (!Helpers.IsEqualWithTolerance(TTo.CreateSaturating(source.Span[i]), destination.Span[i])) + if (!IsEqualWithTolerance(TTo.CreateSaturating(source.Span[i]), destination.Span[i])) { throw new XunitException($"{typeof(TFrom).Name} => {typeof(TTo).Name}. Input: {source.Span[i]}. Actual: {destination.Span[i]}. Expected: {TTo.CreateSaturating(source.Span[i])}."); } @@ -179,7 +177,7 @@ private static void ConvertCheckedImpl() for (int i = 0; i < tensorLength; i++) { - if (!Helpers.IsEqualWithTolerance(TTo.CreateChecked(source.Span[i]), destination.Span[i])) + if (!IsEqualWithTolerance(TTo.CreateChecked(source.Span[i]), destination.Span[i])) { throw new XunitException($"{typeof(TFrom).Name} => {typeof(TTo).Name}. Input: {source.Span[i]}. Actual: {destination.Span[i]}. Expected: {TTo.CreateChecked(source.Span[i])}."); } @@ -201,7 +199,7 @@ private static void ConvertCheckedImpl(TFrom valid, TFrom invalid) TensorPrimitives.ConvertChecked(source.Span, destination.Span); foreach (TTo result in destination.Span) { - Assert.True(Helpers.IsEqualWithTolerance(TTo.CreateChecked(valid), result)); + Assert.True(IsEqualWithTolerance(TTo.CreateChecked(valid), result)); } // Test with at least one invalid @@ -213,6 +211,19 @@ private static void ConvertCheckedImpl(TFrom valid, TFrom invalid) } }; } + + private static bool IsEqualWithTolerance(T expected, T actual, T? tolerance = null) where T : unmanaged, INumber + { + tolerance ??= T.CreateTruncating(0.0001); + + T diff = T.Abs(expected - actual); + if (diff > tolerance && diff > T.Max(T.Abs(expected), T.Abs(actual)) * tolerance) + { + return false; + } + + return true; + } } // The tests for some types have been marked as OuterLoop simply to decrease inner loop testing time. @@ -351,63 +362,48 @@ protected override void SetSpecialValues(Span x, Span y) #region Span -> Destination public static IEnumerable SpanDestinationFunctionsToTest() { - // The current trigonometric algorithm depends on hardware FMA support for best precision. - T? trigTolerance = IsFmaSupported ? null : Helpers.DetermineTolerance(doubleTolerance: 1e-10, floatTolerance: 1e-4f); - - yield return Create(TensorPrimitives.Acosh, T.Acosh); - yield return Create(TensorPrimitives.AcosPi, T.AcosPi); - yield return Create(TensorPrimitives.Acos, T.Acos); - yield return Create(TensorPrimitives.Asinh, T.Asinh); - yield return Create(TensorPrimitives.AsinPi, T.AsinPi); - yield return Create(TensorPrimitives.Asin, T.Asin); - yield return Create(TensorPrimitives.Atanh, T.Atanh); - yield return Create(TensorPrimitives.AtanPi, T.AtanPi); - yield return Create(TensorPrimitives.Atan, T.Atan); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Cbrt, T.Cbrt, Helpers.DetermineTolerance(doubleTolerance: 1e-13)); - yield return Create(TensorPrimitives.Ceiling, T.Ceiling); - yield return Create(TensorPrimitives.Cos, T.Cos, trigTolerance); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Cosh, T.Cosh, Helpers.DetermineTolerance(doubleTolerance: 1e-14)); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.CosPi, T.CosPi, trigTolerance ?? Helpers.DetermineTolerance(floatTolerance: 1e-5f)); - yield return Create(TensorPrimitives.DegreesToRadians, T.DegreesToRadians); - yield return Create(TensorPrimitives.Exp, T.Exp); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Exp2, T.Exp2, Helpers.DetermineTolerance(doubleTolerance: 1e-14, floatTolerance: 1e-5f)); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Exp10, T.Exp10, Helpers.DetermineTolerance(doubleTolerance: 1e-13, floatTolerance: 1e-5f)); - yield return Create(TensorPrimitives.ExpM1, T.ExpM1); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Exp2M1, T.Exp2M1, Helpers.DetermineTolerance(doubleTolerance: 1e-14, floatTolerance: 1e-5f)); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Exp10M1, T.Exp10M1, Helpers.DetermineTolerance(doubleTolerance: 1e-13, floatTolerance: 1e-5f)); - yield return Create(TensorPrimitives.Floor, T.Floor); - yield return Create(TensorPrimitives.Log, T.Log); - yield return Create(TensorPrimitives.Log2, T.Log2); - yield return Create(TensorPrimitives.Log10, T.Log10); - yield return Create(TensorPrimitives.LogP1, T.LogP1); - yield return Create(TensorPrimitives.Log2P1, T.Log2P1); - yield return Create(TensorPrimitives.Log10P1, T.Log10P1); - yield return Create(TensorPrimitives.RadiansToDegrees, T.RadiansToDegrees); - yield return Create(TensorPrimitives.Reciprocal, f => T.One / f); - yield return Create(TensorPrimitives.ReciprocalEstimate, T.ReciprocalEstimate, T.CreateTruncating(Helpers.DefaultToleranceForEstimates)); - yield return Create(TensorPrimitives.ReciprocalSqrt, f => T.One / T.Sqrt(f)); - yield return Create(TensorPrimitives.ReciprocalSqrtEstimate, T.ReciprocalSqrtEstimate, T.CreateTruncating(Helpers.DefaultToleranceForEstimates)); - yield return Create(TensorPrimitives.Round, T.Round); - yield return Create(TensorPrimitives.Sin, T.Sin, trigTolerance); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Sinh, T.Sinh, Helpers.DetermineTolerance(doubleTolerance: 1e-14)); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.SinPi, T.SinPi, Helpers.DetermineTolerance(doubleTolerance: 1e-13, floatTolerance: 1e-4f)); - yield return Create(TensorPrimitives.Sqrt, T.Sqrt); - yield return Create(TensorPrimitives.Tan, T.Tan, trigTolerance); - yield return Create(TensorPrimitives.Tanh, T.Tanh); - yield return Create(TensorPrimitives.TanPi, T.TanPi); - yield return Create(TensorPrimitives.Truncate, T.Truncate); - - static object[] Create(SpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) - => new object[] { tensorPrimitivesMethod, expectedMethod, tolerance }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Acosh), new Func(T.Acosh) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.AcosPi), new Func(T.AcosPi) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Acos), new Func(T.Acos) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Asinh), new Func(T.Asinh) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.AsinPi), new Func(T.AsinPi) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Asin), new Func(T.Asin) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Atanh), new Func(T.Atanh) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.AtanPi), new Func(T.AtanPi) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Atan), new Func(T.Atan) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Cbrt), new Func(T.Cbrt) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Ceiling), new Func(T.Ceiling) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Cos), new Func(T.Cos) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Cosh), new Func(T.Cosh) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.CosPi), new Func(T.CosPi) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.DegreesToRadians), new Func(T.DegreesToRadians) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Exp), new Func(T.Exp) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Exp2), new Func(T.Exp2) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Exp10), new Func(T.Exp10) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.ExpM1), new Func(T.ExpM1) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Exp2M1), new Func(T.Exp2M1) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Exp10M1), new Func(T.Exp10M1) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Floor), new Func(T.Floor) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Log), new Func(T.Log) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Log2), new Func(T.Log2) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Log10), new Func(T.Log10) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.LogP1), new Func(T.LogP1) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Log2P1), new Func(T.Log2P1) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Log10P1), new Func(T.Log10P1) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.RadiansToDegrees), new Func(T.RadiansToDegrees) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Reciprocal), new Func(f => T.One / f) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.ReciprocalEstimate), new Func(T.ReciprocalEstimate), T.CreateTruncating(1.171875e-02) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.ReciprocalSqrt), new Func(f => T.One / T.Sqrt(f)) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.ReciprocalSqrtEstimate), new Func(T.ReciprocalSqrtEstimate), T.CreateTruncating(1.171875e-02) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Round), new Func(T.Round) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Sin), new Func(T.Sin) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Sinh), new Func(T.Sinh) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.SinPi), new Func(T.SinPi) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Sqrt), new Func(T.Sqrt) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Tan), new Func(T.Tan) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Tanh), new Func(T.Tanh) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.TanPi), new Func(T.TanPi) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.Truncate), new Func(T.Truncate) }; } [Theory] @@ -514,22 +510,18 @@ public void SpanDestinationFunctions_ThrowsForOverlapppingInputsWithOutputs(Span #region Span,Span -> Destination public static IEnumerable SpanSpanDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.Atan2, T.Atan2); - yield return Create(TensorPrimitives.Atan2Pi, T.Atan2Pi); - yield return Create(TensorPrimitives.CopySign, T.CopySign); - yield return Create(TensorPrimitives.Hypot, T.Hypot); - yield return Create(TensorPrimitives.Ieee754Remainder, T.Ieee754Remainder); - yield return Create(TensorPrimitives.Log, T.Log); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Pow, T.Pow, Helpers.DetermineTolerance(doubleTolerance: 1e-13, floatTolerance: 1e-4f)); - - static object[] Create(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) - => new object[] { tensorPrimitivesMethod, expectedMethod, tolerance }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Atan2), new Func(T.Atan2) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Atan2Pi), new Func(T.Atan2Pi) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.CopySign), new Func(T.CopySign) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Hypot), new Func(T.Hypot) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Ieee754Remainder), new Func(T.Ieee754Remainder) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Log), new Func(T.Log) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Pow), new Func(T.Pow) }; } [Theory] [MemberData(nameof(SpanSpanDestinationFunctionsToTest))] - public void SpanSpanDestination_AllLengths(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanSpanDestination_AllLengths(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -540,14 +532,14 @@ public void SpanSpanDestination_AllLengths(SpanSpanDestinationDelegate tensorPri tensorPrimitivesMethod(x, y, destination); for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(x[i], y[i]), destination[i], tolerance); + AssertEqualTolerance(expectedMethod(x[i], y[i]), destination[i]); } }); } [Theory] [MemberData(nameof(SpanSpanDestinationFunctionsToTest))] - public void SpanSpanDestination_InPlace(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanSpanDestination_InPlace(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -558,14 +550,14 @@ public void SpanSpanDestination_InPlace(SpanSpanDestinationDelegate tensorPrimit for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(xOrig[i], xOrig[i]), x[i], tolerance); + AssertEqualTolerance(expectedMethod(xOrig[i], xOrig[i]), x[i]); } }); } [Theory] [MemberData(nameof(SpanSpanDestinationFunctionsToTest))] - public void SpanSpanDestination_SpecialValues(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanSpanDestination_SpecialValues(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengths, tensorLength => { @@ -578,7 +570,7 @@ public void SpanSpanDestination_SpecialValues(SpanSpanDestinationDelegate tensor tensorPrimitivesMethod(x.Span, y.Span, destination.Span); for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(x[i], y[i]), destination[i], tolerance); + AssertEqualTolerance(expectedMethod(x[i], y[i]), destination[i]); } }, x); @@ -587,7 +579,7 @@ public void SpanSpanDestination_SpecialValues(SpanSpanDestinationDelegate tensor tensorPrimitivesMethod(x.Span, y.Span, destination.Span); for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(x[i], y[i]), destination[i], tolerance); + AssertEqualTolerance(expectedMethod(x[i], y[i]), destination[i]); } }, y); }); @@ -595,11 +587,8 @@ public void SpanSpanDestination_SpecialValues(SpanSpanDestinationDelegate tensor [Theory] [MemberData(nameof(SpanSpanDestinationFunctionsToTest))] - public void SpanSpanDestination_ThrowsForMismatchedLengths(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanSpanDestination_ThrowsForMismatchedLengths(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -613,11 +602,8 @@ public void SpanSpanDestination_ThrowsForMismatchedLengths(SpanSpanDestinationDe [Theory] [MemberData(nameof(SpanSpanDestinationFunctionsToTest))] - public void SpanSpanDestination_ThrowsForTooShortDestination(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanSpanDestination_ThrowsForTooShortDestination(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -630,11 +616,8 @@ public void SpanSpanDestination_ThrowsForTooShortDestination(SpanSpanDestination [Theory] [MemberData(nameof(SpanSpanDestinationFunctionsToTest))] - public void SpanSpanDestination_ThrowsForOverlapppingInputsWithOutputs(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanSpanDestination_ThrowsForOverlapppingInputsWithOutputs(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - T[] array = new T[10]; AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(array.AsSpan(1, 2), array.AsSpan(5, 2), array.AsSpan(0, 2))); AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(array.AsSpan(1, 2), array.AsSpan(5, 2), array.AsSpan(2, 2))); @@ -646,25 +629,21 @@ public void SpanSpanDestination_ThrowsForOverlapppingInputsWithOutputs(SpanSpanD #region Span,Scalar -> Destination public static IEnumerable SpanScalarDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.Atan2, T.Atan2); - yield return Create(TensorPrimitives.Atan2Pi, T.Atan2Pi); - yield return Create(TensorPrimitives.CopySign, T.CopySign); - yield return Create(TensorPrimitives.Ieee754Remainder, T.Ieee754Remainder); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Pow, T.Pow, Helpers.DetermineTolerance(doubleTolerance: 1e-13, floatTolerance: 1e-4f)); - yield return Create(TensorPrimitives.Log, T.Log); - yield return Create(TensorPrimitives.Max, T.Max); - yield return Create(TensorPrimitives.MaxMagnitude, T.MaxMagnitude); - yield return Create(TensorPrimitives.Min, T.Min); - yield return Create(TensorPrimitives.MinMagnitude, T.MinMagnitude); - - static object[] Create(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) - => new object[] { tensorPrimitivesMethod, expectedMethod, tolerance }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Atan2), new Func(T.Atan2) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Atan2Pi), new Func(T.Atan2Pi) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.CopySign), new Func(T.CopySign) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Ieee754Remainder), new Func(T.Ieee754Remainder) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Pow), new Func(T.Pow) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Log), new Func(T.Log) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Max), new Func(T.Max) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.MaxMagnitude), new Func(T.MaxMagnitude) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Min), new Func(T.Min) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.MinMagnitude), new Func(T.MinMagnitude) }; } [Theory] [MemberData(nameof(SpanScalarDestinationFunctionsToTest))] - public void SpanScalarDestination_AllLengths(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarDestination_AllLengths(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -675,14 +654,14 @@ public void SpanScalarDestination_AllLengths(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarDestination_InPlace(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -694,14 +673,14 @@ public void SpanScalarDestination_InPlace(SpanScalarDestinationDelegate for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(xOrig[i], y), x[i], tolerance); + AssertEqualTolerance(expectedMethod(xOrig[i], y), x[i]); } }); } [Theory] [MemberData(nameof(SpanScalarDestinationFunctionsToTest))] - public void SpanScalarDestination_SpecialValues(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarDestination_SpecialValues(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengths, tensorLength => { @@ -714,7 +693,7 @@ public void SpanScalarDestination_SpecialValues(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarDestination_ThrowsForTooShortDestination(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -739,11 +715,8 @@ public void SpanScalarDestination_ThrowsForTooShortDestination(SpanScalarDestina [Theory] [MemberData(nameof(SpanScalarDestinationFunctionsToTest))] - public void SpanScalarDestination_ThrowsForOverlapppingInputsWithOutputs(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarDestination_ThrowsForOverlapppingInputsWithOutputs(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - T[] array = new T[10]; AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(array.AsSpan(1, 2), default, array.AsSpan(0, 2))); AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(array.AsSpan(1, 2), default, array.AsSpan(2, 2))); @@ -753,19 +726,15 @@ public void SpanScalarDestination_ThrowsForOverlapppingInputsWithOutputs(SpanSca #region Scalar,Span -> Destination public static IEnumerable ScalarSpanFloatDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.Atan2, T.Atan2); - yield return Create(TensorPrimitives.Atan2Pi, T.Atan2Pi); - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.Pow, T.Pow, Helpers.DetermineTolerance(floatTolerance: 1e-4f)); - yield return Create(TensorPrimitives.Ieee754Remainder, T.Ieee754Remainder); - - static object[] Create(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) - => new object[] { tensorPrimitivesMethod, expectedMethod, tolerance }; + yield return new object[] { new ScalarSpanDestinationDelegate(TensorPrimitives.Atan2), new Func(T.Atan2) }; + yield return new object[] { new ScalarSpanDestinationDelegate(TensorPrimitives.Atan2Pi), new Func(T.Atan2Pi) }; + yield return new object[] { new ScalarSpanDestinationDelegate(TensorPrimitives.Pow), new Func(T.Pow) }; + yield return new object[] { new ScalarSpanDestinationDelegate(TensorPrimitives.Ieee754Remainder), new Func(T.Ieee754Remainder) }; } [Theory] [MemberData(nameof(ScalarSpanFloatDestinationFunctionsToTest))] - public void SpanScalarFloatDestination_AllLengths(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarFloatDestination_AllLengths(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -776,14 +745,14 @@ public void SpanScalarFloatDestination_AllLengths(ScalarSpanDestinationDelegate tensorPrimitivesMethod(x, y, destination); for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(x, y[i]), destination[i], tolerance); + AssertEqualTolerance(expectedMethod(x, y[i]), destination[i]); } }); } [Theory] [MemberData(nameof(ScalarSpanFloatDestinationFunctionsToTest))] - public void SpanScalarFloatDestination_InPlace(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarFloatDestination_InPlace(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -795,14 +764,14 @@ public void SpanScalarFloatDestination_InPlace(ScalarSpanDestinationDelegate ten for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(x, yOrig[i]), y[i], tolerance); + AssertEqualTolerance(expectedMethod(x, yOrig[i]), y[i]); } }); } [Theory] [MemberData(nameof(ScalarSpanFloatDestinationFunctionsToTest))] - public void ScalarSpanDestination_SpecialValues(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void ScalarSpanDestination_SpecialValues(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengths, tensorLength => { @@ -815,7 +784,7 @@ public void ScalarSpanDestination_SpecialValues(ScalarSpanDestinationDelegate te tensorPrimitivesMethod(x, y.Span, destination.Span); for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(x, y[i]), destination[i], tolerance); + AssertEqualTolerance(expectedMethod(x, y[i]), destination[i]); } }, y); }); @@ -823,11 +792,8 @@ public void ScalarSpanDestination_SpecialValues(ScalarSpanDestinationDelegate te [Theory] [MemberData(nameof(ScalarSpanFloatDestinationFunctionsToTest))] - public void SpanScalarFloatDestination_ThrowsForTooShortDestination(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarFloatDestination_ThrowsForTooShortDestination(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - Assert.All(Helpers.TensorLengths, tensorLength => { T x = NextRandom(); @@ -840,11 +806,8 @@ public void SpanScalarFloatDestination_ThrowsForTooShortDestination(ScalarSpanDe [Theory] [MemberData(nameof(ScalarSpanFloatDestinationFunctionsToTest))] - public void SpanScalarFloatDestination_ThrowsForOverlapppingInputsWithOutputs(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanScalarFloatDestination_ThrowsForOverlapppingInputsWithOutputs(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - T[] array = new T[10]; AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(default, array.AsSpan(1, 2), array.AsSpan(0, 2))); AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(default, array.AsSpan(1, 2), array.AsSpan(2, 2))); @@ -854,17 +817,13 @@ public void SpanScalarFloatDestination_ThrowsForOverlapppingInputsWithOutputs(Sc #region Span,Int,Span -> Destination public static IEnumerable SpanIntDestinationFunctionsToTest() { - // TODO https://github.com/dotnet/runtime/issues/98861 - yield return Create(TensorPrimitives.RootN, T.RootN, Helpers.DetermineTolerance(doubleTolerance: 1e-13)); - yield return Create(TensorPrimitives.ScaleB, T.ScaleB); - - static object[] Create(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) - => new object[] { tensorPrimitivesMethod, expectedMethod, tolerance }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.RootN), new Func(T.RootN) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.ScaleB), new Func(T.ScaleB) }; } [Theory] [MemberData(nameof(SpanIntDestinationFunctionsToTest))] - public void SpanIntDestination_AllLengths(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanIntDestination_AllLengths(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -875,14 +834,14 @@ public void SpanIntDestination_AllLengths(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanIntDestination_InPlace(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { @@ -894,14 +853,14 @@ public void SpanIntDestination_InPlace(SpanScalarDestinationDelegate for (int i = 0; i < tensorLength; i++) { - AssertEqualTolerance(expectedMethod(xOrig[i], y), x[i], tolerance); + AssertEqualTolerance(expectedMethod(xOrig[i], y), x[i]); } }); } [Theory] [MemberData(nameof(SpanIntDestinationFunctionsToTest))] - public void SpanIntDestination_SpecialValues(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanIntDestination_SpecialValues(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) { Assert.All(Helpers.TensorLengths, tensorLength => { @@ -914,7 +873,7 @@ public void SpanIntDestination_SpecialValues(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanIntDestination_ThrowsForTooShortDestination(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -939,11 +895,8 @@ public void SpanIntDestination_ThrowsForTooShortDestination(SpanScalarDestinatio [Theory] [MemberData(nameof(SpanIntDestinationFunctionsToTest))] - public void SpanIntDestination_ThrowsForOverlapppingInputsWithOutputs(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod, T? tolerance = null) + public void SpanIntDestination_ThrowsForOverlapppingInputsWithOutputs(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func _) { - _ = expectedMethod; - _ = tolerance; - T[] array = new T[10]; AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(array.AsSpan(1, 2), 2, array.AsSpan(0, 2))); AssertExtensions.Throws("destination", () => tensorPrimitivesMethod(array.AsSpan(1, 2), 2, array.AsSpan(2, 2))); @@ -953,12 +906,9 @@ public void SpanIntDestination_ThrowsForOverlapppingInputsWithOutputs(SpanScalar #region Span,Span,Span -> Destination public static IEnumerable SpanSpanSpanDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.FusedMultiplyAdd, T.FusedMultiplyAdd); - yield return Create(TensorPrimitives.Lerp, T.Lerp); - yield return Create(TensorPrimitives.MultiplyAddEstimate, T.FusedMultiplyAdd); // TODO: Change T.FusedMultiplyAdd to T.MultiplyAddEstimate when available - - static object[] Create(SpanSpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanSpanSpanDestinationDelegate(TensorPrimitives.FusedMultiplyAdd), new Func(T.FusedMultiplyAdd) }; + yield return new object[] { new SpanSpanSpanDestinationDelegate(TensorPrimitives.Lerp), new Func(T.Lerp) }; + yield return new object[] { new SpanSpanSpanDestinationDelegate(TensorPrimitives.MultiplyAddEstimate), new Func(T.FusedMultiplyAdd) }; // TODO: Change T.FusedMultiplyAdd to T.MultiplyAddEstimate when available } [Theory] @@ -1089,12 +1039,9 @@ public void SpanSpanSpanDestination_ThrowsForOverlapppingInputsWithOutputs(SpanS #region Span,Span,Scalar -> Destination public static IEnumerable SpanSpanScalarDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.FusedMultiplyAdd, T.FusedMultiplyAdd); - yield return Create(TensorPrimitives.Lerp, T.Lerp); - yield return Create(TensorPrimitives.MultiplyAddEstimate, T.FusedMultiplyAdd); // TODO: Change T.FusedMultiplyAdd to T.MultiplyAddEstimate when available - - static object[] Create(SpanSpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanSpanScalarDestinationDelegate(TensorPrimitives.FusedMultiplyAdd), new Func(T.FusedMultiplyAdd) }; + yield return new object[] { new SpanSpanScalarDestinationDelegate(TensorPrimitives.Lerp), new Func(T.Lerp) }; + yield return new object[] { new SpanSpanScalarDestinationDelegate(TensorPrimitives.MultiplyAddEstimate), new Func(T.FusedMultiplyAdd) }; // TODO: Change T.FusedMultiplyAdd to T.MultiplyAddEstimate when available } [Theory] @@ -1197,12 +1144,9 @@ public void SpanSpanScalarDestination_ThrowsForOverlapppingInputsWithOutputs(Spa #region Span,Scalar,Span -> Destination public static IEnumerable SpanScalarSpanDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.FusedMultiplyAdd, T.FusedMultiplyAdd); - yield return Create(TensorPrimitives.Lerp, T.Lerp); - yield return Create(TensorPrimitives.MultiplyAddEstimate, T.FusedMultiplyAdd); // TODO: Change T.FusedMultiplyAdd to T.MultiplyAddEstimate when available - - static object[] Create(SpanScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanScalarSpanDestinationDelegate(TensorPrimitives.FusedMultiplyAdd), new Func(T.FusedMultiplyAdd) }; + yield return new object[] { new SpanScalarSpanDestinationDelegate(TensorPrimitives.Lerp), new Func(T.Lerp) }; + yield return new object[] { new SpanScalarSpanDestinationDelegate(TensorPrimitives.MultiplyAddEstimate), new Func(T.FusedMultiplyAdd) }; // TODO: Change T.FusedMultiplyAdd to T.MultiplyAddEstimate when available } [Theory] @@ -1305,11 +1249,8 @@ public void SpanScalarSpanDestination_ThrowsForOverlapppingInputsWithOutputs(Spa #region Span -> Destination,Destination public static IEnumerable SpanDestinationDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.SinCos, T.SinCos); - yield return Create(TensorPrimitives.SinCosPi, T.SinCosPi); - - static object[] Create(SpanDestinationDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanDestinationDestinationDelegate(TensorPrimitives.SinCos), new Func(T.SinCos) }; + yield return new object[] { new SpanDestinationDestinationDelegate(TensorPrimitives.SinCosPi), new Func(T.SinCosPi) }; } [Theory] @@ -1656,13 +1597,10 @@ public void Divide_ScalarTensor_ByZero_Throw() #region Span -> Destination public static IEnumerable SpanDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.OnesComplement, i => ~i); - yield return Create(TensorPrimitives.PopCount, T.PopCount); - yield return Create(TensorPrimitives.LeadingZeroCount, T.LeadingZeroCount); - yield return Create(TensorPrimitives.TrailingZeroCount, T.TrailingZeroCount); - - static object[] Create(SpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.OnesComplement), new Func(i => ~i) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.PopCount), new Func(T.PopCount) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.LeadingZeroCount), new Func(T.LeadingZeroCount) }; + yield return new object[] { new SpanDestinationDelegate(TensorPrimitives.TrailingZeroCount), new Func(T.TrailingZeroCount) }; } [Theory] @@ -1727,12 +1665,9 @@ public void SpanDestinationFunctions_ThrowsForOverlapppingInputsWithOutputs(Span #region Span,Span -> Destination public static IEnumerable SpanSpanDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.BitwiseAnd, (x, y) => x & y); - yield return Create(TensorPrimitives.BitwiseOr, (x, y) => x | y); - yield return Create(TensorPrimitives.Xor, (x, y) => x ^ y); - - static object[] Create(SpanSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.BitwiseAnd), new Func((x, y) => x & y) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.BitwiseOr), new Func((x, y) => x | y) }; + yield return new object[] { new SpanSpanDestinationDelegate(TensorPrimitives.Xor), new Func((x, y) => x ^ y) }; } [Theory] @@ -1815,16 +1750,13 @@ public void SpanSpanDestination_ThrowsForOverlapppingInputsWithOutputs(SpanSpanD #region Span,Scalar -> Destination public static IEnumerable SpanScalarDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.BitwiseAnd, (x, y) => x & y); - yield return Create(TensorPrimitives.BitwiseOr, (x, y) => x | y); - yield return Create(TensorPrimitives.Max, T.Max); - yield return Create(TensorPrimitives.MaxMagnitude, T.MaxMagnitude); - yield return Create(TensorPrimitives.Min, T.Min); - yield return Create(TensorPrimitives.MinMagnitude, T.MinMagnitude); - yield return Create(TensorPrimitives.Xor, (x, y) => x ^ y); - - static object[] Create(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.BitwiseAnd), new Func((x, y) => x & y) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.BitwiseOr), new Func((x, y) => x | y) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Max), new Func(T.Max) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.MaxMagnitude), new Func(T.MaxMagnitude) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Min), new Func(T.Min) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.MinMagnitude), new Func(T.MinMagnitude) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.Xor), new Func((x, y) => x ^ y) }; } [Theory] @@ -1892,14 +1824,11 @@ public void SpanScalarDestination_ThrowsForOverlapppingInputWithOutputs(SpanScal #region Shifting/Rotating public static IEnumerable ShiftRotateDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.ShiftLeft, (x, n) => x << n); - yield return Create(TensorPrimitives.ShiftRightArithmetic, (x, n) => x >> n); - yield return Create(TensorPrimitives.ShiftRightLogical, (x, n) => x >>> n); - yield return Create(TensorPrimitives.RotateLeft, T.RotateLeft); - yield return Create(TensorPrimitives.RotateRight, T.RotateRight); - - static object[] Create(SpanScalarDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.ShiftLeft), new Func((x, n) => x << n) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.ShiftRightArithmetic), new Func((x, n) => x >> n) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.ShiftRightLogical), new Func((x, n) => x >>> n) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.RotateLeft), new Func(T.RotateLeft) }; + yield return new object[] { new SpanScalarDestinationDelegate(TensorPrimitives.RotateRight), new Func(T.RotateRight) }; } [Theory] @@ -2067,7 +1996,6 @@ public void CopySign_ThrowsForOverlapppingInputsWithOutputs() public unsafe abstract class GenericNumberTensorPrimitivesTests : TensorPrimitivesTests where T : unmanaged, INumber, IMinMaxValue { - protected static bool IsFmaSupported => Fma.IsSupported || AdvSimd.Arm64.IsSupported || (AdvSimd.IsSupported && typeof(T) == typeof(float)); protected override void Abs(ReadOnlySpan x, Span destination) => TensorPrimitives.Abs(x, destination); protected override T Abs(T x) => T.Abs(x); protected override void Add(ReadOnlySpan x, ReadOnlySpan y, Span destination) => TensorPrimitives.Add(x, y, destination); @@ -2143,7 +2071,10 @@ protected override T NextRandom() protected override void AssertEqualTolerance(T expected, T actual, T? tolerance = null) { - if (!Helpers.IsEqualWithTolerance(expected, actual, tolerance)) + tolerance ??= T.CreateTruncating(0.0001); + + T diff = T.Abs(expected - actual); + if (diff > tolerance && diff > T.Max(T.Abs(expected), T.Abs(actual)) * tolerance) { throw EqualException.ForMismatchedValues(expected, actual); } @@ -2174,11 +2105,8 @@ protected override void SetSpecialValues(Span x, Span y) { } #region Scalar,Span -> Destination public static IEnumerable ScalarSpanDestinationFunctionsToTest() { - yield return Create(TensorPrimitives.Divide, (x, y) => x / y); - yield return Create(TensorPrimitives.Subtract, (x, y) => x - y); - - static object[] Create(ScalarSpanDestinationDelegate tensorPrimitivesMethod, Func expectedMethod) - => new object[] { tensorPrimitivesMethod, expectedMethod }; + yield return new object[] { new ScalarSpanDestinationDelegate(TensorPrimitives.Divide), new Func((x, y) => x / y) }; + yield return new object[] { new ScalarSpanDestinationDelegate(TensorPrimitives.Subtract), new Func((x, y) => x - y) }; } [Theory] diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.NonGeneric.Single.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.NonGeneric.Single.cs index d6df8365c59db6..ac883851299d8f 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.NonGeneric.Single.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitives.NonGeneric.Single.cs @@ -106,7 +106,10 @@ protected override float MinMagnitude(float x, float y) protected override void AssertEqualTolerance(float expected, float actual, float? tolerance = null) { - if (!Helpers.IsEqualWithTolerance(expected, actual, tolerance)) + tolerance ??= 0.0001f; + + double diff = Math.Abs((double)expected - (double)actual); + if (diff > tolerance && diff > Math.Max(Math.Abs(expected), Math.Abs(actual)) * tolerance) { throw EqualException.ForMismatchedValues(expected, actual); } diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index 173e57649cf8ee..b0b44fddb0aef5 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -663,9 +663,6 @@ public void Cosh_ValueRange() { if (!IsFloatingPoint) return; - // TODO https://github.com/dotnet/runtime/issues/98861 - T? tolerance = Helpers.DetermineTolerance(doubleTolerance: 1e-14); - Assert.All(VectorLengthAndIteratedRange(ConvertFromSingle(-100f), ConvertFromSingle(100f), ConvertFromSingle(3f)), arg => { T[] x = new T[arg.Length]; @@ -677,7 +674,7 @@ public void Cosh_ValueRange() T expected = Cosh(arg.Element); foreach (T actual in dest) { - AssertEqualTolerance(expected, actual, tolerance); + AssertEqualTolerance(expected, actual); } }); } @@ -955,9 +952,6 @@ public void Dot_ThrowsForMismatchedLengths_x_y() [Fact] public void Dot_AllLengths() { - // TODO https://github.com/dotnet/runtime/issues/98861 - T? tolerance = Helpers.DetermineTolerance(doubleTolerance: 1e-14f, floatTolerance: 1e-3f); - Assert.All(Helpers.TensorLengthsIncluding0, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -969,7 +963,7 @@ public void Dot_AllLengths() dot = Add(dot, Multiply(x[i], y[i])); } - AssertEqualTolerance(dot, Dot(x, y), tolerance); + AssertEqualTolerance(dot, Dot(x, y)); }); } #endregion @@ -2885,9 +2879,6 @@ public void Sinh_ValueRange() { if (!IsFloatingPoint) return; - // TODO https://github.com/dotnet/runtime/issues/98861 - T? tolerance = Helpers.DetermineTolerance(doubleTolerance: 1e-14); - Assert.All(VectorLengthAndIteratedRange(ConvertFromSingle(-100f), ConvertFromSingle(100f), ConvertFromSingle(3f)), args => { T[] x = new T[args.Length]; @@ -2899,7 +2890,7 @@ public void Sinh_ValueRange() T expected = Sinh(args.Element); foreach (T actual in dest) { - AssertEqualTolerance(expected, actual, tolerance); + AssertEqualTolerance(expected, actual); } }); } @@ -3148,9 +3139,6 @@ public void Subtract_TensorScalar_ThrowsForOverlapppingInputsWithOutputs() [Fact] public void Sum_AllLengths() { - // TODO https://github.com/dotnet/runtime/issues/98861 - T? tolerance = Helpers.DetermineTolerance(doubleTolerance: 1e-13, floatTolerance: 1e-5f); - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -3160,7 +3148,7 @@ public void Sum_AllLengths() { sum = Add(sum, value); } - AssertEqualTolerance(sum, Sum(x), tolerance); + AssertEqualTolerance(sum, Sum(x)); }); } #endregion @@ -3169,9 +3157,6 @@ public void Sum_AllLengths() [Fact] public void SumOfMagnitudes_AllLengths() { - // TODO https://github.com/dotnet/runtime/issues/98861 - T? tolerance = Helpers.DetermineTolerance(doubleTolerance: 1e-12, floatTolerance: 1e-6f); - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateTensor(tensorLength); @@ -3182,7 +3167,7 @@ public void SumOfMagnitudes_AllLengths() { sum = Add(sum, Abs(value)); } - AssertEqualTolerance(sum, SumOfMagnitudes(x), tolerance); + AssertEqualTolerance(sum, SumOfMagnitudes(x)); }); } #endregion @@ -3191,9 +3176,6 @@ public void SumOfMagnitudes_AllLengths() [Fact] public void SumOfSquares_AllLengths() { - // TODO https://github.com/dotnet/runtime/issues/98861 - T? tolerance = Helpers.DetermineTolerance(doubleTolerance: 1e-12, floatTolerance: 1e-6f); - Assert.All(Helpers.TensorLengths, tensorLength => { using BoundedMemory x = CreateAndFillTensor(tensorLength); @@ -3203,7 +3185,7 @@ public void SumOfSquares_AllLengths() { sum = Add(sum, Multiply(value, value)); } - AssertEqualTolerance(sum, SumOfSquares(x), tolerance); + AssertEqualTolerance(sum, SumOfSquares(x)); }); } #endregion diff --git a/src/libraries/System.Private.CoreLib/gen/IntrinsicsInSystemPrivateCoreLibAnalyzer.cs b/src/libraries/System.Private.CoreLib/gen/IntrinsicsInSystemPrivateCoreLibAnalyzer.cs index f9a5eff99648cb..75f702eaca8edf 100644 --- a/src/libraries/System.Private.CoreLib/gen/IntrinsicsInSystemPrivateCoreLibAnalyzer.cs +++ b/src/libraries/System.Private.CoreLib/gen/IntrinsicsInSystemPrivateCoreLibAnalyzer.cs @@ -293,12 +293,6 @@ private static INamedTypeSymbol[][] DecomposePropertySymbolForIsSupportedGroups_ if (propertyDefiningSyntax is PropertyDeclarationSyntax propertyDeclaration && propertyDeclaration.ExpressionBody is ArrowExpressionClauseSyntax arrowExpression) { - if (model.SyntaxTree != arrowExpression.SyntaxTree) - { -#pragma warning disable RS1030 - model = model.Compilation.GetSemanticModel(arrowExpression.SyntaxTree); -#pragma warning restore RS1030 - } return DecomposeConditionForIsSupportedGroups(context, model, arrowExpression.Expression); } } diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index 2f58fae3471e9e..b6d8fd00aa4049 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -3346,7 +3346,7 @@ Object type {0} does not match target type {1}. - + Non-static field requires a target. @@ -4304,7 +4304,10 @@ Cannot set initonly static field '{0}' after type '{1}' is initialized. + + This AssemblyBuilder instance doesn't support saving. Use AssemblyBuilder.DefinePersistedAssembly to create an AssemblyBuilder instance that supports saving. + Blocking wait is not supported on the JS interop threads. - \ No newline at end of file + diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 0a2eb4515d7589..a73e8247a58e72 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -268,8 +268,6 @@ - - @@ -874,7 +872,6 @@ - @@ -1031,7 +1028,6 @@ - @@ -1088,7 +1084,6 @@ - @@ -1123,7 +1118,6 @@ - @@ -1525,7 +1519,6 @@ - @@ -1539,7 +1532,6 @@ - @@ -2148,6 +2140,7 @@ + @@ -2465,6 +2458,7 @@ + @@ -2766,4 +2760,4 @@ - + \ No newline at end of file diff --git a/src/libraries/System.Private.CoreLib/src/System/Array.cs b/src/libraries/System.Private.CoreLib/src/System/Array.cs index 84bd5ed20eed1f..c21caa8cc1d70a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Array.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Array.cs @@ -58,7 +58,7 @@ public static void Resize([NotNull] ref T[]? array, int newSize) // actually of type U[], where U:T; or that an int[] <-> uint[] or // similar cast has occurred. In any case, since it's always legal // to reinterpret U as T in this scenario (but not necessarily the - // other way around), we can use SpanHelpers.Memmove here. + // other way around), we can use Buffer.Memmove here. T[] newArray = new T[newSize]; Buffer.Memmove( @@ -377,7 +377,7 @@ public static unsafe void Copy(Array sourceArray, Array destinationArray, int le if (pMT->ContainsGCPointers) Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount); else - SpanHelpers.Memmove(ref dst, ref src, byteCount); + Buffer.Memmove(ref dst, ref src, byteCount); // GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray return; @@ -408,7 +408,7 @@ public static unsafe void Copy(Array sourceArray, int sourceIndex, Array destina if (pMT->ContainsGCPointers) Buffer.BulkMoveWithWriteBarrier(ref dst, ref src, byteCount); else - SpanHelpers.Memmove(ref dst, ref src, byteCount); + Buffer.Memmove(ref dst, ref src, byteCount); // GC.KeepAlive(sourceArray) not required. pMT kept alive via sourceArray return; diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.Unix.cs new file mode 100644 index 00000000000000..008bc9310a2417 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Buffer.Unix.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System +{ + public static partial class Buffer + { +#if TARGET_ARM64 || TARGET_LOONGARCH64 + // Managed code is currently faster than glibc unoptimized memmove + // TODO-ARM64-UNIX-OPT revisit when glibc optimized memmove is in Linux distros + // https://github.com/dotnet/runtime/issues/8897 + private static nuint MemmoveNativeThreshold => nuint.MaxValue; +#elif TARGET_ARM + private const nuint MemmoveNativeThreshold = 512; +#else + private const nuint MemmoveNativeThreshold = 2048; +#endif + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.Windows.cs new file mode 100644 index 00000000000000..4dea08790b91a1 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Buffer.Windows.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System +{ + public static partial class Buffer + { +#if TARGET_ARM64 + // Determine optimal value for Windows. + // https://github.com/dotnet/runtime/issues/8896 + private static nuint MemmoveNativeThreshold => nuint.MaxValue; +#else + private const nuint MemmoveNativeThreshold = 2048; +#endif + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs index 24f8794d852afd..51ec733aaef590 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs @@ -1,9 +1,14 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#if TARGET_AMD64 || TARGET_ARM64 || (TARGET_32BIT && !TARGET_ARM) || TARGET_LOONGARCH64 +#define HAS_CUSTOM_BLOCKS +#endif + using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; namespace System { @@ -123,16 +128,227 @@ public static unsafe void MemoryCopy(void* source, void* destination, ulong dest Memmove(ref *(byte*)destination, ref *(byte*)source, checked((nuint)sourceBytesToCopy)); } + [Intrinsic] // Unrolled for small constant lengths + internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) + { + // P/Invoke into the native version when the buffers are overlapping. + if (((nuint)(nint)Unsafe.ByteOffset(ref src, ref dest) < len) || ((nuint)(nint)Unsafe.ByteOffset(ref dest, ref src) < len)) + { + goto BuffersOverlap; + } + + // Use "(IntPtr)(nint)len" to avoid overflow checking on the explicit cast to IntPtr + + ref byte srcEnd = ref Unsafe.Add(ref src, (IntPtr)(nint)len); + ref byte destEnd = ref Unsafe.Add(ref dest, (IntPtr)(nint)len); + + if (len <= 16) + goto MCPY02; + if (len > 64) + goto MCPY05; + + MCPY00: + // Copy bytes which are multiples of 16 and leave the remainder for MCPY01 to handle. + Debug.Assert(len > 16 && len <= 64); +#if HAS_CUSTOM_BLOCKS + Unsafe.As(ref dest) = Unsafe.As(ref src); // [0,16] +#elif TARGET_64BIT + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); // [0,16] +#else + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); + Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); + Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); // [0,16] +#endif + if (len <= 32) + goto MCPY01; +#if HAS_CUSTOM_BLOCKS + Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); // [0,32] +#elif TARGET_64BIT + Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); + Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); // [0,32] +#else + Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); + Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); + Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); + Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); // [0,32] +#endif + if (len <= 48) + goto MCPY01; +#if HAS_CUSTOM_BLOCKS + Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); // [0,48] +#elif TARGET_64BIT + Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); + Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); // [0,48] +#else + Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); + Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); + Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); + Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); // [0,48] +#endif + + MCPY01: + // Unconditionally copy the last 16 bytes using destEnd and srcEnd and return. + Debug.Assert(len > 16 && len <= 64); +#if HAS_CUSTOM_BLOCKS + Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); +#elif TARGET_64BIT + Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); +#else + Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); +#endif + return; + + MCPY02: + // Copy the first 8 bytes and then unconditionally copy the last 8 bytes and return. + if ((len & 24) == 0) + goto MCPY03; + Debug.Assert(len >= 8 && len <= 16); +#if TARGET_64BIT + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); +#else + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); +#endif + return; + + MCPY03: + // Copy the first 4 bytes and then unconditionally copy the last 4 bytes and return. + if ((len & 4) == 0) + goto MCPY04; + Debug.Assert(len >= 4 && len < 8); + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); + return; + + MCPY04: + // Copy the first byte. For pending bytes, do an unconditionally copy of the last 2 bytes and return. + Debug.Assert(len < 4); + if (len == 0) + return; + dest = src; + if ((len & 2) == 0) + return; + Unsafe.As(ref Unsafe.Add(ref destEnd, -2)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -2)); + return; + + MCPY05: + // PInvoke to the native version when the copy length exceeds the threshold. + if (len > MemmoveNativeThreshold) + { + goto PInvoke; + } + +#if HAS_CUSTOM_BLOCKS + if (len >= 256) + { + // Try to opportunistically align the destination below. The input isn't pinned, so the GC + // is free to move the references. We're therefore assuming that reads may still be unaligned. + // + // dest is more important to align than src because an unaligned store is more expensive + // than an unaligned load. + nuint misalignedElements = 64 - (nuint)Unsafe.AsPointer(ref dest) & 63; + Unsafe.As(ref dest) = Unsafe.As(ref src); + src = ref Unsafe.Add(ref src, misalignedElements); + dest = ref Unsafe.Add(ref dest, misalignedElements); + len -= misalignedElements; + } +#endif + + // Copy 64-bytes at a time until the remainder is less than 64. + // If remainder is greater than 16 bytes, then jump to MCPY00. Otherwise, unconditionally copy the last 16 bytes and return. + Debug.Assert(len > 64 && len <= MemmoveNativeThreshold); + nuint n = len >> 6; + + MCPY06: +#if HAS_CUSTOM_BLOCKS + Unsafe.As(ref dest) = Unsafe.As(ref src); +#elif TARGET_64BIT + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); + Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); + Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); + Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); + Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); + Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); + Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); +#else + Unsafe.As(ref dest) = Unsafe.As(ref src); + Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); + Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); + Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); + Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); + Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); + Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); + Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); + Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); + Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); + Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); + Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); + Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); + Unsafe.As(ref Unsafe.Add(ref dest, 52)) = Unsafe.As(ref Unsafe.Add(ref src, 52)); + Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); + Unsafe.As(ref Unsafe.Add(ref dest, 60)) = Unsafe.As(ref Unsafe.Add(ref src, 60)); +#endif + dest = ref Unsafe.Add(ref dest, 64); + src = ref Unsafe.Add(ref src, 64); + n--; + if (n != 0) + goto MCPY06; + + len %= 64; + if (len > 16) + goto MCPY00; +#if HAS_CUSTOM_BLOCKS + Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); +#elif TARGET_64BIT + Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); +#else + Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); + Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); +#endif + return; + + BuffersOverlap: + // If the buffers overlap perfectly, there's no point to copying the data. + if (Unsafe.AreSame(ref dest, ref src)) + { + return; + } + + PInvoke: + _Memmove(ref dest, ref src, len); + } + // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)] - internal static unsafe void _Memmove(ref byte dest, ref byte src, nuint len) + private static unsafe void _Memmove(ref byte dest, ref byte src, nuint len) { fixed (byte* pDest = &dest) fixed (byte* pSrc = &src) __Memmove(pDest, pSrc, len); } +#if HAS_CUSTOM_BLOCKS + [StructLayout(LayoutKind.Sequential, Size = 16)] + private struct Block16 { } + + [StructLayout(LayoutKind.Sequential, Size = 64)] + private struct Block64 { } +#endif // HAS_CUSTOM_BLOCKS + // Non-inlinable wrapper around the QCall that avoids polluting the fast path // with P/Invoke prolog/epilog. [MethodImpl(MethodImplOptions.NoInlining)] @@ -154,7 +370,7 @@ internal static unsafe void Memmove(ref T destination, ref T source, nuint el if (!RuntimeHelpers.IsReferenceOrContainsReferences()) { // Blittable memmove - SpanHelpers.Memmove( + Memmove( ref Unsafe.As(ref destination), ref Unsafe.As(ref source), elementCount * (nuint)sizeof(T)); @@ -185,6 +401,7 @@ internal static void BulkMoveWithWriteBarrier(ref byte destination, ref byte sou _BulkMoveWithWriteBarrier(ref destination, ref source, byteCount); } +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 // Non-inlinable wrapper around the loop for copying large blocks in chunks [MethodImpl(MethodImplOptions.NoInlining)] private static void _BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) @@ -219,6 +436,7 @@ private static void _BulkMoveWithWriteBarrier(ref byte destination, ref byte sou } __BulkMoveWithWriteBarrier(ref destination, ref source, byteCount); } +#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 #endif // !MONO } diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs index b63c711e410326..08ca62b533f510 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs @@ -85,15 +85,6 @@ public static unsafe OperationStatus EncodeToUtf8(ReadOnlySpan bytes, Span goto DoneExit; } - end = srcMax - 48; - if (AdvSimd.Arm64.IsSupported && (end >= src)) - { - AdvSimdEncode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - goto DoneExit; - } - end = srcMax - 16; if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian && (end >= src)) { @@ -489,64 +480,6 @@ private static unsafe void Avx2Encode(ref byte* srcBytes, ref byte* destBytes, b destBytes = dest; } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - private static unsafe void AdvSimdEncode(ref byte* srcBytes, ref byte* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, byte* destStart) - { - // C# implementation of https://github.com/aklomp/base64/blob/3a5add8652076612a8407627a42c768736a4263f/lib/arch/neon64/enc_loop.c - Vector128 str1; - Vector128 str2; - Vector128 str3; - Vector128 res1; - Vector128 res2; - Vector128 res3; - Vector128 res4; - Vector128 tblEnc1 = Vector128.Create("ABCDEFGHIJKLMNOP"u8).AsByte(); - Vector128 tblEnc2 = Vector128.Create("QRSTUVWXYZabcdef"u8).AsByte(); - Vector128 tblEnc3 = Vector128.Create("ghijklmnopqrstuv"u8).AsByte(); - Vector128 tblEnc4 = Vector128.Create("wxyz0123456789+/"u8).AsByte(); - byte* src = srcBytes; - byte* dest = destBytes; - - // If we have Neon support, pick off 48 bytes at a time for as long as we can. - do - { - // Load 48 bytes and deinterleave: - AssertRead>(src, srcStart, sourceLength); - (str1, str2, str3) = AdvSimd.Arm64.LoadVector128x3AndUnzip(src); - - // Divide bits of three input bytes over four output bytes: - res1 = AdvSimd.ShiftRightLogical(str1, 2); - res2 = AdvSimd.ShiftRightLogical(str2, 4); - res3 = AdvSimd.ShiftRightLogical(str3, 6); - res2 = AdvSimd.ShiftLeftAndInsert(res2, str1, 4); - res3 = AdvSimd.ShiftLeftAndInsert(res3, str2, 2); - - // Clear top two bits: - res2 &= AdvSimd.DuplicateToVector128((byte)0x3F); - res3 &= AdvSimd.DuplicateToVector128((byte)0x3F); - res4 = str3 & AdvSimd.DuplicateToVector128((byte)0x3F); - - // The bits have now been shifted to the right locations; - // translate their values 0..63 to the Base64 alphabet. - // Use a 64-byte table lookup: - res1 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res1); - res2 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res2); - res3 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res3); - res4 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res4); - - // Interleave and store result: - AssertWrite>(dest, destStart, destLength); - AdvSimd.Arm64.StoreVector128x4AndZip(dest, (res1, res2, res3, res4)); - - src += 48; - dest += 64; - } while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Ssse3))] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] diff --git a/src/libraries/System.Private.CoreLib/src/System/DateTime.cs b/src/libraries/System.Private.CoreLib/src/System/DateTime.cs index 3eeaaabbd358a4..efcf7155c0f1cb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/DateTime.cs +++ b/src/libraries/System.Private.CoreLib/src/System/DateTime.cs @@ -332,7 +332,6 @@ public DateTime(int year, int month, int day, int hour, int minute, int second) else { // if we have a leap second, then we adjust it to 59 so that DateTime will consider it the last in the specified minute. - // codeql[cs/leap-year/unsafe-date-construction-from-two-elements] - DateTime is constructed using the user specified values, not a combination of different sources. It would be intentional to throw if an invalid combination occurred. this = new DateTime(year, month, day, hour, minute, 59); ValidateLeapSecond(); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/FeatureGuardAttribute.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/FeatureGuardAttribute.cs deleted file mode 100644 index f0ac084e94b0e0..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/FeatureGuardAttribute.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Diagnostics.CodeAnalysis -{ - /// - /// Indicates that the specified public static boolean get-only property - /// guards access to the specified feature. - /// - /// - /// Analyzers can use this to prevent warnings on calls to code that is - /// annotated as requiring that feature, when the callsite is guarded by a - /// call to the property. - /// - [AttributeUsage(AttributeTargets.Property, Inherited = false, AllowMultiple = true)] -#if SYSTEM_PRIVATE_CORELIB - public -#else - internal -#endif - sealed class FeatureGuardAttribute : Attribute - { - /// - /// Initializes a new instance of the class - /// with the specified feature type. - /// - /// - /// The type that represents the feature guarded by the property. - /// - public FeatureGuardAttribute(Type featureType) - { - FeatureType = featureType; - } - - /// - /// The type that represents the feature guarded by the property. - /// - public Type FeatureType { get; } - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/FeatureSwitchDefinitionAttribute.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/FeatureSwitchDefinitionAttribute.cs deleted file mode 100644 index 2089d87ef5d06f..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/CodeAnalysis/FeatureSwitchDefinitionAttribute.cs +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Diagnostics.CodeAnalysis -{ - /// - /// Indicates that the specified public static boolean get-only property - /// corresponds to the feature switch specified by name. - /// - /// - /// IL rewriters and compilers can use this to substitute the return value - /// of the specified property with the value of the feature switch. - /// - [AttributeUsage(AttributeTargets.Property, Inherited = false)] -#if SYSTEM_PRIVATE_CORELIB - public -#else - internal -#endif - sealed class FeatureSwitchDefinitionAttribute : Attribute - { - /// - /// Initializes a new instance of the class - /// with the specified feature switch name. - /// - /// - /// The name of the feature switch that provides the value for the specified property. - /// - public FeatureSwitchDefinitionAttribute(string switchName) - { - SwitchName = switchName; - } - - /// - /// The name of the feature switch that provides the value for the specified property. - /// - public string SwitchName { get; } - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.cs index 971451981044f8..74acfef13fccfe 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipe.cs @@ -5,6 +5,8 @@ using System.Runtime.InteropServices; using System.Threading; +#if FEATURE_PERFTRACING + namespace System.Diagnostics.Tracing { [StructLayout(LayoutKind.Sequential)] @@ -148,3 +150,5 @@ internal static unsafe ulong Enable( } } } + +#endif // FEATURE_PERFTRACING diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeEventDispatcher.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeEventDispatcher.cs index 548f792b524311..030560b2002145 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeEventDispatcher.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeEventDispatcher.cs @@ -7,6 +7,7 @@ namespace System.Diagnostics.Tracing { +#if FEATURE_PERFTRACING internal sealed class EventPipeEventDispatcher { internal sealed class EventListenerSubscription @@ -226,4 +227,5 @@ private static DateTime TimeStampToDateTime(long timeStamp, DateTime syncTimeUtc return new DateTime(inTicks, DateTimeKind.Utc); } } +#endif // FEATURE_PERFTRACING } diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeMetadataGenerator.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeMetadataGenerator.cs index ed19ea2178e38d..4ffa0a5895d429 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeMetadataGenerator.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipeMetadataGenerator.cs @@ -5,6 +5,7 @@ namespace System.Diagnostics.Tracing { +#if FEATURE_PERFTRACING internal sealed class EventPipeMetadataGenerator { private enum MetadataTag @@ -760,4 +761,6 @@ private static bool GetMetadataLengthForNamedTypeV2(string name, TraceLoggingTyp return true; } } + +#endif // FEATURE_PERFTRACING } diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipePayloadDecoder.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipePayloadDecoder.cs index 661910025d8b1b..ae972715684588 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipePayloadDecoder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventPipePayloadDecoder.cs @@ -7,6 +7,7 @@ namespace System.Diagnostics.Tracing { +#if FEATURE_PERFTRACING internal static class EventPipePayloadDecoder { /// @@ -137,4 +138,5 @@ internal static object[] DecodePayload(ref EventSource.EventMetadata metadata, R return decodedFields; } } +#endif // FEATURE_PERFTRACING } diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource.cs index 61c7d5218bc67a..7a292e019fddac 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/EventSource.cs @@ -1120,14 +1120,6 @@ protected unsafe void WriteEvent(int eventId, long arg1, byte[]? arg2) } } - // Returns the object as a IntPtr - safe when only used for logging - internal static unsafe nint ObjectIDForEvents(object? o) - { -#pragma warning disable CS8500 // takes address of managed type - return *(nint*)&o; -#pragma warning restore CS8500 - } - #pragma warning restore 1591 /// @@ -5498,7 +5490,7 @@ private string CreateManifestString() if (channelInfo.Attribs != null) { EventChannelAttribute attribs = channelInfo.Attribs; - if (Enum.IsDefined(attribs.EventChannelType)) + if (Enum.IsDefined(typeof(EventChannelType), attribs.EventChannelType)) channelType = attribs.EventChannelType.ToString(); enabled = attribs.Enabled; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs index 432540c79ff151..5ba348edd7fe20 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.NativeSinks.cs @@ -94,7 +94,7 @@ private void ContentionLockCreated(nint LockID, nint AssociatedObjectID, ushort [NonEvent] [MethodImpl(MethodImplOptions.NoInlining)] - public void ContentionLockCreated(Lock lockObj) => ContentionLockCreated(lockObj.LockIdForEvents, ObjectIDForEvents(lockObj)); + public void ContentionLockCreated(Lock lockObj) => ContentionLockCreated(lockObj.LockIdForEvents, lockObj.ObjectIdForEvents); [Event(81, Level = EventLevel.Informational, Message = Messages.ContentionStart, Task = Tasks.Contention, Opcode = EventOpcode.Start, Version = 2, Keywords = Keywords.ContentionKeyword)] private void ContentionStart( @@ -115,7 +115,7 @@ public void ContentionStart(Lock lockObj) => ContentionFlagsMap.Managed, DefaultClrInstanceId, lockObj.LockIdForEvents, - ObjectIDForEvents(lockObj), + lockObj.ObjectIdForEvents, lockObj.OwningThreadId); [Event(91, Level = EventLevel.Informational, Message = Messages.ContentionStop, Task = Tasks.Contention, Opcode = EventOpcode.Stop, Version = 1, Keywords = Keywords.ContentionKeyword)] @@ -360,7 +360,7 @@ private void WaitHandleWaitStart( public unsafe void WaitHandleWaitStart( WaitHandleWaitSourceMap waitSource = WaitHandleWaitSourceMap.Unknown, object? associatedObject = null) => - WaitHandleWaitStart(waitSource, ObjectIDForEvents(associatedObject)); + WaitHandleWaitStart(waitSource, *(nint*)Unsafe.AsPointer(ref associatedObject)); [Event(302, Level = EventLevel.Verbose, Message = Messages.WaitHandleWaitStop, Task = Tasks.WaitHandleWait, Opcode = EventOpcode.Stop, Version = 0, Keywords = Keywords.WaitHandleKeyword)] public void WaitHandleWaitStop(ushort ClrInstanceID = DefaultClrInstanceId) diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs index 8c85770e1f0aa2..11c4dfcfe6c539 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/NativeRuntimeEventSource.Threading.cs @@ -107,7 +107,7 @@ private unsafe void ContentionLockCreated(nint LockID, nint AssociatedObjectID, [NonEvent] [MethodImpl(MethodImplOptions.NoInlining)] - public void ContentionLockCreated(Lock lockObj) => ContentionLockCreated(lockObj.LockIdForEvents, ObjectIDForEvents(lockObj)); + public void ContentionLockCreated(Lock lockObj) => ContentionLockCreated(lockObj.LockIdForEvents, lockObj.ObjectIdForEvents); [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:UnrecognizedReflectionPattern", Justification = "Parameters to this method are primitive and are trimmer safe")] [Event(81, Level = EventLevel.Informational, Message = Messages.ContentionStart, Task = Tasks.Contention, Opcode = EventOpcode.Start, Version = 2, Keywords = Keywords.ContentionKeyword)] @@ -146,7 +146,7 @@ public void ContentionStart(Lock lockObj) => ContentionFlagsMap.Managed, DefaultClrInstanceId, lockObj.LockIdForEvents, - ObjectIDForEvents(lockObj), + lockObj.ObjectIdForEvents, lockObj.OwningThreadId); [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:UnrecognizedReflectionPattern", Justification = "Parameters to this method are primitive and are trimmer safe")] @@ -557,7 +557,7 @@ private unsafe void WaitHandleWaitStart( public unsafe void WaitHandleWaitStart( WaitHandleWaitSourceMap waitSource = WaitHandleWaitSourceMap.Unknown, object? associatedObject = null) => - WaitHandleWaitStart(waitSource, ObjectIDForEvents(associatedObject)); + WaitHandleWaitStart(waitSource, *(nint*)Unsafe.AsPointer(ref associatedObject)); [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026:UnrecognizedReflectionPattern", Justification = "Parameters to this method are primitive and are trimmer safe")] [Event(302, Level = EventLevel.Verbose, Message = Messages.WaitHandleWaitStop, Task = Tasks.WaitHandleWait, Opcode = EventOpcode.Stop, Version = 0, Keywords = Keywords.WaitHandleKeyword)] diff --git a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/TraceLogging/XplatEventLogger.cs b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/TraceLogging/XplatEventLogger.cs index 17da3860ec2b01..25e19b88db5263 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/TraceLogging/XplatEventLogger.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Diagnostics/Tracing/TraceLogging/XplatEventLogger.cs @@ -1,10 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using System.Runtime.Versioning; using System.Text; #if FEATURE_EVENTSOURCE_XPLAT @@ -15,13 +18,8 @@ internal sealed partial class XplatEventLogger : EventListener { public XplatEventLogger() { } - private static readonly string s_eventSourceNameFilter = GetClrConfig("EventSourceFilter"); - private static readonly string s_eventSourceEventFilter = GetClrConfig("EventNameFilter"); - - private static unsafe string GetClrConfig(string configName) => new string(EventSource_GetClrConfig(configName)); - - [LibraryImport(RuntimeHelpers.QCall, StringMarshalling = StringMarshalling.Utf16)] - private static unsafe partial char* EventSource_GetClrConfig(string configName); + private static readonly Lazy eventSourceNameFilter = new Lazy(() => CompatibilitySwitch.GetValueInternal("EventSourceFilter")); + private static readonly Lazy eventSourceEventFilter = new Lazy(() => CompatibilitySwitch.GetValueInternal("EventNameFilter")); private static bool initializedPersistentListener; @@ -155,7 +153,9 @@ private static void AppendByteArrayAsHexString(ref ValueStringBuilder builder, b protected internal override void OnEventSourceCreated(EventSource eventSource) { - if (string.IsNullOrEmpty(s_eventSourceNameFilter) || (eventSource.Name.Contains(s_eventSourceNameFilter, StringComparison.OrdinalIgnoreCase))) + + string? eventSourceFilter = eventSourceNameFilter.Value; + if (string.IsNullOrEmpty(eventSourceFilter) || (eventSource.Name.Contains(eventSourceFilter, StringComparison.OrdinalIgnoreCase))) { EnableEvents(eventSource, EventLevel.LogAlways, EventKeywords.All, null); } @@ -173,7 +173,8 @@ protected internal override void OnEventWritten(EventWrittenEventArgs eventData) return; } - if (string.IsNullOrEmpty(s_eventSourceEventFilter) || (eventData.EventName!.Contains(s_eventSourceEventFilter, StringComparison.OrdinalIgnoreCase))) + string? eventFilter = eventSourceEventFilter.Value; + if (string.IsNullOrEmpty(eventFilter) || (eventData.EventName!.Contains(eventFilter, StringComparison.OrdinalIgnoreCase))) { LogOnEventWritten(eventData); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.Icu.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.Icu.cs index 26919ba0d50a8c..4d3314f22ec592 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.Icu.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/CalendarData.Icu.cs @@ -135,16 +135,14 @@ private static unsafe bool GetCalendarInfo(string localeName, CalendarId calenda out calendarString); } - private static unsafe bool EnumDatePatterns(string localeName, CalendarId calendarId, CalendarDataType dataType, out string[]? datePatterns) + private static bool EnumDatePatterns(string localeName, CalendarId calendarId, CalendarDataType dataType, out string[]? datePatterns) { datePatterns = null; IcuEnumCalendarsData callbackContext = default; callbackContext.Results = new List(); callbackContext.DisallowDuplicates = true; -#pragma warning disable CS8500 // takes address of managed type - bool result = EnumCalendarInfo(localeName, calendarId, dataType, &callbackContext); -#pragma warning restore CS8500 + bool result = EnumCalendarInfo(localeName, calendarId, dataType, ref callbackContext); if (result) { List datePatternsList = callbackContext.Results; @@ -364,15 +362,13 @@ private static int CountOccurrences(string input, char value, ref int index) return index - startIndex; } - private static unsafe bool EnumMonthNames(string localeName, CalendarId calendarId, CalendarDataType dataType, out string[]? monthNames, ref string? leapHebrewMonthName) + private static bool EnumMonthNames(string localeName, CalendarId calendarId, CalendarDataType dataType, out string[]? monthNames, ref string? leapHebrewMonthName) { monthNames = null; IcuEnumCalendarsData callbackContext = default; callbackContext.Results = new List(); -#pragma warning disable CS8500 // takes address of managed type - bool result = EnumCalendarInfo(localeName, calendarId, dataType, &callbackContext); -#pragma warning restore CS8500 + bool result = EnumCalendarInfo(localeName, calendarId, dataType, ref callbackContext); if (result) { // the month-name arrays are expected to have 13 elements. If ICU only returns 12, add an @@ -414,15 +410,13 @@ private static bool EnumEraNames(string localeName, CalendarId calendarId, Calen return result; } - internal static unsafe bool EnumCalendarInfo(string localeName, CalendarId calendarId, CalendarDataType dataType, out string[]? calendarData) + internal static bool EnumCalendarInfo(string localeName, CalendarId calendarId, CalendarDataType dataType, out string[]? calendarData) { calendarData = null; IcuEnumCalendarsData callbackContext = default; callbackContext.Results = new List(); -#pragma warning disable CS8500 // takes address of managed type - bool result = EnumCalendarInfo(localeName, calendarId, dataType, &callbackContext); -#pragma warning restore CS8500 + bool result = EnumCalendarInfo(localeName, calendarId, dataType, ref callbackContext); if (result) { calendarData = callbackContext.Results.ToArray(); @@ -431,12 +425,10 @@ internal static unsafe bool EnumCalendarInfo(string localeName, CalendarId calen return result; } -#pragma warning disable CS8500 // takes address of managed type - private static unsafe bool EnumCalendarInfo(string localeName, CalendarId calendarId, CalendarDataType dataType, IcuEnumCalendarsData* callbackContext) + private static unsafe bool EnumCalendarInfo(string localeName, CalendarId calendarId, CalendarDataType dataType, ref IcuEnumCalendarsData callbackContext) { - return Interop.Globalization.EnumCalendarInfo(&EnumCalendarInfoCallback, localeName, calendarId, dataType, (IntPtr)callbackContext); + return Interop.Globalization.EnumCalendarInfo(&EnumCalendarInfoCallback, localeName, calendarId, dataType, (IntPtr)Unsafe.AsPointer(ref callbackContext)); } -#pragma warning restore CS8500 [UnmanagedCallersOnly] private static unsafe void EnumCalendarInfoCallback(char* calendarStringPtr, IntPtr context) diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/GregorianCalendarHelper.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/GregorianCalendarHelper.cs index 04298c12e7f6f7..b100b633e9ac06 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/GregorianCalendarHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/GregorianCalendarHelper.cs @@ -35,7 +35,6 @@ internal EraInfo(int era, int startYear, int startMonth, int startDay, int yearO this.yearOffset = yearOffset; this.minEraYear = minEraYear; this.maxEraYear = maxEraYear; - // codeql[cs/leap-year/unsafe-date-construction-from-two-elements] - A DateTime object is created using values obtained from the machine configuration. this.ticks = new DateTime(startYear, startMonth, startDay).Ticks; this.eraName = eraName; this.abbrevEraName = abbrevEraName; diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/TimeSpanFormat.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/TimeSpanFormat.cs index 8c8bea12bbf6b2..4e11f14c80903b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/TimeSpanFormat.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/TimeSpanFormat.cs @@ -268,11 +268,12 @@ internal static unsafe bool TryFormatStandard(TimeSpan value, StandardFor // Write fraction and separator, if necessary if (fractionDigits != 0) { + Debug.Assert(format == StandardFormat.C || decimalSeparator != null); if (format == StandardFormat.C) { *p++ = TChar.CastFrom('.'); } - else if (decimalSeparator.Length == 1) + else if (decimalSeparator!.Length == 1) { *p++ = decimalSeparator[0]; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Guid.cs b/src/libraries/System.Private.CoreLib/src/System/Guid.cs index 911fb9f7184a30..b6946be5ec578b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Guid.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Guid.cs @@ -841,6 +841,10 @@ private static bool IsHexPrefix(ReadOnlySpan str, int i) => str[i] == '0' && (str[i + 1] | 0x20) == 'x'; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe ReadOnlySpan AsBytes(in Guid source) => + new ReadOnlySpan(Unsafe.AsPointer(ref Unsafe.AsRef(in source)), sizeof(Guid)); + // Returns an unsigned byte array containing the GUID. public byte[] ToByteArray() { @@ -852,7 +856,7 @@ public byte[] ToByteArray() else { // slower path for BigEndian - Guid guid = new Guid(MemoryMarshal.AsBytes(new ReadOnlySpan(in this)), false); + Guid guid = new Guid(AsBytes(this), false); MemoryMarshal.TryWrite(g, in guid); } return g; @@ -870,7 +874,7 @@ public byte[] ToByteArray(bool bigEndian) else { // slower path for Reverse - Guid guid = new Guid(MemoryMarshal.AsBytes(new ReadOnlySpan(in this)), bigEndian); + Guid guid = new Guid(AsBytes(this), bigEndian); MemoryMarshal.TryWrite(g, in guid); } return g; @@ -889,7 +893,7 @@ public bool TryWriteBytes(Span destination) else { // slower path for BigEndian - Guid guid = new Guid(MemoryMarshal.AsBytes(new ReadOnlySpan(in this)), false); + Guid guid = new Guid(AsBytes(this), false); MemoryMarshal.TryWrite(destination, in guid); } return true; @@ -911,7 +915,7 @@ public bool TryWriteBytes(Span destination, bool bigEndian, out int bytesW else { // slower path for Reverse - Guid guid = new Guid(MemoryMarshal.AsBytes(new ReadOnlySpan(in this)), bigEndian); + Guid guid = new Guid(AsBytes(this), bigEndian); MemoryMarshal.TryWrite(destination, in guid); } bytesWritten = 16; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/UnmanagedMemoryStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/UnmanagedMemoryStream.cs index 68adbf72bc6b96..b1b18a2c343135 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/UnmanagedMemoryStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/UnmanagedMemoryStream.cs @@ -390,7 +390,7 @@ internal int ReadCore(Span buffer) try { _buffer.AcquirePointer(ref pointer); - SpanHelpers.Memmove(ref MemoryMarshal.GetReference(buffer), ref *(pointer + pos + _offset), (nuint)nInt); + Buffer.Memmove(ref MemoryMarshal.GetReference(buffer), ref *(pointer + pos + _offset), (nuint)nInt); } finally { @@ -402,7 +402,7 @@ internal int ReadCore(Span buffer) } else { - SpanHelpers.Memmove(ref MemoryMarshal.GetReference(buffer), ref *(_mem + pos), (nuint)nInt); + Buffer.Memmove(ref MemoryMarshal.GetReference(buffer), ref *(_mem + pos), (nuint)nInt); } } @@ -669,7 +669,7 @@ internal unsafe void WriteCore(ReadOnlySpan buffer) try { _buffer.AcquirePointer(ref pointer); - SpanHelpers.Memmove(ref *(pointer + pos + _offset), ref MemoryMarshal.GetReference(buffer), (nuint)buffer.Length); + Buffer.Memmove(ref *(pointer + pos + _offset), ref MemoryMarshal.GetReference(buffer), (nuint)buffer.Length); } finally { @@ -681,7 +681,7 @@ internal unsafe void WriteCore(ReadOnlySpan buffer) } else { - SpanHelpers.Memmove(ref *(_mem + pos), ref MemoryMarshal.GetReference(buffer), (nuint)buffer.Length); + Buffer.Memmove(ref *(_mem + pos), ref MemoryMarshal.GetReference(buffer), (nuint)buffer.Length); } _position = n; diff --git a/src/libraries/System.Private.CoreLib/src/System/Math.cs b/src/libraries/System.Private.CoreLib/src/System/Math.cs index 60a638198f8f60..1d1c50a4e2b552 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Math.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Math.cs @@ -235,16 +235,6 @@ public static long BigMul(long a, long b, out long low) return (long)high - ((a >> 63) & b) - ((b >> 63) & a); } - /// Produces the full product of two 64-bit numbers. - /// The first number to multiply. - /// The second number to multiply. - /// The full product of the specified numbers. - internal static Int128 BigMul(long a, long b) - { - long high = Math.BigMul(a, b, out long low); - return new Int128((ulong)high, (ulong)low); - } - public static double BitDecrement(double x) { ulong bits = BitConverter.DoubleToUInt64Bits(x); diff --git a/src/libraries/System.Private.CoreLib/src/System/Memory.cs b/src/libraries/System.Private.CoreLib/src/System/Memory.cs index 989cac29c57c1f..25e9778d66b538 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Memory.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Memory.cs @@ -398,7 +398,6 @@ public unsafe MemoryHandle Pin() { if (typeof(T) == typeof(char) && tmpObject is string s) { - // Unsafe.AsPointer is safe since the handle pins it GCHandle handle = GCHandle.Alloc(tmpObject, GCHandleType.Pinned); ref char stringData = ref Unsafe.Add(ref s.GetRawStringData(), _index); return new MemoryHandle(Unsafe.AsPointer(ref stringData), handle); @@ -411,13 +410,11 @@ public unsafe MemoryHandle Pin() // Array is already pre-pinned if (_index < 0) { - // Unsafe.AsPointer is safe since it's pinned void* pointer = Unsafe.Add(Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(tmpObject))), _index & ReadOnlyMemory.RemoveFlagsBitMask); return new MemoryHandle(pointer); } else { - // Unsafe.AsPointer is safe since the handle pins it GCHandle handle = GCHandle.Alloc(tmpObject, GCHandleType.Pinned); void* pointer = Unsafe.Add(Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(tmpObject))), _index); return new MemoryHandle(pointer, handle); diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs index fac474c1b36913..4039f244d77bd6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.Globalization.cs @@ -294,7 +294,6 @@ public static int ToUpperInvariant(this ReadOnlySpan source, Span de /// The source span. /// The sequence to compare to the end of the source span. /// One of the enumeration values that determines how the and are compared. - [Intrinsic] // Unrolled and vectorized for half-constant input (Ordinal) public static bool EndsWith(this ReadOnlySpan span, ReadOnlySpan value, StringComparison comparisonType) { string.CheckStringComparison(comparisonType); diff --git a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs index 83ea307fc8570a..bffab304afb0c8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs +++ b/src/libraries/System.Private.CoreLib/src/System/MemoryExtensions.cs @@ -2573,7 +2573,6 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(value)), /// Determines whether the specified sequence appears at the end of the span. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - [Intrinsic] // Unrolled and vectorized for half-constant input public static unsafe bool EndsWith(this Span span, ReadOnlySpan value) where T : IEquatable? { int spanLength = span.Length; @@ -2598,7 +2597,6 @@ ref MemoryMarshal.GetReference(value), /// Determines whether the specified sequence appears at the end of the span. /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - [Intrinsic] // Unrolled and vectorized for half-constant input public static unsafe bool EndsWith(this ReadOnlySpan span, ReadOnlySpan value) where T : IEquatable? { int spanLength = span.Length; diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs b/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs index 9dba42def297f8..3bb61ebcf736f6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.Formatting.cs @@ -407,7 +407,7 @@ public static unsafe bool TryFormatDecimal(decimal value, ReadOnlySpan= 0) { *dst++ = *p++; @@ -1627,7 +1627,7 @@ private static unsafe void Int32ToNumber(int value, ref NumberBuffer number) value = -value; } - byte* buffer = number.DigitsPtr; + byte* buffer = number.GetDigitsPointer(); byte* p = UInt32ToDecChars(buffer + Int32Precision, (uint)value, 0); int i = (int)(buffer + Int32Precision - p); @@ -1635,7 +1635,7 @@ private static unsafe void Int32ToNumber(int value, ref NumberBuffer number) number.DigitsCount = i; number.Scale = i; - byte* dst = number.DigitsPtr; + byte* dst = number.GetDigitsPointer(); while (--i >= 0) { *dst++ = *p++; @@ -1824,7 +1824,7 @@ private static unsafe void UInt32ToNumber(uint value, ref NumberBuffer number) number.DigitsCount = UInt32Precision; number.IsNegative = false; - byte* buffer = number.DigitsPtr; + byte* buffer = number.GetDigitsPointer(); byte* p = UInt32ToDecChars(buffer + UInt32Precision, value, 0); int i = (int)(buffer + UInt32Precision - p); @@ -1832,7 +1832,7 @@ private static unsafe void UInt32ToNumber(uint value, ref NumberBuffer number) number.DigitsCount = i; number.Scale = i; - byte* dst = number.DigitsPtr; + byte* dst = number.GetDigitsPointer(); while (--i >= 0) { *dst++ = *p++; @@ -2058,7 +2058,7 @@ private static unsafe void Int64ToNumber(long value, ref NumberBuffer number) value = -value; } - byte* buffer = number.DigitsPtr; + byte* buffer = number.GetDigitsPointer(); byte* p = UInt64ToDecChars(buffer + Int64Precision, (ulong)value, 0); int i = (int)(buffer + Int64Precision - p); @@ -2066,7 +2066,7 @@ private static unsafe void Int64ToNumber(long value, ref NumberBuffer number) number.DigitsCount = i; number.Scale = i; - byte* dst = number.DigitsPtr; + byte* dst = number.GetDigitsPointer(); while (--i >= 0) { *dst++ = *p++; @@ -2289,7 +2289,7 @@ private static unsafe void UInt64ToNumber(ulong value, ref NumberBuffer number) number.DigitsCount = UInt64Precision; number.IsNegative = false; - byte* buffer = number.DigitsPtr; + byte* buffer = number.GetDigitsPointer(); byte* p = UInt64ToDecChars(buffer + UInt64Precision, value, 0); int i = (int)(buffer + UInt64Precision - p); @@ -2297,7 +2297,7 @@ private static unsafe void UInt64ToNumber(ulong value, ref NumberBuffer number) number.DigitsCount = i; number.Scale = i; - byte* dst = number.DigitsPtr; + byte* dst = number.GetDigitsPointer(); while (--i >= 0) { *dst++ = *p++; @@ -2484,7 +2484,7 @@ private static unsafe void Int128ToNumber(Int128 value, ref NumberBuffer number) value = -value; } - byte* buffer = number.DigitsPtr; + byte* buffer = number.GetDigitsPointer(); byte* p = UInt128ToDecChars(buffer + Int128Precision, (UInt128)value, 0); int i = (int)(buffer + Int128Precision - p); @@ -2492,7 +2492,7 @@ private static unsafe void Int128ToNumber(Int128 value, ref NumberBuffer number) number.DigitsCount = i; number.Scale = i; - byte* dst = number.DigitsPtr; + byte* dst = number.GetDigitsPointer(); while (--i >= 0) { *dst++ = *p++; @@ -2701,7 +2701,7 @@ private static unsafe void UInt128ToNumber(UInt128 value, ref NumberBuffer numbe number.DigitsCount = UInt128Precision; number.IsNegative = false; - byte* buffer = number.DigitsPtr; + byte* buffer = number.GetDigitsPointer(); byte* p = UInt128ToDecChars(buffer + UInt128Precision, value, 0); int i = (int)(buffer + UInt128Precision - p); @@ -2709,7 +2709,7 @@ private static unsafe void UInt128ToNumber(UInt128 value, ref NumberBuffer numbe number.DigitsCount = i; number.Scale = i; - byte* dst = number.DigitsPtr; + byte* dst = number.GetDigitsPointer(); while (--i >= 0) { *dst++ = *p++; @@ -3050,7 +3050,7 @@ internal static unsafe void NumberToStringFormat(ref ValueListBuilder( Debug.Assert(typeof(TChar) == typeof(char) || typeof(TChar) == typeof(byte)); int digPos = number.Scale; - byte* dig = number.DigitsPtr; + byte* dig = number.GetDigitsPointer(); if (digPos > 0) { if (groupDigits != null) { + Debug.Assert(sGroup != null, "Must be null when groupDigits != null"); int groupSizeIndex = 0; // Index into the groupDigits array. int bufferSize = digPos; // The length of the result buffer string. int groupSize = 0; // The current group size. @@ -3582,6 +3583,7 @@ private static unsafe void FormatFixed( if (nMaxDigits > 0) { + Debug.Assert(sDecimal != null); vlb.Append(sDecimal); if ((digPos < 0) && (nMaxDigits > 0)) { @@ -3657,7 +3659,7 @@ private static unsafe void FormatScientific(ref ValueListBuilder v { Debug.Assert(typeof(TChar) == typeof(char) || typeof(TChar) == typeof(byte)); - byte* dig = number.DigitsPtr; + byte* dig = number.GetDigitsPointer(); vlb.Append(TChar.CastFrom((*dig != 0) ? (char)(*dig++) : '0')); @@ -3716,7 +3718,7 @@ private static unsafe void FormatGeneral(ref ValueListBuilder vlb, } } - byte* dig = number.DigitsPtr; + byte* dig = number.GetDigitsPointer(); if (digPos > 0) { @@ -3786,7 +3788,7 @@ private static void FormatPercent(ref ValueListBuilder vlb, ref Nu internal static unsafe void RoundNumber(ref NumberBuffer number, int pos, bool isCorrectlyRounded) { - byte* dig = number.DigitsPtr; + byte* dig = number.GetDigitsPointer(); int i = 0; while (i < pos && dig[i] != '\0') diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.NumberToFloatingPointBits.cs b/src/libraries/System.Private.CoreLib/src/System/Number.NumberToFloatingPointBits.cs index 303ae2f43c9222..8043171f596dfb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.NumberToFloatingPointBits.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.NumberToFloatingPointBits.cs @@ -700,7 +700,7 @@ private static void AccumulateDecimalDigitsIntoBigInteger(scoped ref NumberBuffe { BigInteger.SetZero(out result); - byte* src = number.DigitsPtr + firstIndex; + byte* src = number.GetDigitsPointer() + firstIndex; uint remaining = lastIndex - firstIndex; while (remaining != 0) @@ -974,7 +974,7 @@ private static ulong NumberToFloatingPointBits(ref NumberBuffer number) { Debug.Assert(TFloat.DenormalMantissaBits <= FloatingPointMaxDenormalMantissaBits); - Debug.Assert(number.DigitsPtr[0] != '0'); + Debug.Assert(number.GetDigitsPointer()[0] != '0'); Debug.Assert(number.Scale <= FloatingPointMaxExponent); Debug.Assert(number.Scale >= FloatingPointMinExponent); @@ -998,7 +998,7 @@ private static ulong NumberToFloatingPointBits(ref NumberBuffer number) // Above 19 digits, we rely on slow path if (totalDigits <= 19) { - byte* src = number.DigitsPtr; + byte* src = number.GetDigitsPointer(); ulong mantissa = DigitsToUInt64(src, (int)(totalDigits)); diff --git a/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs b/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs index 935328a21d20ed..952733c9268df9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Number.Parsing.cs @@ -113,7 +113,7 @@ private static unsafe bool TryNumberBufferToBinaryInteger(ref NumberBu return false; } - byte* p = number.DigitsPtr; + byte* p = number.GetDigitsPointer(); Debug.Assert(p != null); TInteger n = TInteger.Zero; @@ -725,7 +725,7 @@ internal static unsafe bool TryNumberToDecimal(ref NumberBuffer number, ref deci { number.CheckConsistency(); - byte* p = number.DigitsPtr; + byte* p = number.GetDigitsPointer(); int e = number.Scale; bool sign = number.IsNegative; uint c = *p; diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumber.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumber.cs index 69273c1668764d..e41d0b999850b8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumber.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumber.cs @@ -37,7 +37,7 @@ static virtual TSelf Clamp(TSelf value, TSelf min, TSelf max) return result; } - /// Copies the sign of a value to the sign of another value. + /// Copies the sign of a value to the sign of another value.. /// The value whose magnitude is used in the result. /// The value whose sign is used in the result. /// A value with the magnitude of and the sign of . @@ -154,7 +154,7 @@ static virtual TSelf MinNumber(TSelf x, TSelf y) /// Computes the sign of a value. /// The value whose sign is to be computed. - /// A positive value if is positive, 0 if is zero, and a negative value if is negative. + /// A positive value if is positive, if is zero, and a negative value if is negative. /// It is recommended that a function return 1, 0, and -1, respectively. static virtual int Sign(TSelf value) { diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs index 7e5acfafe75aeb..cd31d571ca3de0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/INumberBase.cs @@ -195,7 +195,7 @@ static virtual TSelf CreateTruncating(TOther value) /// true if is an odd integer; otherwise, false. /// /// This correctly handles floating-point values and so 3.0 will return true while 3.3 will return false. - /// This functioning returning false does not imply that will return true. A number with a fractional portion, 3.3, is neither even nor odd. + /// This functioning returning false does not imply that will return true. A number with a fractional portion, 3.3, is neither even nor odd. /// static abstract bool IsOddInteger(TSelf value); diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/IUnaryPlusOperators.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/IUnaryPlusOperators.cs index 3e3d9c25704d73..5ba2d43f6a94c9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/IUnaryPlusOperators.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/IUnaryPlusOperators.cs @@ -5,7 +5,7 @@ namespace System.Numerics { /// Defines a mechanism for computing the unary plus of a value. /// The type that implements this interface. - /// The type that contains the result of converting . + /// The type that contains the result of negating . public interface IUnaryPlusOperators where TSelf : IUnaryPlusOperators? { diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs index 4ad327aebc9ee4..efedc7f7d18db9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs @@ -181,7 +181,6 @@ public static Vector Indices /// true if is supported; otherwise, false. public static bool IsSupported { - [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] get => (typeof(T) == typeof(byte)) || (typeof(T) == typeof(double)) || diff --git a/src/libraries/System.Private.CoreLib/src/System/ReadOnlyMemory.cs b/src/libraries/System.Private.CoreLib/src/System/ReadOnlyMemory.cs index 6b59ac75e57663..9037e4110817e1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ReadOnlyMemory.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ReadOnlyMemory.cs @@ -313,7 +313,6 @@ public unsafe MemoryHandle Pin() { if (typeof(T) == typeof(char) && tmpObject is string s) { - // Unsafe.AsPointer is safe since the handle pins it GCHandle handle = GCHandle.Alloc(tmpObject, GCHandleType.Pinned); ref char stringData = ref Unsafe.Add(ref s.GetRawStringData(), _index); return new MemoryHandle(Unsafe.AsPointer(ref stringData), handle); @@ -326,13 +325,11 @@ public unsafe MemoryHandle Pin() // Array is already pre-pinned if (_index < 0) { - // Unsafe.AsPointer is safe since it's pinned void* pointer = Unsafe.Add(Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(tmpObject))), _index & RemoveFlagsBitMask); return new MemoryHandle(pointer); } else { - // Unsafe.AsPointer is safe since the handle pins it GCHandle handle = GCHandle.Alloc(tmpObject, GCHandleType.Pinned); void* pointer = Unsafe.Add(Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(Unsafe.As(tmpObject))), _index); return new MemoryHandle(pointer, handle); diff --git a/src/libraries/System.Private.CoreLib/src/System/ReadOnlySpan.cs b/src/libraries/System.Private.CoreLib/src/System/ReadOnlySpan.cs index b182d42b66ecd9..d198fed12fa33d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ReadOnlySpan.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ReadOnlySpan.cs @@ -120,6 +120,7 @@ public ReadOnlySpan(ref readonly T reference) _length = 1; } +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 // Constructor for internal use only. It is not safe to expose publicly, and is instead exposed via the unsafe MemoryMarshal.CreateReadOnlySpan. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal ReadOnlySpan(ref T reference, int length) @@ -129,6 +130,7 @@ internal ReadOnlySpan(ref T reference, int length) _reference = ref reference; _length = length; } +#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 /// /// Returns the specified element of the read-only span. @@ -215,18 +217,6 @@ public static implicit operator ReadOnlySpan(ArraySegment segment) /// public static ReadOnlySpan Empty => default; - /// - /// Casts a read-only span of to a read-only span of . - /// - /// The element type of the source read-only span, which must be derived from . - /// The source read-only span. No copy is made. - /// A read-only span with elements cast to the new type. - /// This method uses a covariant cast, producing a read-only span that shares the same memory as the source. The relationships expressed in the type constraints ensure that the cast is a safe operation. - public static ReadOnlySpan CastUp(ReadOnlySpan items) where TDerived : class?, T - { - return new ReadOnlySpan(ref Unsafe.As(ref items._reference), items.Length); - } - /// Gets an enumerator for this span. public Enumerator GetEnumerator() => new Enumerator(this); diff --git a/src/libraries/System.Private.CoreLib/src/System/Reflection/Emit/AssemblyBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/Reflection/Emit/AssemblyBuilder.cs index 1b1220536b9e37..8599d39aa4b166 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Reflection/Emit/AssemblyBuilder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Reflection/Emit/AssemblyBuilder.cs @@ -33,8 +33,56 @@ public ModuleBuilder DefineDynamicModule(string name) return GetDynamicModuleCore(name); } + /// + /// Defines an that can be saved to a file or stream. + /// + /// The name of the assembly. + /// The assembly that denotes the "system assembly" that houses the well-known types such as + /// A collection that contains the attributes of the assembly. + /// An that can be persisted. + /// The or or is null. + /// Currently the persisted assembly doesn't support running, need to save it and load back to run. + public static AssemblyBuilder DefinePersistedAssembly(AssemblyName name, Assembly coreAssembly, IEnumerable? assemblyAttributes = null) + { + ArgumentNullException.ThrowIfNull(name); + ArgumentException.ThrowIfNullOrEmpty(name.Name, "AssemblyName.Name"); + ArgumentNullException.ThrowIfNull(coreAssembly); + + Type assemblyType = Type.GetType("System.Reflection.Emit.AssemblyBuilderImpl, System.Reflection.Emit", throwOnError: true)!; + ConstructorInfo con = assemblyType.GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, [typeof(AssemblyName), typeof(Assembly), typeof(IEnumerable)])!; + return (AssemblyBuilder)con.Invoke([name, coreAssembly, assemblyAttributes]); + } + protected abstract ModuleBuilder? GetDynamicModuleCore(string name); + /// + /// Serializes the assembly to . + /// + /// The to which the assembly serialized. + /// is null. + /// The AssemblyBuilder instance doesn't support saving. + public void Save(Stream stream) => SaveCore(stream); + + /// + /// Saves the assembly to disk. + /// + /// The file name of the assembly. + /// is null. + /// The AssemblyBuilder instance doesn't support saving. + public void Save(string assemblyFileName) + { + ArgumentNullException.ThrowIfNull(assemblyFileName); + + using var peStream = new FileStream(assemblyFileName, FileMode.Create, FileAccess.Write); + SaveCore(peStream); + } + + /// + /// When implemented in a derived type, serializes the assembly to a stream. + /// + /// The stream to which the assembly serialized. + protected virtual void SaveCore(Stream stream) => throw new NotSupportedException(SR.NotSupported_AssemblySave); + public void SetCustomAttribute(ConstructorInfo con, byte[] binaryAttribute) { ArgumentNullException.ThrowIfNull(con); diff --git a/src/libraries/System.Private.CoreLib/src/System/Reflection/FieldAccessor.cs b/src/libraries/System.Private.CoreLib/src/System/Reflection/FieldAccessor.cs index f8d9f6b1ba1761..cbe7f96a9ef36d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Reflection/FieldAccessor.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Reflection/FieldAccessor.cs @@ -13,7 +13,7 @@ internal sealed class FieldAccessor private readonly RtFieldInfo _fieldInfo; private IntPtr _addressOrOffset; private unsafe MethodTable* _methodTable; - private volatile FieldAccessorType _fieldAccessType; + private FieldAccessorType _fieldAccessType; internal FieldAccessor(FieldInfo fieldInfo) { @@ -52,19 +52,19 @@ private void Initialize() { if (fieldType.IsEnum) { - _methodTable = (MethodTable*)fieldType.TypeHandle.Value; _fieldAccessType = GetPrimitiveAccessorTypeForStatic(fieldType.GetEnumUnderlyingType()); + _methodTable = (MethodTable*)fieldType.TypeHandle.Value; } else if (RuntimeTypeHandle.GetCorElementType(fieldType) == CorElementType.ELEMENT_TYPE_VALUETYPE) { // The runtime stores non-primitive value types as a boxed value. - _methodTable = (MethodTable*)fieldType.TypeHandle.Value; _fieldAccessType = FieldAccessorType.StaticValueTypeBoxed; + _methodTable = (MethodTable*)fieldType.TypeHandle.Value; } else { - _methodTable = (MethodTable*)fieldType.TypeHandle.Value; _fieldAccessType = GetPrimitiveAccessorTypeForStatic(fieldType); + _methodTable = (MethodTable*)fieldType.TypeHandle.Value; } } else if (fieldType.IsPointer) @@ -73,8 +73,8 @@ private void Initialize() } else if (fieldType.IsFunctionPointer) { - _methodTable = (MethodTable*)typeof(IntPtr).TypeHandle.Value; _fieldAccessType = GetIntPtrAccessorTypeForStatic(); + _methodTable = (MethodTable*)typeof(IntPtr).TypeHandle.Value; } else { @@ -87,13 +87,13 @@ private void Initialize() if (fieldType.IsEnum) { - _methodTable = (MethodTable*)fieldType.TypeHandle.Value; _fieldAccessType = GetPrimitiveAccessorTypeForInstance(fieldType.GetEnumUnderlyingType()); + _methodTable = (MethodTable*)fieldType.TypeHandle.Value; } else if (fieldType.IsValueType) { - _methodTable = (MethodTable*)fieldType.TypeHandle.Value; _fieldAccessType = GetPrimitiveAccessorTypeForInstance(fieldType); + _methodTable = (MethodTable*)fieldType.TypeHandle.Value; } else if (fieldType.IsPointer) { @@ -101,8 +101,8 @@ private void Initialize() } else if (fieldType.IsFunctionPointer) { - _methodTable = (MethodTable*)typeof(IntPtr).TypeHandle.Value; _fieldAccessType = GetIntPtrAccessorTypeForInstance(); + _methodTable = (MethodTable*)typeof(IntPtr).TypeHandle.Value; } else { diff --git a/src/libraries/System.Private.CoreLib/src/System/Reflection/GenericParameterAttributes.cs b/src/libraries/System.Private.CoreLib/src/System/Reflection/GenericParameterAttributes.cs index 26bb5d929a4fdd..c05d0ec6f2000d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Reflection/GenericParameterAttributes.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Reflection/GenericParameterAttributes.cs @@ -14,6 +14,5 @@ public enum GenericParameterAttributes ReferenceTypeConstraint = 0x0004, NotNullableValueTypeConstraint = 0x0008, DefaultConstructorConstraint = 0x0010, - AllowByRefLike = 0x0020, } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.cs b/src/libraries/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.cs index 7cfe087de94cbe..e4aa24dd7eddf5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.cs @@ -269,11 +269,6 @@ internal ResourceSet CreateResourceSet(Stream store, Assembly assembly) } } - private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, CultureInfo culture, Version? version) - { - return RuntimeAssembly.InternalGetSatelliteAssembly(mainAssembly, culture, version, throwOnFileNotFound: false); - } - [RequiresUnreferencedCode("The CustomResourceTypesSupport feature switch has been enabled for this app which is being trimmed. " + "Custom readers as well as custom objects on the resources file are not observable by the trimmer and so required assemblies, types and members may be removed.")] private static ResourceSet InternalGetResourceSetFromSerializedData(Stream store, string readerTypeName, string? resSetTypeName, ResourceManager.ResourceManagerMediator mediator) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs index 0caebcee620366..c4719b86132779 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncVoidMethodBuilder.cs @@ -80,27 +80,13 @@ public void SetResult() TplEventSource.Log.TraceOperationEnd(this.Task.Id, AsyncCausalityStatus.Completed); } - // Grab the context. Calling SetResult will complete the builder which can cause the state - // to be cleared out of the builder, so we can't touch anything on this builder after calling Set*. - // This clearing is done as part of the AsyncStateMachineBox.MoveNext method after it calls - // MoveNext on the state machine: it's possible to have a chain of events like this: - // Thread 1: Calls AsyncStateMachineBox.MoveNext, which calls StateMachine.MoveNext. - // Thread 1: StateMachine.MoveNext hooks up a continuation and returns - // Thread 2: That continuation runs and calls AsyncStateMachineBox.MoveNext, which calls SetResult on the builder (below) - // which will result in the state machine task being marked completed. - // Thread 1: The original AsyncStateMachineBox.MoveNext call continues and sees that the task is now completed - // Thread 1: Clears the builder - // Thread 2: Continues in this call to AsyncVoidMethodBuilder. If it touches anything on this instance, it will be cleared. - SynchronizationContext? context = _synchronizationContext; - // Mark the builder as completed. As this is a void-returning method, this mostly // doesn't matter, but it can affect things like debug events related to finalization. - // Marking the task completed will also then enable the MoveNext code to clear state. _builder.SetResult(); - if (context != null) + if (_synchronizationContext != null) { - NotifySynchronizationContextOfCompletion(context); + NotifySynchronizationContextOfCompletion(); } } @@ -120,18 +106,17 @@ public void SetException(Exception exception) TplEventSource.Log.TraceOperationEnd(this.Task.Id, AsyncCausalityStatus.Error); } - SynchronizationContext? context = _synchronizationContext; - if (context != null) + if (_synchronizationContext != null) { // If we captured a synchronization context, Post the throwing of the exception to it // and decrement its outstanding operation count. try { - Task.ThrowAsync(exception, targetContext: context); + Task.ThrowAsync(exception, targetContext: _synchronizationContext); } finally { - NotifySynchronizationContextOfCompletion(context); + NotifySynchronizationContextOfCompletion(); } } else @@ -147,12 +132,12 @@ public void SetException(Exception exception) } /// Notifies the current synchronization context that the operation completed. - private static void NotifySynchronizationContextOfCompletion(SynchronizationContext context) + private void NotifySynchronizationContextOfCompletion() { - Debug.Assert(context != null, "Must only be used with a non-null context."); + Debug.Assert(_synchronizationContext != null, "Must only be used with a non-null context."); try { - context.OperationCompleted(); + _synchronizationContext.OperationCompleted(); } catch (Exception exc) { diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/ParamCollectionAttribute.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/ParamCollectionAttribute.cs deleted file mode 100644 index 5ca0d96ecb6217..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/ParamCollectionAttribute.cs +++ /dev/null @@ -1,13 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Runtime.CompilerServices -{ - /// - /// Indicates that a method will allow a variable number of arguments in its invocation. - /// - [AttributeUsage(AttributeTargets.Parameter, Inherited = true, AllowMultiple = false)] - public sealed class ParamCollectionAttribute : Attribute - { - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NonNativeAot.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NonNativeAot.cs index 4495e552342c9b..1ae0005d941e04 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NonNativeAot.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.NonNativeAot.cs @@ -1,13 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics.CodeAnalysis; - namespace System.Runtime.CompilerServices { public static partial class RuntimeFeature { - [FeatureSwitchDefinition("System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeSupported")] public static bool IsDynamicCodeSupported { #if MONO @@ -16,7 +13,6 @@ public static bool IsDynamicCodeSupported get; } = AppContext.TryGetSwitch("System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeSupported", out bool isDynamicCodeSupported) ? isDynamicCodeSupported : true; - [FeatureGuard(typeof(RequiresDynamicCodeAttribute))] public static bool IsDynamicCodeCompiled { #if MONO diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.cs index 392a4902337ab2..e694364942513f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeFeature.cs @@ -32,11 +32,6 @@ public static partial class RuntimeFeature /// public const string ByRefFields = nameof(ByRefFields); - /// - /// Represents a runtime feature where byref-like types can be used in Generic parameters. - /// - public const string ByRefLikeGenerics = nameof(ByRefLikeGenerics); - /// /// Indicates that this version of runtime supports virtual static members of interfaces. /// @@ -57,7 +52,6 @@ public static bool IsSupported(string feature) case PortablePdb: case CovariantReturnsOfClasses: case ByRefFields: - case ByRefLikeGenerics: case UnmanagedSignatureCallingConvention: case DefaultImplementationsOfInterfaces: case VirtualStaticsInInterfaces: diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs index d33471813491ac..cf213590851497 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.cs @@ -9,11 +9,7 @@ namespace System.Runtime.CompilerServices public static partial class RuntimeHelpers { // The special dll name to be used for DllImport of QCalls -#if NATIVEAOT - internal const string QCall = "*"; -#else internal const string QCall = "QCall"; -#endif public delegate void TryCode(object? userData); @@ -54,7 +50,7 @@ public static T[] GetSubArray(T[] array, Range range) } // In either case, the newly-allocated array is the exact same type as the - // original incoming array. It's safe for us to SpanHelpers.Memmove the contents + // original incoming array. It's safe for us to Buffer.Memmove the contents // from the source array to the destination array, otherwise the contents // wouldn't have been valid for the source array in the first place. @@ -129,6 +125,9 @@ internal static bool IsPrimitiveType(this CorElementType et) [Intrinsic] internal static bool IsKnownConstant(char t) => false; + + [Intrinsic] + internal static bool IsKnownConstant(int t) => false; #pragma warning restore IDE0060 } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs index 901d354cfc7c84..b6dc25bb436438 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/Unsafe.cs @@ -3,7 +3,6 @@ #pragma warning disable IDE0060 // implementations provided as intrinsics using System; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.Versioning; @@ -908,32 +907,5 @@ public static ref T Unbox(object box) // unbox !!T // ret } - - - // Internal helper methods: - - // Determines if the address is aligned at least to `alignment` bytes. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool IsOpportunisticallyAligned(ref readonly T address, nuint alignment) - { - // `alignment` is expected to be a power of 2 in bytes. - // We use Unsafe.AsPointer to convert to a pointer, - // GC will keep alignment when moving objects (up to sizeof(void*)), - // otherwise alignment should be considered a hint if not pinned. - Debug.Assert(nuint.IsPow2(alignment)); - return ((nuint)AsPointer(ref AsRef(in address)) & (alignment - 1)) == 0; - } - - // Determines the misalignment of the address with respect to the specified `alignment`. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static nuint OpportunisticMisalignment(ref readonly T address, nuint alignment) - { - // `alignment` is expected to be a power of 2 in bytes. - // We use Unsafe.AsPointer to convert to a pointer, - // GC will keep alignment when moving objects (up to sizeof(void*)), - // otherwise alignment should be considered a hint if not pinned. - Debug.Assert(nuint.IsPow2(alignment)); - return (nuint)AsPointer(ref AsRef(in address)) & (alignment - 1); - } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/GCHandle.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/GCHandle.cs index 14151c1c02705a..7a45e868a3d02e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/GCHandle.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/GCHandle.cs @@ -127,7 +127,6 @@ public readonly IntPtr AddrOfPinnedObject() unsafe { - // Unsafe.AsPointer calls are safe since object is pinned. if (RuntimeHelpers.ObjectHasComponentSize(target)) { if (target.GetType() == typeof(string)) diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.cs index 74ef8257749aa1..109efc9da4471e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshal.cs @@ -165,7 +165,6 @@ public static unsafe IntPtr UnsafeAddrOfPinnedArrayElement(Array arr, int index) { ArgumentNullException.ThrowIfNull(arr); - // Unsafe.AsPointer is safe since array must be pinned void* pRawData = Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(arr)); return (IntPtr)((byte*)pRawData + (uint)index * (nuint)arr.GetElementSize()); } @@ -174,7 +173,6 @@ public static unsafe IntPtr UnsafeAddrOfPinnedArrayElement(T[] arr, int index { ArgumentNullException.ThrowIfNull(arr); - // Unsafe.AsPointer is safe since array must be pinned void* pRawData = Unsafe.AsPointer(ref MemoryMarshal.GetArrayDataReference(arr)); #pragma warning disable 8500 // sizeof of managed types return (IntPtr)((byte*)pRawData + (uint)index * (nuint)sizeof(T)); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/AnsiStringMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/AnsiStringMarshaller.cs index 34cfccd08755d5..4588f6912efd89 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/AnsiStringMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/AnsiStringMarshaller.cs @@ -87,7 +87,6 @@ public void FromManaged(string? managed, Span buffer) } } - // Unsafe.AsPointer is safe since buffer must be pinned _unmanagedValue = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(buffer)); Marshal.GetAnsiStringBytes(managed, buffer); // Includes null terminator diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ArrayMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ArrayMarshaller.cs index 823c31917e764f..0800b623d449ee 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ArrayMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ArrayMarshaller.cs @@ -173,11 +173,7 @@ public void FromManaged(T[]? array, Span buffer) /// Returns the unmanaged value representing the array. /// /// A pointer to the beginning of the unmanaged value. - public TUnmanagedElement* ToUnmanaged() - { - // Unsafe.AsPointer is safe since buffer must be pinned - return (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); - } + public TUnmanagedElement* ToUnmanaged() => (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); /// /// Frees resources. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/BStrStringMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/BStrStringMarshaller.cs index 561f8ce8de3121..38b033d0a9bd79 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/BStrStringMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/BStrStringMarshaller.cs @@ -86,7 +86,6 @@ public void FromManaged(string? managed, Span buffer) else { // Set length and update buffer target - // Unsafe.AsPointer is safe since buffer must be pinned byte* pBuffer = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(buffer)); *((uint*)pBuffer) = (uint)lengthInBytes; ptrToFirstChar = (ushort*)(pBuffer + sizeof(uint)); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/PointerArrayMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/PointerArrayMarshaller.cs index 846879583d5ac4..ee7a3a134229cd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/PointerArrayMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/PointerArrayMarshaller.cs @@ -174,11 +174,7 @@ public void FromManaged(T*[]? array, Span buffer) /// Returns the unmanaged value representing the array. /// /// A pointer to the beginning of the unmanaged value. - public TUnmanagedElement* ToUnmanaged() - { - // Unsafe.AsPointer is safe since buffer must be pinned - return (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); - } + public TUnmanagedElement* ToUnmanaged() => (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); /// /// Frees resources. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ReadOnlySpanMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ReadOnlySpanMarshaller.cs index bab60b629e319e..8fe502608dce6b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ReadOnlySpanMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/ReadOnlySpanMarshaller.cs @@ -142,11 +142,7 @@ public void FromManaged(ReadOnlySpan managed, Span buffer) /// /// Returns the unmanaged value representing the array. /// - public TUnmanagedElement* ToUnmanaged() - { - // Unsafe.AsPointer is safe since buffer must be pinned - return (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); - } + public TUnmanagedElement* ToUnmanaged() => (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); /// /// Frees resources. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/SpanMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/SpanMarshaller.cs index fb8aa49b12b617..a9d42299848df6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/SpanMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/SpanMarshaller.cs @@ -170,11 +170,7 @@ public void FromManaged(Span managed, Span buffer) /// /// Returns the unmanaged value representing the array. /// - public TUnmanagedElement* ToUnmanaged() - { - // Unsafe.AsPointer is safe since buffer must be pinned - return (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); - } + public TUnmanagedElement* ToUnmanaged() => (TUnmanagedElement*)Unsafe.AsPointer(ref GetPinnableReference()); /// /// Frees resources. diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/Utf8StringMarshaller.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/Utf8StringMarshaller.cs index ee231616eaad23..e6d529392bc9e0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/Utf8StringMarshaller.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/Marshalling/Utf8StringMarshaller.cs @@ -91,7 +91,6 @@ public void FromManaged(string? managed, Span buffer) } } - // Unsafe.AsPointer is safe since buffer must be pinned _unmanagedValue = (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(buffer)); int byteCount = Encoding.UTF8.GetBytes(managed, buffer); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.cs index 0b0da448e4eba9..2fc9946f03803b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/MemoryMarshal.cs @@ -87,6 +87,7 @@ public static Memory AsMemory(ReadOnlyMemory memory) => /// public static ref T GetReference(ReadOnlySpan span) => ref span._reference; +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 /// /// Returns a reference to the 0th element of the Span. If the Span is empty, returns a reference to fake non-null pointer. Such a reference can be used /// for pinning but must never be dereferenced. This is useful for interop with methods that do not accept null pointers for zero-sized buffers. @@ -100,6 +101,7 @@ public static Memory AsMemory(ReadOnlyMemory memory) => /// [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static unsafe ref T GetNonNullPinnableReference(ReadOnlySpan span) => ref (span.Length != 0) ? ref Unsafe.AsRef(in span._reference) : ref Unsafe.AsRef((void*)1); +#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 /// /// Casts a Span of one primitive type to another primitive type . diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeMemory.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeMemory.cs index 7fb4af35480a9b..069d67e5e4621e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeMemory.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/NativeMemory.cs @@ -61,7 +61,7 @@ public static unsafe void Clear(void* ptr, nuint byteCount) [CLSCompliant(false)] public static void Copy(void* source, void* destination, nuint byteCount) { - SpanHelpers.Memmove(ref *(byte*)destination, ref *(byte*)source, byteCount); + Buffer.Memmove(ref *(byte*)destination, ref *(byte*)source, byteCount); } /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeBuffer.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeBuffer.cs index 76858298feb217..d35b5dd174fc73 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeBuffer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeBuffer.cs @@ -194,7 +194,7 @@ public T Read(ulong byteOffset) where T : struct { DangerousAddRef(ref mustCallRelease); - SpanHelpers.Memmove(ref Unsafe.As(ref value), ref *ptr, sizeofT); + Buffer.Memmove(ref Unsafe.As(ref value), ref *ptr, sizeofT); } finally { @@ -281,7 +281,7 @@ public void Write(ulong byteOffset, T value) where T : struct { DangerousAddRef(ref mustCallRelease); - SpanHelpers.Memmove(ref *ptr, ref Unsafe.As(ref value), sizeofT); + Buffer.Memmove(ref *ptr, ref Unsafe.As(ref value), sizeofT); } finally { diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.PlatformNotSupported.cs index 3202d6dc94a628..7ccb2b70e6eb15 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.PlatformNotSupported.cs @@ -3776,6 +3776,9 @@ internal Arm64() { } /// public static unsafe void StorePairScalarNonTemporal(uint* address, Vector64 value1, Vector64 value2) { throw new PlatformNotSupportedException(); } +#if false + // Should be disabled until Mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 + /// /// void vst2_lane_s8 (int8_t * ptr, int8x16x2_t val, const int lane) /// A64: ST2 { Vt.16B, Vt+1.16B }[index], [Xn] @@ -3946,6 +3949,7 @@ internal Arm64() { } /// A64: ST3 { Vt.2D, Vt+1.2D, Vt+2.2D, Vt+3.2D }[index], [Xn] /// public static unsafe void StoreSelectedScalar(double* address, (Vector128 value1, Vector128 value2, Vector128 value3, Vector128 value4) value, [ConstantExpected(Max = (byte)(1))] byte index) { throw new PlatformNotSupportedException(); } +#endif /// /// A64: ST2 { Vn.16B, Vn+1.16B }, [Xn] @@ -15928,6 +15932,9 @@ internal Arm64() { } /// public static unsafe void StoreSelectedScalar(ulong* address, Vector128 value, [ConstantExpected(Max = (byte)(1))] byte index) { throw new PlatformNotSupportedException(); } +#if false + // Should be disabled until Mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 + /// /// A64: ST2 { Vt.8B, Vt+1.8B }[index], [Xn] /// @@ -16032,6 +16039,7 @@ internal Arm64() { } /// A64: ST4 { Vt.2S, Vt+1.2S, Vt+2.2S, Vt+3.2S }[index], [Xn] /// public static unsafe void StoreSelectedScalar(float* address, (Vector64 value1, Vector64 value2, Vector64 value3, Vector64 value4) value, [ConstantExpected(Max = (byte)(1))] byte index) { throw new PlatformNotSupportedException(); } +#endif /// /// A64: ST2 { Vn.8B, Vn+1.8B }, [Xn] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.cs index d343750838cd6e..8ead33cfde5afc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/AdvSimd.cs @@ -3774,6 +3774,9 @@ internal Arm64() { } /// public static unsafe void StorePairScalarNonTemporal(uint* address, Vector64 value1, Vector64 value2) => StorePairScalarNonTemporal(address, value1, value2); +#if false + // Should be disabled until Mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 + /// /// void vst2_lane_s8 (int8_t * ptr, int8x16x2_t val, const int lane) /// A64: ST2 { Vt.16B, Vt+1.16B }[index], [Xn] @@ -3944,6 +3947,7 @@ internal Arm64() { } /// A64: ST4 { Vt.2D, Vt+1.2D, Vt+2.2D, Vt+3.2D }[index], [Xn] /// public static unsafe void StoreSelectedScalar(double* address, (Vector128 value1, Vector128 value2, Vector128 value3, Vector128 value4) value, [ConstantExpected(Max = (byte)(1))] byte index) => StoreSelectedScalar(address, value, index); +#endif /// /// A64: ST2 { Vn.16B, Vn+1.16B }, [Xn] @@ -15925,6 +15929,9 @@ internal Arm64() { } /// public static unsafe void StoreSelectedScalar(ulong* address, Vector128 value, [ConstantExpected(Max = (byte)(1))] byte index) => StoreSelectedScalar(address, value, index); +#if false + // Should be disabled until Mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 + /// /// A64: ST2 { Vt.8B, Vt+1.8B }[index], [Xn] /// @@ -16029,6 +16036,7 @@ internal Arm64() { } /// A64: ST4 { Vt.2S, Vt+1.2S, Vt+2.2S, Vt+3.2S }[index], [Xn] /// public static unsafe void StoreSelectedScalar(float* address, (Vector64 value1, Vector64 value2, Vector64 value3, Vector64 value4) value, [ConstantExpected(Max = (byte)(1))] byte index) => StoreSelectedScalar(address, value, index); +#endif /// /// A64: ST2 { Vn.8B, Vn+1.8B }, [Xn] diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs deleted file mode 100644 index 868300bf14acaa..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Enums.cs +++ /dev/null @@ -1,95 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Runtime.Intrinsics.Arm -{ - // Used to specify or limit the number of elements used within an method. - // Matches the field "pattern" within the Arm Architecture Reference Manual - public enum SveMaskPattern : byte - { - /// - /// POW2 - /// - LargestPowerOf2 = 0, // The largest power of 2. - - /// - /// VL1 - /// - VectorCount1 = 1, // Exactly 1 element. - - /// - /// VL2 - /// - VectorCount2 = 2, // Exactly 2 elements. - - /// - /// VL3 - /// - VectorCount3 = 3, // Exactly 3 elements. - - /// - /// VL4 - /// - VectorCount4 = 4, // Exactly 4 elements. - - /// - /// VL5 - /// - VectorCount5 = 5, // Exactly 5 elements. - - /// - /// VL6 - /// - VectorCount6 = 6, // Exactly 6 elements. - - /// - /// VL7 - /// - VectorCount7 = 7, // Exactly 7 elements. - - /// - /// VL8 - /// - VectorCount8 = 8, // Exactly 8 elements. - - /// - /// VL16 - /// - VectorCount16 = 9, // Exactly 16 elements. - - /// - /// VL32 - /// - VectorCount32 = 10, // Exactly 32 elements. - - /// - /// VL64 - /// - VectorCount64 = 11, // Exactly 64 elements. - - /// - /// VL128 - /// - VectorCount128 = 12, // Exactly 128 elements. - - /// - /// VL256 - /// - VectorCount256 = 13, // Exactly 256 elements. - - /// - /// MUL4 - /// - LargestMultipleOf4 = 29, // The largest multiple of 4. - - /// - /// MUL3 - /// - LargestMultipleOf3 = 30, // The largest multiple of 3. - - /// - /// ALL - /// - All = 31 // All available (implicitly a multiple of two). - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index fbd5ee65ca748f..3eeb40d5d9de19 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -30,170 +30,5 @@ internal Arm64() { } public static new bool IsSupported { [Intrinsic] get { return false; } } } - - /// CreateTrueMaskByte : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskDouble : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskInt16 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskInt32 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskInt64 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskSByte : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskSByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskSingle : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskSingle([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskUInt16 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b16(enum svpattern pattern) - /// PTRUE Presult.H, pattern - /// - public static unsafe Vector CreateTrueMaskUInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskUInt32 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b32(enum svpattern pattern) - /// PTRUE Presult.S, pattern - /// - public static unsafe Vector CreateTrueMaskUInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - /// CreateTrueMaskUInt64 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b64(enum svpattern pattern) - /// PTRUE Presult.D, pattern - /// - public static unsafe Vector CreateTrueMaskUInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw new PlatformNotSupportedException(); } - - - - /// LoadVector : Unextended load - - /// - /// svint8_t svld1[_s8](svbool_t pg, const int8_t *base) - /// LD1B Zresult.B, Pg/Z, [Xarray, Xindex] - /// LD1B Zresult.B, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, sbyte* address) { throw new PlatformNotSupportedException(); } - - /// - /// svint16_t svld1[_s16](svbool_t pg, const int16_t *base) - /// LD1H Zresult.H, Pg/Z, [Xarray, Xindex, LSL #1] - /// LD1H Zresult.H, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, short* address) { throw new PlatformNotSupportedException(); } - - /// - /// svint32_t svld1[_s32](svbool_t pg, const int32_t *base) - /// LD1W Zresult.S, Pg/Z, [Xarray, Xindex, LSL #2] - /// LD1W Zresult.S, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, int* address) { throw new PlatformNotSupportedException(); } - - /// - /// svint64_t svld1[_s64](svbool_t pg, const int64_t *base) - /// LD1D Zresult.D, Pg/Z, [Xarray, Xindex, LSL #3] - /// LD1D Zresult.D, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, long* address) { throw new PlatformNotSupportedException(); } - - /// - /// svuint8_t svld1[_u8](svbool_t pg, const uint8_t *base) - /// LD1B Zresult.B, Pg/Z, [Xarray, Xindex] - /// LD1B Zresult.B, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, byte* address) { throw new PlatformNotSupportedException(); } - - /// - /// svuint16_t svld1[_u16](svbool_t pg, const uint16_t *base) - /// LD1H Zresult.H, Pg/Z, [Xarray, Xindex, LSL #1] - /// LD1H Zresult.H, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, ushort* address) { throw new PlatformNotSupportedException(); } - - /// - /// svuint32_t svld1[_u32](svbool_t pg, const uint32_t *base) - /// LD1W Zresult.S, Pg/Z, [Xarray, Xindex, LSL #2] - /// LD1W Zresult.S, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, uint* address) { throw new PlatformNotSupportedException(); } - - /// - /// svuint64_t svld1[_u64](svbool_t pg, const uint64_t *base) - /// LD1D Zresult.D, Pg/Z, [Xarray, Xindex, LSL #3] - /// LD1D Zresult.D, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, ulong* address) { throw new PlatformNotSupportedException(); } - - /// - /// svfloat32_t svld1[_f32](svbool_t pg, const float32_t *base) - /// LD1W Zresult.S, Pg/Z, [Xarray, Xindex, LSL #2] - /// LD1W Zresult.S, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, float* address) { throw new PlatformNotSupportedException(); } - - /// - /// svfloat64_t svld1[_f64](svbool_t pg, const float64_t *base) - /// LD1D Zresult.D, Pg/Z, [Xarray, Xindex, LSL #3] - /// LD1D Zresult.D, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, double* address) { throw new PlatformNotSupportedException(); } - - } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index 6ba2a2c67bc8a7..7a71144e0bc33f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -27,170 +27,5 @@ internal Arm64() { } public static new bool IsSupported { get => IsSupported; } } - - - /// CreateTrueMaskByte : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskByte(pattern); - - - /// CreateTrueMaskDouble : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskDouble(pattern); - - - /// CreateTrueMaskInt16 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskInt16(pattern); - - - /// CreateTrueMaskInt32 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskInt32(pattern); - - - /// CreateTrueMaskInt64 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskInt64(pattern); - - - /// CreateTrueMaskSByte : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskSByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskSByte(pattern); - - - /// CreateTrueMaskSingle : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b8(enum svpattern pattern) - /// PTRUE Presult.B, pattern - /// - public static unsafe Vector CreateTrueMaskSingle([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskSingle(pattern); - - - /// CreateTrueMaskUInt16 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b16(enum svpattern pattern) - /// PTRUE Presult.H, pattern - /// - public static unsafe Vector CreateTrueMaskUInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskUInt16(pattern); - - - /// CreateTrueMaskUInt32 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b32(enum svpattern pattern) - /// PTRUE Presult.S, pattern - /// - public static unsafe Vector CreateTrueMaskUInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskUInt32(pattern); - - - /// CreateTrueMaskUInt64 : Set predicate elements to true - - /// - /// svbool_t svptrue_pat_b64(enum svpattern pattern) - /// PTRUE Presult.D, pattern - /// - public static unsafe Vector CreateTrueMaskUInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) => CreateTrueMaskUInt64(pattern); - - - - /// LoadVector : Unextended load - - /// - /// svint8_t svld1[_s8](svbool_t pg, const int8_t *base) - /// LD1B Zresult.B, Pg/Z, [Xarray, Xindex] - /// LD1B Zresult.B, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, sbyte* address) => LoadVector(mask, address); - - /// - /// svint16_t svld1[_s16](svbool_t pg, const int16_t *base) - /// LD1H Zresult.H, Pg/Z, [Xarray, Xindex, LSL #1] - /// LD1H Zresult.H, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, short* address) => LoadVector(mask, address); - - /// - /// svint32_t svld1[_s32](svbool_t pg, const int32_t *base) - /// LD1W Zresult.S, Pg/Z, [Xarray, Xindex, LSL #2] - /// LD1W Zresult.S, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, int* address) => LoadVector(mask, address); - - /// - /// svint64_t svld1[_s64](svbool_t pg, const int64_t *base) - /// LD1D Zresult.D, Pg/Z, [Xarray, Xindex, LSL #3] - /// LD1D Zresult.D, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, long* address) => LoadVector(mask, address); - - /// - /// svuint8_t svld1[_u8](svbool_t pg, const uint8_t *base) - /// LD1B Zresult.B, Pg/Z, [Xarray, Xindex] - /// LD1B Zresult.B, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, byte* address) => LoadVector(mask, address); - - /// - /// svuint16_t svld1[_u16](svbool_t pg, const uint16_t *base) - /// LD1H Zresult.H, Pg/Z, [Xarray, Xindex, LSL #1] - /// LD1H Zresult.H, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, ushort* address) => LoadVector(mask, address); - - /// - /// svuint32_t svld1[_u32](svbool_t pg, const uint32_t *base) - /// LD1W Zresult.S, Pg/Z, [Xarray, Xindex, LSL #2] - /// LD1W Zresult.S, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, uint* address) => LoadVector(mask, address); - - /// - /// svuint64_t svld1[_u64](svbool_t pg, const uint64_t *base) - /// LD1D Zresult.D, Pg/Z, [Xarray, Xindex, LSL #3] - /// LD1D Zresult.D, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, ulong* address) => LoadVector(mask, address); - - /// - /// svfloat32_t svld1[_f32](svbool_t pg, const float32_t *base) - /// LD1W Zresult.S, Pg/Z, [Xarray, Xindex, LSL #2] - /// LD1W Zresult.S, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, float* address) => LoadVector(mask, address); - - /// - /// svfloat64_t svld1[_f64](svbool_t pg, const float64_t *base) - /// LD1D Zresult.D, Pg/Z, [Xarray, Xindex, LSL #3] - /// LD1D Zresult.D, Pg/Z, [Xbase, #0, MUL VL] - /// - public static unsafe Vector LoadVector(Vector mask, double* address) => LoadVector(mask, address); - } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs index 1347a082afab97..f2eae1dad34916 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector256.cs @@ -36,9 +36,6 @@ public static unsafe class Vector256 internal const int Alignment = 8; #elif TARGET_ARM64 internal const int Alignment = 16; -#elif TARGET_RISCV64 - // TODO-RISCV64: Update alignment to proper value when we implement RISC-V intrinsic. - internal const int Alignment = 16; #else internal const int Alignment = 32; #endif diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs index edb84585a987de..d18e705c3b1314 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Vector512.cs @@ -36,9 +36,6 @@ public static unsafe class Vector512 internal const int Alignment = 8; #elif TARGET_ARM64 internal const int Alignment = 16; -#elif TARGET_RISCV64 - // TODO-RISCV64: Update alignment to proper value when we implement RISC-V intrinsic. - internal const int Alignment = 16; #else internal const int Alignment = 64; #endif diff --git a/src/libraries/System.Private.CoreLib/src/System/RuntimeType.cs b/src/libraries/System.Private.CoreLib/src/System/RuntimeType.cs index 17fbb2950c6c20..bd6a6f89bb1353 100644 --- a/src/libraries/System.Private.CoreLib/src/System/RuntimeType.cs +++ b/src/libraries/System.Private.CoreLib/src/System/RuntimeType.cs @@ -692,7 +692,7 @@ public override bool IsAssignableFrom([NotNullWhen(true)] Type? c) if (constraint.IsGenericParameter) { - GenericParameterAttributes special = constraint.GenericParameterAttributes; + GenericParameterAttributes special = constraint.GenericParameterAttributes & GenericParameterAttributes.SpecialConstraintMask; if ((special & GenericParameterAttributes.ReferenceTypeConstraint) == 0 && (special & GenericParameterAttributes.NotNullableValueTypeConstraint) == 0) @@ -704,7 +704,7 @@ public override bool IsAssignableFrom([NotNullWhen(true)] Type? c) if (baseType == ObjectType) { - GenericParameterAttributes special = GenericParameterAttributes; + GenericParameterAttributes special = GenericParameterAttributes & GenericParameterAttributes.SpecialConstraintMask; if ((special & GenericParameterAttributes.NotNullableValueTypeConstraint) != 0) baseType = ValueType; } diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs index deccdddb8feced..bc8b3fd0c6c8f4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/IndexOfAnyAsciiSearcher.cs @@ -10,6 +10,7 @@ using System.Runtime.Intrinsics.X86; #pragma warning disable 8500 // sizeof of managed types +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 namespace System.Buffers { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs index 076340bebebad9..150372914d8b2e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/ProbabilisticMap.cs @@ -10,6 +10,7 @@ using System.Runtime.Intrinsics.Wasm; using System.Runtime.Intrinsics.X86; +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 namespace System.Buffers { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs index d8970655cb31b7..d420c70d16c198 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/Helpers/RabinKarp.cs @@ -64,12 +64,6 @@ public RabinKarp(ReadOnlySpan values) foreach (string value in values) { - if (value.Length > MaxInputLength) - { - // This value can never match. There's no point in including it in the buckets. - continue; - } - nuint hash = 0; for (int i = 0; i < minimumLength; i++) { diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs index 2bff05214c518a..86a13dd04b9b0e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Strings/StringSearchValues.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; @@ -15,8 +14,6 @@ namespace System.Buffers { internal static class StringSearchValues { - private const int TeddyBucketCount = 8; - private static readonly SearchValues s_asciiLetters = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"); @@ -251,18 +248,6 @@ static SearchValues PickAhoCorasickImplementation(AhoC Debug.Assert(!(asciiStartLettersOnly && asciiStartUnaffectedByCaseConversion)); - // If we still have empty buckets we could use and we're ignoring case, we may be able to - // generate all possible permutations of the first N characters and switch to case-sensitive searching. - // E.g. ["ab", "c!"] => ["ab", "Ab" "aB", "AB", "c!", "C!"]. - // This won't apply to inputs with many letters (e.g. "abc" => 8 permutations on its own). - if (!asciiStartUnaffectedByCaseConversion && - values.Length < TeddyBucketCount && - TryGenerateAllCasePermutationsForPrefixes(values, n, TeddyBucketCount, out string[]? newValues)) - { - asciiStartUnaffectedByCaseConversion = true; - values = newValues; - } - if (asciiStartUnaffectedByCaseConversion) { return nonAsciiAffectedByCaseConversion @@ -293,9 +278,9 @@ private static SearchValues PickTeddyImplementation 1); Debug.Assert(n is 2 or 3); - if (values.Length > TeddyBucketCount) + if (values.Length > 8) { - string[][] buckets = TeddyBucketizer.Bucketize(values, TeddyBucketCount, n); + string[][] buckets = TeddyBucketizer.Bucketize(values, bucketCount: 8, n); // Potential optimization: We don't have to pick the first N characters for the fingerprint. // Different offset selection can noticeably improve throughput (e.g. 2x). @@ -312,68 +297,6 @@ private static SearchValues PickTeddyImplementation values, int n, int maxValues, [NotNullWhen(true)] out string[]? newValues) - { - Debug.Assert(n is 2 or 3); - Debug.Assert(values.Length < maxValues); - - // Count how many possible permutations there are. - int newValuesCount = 0; - - foreach (string value in values) - { - int permutations = 1; - - foreach (char c in value.AsSpan(0, n)) - { - Debug.Assert(char.IsAscii(c)); - - if (char.IsAsciiLetter(c)) - { - permutations *= 2; - } - } - - newValuesCount += permutations; - } - - Debug.Assert(newValuesCount > values.Length, "Shouldn't have been called if there were no letters present"); - - if (newValuesCount > maxValues) - { - newValues = null; - return false; - } - - // Generate the permutations. - newValues = new string[newValuesCount]; - newValuesCount = 0; - - foreach (string value in values) - { - int start = newValuesCount; - - newValues[newValuesCount++] = value; - - for (int i = 0; i < n; i++) - { - char c = value[i]; - - if (char.IsAsciiLetter(c)) - { - // Copy all the previous permutations of this value but change the casing of the i-th character. - foreach (string previous in newValues.AsSpan(start, newValuesCount - start)) - { - newValues[newValuesCount++] = $"{previous.AsSpan(0, i)}{(char)(c ^ 0x20)}{previous.AsSpan(i + 1)}"; - } - } - } - } - - Debug.Assert(newValuesCount == newValues.Length); - return true; - } - private static SearchValues CreateForSingleValue( string value, HashSet? uniqueValues, diff --git a/src/libraries/System.Private.CoreLib/src/System/Span.cs b/src/libraries/System.Private.CoreLib/src/System/Span.cs index 38b94e872b0cb3..aaf3763d81b755 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Span.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Span.cs @@ -126,6 +126,7 @@ public Span(ref T reference) _length = 1; } +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 // Constructor for internal use only. It is not safe to expose publicly, and is instead exposed via the unsafe MemoryMarshal.CreateSpan. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Span(ref T reference, int length) @@ -135,6 +136,7 @@ internal Span(ref T reference, int length) _reference = ref reference; _length = length; } +#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 /// /// Returns a reference to specified element of the Span. @@ -298,7 +300,19 @@ public unsafe void Clear() [MethodImpl(MethodImplOptions.AggressiveInlining)] public unsafe void Fill(T value) { - SpanHelpers.Fill(ref _reference, (uint)_length, value); + if (sizeof(T) == 1) + { + // Special-case single-byte types like byte / sbyte / bool. + // The runtime eventually calls memset, which can efficiently support large buffers. + // We don't need to check IsReferenceOrContainsReferences because no references + // can ever be stored in types this small. + Unsafe.InitBlockUnaligned(ref Unsafe.As(ref _reference), *(byte*)&value, (uint)_length); + } + else + { + // Call our optimized workhorse method for all other types. + SpanHelpers.Fill(ref _reference, (uint)_length, value); + } } /// diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs deleted file mode 100644 index b4aa563b277478..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.ByteMemOps.cs +++ /dev/null @@ -1,537 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -#if TARGET_AMD64 || TARGET_ARM64 || (TARGET_32BIT && !TARGET_ARM) || TARGET_LOONGARCH64 -// JIT is guaranteed to unroll blocks up to 64 bytes in size -#define HAS_CUSTOM_BLOCKS -#endif - -using System.Diagnostics; -using System.Numerics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System -{ - internal static partial class SpanHelpers // .ByteMemOps - { -#if TARGET_ARM64 || TARGET_LOONGARCH64 - private const ulong MemmoveNativeThreshold = ulong.MaxValue; -#elif TARGET_ARM - private const nuint MemmoveNativeThreshold = 512; -#else - private const nuint MemmoveNativeThreshold = 2048; -#endif - private const nuint ZeroMemoryNativeThreshold = 1024; - - -#if HAS_CUSTOM_BLOCKS - [StructLayout(LayoutKind.Sequential, Size = 16)] - private struct Block16 {} - - [StructLayout(LayoutKind.Sequential, Size = 64)] - private struct Block64 {} -#endif // HAS_CUSTOM_BLOCKS - -#if NATIVEAOT - [System.Runtime.RuntimeExport("RhSpanHelpers_MemCopy")] -#endif - [Intrinsic] // Unrolled for small constant lengths - internal static unsafe void Memmove(ref byte dest, ref byte src, nuint len) - { - // P/Invoke into the native version when the buffers are overlapping. - if ((nuint)Unsafe.ByteOffset(ref src, ref dest) < len || - (nuint)Unsafe.ByteOffset(ref dest, ref src) < len) - { - goto BuffersOverlap; - } - - ref byte srcEnd = ref Unsafe.Add(ref src, len); - ref byte destEnd = ref Unsafe.Add(ref dest, len); - - if (len <= 16) - goto MCPY02; - if (len > 64) - goto MCPY05; - - MCPY00: - // Copy bytes which are multiples of 16 and leave the remainder for MCPY01 to handle. - Debug.Assert(len > 16 && len <= 64); -#if HAS_CUSTOM_BLOCKS - Unsafe.As(ref dest) = Unsafe.As(ref src); // [0,16] -#elif TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); // [0,16] -#else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); // [0,16] -#endif - if (len <= 32) - goto MCPY01; -#if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); // [0,32] -#elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); // [0,32] -#else - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); // [0,32] -#endif - if (len <= 48) - goto MCPY01; -#if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); // [0,48] -#elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); // [0,48] -#else - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); // [0,48] -#endif - - MCPY01: - // Unconditionally copy the last 16 bytes using destEnd and srcEnd and return. - Debug.Assert(len > 16 && len <= 64); -#if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); -#elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); -#else - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); -#endif - return; - - MCPY02: - // Copy the first 8 bytes and then unconditionally copy the last 8 bytes and return. - if ((len & 24) == 0) - goto MCPY03; - Debug.Assert(len >= 8 && len <= 16); -#if TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); -#else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); -#endif - return; - - MCPY03: - // Copy the first 4 bytes and then unconditionally copy the last 4 bytes and return. - if ((len & 4) == 0) - goto MCPY04; - Debug.Assert(len >= 4 && len < 8); - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); - return; - - MCPY04: - // Copy the first byte. For pending bytes, do an unconditionally copy of the last 2 bytes and return. - Debug.Assert(len < 4); - if (len == 0) - return; - dest = src; - if ((len & 2) == 0) - return; - Unsafe.As(ref Unsafe.Add(ref destEnd, -2)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -2)); - return; - - MCPY05: - // PInvoke to the native version when the copy length exceeds the threshold. - if (len > MemmoveNativeThreshold) - { - goto PInvoke; - } - -#if HAS_CUSTOM_BLOCKS - if (len >= 256) - { - // Try to opportunistically align the destination below. The input isn't pinned, so the GC - // is free to move the references. We're therefore assuming that reads may still be unaligned. - // - // dest is more important to align than src because an unaligned store is more expensive - // than an unaligned load. - nuint misalignedElements = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); - Unsafe.As(ref dest) = Unsafe.As(ref src); - src = ref Unsafe.Add(ref src, misalignedElements); - dest = ref Unsafe.Add(ref dest, misalignedElements); - len -= misalignedElements; - } -#endif - - // Copy 64-bytes at a time until the remainder is less than 64. - // If remainder is greater than 16 bytes, then jump to MCPY00. Otherwise, unconditionally copy the last 16 bytes and return. - Debug.Assert(len > 64 && len <= MemmoveNativeThreshold); - nuint n = len >> 6; - - MCPY06: -#if HAS_CUSTOM_BLOCKS - Unsafe.As(ref dest) = Unsafe.As(ref src); -#elif TARGET_64BIT - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); - Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); -#else - Unsafe.As(ref dest) = Unsafe.As(ref src); - Unsafe.As(ref Unsafe.Add(ref dest, 4)) = Unsafe.As(ref Unsafe.Add(ref src, 4)); - Unsafe.As(ref Unsafe.Add(ref dest, 8)) = Unsafe.As(ref Unsafe.Add(ref src, 8)); - Unsafe.As(ref Unsafe.Add(ref dest, 12)) = Unsafe.As(ref Unsafe.Add(ref src, 12)); - Unsafe.As(ref Unsafe.Add(ref dest, 16)) = Unsafe.As(ref Unsafe.Add(ref src, 16)); - Unsafe.As(ref Unsafe.Add(ref dest, 20)) = Unsafe.As(ref Unsafe.Add(ref src, 20)); - Unsafe.As(ref Unsafe.Add(ref dest, 24)) = Unsafe.As(ref Unsafe.Add(ref src, 24)); - Unsafe.As(ref Unsafe.Add(ref dest, 28)) = Unsafe.As(ref Unsafe.Add(ref src, 28)); - Unsafe.As(ref Unsafe.Add(ref dest, 32)) = Unsafe.As(ref Unsafe.Add(ref src, 32)); - Unsafe.As(ref Unsafe.Add(ref dest, 36)) = Unsafe.As(ref Unsafe.Add(ref src, 36)); - Unsafe.As(ref Unsafe.Add(ref dest, 40)) = Unsafe.As(ref Unsafe.Add(ref src, 40)); - Unsafe.As(ref Unsafe.Add(ref dest, 44)) = Unsafe.As(ref Unsafe.Add(ref src, 44)); - Unsafe.As(ref Unsafe.Add(ref dest, 48)) = Unsafe.As(ref Unsafe.Add(ref src, 48)); - Unsafe.As(ref Unsafe.Add(ref dest, 52)) = Unsafe.As(ref Unsafe.Add(ref src, 52)); - Unsafe.As(ref Unsafe.Add(ref dest, 56)) = Unsafe.As(ref Unsafe.Add(ref src, 56)); - Unsafe.As(ref Unsafe.Add(ref dest, 60)) = Unsafe.As(ref Unsafe.Add(ref src, 60)); -#endif - dest = ref Unsafe.Add(ref dest, 64); - src = ref Unsafe.Add(ref src, 64); - n--; - if (n != 0) - goto MCPY06; - - len %= 64; - if (len > 16) - goto MCPY00; -#if HAS_CUSTOM_BLOCKS - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); -#elif TARGET_64BIT - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); -#else - Unsafe.As(ref Unsafe.Add(ref destEnd, -16)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -16)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -12)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -12)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -8)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -8)); - Unsafe.As(ref Unsafe.Add(ref destEnd, -4)) = Unsafe.As(ref Unsafe.Add(ref srcEnd, -4)); -#endif - return; - - BuffersOverlap: - Debug.Assert(len > 0); - // If the buffers overlap perfectly, there's no point to copying the data. - if (Unsafe.AreSame(ref dest, ref src)) - { - // Both could be null with a non-zero length, perform an implicit null check. - _ = Unsafe.ReadUnaligned(ref dest); - return; - } - - PInvoke: - // Implicit nullchecks - Debug.Assert(len > 0); - _ = Unsafe.ReadUnaligned(ref dest); - _ = Unsafe.ReadUnaligned(ref src); - Buffer._Memmove(ref dest, ref src, len); - } - -#if NATIVEAOT - [System.Runtime.RuntimeExport("RhSpanHelpers_MemZero")] -#endif - [Intrinsic] // Unrolled for small sizes - public static unsafe void ClearWithoutReferences(ref byte dest, nuint len) - { - if (len == 0) - return; - - ref byte destEnd = ref Unsafe.Add(ref dest, len); - - if (len <= 16) - goto MZER02; - if (len > 64) - goto MZER05; - - MZER00: - // Clear bytes which are multiples of 16 and leave the remainder for MZER01 to handle. - Debug.Assert(len > 16 && len <= 64); -#if HAS_CUSTOM_BLOCKS - Unsafe.WriteUnaligned(ref dest, default); -#elif TARGET_64BIT - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), 0); -#else - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 12), 0); -#endif - if (len <= 32) - goto MZER01; -#if HAS_CUSTOM_BLOCKS - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), default); -#elif TARGET_64BIT - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), 0); -#else - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 20), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 28), 0); -#endif - if (len <= 48) - goto MZER01; -#if HAS_CUSTOM_BLOCKS - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), default); -#elif TARGET_64BIT - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), 0); -#else - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 36), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 44), 0); -#endif - - MZER01: - // Unconditionally clear the last 16 bytes using destEnd and return. - Debug.Assert(len > 16 && len <= 64); -#if HAS_CUSTOM_BLOCKS - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), default); -#elif TARGET_64BIT - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), 0); -#else - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -12), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), 0); -#endif - return; - - MZER02: - // Clear the first 8 bytes and then unconditionally clear the last 8 bytes and return. - if ((len & 24) == 0) - goto MZER03; - Debug.Assert(len >= 8 && len <= 16); -#if TARGET_64BIT - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), 0); -#else - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), 0); -#endif - return; - - MZER03: - // Clear the first 4 bytes and then unconditionally clear the last 4 bytes and return. - if ((len & 4) == 0) - goto MZER04; - Debug.Assert(len >= 4 && len < 8); - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), 0); - return; - - MZER04: - // Clear the first byte. For pending bytes, do an unconditionally clear of the last 2 bytes and return. - Debug.Assert(len < 4); - if (len == 0) - return; - dest = 0; - if ((len & 2) == 0) - return; - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -2), 0); - return; - - MZER05: - // PInvoke to the native version when the clear length exceeds the threshold. - if (len > ZeroMemoryNativeThreshold) - { - goto PInvoke; - } - -#if HAS_CUSTOM_BLOCKS - if (len >= 256) - { - // Try to opportunistically align the destination below. The input isn't pinned, so the GC - // is free to move the references. We're therefore assuming that reads may still be unaligned. - nuint misalignedElements = 64 - Unsafe.OpportunisticMisalignment(ref dest, 64); - Unsafe.WriteUnaligned(ref dest, default); - dest = ref Unsafe.Add(ref dest, misalignedElements); - len -= misalignedElements; - } -#endif - // Clear 64-bytes at a time until the remainder is less than 64. - // If remainder is greater than 16 bytes, then jump to MZER00. Otherwise, unconditionally clear the last 16 bytes and return. - Debug.Assert(len > 64 && len <= ZeroMemoryNativeThreshold); - nuint n = len >> 6; - - MZER06: -#if HAS_CUSTOM_BLOCKS - Unsafe.WriteUnaligned(ref dest, default); -#elif TARGET_64BIT - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 48), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 56), 0); -#else - Unsafe.WriteUnaligned(ref dest, 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 4), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 8), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 12), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 20), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 24), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 28), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 32), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 36), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 40), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 44), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 48), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 52), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 56), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref dest, 60), 0); -#endif - dest = ref Unsafe.Add(ref dest, 64); - n--; - if (n != 0) - goto MZER06; - - len %= 64; - if (len > 16) - goto MZER00; -#if HAS_CUSTOM_BLOCKS - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), default); -#elif TARGET_64BIT - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), 0); -#else - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -16), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -12), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -8), 0); - Unsafe.WriteUnaligned(ref Unsafe.Add(ref destEnd, -4), 0); -#endif - return; - - PInvoke: - // Implicit nullchecks - _ = Unsafe.ReadUnaligned(ref dest); - Buffer._ZeroMemory(ref dest, len); - } - -#if NATIVEAOT - [System.Runtime.RuntimeExport("RhSpanHelpers_MemSet")] -#endif - internal static void Fill(ref byte dest, byte value, nuint len) - { - if (!Vector.IsHardwareAccelerated) - { - goto CannotVectorize; - } - - if (len >= (nuint)Vector.Count) - { - // We have enough data for at least one vectorized write. - Vector vector = new (value); - nuint stopLoopAtOffset = len & (nuint)(nint)(2 * (int)-Vector.Count); // intentional sign extension carries the negative bit - nuint offset = 0; - - // Loop, writing 2 vectors at a time. - // Compare 'numElements' rather than 'stopLoopAtOffset' because we don't want a dependency - // on the very recently calculated 'stopLoopAtOffset' value. - if (len >= (uint)(2 * Vector.Count)) - { - do - { - Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref dest, offset), vector); - Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref dest, offset + (nuint)Vector.Count), vector); - offset += (uint)(2 * Vector.Count); - } while (offset < stopLoopAtOffset); - } - - // At this point, if any data remains to be written, it's strictly less than - // 2 * sizeof(Vector) bytes. The loop above had us write an even number of vectors. - // If the total byte length instead involves us writing an odd number of vectors, write - // one additional vector now. The bit check below tells us if we're in an "odd vector - // count" situation. - if ((len & (nuint)Vector.Count) != 0) - { - Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref dest, offset), vector); - } - - // It's possible that some small buffer remains to be populated - something that won't - // fit an entire vector's worth of data. Instead of falling back to a loop, we'll write - // a vector at the very end of the buffer. This may involve overwriting previously - // populated data, which is fine since we're splatting the same value for all entries. - // There's no need to perform a length check here because we already performed this - // check before entering the vectorized code path. - Unsafe.WriteUnaligned(ref Unsafe.AddByteOffset(ref dest, len - (nuint)Vector.Count), vector); - - // And we're done! - return; - } - - CannotVectorize: - - // If we reached this point, we cannot vectorize this T, or there are too few - // elements for us to vectorize. Fall back to an unrolled loop. - nuint i = 0; - - // Write 8 elements at a time - if (len >= 8) - { - nuint stopLoopAtOffset = len & ~(nuint)7; - do - { - Unsafe.Add(ref dest, (nint)i + 0) = value; - Unsafe.Add(ref dest, (nint)i + 1) = value; - Unsafe.Add(ref dest, (nint)i + 2) = value; - Unsafe.Add(ref dest, (nint)i + 3) = value; - Unsafe.Add(ref dest, (nint)i + 4) = value; - Unsafe.Add(ref dest, (nint)i + 5) = value; - Unsafe.Add(ref dest, (nint)i + 6) = value; - Unsafe.Add(ref dest, (nint)i + 7) = value; - } while ((i += 8) < stopLoopAtOffset); - } - - // Write next 4 elements if needed - if ((len & 4) != 0) - { - Unsafe.Add(ref dest, (nint)i + 0) = value; - Unsafe.Add(ref dest, (nint)i + 1) = value; - Unsafe.Add(ref dest, (nint)i + 2) = value; - Unsafe.Add(ref dest, (nint)i + 3) = value; - i += 4; - } - - // Write next 2 elements if needed - if ((len & 2) != 0) - { - Unsafe.Add(ref dest, (nint)i + 0) = value; - Unsafe.Add(ref dest, (nint)i + 1) = value; - i += 2; - } - - // Write final element if needed - if ((len & 1) != 0) - { - Unsafe.Add(ref dest, (nint)i) = value; - } - } - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 37f90c695090c2..69ce8c1f7fada9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -7,6 +7,7 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 #pragma warning disable 8500 // sizeof of managed types diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index ee378b7646b5cb..da77d42320a98a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -8,6 +8,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 #pragma warning disable 8500 // sizeof of managed types @@ -15,7 +16,6 @@ namespace System { internal static partial class SpanHelpers // .T { - [Intrinsic] // Unrolled for small sizes public static unsafe void Fill(ref T refData, nuint numElements, T value) { // Early checks to see if it's even possible to vectorize - JIT will turn these checks into consts. diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs index 7776024aaeb1fd..a7e5f48d63180d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.cs @@ -12,9 +12,330 @@ namespace System { internal static partial class SpanHelpers { + public static unsafe void ClearWithoutReferences(ref byte b, nuint byteLength) + { + if (byteLength == 0) + return; + +#if TARGET_AMD64 || TARGET_ARM64 || TARGET_LOONGARCH64 + // The exact matrix on when ZeroMemory is faster than InitBlockUnaligned is very complex. The factors to consider include + // type of hardware and memory alignment. This threshold was chosen as a good balance across different configurations. + if (byteLength > 768) + goto PInvoke; + Unsafe.InitBlockUnaligned(ref b, 0, (uint)byteLength); + return; +#else + // TODO: Optimize other platforms to be on par with AMD64 CoreCLR + // Note: It's important that this switch handles lengths at least up to 22. + // See notes below near the main loop for why. + + // The switch will be very fast since it can be implemented using a jump + // table in assembly. See http://stackoverflow.com/a/449297/4077294 for more info. + + switch (byteLength) + { + case 1: + b = 0; + return; + case 2: + Unsafe.As(ref b) = 0; + return; + case 3: + Unsafe.As(ref b) = 0; + Unsafe.Add(ref b, 2) = 0; + return; + case 4: + Unsafe.As(ref b) = 0; + return; + case 5: + Unsafe.As(ref b) = 0; + Unsafe.Add(ref b, 4) = 0; + return; + case 6: + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + return; + case 7: + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.Add(ref b, 6) = 0; + return; + case 8: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + return; + case 9: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.Add(ref b, 8) = 0; + return; + case 10: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + return; + case 11: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.Add(ref b, 10) = 0; + return; + case 12: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + return; + case 13: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.Add(ref b, 12) = 0; + return; + case 14: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; + return; + case 15: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; + Unsafe.Add(ref b, 14) = 0; + return; + case 16: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + return; + case 17: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + Unsafe.Add(ref b, 16) = 0; + return; + case 18: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 16)) = 0; + return; + case 19: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 16)) = 0; + Unsafe.Add(ref b, 18) = 0; + return; + case 20: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 16)) = 0; + return; + case 21: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 16)) = 0; + Unsafe.Add(ref b, 20) = 0; + return; + case 22: +#if TARGET_64BIT + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; +#else + Unsafe.As(ref b) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 4)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 8)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 12)) = 0; +#endif + Unsafe.As(ref Unsafe.Add(ref b, 16)) = 0; + Unsafe.As(ref Unsafe.Add(ref b, 20)) = 0; + return; + } + + // P/Invoke into the native version for large lengths + if (byteLength >= 512) goto PInvoke; + + nuint i = 0; // byte offset at which we're copying + + if (((nuint)Unsafe.AsPointer(ref b) & 3) != 0) + { + if (((nuint)Unsafe.AsPointer(ref b) & 1) != 0) + { + b = 0; + i += 1; + if (((nuint)Unsafe.AsPointer(ref b) & 2) != 0) + goto IntAligned; + } + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + i += 2; + } + + IntAligned: + + // On 64-bit IntPtr.Size == 8, so we want to advance to the next 8-aligned address. If + // (int)b % 8 is 0, 5, 6, or 7, we will already have advanced by 0, 3, 2, or 1 + // bytes to the next aligned address (respectively), so do nothing. On the other hand, + // if it is 1, 2, 3, or 4 we will want to copy-and-advance another 4 bytes until + // we're aligned. + // The thing 1, 2, 3, and 4 have in common that the others don't is that if you + // subtract one from them, their 3rd lsb will not be set. Hence, the below check. + + if ((((nuint)Unsafe.AsPointer(ref b) - 1) & 4) == 0) + { + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + i += 4; + } + + nuint end = byteLength - 16; + byteLength -= i; // lower 4 bits of byteLength represent how many bytes are left *after* the unrolled loop + + // We know due to the above switch-case that this loop will always run 1 iteration; max + // bytes we clear before checking is 23 (7 to align the pointers, 16 for 1 iteration) so + // the switch handles lengths 0-22. + Debug.Assert(end >= 7 && i <= end); + + // This is separated out into a different variable, so the i + 16 addition can be + // performed at the start of the pipeline and the loop condition does not have + // a dependency on the writes. + nuint counter; + + do + { + counter = i + 16; + + // This loop looks very costly since there appear to be a bunch of temporary values + // being created with the adds, but the jit (for x86 anyways) will convert each of + // these to use memory addressing operands. + + // So the only cost is a bit of code size, which is made up for by the fact that + // we save on writes to b. + +#if TARGET_64BIT + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i + 8)) = 0; +#else + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i + 4)) = 0; + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i + 8)) = 0; + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i + 12)) = 0; +#endif + + i = counter; + + // See notes above for why this wasn't used instead + // i += 16; + } + while (counter <= end); + + if ((byteLength & 8) != 0) + { +#if TARGET_64BIT + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; +#else + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i + 4)) = 0; +#endif + i += 8; + } + if ((byteLength & 4) != 0) + { + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + i += 4; + } + if ((byteLength & 2) != 0) + { + Unsafe.As(ref Unsafe.AddByteOffset(ref b, i)) = 0; + i += 2; + } + if ((byteLength & 1) != 0) + { + Unsafe.AddByteOffset(ref b, i) = 0; + // We're not using i after this, so not needed + // i += 1; + } + + return; +#endif + + PInvoke: + Buffer._ZeroMemory(ref b, byteLength); + } + public static unsafe void ClearWithReferences(ref IntPtr ip, nuint pointerSizeLength) { - Debug.Assert(Unsafe.IsOpportunisticallyAligned(ref ip, (uint)sizeof(IntPtr)), "Should've been aligned on natural word boundary."); + Debug.Assert((int)Unsafe.AsPointer(ref ip) % sizeof(IntPtr) == 0, "Should've been aligned on natural word boundary."); // First write backward 8 natural words at a time. // Writing backward allows us to get away with only simple modifications to the @@ -329,6 +650,7 @@ public static unsafe void Reverse(ref T elements, nuint length) ReverseInner(ref elements, length); } +#pragma warning disable IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 [MethodImpl(MethodImplOptions.AggressiveInlining)] private static void ReverseInner(ref T elements, nuint length) { @@ -345,5 +667,6 @@ private static void ReverseInner(ref T elements, nuint length) last = ref Unsafe.Subtract(ref last, 1); } while (Unsafe.IsAddressLessThan(ref first, ref last)); } +#pragma warning restore IDE0060 // https://github.com/dotnet/roslyn-analyzers/issues/6228 } } diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Comparison.cs b/src/libraries/System.Private.CoreLib/src/System/String.Comparison.cs index 356903063d1561..66b9432dfe24fd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Comparison.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Comparison.cs @@ -533,7 +533,6 @@ public bool EndsWith(string value) return EndsWith(value, StringComparison.CurrentCulture); } - [Intrinsic] // Unrolled and vectorized for half-constant input (Ordinal) public bool EndsWith(string value, StringComparison comparisonType) { ArgumentNullException.ThrowIfNull(value); diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs index 5507aed9c77004..85801e101a1735 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Ascii.cs @@ -141,7 +141,7 @@ private static unsafe bool IsValidCore(ref T searchSpace, int length) where T // Try to opportunistically align the reads below. The input isn't pinned, so the GC // is free to move the references. We're therefore assuming that reads may still be unaligned. // They may also be unaligned if the input chars aren't 2-byte aligned. - nuint misalignedElements = Unsafe.OpportunisticMisalignment(ref searchSpace, (uint)Vector256.Count) / (nuint)sizeof(T); + nuint misalignedElements = ((nuint)Unsafe.AsPointer(ref searchSpace) & (nuint)(Vector256.Count - 1)) / (nuint)sizeof(T); i -= misalignedElements; Debug.Assert((int)i > 3 * Vector256.Count); @@ -193,7 +193,7 @@ private static unsafe bool IsValidCore(ref T searchSpace, int length) where T // Try to opportunistically align the reads below. The input isn't pinned, so the GC // is free to move the references. We're therefore assuming that reads may still be unaligned. // They may also be unaligned if the input chars aren't 2-byte aligned. - nuint misalignedElements = Unsafe.OpportunisticMisalignment(ref searchSpace, (uint)Vector128.Count) / (nuint)sizeof(T); + nuint misalignedElements = ((nuint)Unsafe.AsPointer(ref searchSpace) & (nuint)(Vector128.Count - 1)) / (nuint)sizeof(T); i -= misalignedElements; Debug.Assert((int)i > 3 * Vector128.Count); diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs index 95ebfdc7737eb6..5e4e6c696739e0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/Encoding.cs @@ -1297,7 +1297,7 @@ internal unsafe bool AddChar(char ch, int numBytes) { // Throw maybe _bytes -= numBytes; // Didn't encode these bytes - _enc.ThrowCharsOverflow(_decoder, _chars == _charStart); // Throw? + _enc.ThrowCharsOverflow(_decoder, _bytes <= _byteStart); // Throw? return false; // No throw, but no store either } @@ -1316,7 +1316,7 @@ internal unsafe bool AddChar(char ch1, char ch2, int numBytes) { // Throw maybe _bytes -= numBytes; // Didn't encode these bytes - _enc.ThrowCharsOverflow(_decoder, _chars == _charStart); // Throw? + _enc.ThrowCharsOverflow(_decoder, _bytes <= _byteStart); // Throw? return false; // No throw, but no store either } return AddChar(ch1, numBytes) && AddChar(ch2, numBytes); diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/UTF32Encoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/UTF32Encoding.cs index 022c87f070feba..83d4e2ee6daa21 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/UTF32Encoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/UTF32Encoding.cs @@ -934,7 +934,7 @@ internal override unsafe int GetChars(byte* bytes, int byteCount, if (iChar >= 0x10000) { // Surrogates take 2 - if (charEnd - chars < 2) + if (chars >= charEnd - 1) { // Throwing or stopping // We either read enough bytes for bytes-=4 to work, or we're diff --git a/src/libraries/System.Private.CoreLib/src/System/Text/UnicodeEncoding.cs b/src/libraries/System.Private.CoreLib/src/System/Text/UnicodeEncoding.cs index 00fe83247e44dd..6bee6e9d6aae4c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Text/UnicodeEncoding.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Text/UnicodeEncoding.cs @@ -1535,7 +1535,7 @@ internal sealed override unsafe int GetChars( } // Valid surrogate pair, add our lastChar (will need 2 chars) - if (charEnd - chars < 2) + if (chars >= charEnd - 1) { // couldn't find room for this surrogate pair // We either advanced bytes or chars should == charStart and throw below diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs index f9f34ecb19cf04..66cf6a03f607a1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Lock.cs @@ -38,7 +38,10 @@ public sealed partial class Lock private uint _state; // see State for layout private uint _recursionCount; private short _spinCount; - private ushort _waiterStartTimeMs; + + // The lowest bit is a flag, when set it indicates that the lock should use trivial waits + private ushort _waiterStartTimeMsAndFlags; + private AutoResetEvent? _waitEvent; #if NATIVEAOT // The method needs to be public in NativeAOT so that other private libraries can access it @@ -48,7 +51,10 @@ internal Lock(bool useTrivialWaits) #endif : this() { - State.InitializeUseTrivialWaits(this, useTrivialWaits); + if (useTrivialWaits) + { + _waiterStartTimeMsAndFlags = 1; + } } /// @@ -482,7 +488,7 @@ private ThreadId TryEnterSlow(int timeoutMs, ThreadId currentThreadId) int remainingTimeoutMs = timeoutMs; while (true) { - if (!waitEvent.WaitOneNoCheck(remainingTimeoutMs, new State(this).UseTrivialWaits)) + if (!waitEvent.WaitOneNoCheck(remainingTimeoutMs, UseTrivialWaits)) { break; } @@ -561,7 +567,19 @@ private ThreadId TryEnterSlow(int timeoutMs, ThreadId currentThreadId) return new ThreadId(0); } - private void ResetWaiterStartTime() => _waiterStartTimeMs = 0; + // Trivial waits are: + // - Not interruptible by Thread.Interrupt + // - Don't allow reentrance through APCs or message pumping + // - Not forwarded to SynchronizationContext wait overrides + private bool UseTrivialWaits => (_waiterStartTimeMsAndFlags & 1) != 0; + + private ushort WaiterStartTimeMs + { + get => (ushort)(_waiterStartTimeMsAndFlags >> 1); + set => _waiterStartTimeMsAndFlags = (ushort)((value << 1) | (_waiterStartTimeMsAndFlags & 1)); + } + + private void ResetWaiterStartTime() => WaiterStartTimeMs = 0; [MethodImpl(MethodImplOptions.AggressiveInlining)] private void RecordWaiterStartTime() @@ -572,7 +590,7 @@ private void RecordWaiterStartTime() // Don't record zero, that value is reserved for indicating that a time is not recorded currentTimeMs--; } - _waiterStartTimeMs = currentTimeMs; + WaiterStartTimeMs = currentTimeMs; } private bool ShouldStopPreemptingWaiters @@ -581,10 +599,10 @@ private bool ShouldStopPreemptingWaiters get { // If the recorded time is zero, a time has not been recorded yet - ushort waiterStartTimeMs = _waiterStartTimeMs; + ushort waiterStartTimeMs = WaiterStartTimeMs; return waiterStartTimeMs != 0 && - (ushort)(Environment.TickCount - waiterStartTimeMs) >= MaxDurationMsForPreemptingWaiters; + (ushort)Environment.TickCount - waiterStartTimeMs >= MaxDurationMsForPreemptingWaiters; } } @@ -646,6 +664,15 @@ internal nint LockIdForEvents } } + internal unsafe nint ObjectIdForEvents + { + get + { + Lock lockObj = this; + return *(nint*)Unsafe.AsPointer(ref lockObj); + } + } + internal ulong OwningThreadId => _owningThreadId; private static short DetermineMaxSpinCount() => @@ -680,8 +707,8 @@ private struct State : IEquatable private const uint SpinnerCountIncrement = (uint)1 << 2; // bits 2-4 private const uint SpinnerCountMask = (uint)0x7 << 2; private const uint IsWaiterSignaledToWakeMask = (uint)1 << 5; // bit 5 - private const uint UseTrivialWaitsMask = (uint)1 << 6; // bit 6 - private const uint WaiterCountIncrement = (uint)1 << 7; // bits 7-31 + private const byte WaiterCountShift = 6; + private const uint WaiterCountIncrement = (uint)1 << WaiterCountShift; // bits 6-31 private uint _state; @@ -760,22 +787,6 @@ private void ClearIsWaiterSignaledToWake() _state -= IsWaiterSignaledToWakeMask; } - // Trivial waits are: - // - Not interruptible by Thread.Interrupt - // - Don't allow reentrance through APCs or message pumping - // - Not forwarded to SynchronizationContext wait overrides - public bool UseTrivialWaits => (_state & UseTrivialWaitsMask) != 0; - - public static void InitializeUseTrivialWaits(Lock lockObj, bool useTrivialWaits) - { - Debug.Assert(lockObj._state == 0); - - if (useTrivialWaits) - { - lockObj._state = UseTrivialWaitsMask; - } - } - public bool HasAnyWaiters => _state >= WaiterCountIncrement; private bool TryIncrementWaiterCount() diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Future.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Future.cs index ce4f18b6099baf..827d6915dd3252 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Future.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Future.cs @@ -66,6 +66,17 @@ public class Task : Task // The value itself, if set. internal TResult? m_result; + // Extract rarely used helper for a static method in a separate type so that the Func, Task> + // generic instantiations don't contribute to all Task instantiations, but only those where WhenAny is used. + internal static class TaskWhenAnyCast + { + // Delegate used by: + // public static Task> WhenAny(IEnumerable> tasks); + // public static Task> WhenAny(params Task[] tasks); + // Used to "cast" from Task to Task>. + internal static readonly Func, Task> Value = completed => (Task)completed.Result; + } + // Construct a promise-style task without any options. internal Task() { diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs index e6d08b71f58814..09eb579f4edd3e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/Task.cs @@ -3473,16 +3473,15 @@ private void RunContinuations(object continuationObject) // separated out of Fin } // Not a single; it must be a list. - List list = (List)continuationObject; + List continuations = (List)continuationObject; // // Begin processing of continuation list // // Wait for any concurrent adds or removes to be retired - Monitor.Enter(list); - Monitor.Exit(list); - Span continuations = CollectionsMarshal.AsSpan(list); + lock (continuations) { } + int continuationCount = continuations.Count; // Fire the asynchronous continuations first. However, if we're not able to run any continuations synchronously, // then we can skip this first pass, since the second pass that tries to run everything synchronously will instead @@ -3490,7 +3489,7 @@ private void RunContinuations(object continuationObject) // separated out of Fin if (canInlineContinuations) { bool forceContinuationsAsync = false; - for (int i = 0; i < continuations.Length; i++) + for (int i = 0; i < continuationCount; i++) { // For StandardTaskContinuations, we respect the TaskContinuationOptions.ExecuteSynchronously option, // as the developer needs to explicitly opt-into running the continuation synchronously, and if they do, @@ -3544,7 +3543,7 @@ private void RunContinuations(object continuationObject) // separated out of Fin } // ... and then fire the synchronous continuations (if there are any). - for (int i = 0; i < continuations.Length; i++) + for (int i = 0; i < continuationCount; i++) { object? currentContinuation = continuations[i]; if (currentContinuation == null) @@ -4511,79 +4510,62 @@ internal void AddCompletionAction(ITaskCompletionAction action, bool addBeforeOt // Support method for AddTaskContinuation that takes care of multi-continuation logic. // Returns true if and only if the continuation was successfully queued. + // THIS METHOD ASSUMES THAT m_continuationObject IS NOT NULL. That case was taken + // care of in the calling method, AddTaskContinuation(). private bool AddTaskContinuationComplex(object tc, bool addBeforeOthers) { Debug.Assert(tc != null, "Expected non-null tc object in AddTaskContinuationComplex"); object? oldValue = m_continuationObject; - Debug.Assert(oldValue is not null, "Expected non-null m_continuationObject object"); - if (oldValue == s_taskCompletionSentinel) - { - return false; - } // Logic for the case where we were previously storing a single continuation - List? list = oldValue as List; - if (list is null) + if ((oldValue != s_taskCompletionSentinel) && (!(oldValue is List))) { // Construct a new TaskContinuation list and CAS it in. - list = new List(); - if (addBeforeOthers) - { - list.Add(tc); - list.Add(oldValue); - } - else - { - list.Add(oldValue); - list.Add(tc); - } - - object? expected = oldValue; - oldValue = Interlocked.CompareExchange(ref m_continuationObject, list, expected); - if (oldValue == expected) - { - // We successfully stored the new list with both continuations in it, so we're done. - return true; - } + Interlocked.CompareExchange(ref m_continuationObject, new List { oldValue }, oldValue); // We might be racing against another thread converting the single into - // a list, or we might be racing against task completion, so recheck for list again. - list = oldValue as List; - if (list is null) - { - Debug.Assert(oldValue == s_taskCompletionSentinel, "Expected m_continuationObject to be list or sentinel"); - return false; - } + // a list, or we might be racing against task completion, so resample "list" + // below. } - lock (list) + // m_continuationObject is guaranteed at this point to be either a List or + // s_taskCompletionSentinel. + List? list = m_continuationObject as List; + Debug.Assert((list != null) || (m_continuationObject == s_taskCompletionSentinel), + "Expected m_continuationObject to be list or sentinel"); + + // If list is null, it can only mean that s_taskCompletionSentinel has been exchanged + // into m_continuationObject. Thus, the task has completed and we should return false + // from this method, as we will not be queuing up the continuation. + if (list != null) { - // It is possible for the task to complete right after we snap the copy of - // the list. If so, then return false without queuing the continuation. - if (m_continuationObject == s_taskCompletionSentinel) + lock (list) { - return false; - } + // It is possible for the task to complete right after we snap the copy of + // the list. If so, then fall through and return false without queuing the + // continuation. + if (m_continuationObject != s_taskCompletionSentinel) + { + // Before growing the list we remove possible null entries that are the + // result from RemoveContinuations() + if (list.Count == list.Capacity) + { + list.RemoveAll(l => l == null); + } - // Before growing the list we remove possible null entries that are the - // result from RemoveContinuations() - if (list.Count == list.Capacity) - { - list.RemoveAll(l => l == null); - } + if (addBeforeOthers) + list.Insert(0, tc); + else + list.Add(tc); - if (addBeforeOthers) - { - list.Insert(0, tc); - } - else - { - list.Add(tc); + return true; // continuation successfully queued, so return true. + } } } - return true; // continuation successfully queued, so return true. + // We didn't succeed in queuing the continuation, so return false. + return false; } // Record a continuation task or action. @@ -4621,15 +4603,12 @@ internal void RemoveContinuation(object continuationObject) // could be TaskCont { // This is not a list. If we have a single object (the one we want to remove) we try to replace it with an empty list. // Note we cannot go back to a null state, since it will mess up the AddTaskContinuation logic. - continuationsLocalRef = Interlocked.CompareExchange(ref m_continuationObject, new List(), continuationObject); - if (continuationsLocalRef != continuationObject) + if (Interlocked.CompareExchange(ref m_continuationObject, new List(), continuationObject) != continuationObject) { // If we fail it means that either AddContinuationComplex won the race condition and m_continuationObject is now a List // that contains the element we want to remove. Or FinishContinuations set the s_taskCompletionSentinel. - // So we should try to get a list one more time and if it's null then there is nothing else to do. - continuationsLocalListRef = continuationsLocalRef as List; - if (continuationsLocalListRef is null) - return; + // So we should try to get a list one more time + continuationsLocalListRef = m_continuationObject as List; } else { @@ -4638,20 +4617,24 @@ internal void RemoveContinuation(object continuationObject) // could be TaskCont } } - lock (continuationsLocalListRef) + // if continuationsLocalRef == null it means s_taskCompletionSentinel has been set already and there is nothing else to do. + if (continuationsLocalListRef != null) { - // There is a small chance that this task completed since we took a local snapshot into - // continuationsLocalRef. In that case, just return; we don't want to be manipulating the - // continuation list as it is being processed. - if (m_continuationObject == s_taskCompletionSentinel) return; + lock (continuationsLocalListRef) + { + // There is a small chance that this task completed since we took a local snapshot into + // continuationsLocalRef. In that case, just return; we don't want to be manipulating the + // continuation list as it is being processed. + if (m_continuationObject == s_taskCompletionSentinel) return; - // Find continuationObject in the continuation list - int index = continuationsLocalListRef.IndexOf(continuationObject); + // Find continuationObject in the continuation list + int index = continuationsLocalListRef.IndexOf(continuationObject); - if (index >= 0) - { - // null out that TaskContinuation entry, which will be interpreted as "to be cleaned up" - continuationsLocalListRef[index] = null; + if (index >= 0) + { + // null out that TaskContinuation entry, which will be interpreted as "to be cleaned up" + continuationsLocalListRef[index] = null; + } } } } @@ -5919,14 +5902,14 @@ internal static Task WhenAll(ReadOnlySpan tasks) => // TODO https://github /// A Task that gets completed when all of its constituent tasks complete. private sealed class WhenAllPromise : Task, ITaskCompletionAction { + /// Either a single faulted/canceled task, or a list of faulted/canceled tasks. + private object? _failedOrCanceled; /// The number of tasks remaining to complete. private int _remainingToComplete; internal WhenAllPromise(ReadOnlySpan tasks) { Debug.Assert(tasks.Length != 0, "Expected a non-zero length task array"); - Debug.Assert(m_stateObject is null, "Expected to be able to use the state object field for faulted/canceled tasks."); - m_stateFlags |= (int)InternalTaskOptions.HiddenState; // Throw if any of the provided tasks is null. This is best effort to inform the caller // they've made a mistake. If between the time we check for nulls and the time we hook @@ -5983,14 +5966,16 @@ public void Invoke(Task? completedTask) if (!completedTask.IsCompletedSuccessfully) { // Try to store the completed task as the first that's failed or faulted. - object? failedOrCanceled = Interlocked.CompareExchange(ref m_stateObject, completedTask, null); - if (failedOrCanceled != null) + if (Interlocked.CompareExchange(ref _failedOrCanceled, completedTask, null) != null) { // There was already something there. while (true) { + object? failedOrCanceled = _failedOrCanceled; + Debug.Assert(failedOrCanceled is not null); + // If it was a list, add it to the list. - if (failedOrCanceled is List list) + if (_failedOrCanceled is List list) { lock (list) { @@ -6001,15 +5986,13 @@ public void Invoke(Task? completedTask) // Otherwise, it was a Task. Create a new list containing that task and this one, and store it in. Debug.Assert(failedOrCanceled is Task, $"Expected Task, got {failedOrCanceled}"); - Task first = (Task)failedOrCanceled; - failedOrCanceled = Interlocked.CompareExchange(ref m_stateObject, new List { first, completedTask }, first); - if (failedOrCanceled == first) + if (Interlocked.CompareExchange(ref _failedOrCanceled, new List { (Task)failedOrCanceled, completedTask }, failedOrCanceled) == failedOrCanceled) { break; } // We lost the race, which means we should loop around one more time and it'll be a list. - Debug.Assert(failedOrCanceled is List); + Debug.Assert(_failedOrCanceled is List); } } } @@ -6018,7 +6001,7 @@ public void Invoke(Task? completedTask) // Decrement the count, and only continue to complete the promise if we're the last one. if (Interlocked.Decrement(ref _remainingToComplete) == 0) { - object? failedOrCanceled = m_stateObject; + object? failedOrCanceled = _failedOrCanceled; if (failedOrCanceled is null) { if (TplEventSource.Log.IsEnabled()) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TplEventSource.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TplEventSource.cs index 048306cdf50763..c80e3f300e5d89 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TplEventSource.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Tasks/TplEventSource.cs @@ -502,7 +502,7 @@ public void TraceSynchronousWorkEnd(CausalitySynchronousWork Work) } [NonEvent] - public unsafe void RunningContinuation(int TaskID, object Object) => RunningContinuation(TaskID, ObjectIDForEvents(Object)); + public unsafe void RunningContinuation(int TaskID, object Object) { RunningContinuation(TaskID, (long)*((void**)Unsafe.AsPointer(ref Object))); } [Event(20, Keywords = Keywords.Debug)] private void RunningContinuation(int TaskID, long Object) { @@ -511,7 +511,7 @@ private void RunningContinuation(int TaskID, long Object) } [NonEvent] - public unsafe void RunningContinuationList(int TaskID, int Index, object Object) => RunningContinuationList(TaskID, Index, ObjectIDForEvents(Object)); + public unsafe void RunningContinuationList(int TaskID, int Index, object Object) { RunningContinuationList(TaskID, Index, (long)*((void**)Unsafe.AsPointer(ref Object))); } [Event(21, Keywords = Keywords.Debug)] public void RunningContinuationList(int TaskID, int Index, long Object) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs index 9d3fd7a0466d72..1e400eec097bed 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/Thread.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; @@ -543,93 +542,41 @@ public void SetCompressedStack(CompressedStack stack) public static void MemoryBarrier() => Interlocked.MemoryBarrier(); public static void Sleep(TimeSpan timeout) => Sleep(WaitHandle.ToTimeoutMilliseconds(timeout)); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static byte VolatileRead(ref byte address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static double VolatileRead(ref double address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static short VolatileRead(ref short address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static int VolatileRead(ref int address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static long VolatileRead(ref long address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static IntPtr VolatileRead(ref IntPtr address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [return: NotNullIfNotNull(nameof(address))] public static object? VolatileRead([NotNullIfNotNull(nameof(address))] ref object? address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static sbyte VolatileRead(ref sbyte address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static float VolatileRead(ref float address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static ushort VolatileRead(ref ushort address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static uint VolatileRead(ref uint address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static ulong VolatileRead(ref ulong address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static UIntPtr VolatileRead(ref UIntPtr address) => Volatile.Read(ref address); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref byte address, byte value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref double address, double value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref short address, short value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref int address, int value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref long address, long value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref IntPtr address, IntPtr value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite([NotNullIfNotNull(nameof(value))] ref object? address, object? value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static void VolatileWrite(ref sbyte address, sbyte value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] public static void VolatileWrite(ref float address, float value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static void VolatileWrite(ref ushort address, ushort value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static void VolatileWrite(ref uint address, uint value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static void VolatileWrite(ref ulong address, ulong value) => Volatile.Write(ref address, value); - [Obsolete(Obsoletions.ThreadVolatileReadWriteMessage, DiagnosticId = Obsoletions.ThreadVolatileReadWriteDiagId, UrlFormat = Obsoletions.SharedUrlFormat)] - [EditorBrowsable(EditorBrowsableState.Never)] [CLSCompliant(false)] public static void VolatileWrite(ref UIntPtr address, UIntPtr value) => Volatile.Write(ref address, value); diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/WaitSubsystem.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/WaitSubsystem.Unix.cs index 4a349d8b303117..d8cb5da15cd935 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/WaitSubsystem.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/WaitSubsystem.Unix.cs @@ -344,6 +344,7 @@ public static int Wait( bool waitForAll, int timeoutMilliseconds) { + Debug.Assert(waitHandles != null); Debug.Assert(waitHandles.Length > 0); Debug.Assert(waitHandles.Length <= WaitHandle.MaxWaitHandles); Debug.Assert(timeoutMilliseconds >= -1); diff --git a/src/libraries/System.Private.CoreLib/src/System/TimeSpan.cs b/src/libraries/System.Private.CoreLib/src/System/TimeSpan.cs index 98d6b9523f229c..ed952a89499d51 100644 --- a/src/libraries/System.Private.CoreLib/src/System/TimeSpan.cs +++ b/src/libraries/System.Private.CoreLib/src/System/TimeSpan.cs @@ -308,195 +308,6 @@ public TimeSpan Duration() public override int GetHashCode() => _ticks.GetHashCode(); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static TimeSpan FromUnits(long units, long ticksPerUnit, long minUnits, long maxUnits) - { - System.Diagnostics.Debug.Assert(minUnits < 0); - System.Diagnostics.Debug.Assert(maxUnits > 0); - - if (units > maxUnits || units < minUnits) - { - ThrowHelper.ThrowArgumentOutOfRange_TimeSpanTooLong(); - } - return TimeSpan.FromTicks(units * ticksPerUnit); - } - - /// - /// Initializes a new instance of the structure to a specified number of - /// days. - /// - /// Number of days. - /// Returns a that represents a specified number of days. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromDays(int days) => FromUnits(days, TicksPerDay, MinDays, MaxDays); - - /// - /// Initializes a new instance of the structure to a specified number of - /// days, hours, minutes, seconds, milliseconds, and microseconds. - /// - /// Number of days. - /// Number of hours. - /// Number of minutes. - /// Number of seconds. - /// Number of milliseconds. - /// Number of microseconds. - /// Returns a that represents a specified number of days, hours, minutes, seconds, milliseconds, and microseconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromDays(int days, int hours = 0, long minutes = 0, long seconds = 0, long milliseconds = 0, long microseconds = 0) - { - Int128 totalMicroseconds = Math.BigMul(days, MicrosecondsPerDay) - + Math.BigMul(hours, MicrosecondsPerHour) - + Math.BigMul(minutes, MicrosecondsPerMinute) - + Math.BigMul(seconds, MicrosecondsPerSecond) - + Math.BigMul(milliseconds, MicrosecondsPerMillisecond) - + microseconds; - - return FromMicroseconds(totalMicroseconds); - } - - /// - /// Initializes a new instance of the structure to a specified number of - /// hours. - /// - /// Number of hours. - /// Returns a that represents a specified number of hours. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromHours(int hours) => FromUnits(hours, TicksPerHour, MinHours, MaxHours); - - /// - /// Initializes a new instance of the structure to a specified number of - /// hours, minutes, seconds, milliseconds, and microseconds. - /// - /// Number of hours. - /// Number of minutes. - /// Number of seconds. - /// Number of milliseconds. - /// Number of microseconds. - /// Returns a that represents a specified number of hours, minutes, seconds, milliseconds, and microseconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromHours(int hours, long minutes = 0, long seconds = 0, long milliseconds = 0, long microseconds = 0) - { - Int128 totalMicroseconds = Math.BigMul(hours, MicrosecondsPerHour) - + Math.BigMul(minutes, MicrosecondsPerMinute) - + Math.BigMul(seconds, MicrosecondsPerSecond) - + Math.BigMul(milliseconds, MicrosecondsPerMillisecond) - + microseconds; - - return FromMicroseconds(totalMicroseconds); - } - - /// - /// Initializes a new instance of the structure to a specified number of - /// minutes. - /// - /// Number of minutes. - /// Returns a that represents a specified number of minutes. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromMinutes(long minutes) => FromUnits(minutes, TicksPerMinute, MinMinutes, MaxMinutes); - - /// - /// Initializes a new instance of the structure to a specified number of - /// minutes, seconds, milliseconds, and microseconds. - /// - /// Number of minutes. - /// Number of seconds. - /// Number of milliseconds. - /// Number of microseconds. - /// Returns a that represents a specified number of minutes, seconds, milliseconds, and microseconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromMinutes(long minutes, long seconds = 0, long milliseconds = 0, long microseconds = 0) - { - Int128 totalMicroseconds = Math.BigMul(minutes, MicrosecondsPerMinute) - + Math.BigMul(seconds, MicrosecondsPerSecond) - + Math.BigMul(milliseconds, MicrosecondsPerMillisecond) - + microseconds; - - return FromMicroseconds(totalMicroseconds); - } - - /// - /// Initializes a new instance of the structure to a specified number of - /// seconds. - /// - /// Number of seconds. - /// Returns a that represents a specified number of seconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromSeconds(long seconds) => FromUnits(seconds, TicksPerSecond, MinSeconds, MaxSeconds); - - /// - /// Initializes a new instance of the structure to a specified number of - /// seconds, milliseconds, and microseconds. - /// - /// Number of seconds. - /// Number of milliseconds. - /// Number of microseconds. - /// Returns a that represents a specified number of seconds, milliseconds, and microseconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromSeconds(long seconds, long milliseconds = 0, long microseconds = 0) - { - Int128 totalMicroseconds = Math.BigMul(seconds, MicrosecondsPerSecond) - + Math.BigMul(milliseconds, MicrosecondsPerMillisecond) - + microseconds; - - return FromMicroseconds(totalMicroseconds); - } - - /// - /// Initializes a new instance of the structure to a specified number of - /// milliseconds, and microseconds. - /// - /// Number of milliseconds. - /// Number of microseconds. - /// Returns a that represents a specified number of milliseconds, and microseconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromMilliseconds(long milliseconds, long microseconds = 0) - { - Int128 totalMicroseconds = Math.BigMul(milliseconds, MicrosecondsPerMillisecond) - + microseconds; - - return FromMicroseconds(totalMicroseconds); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static TimeSpan FromMicroseconds(Int128 microseconds) - { - if ((microseconds > MaxMicroseconds) || (microseconds < MinMicroseconds)) - { - ThrowHelper.ThrowArgumentOutOfRange_TimeSpanTooLong(); - } - long ticks = (long)microseconds * TicksPerMicrosecond; - return TimeSpan.FromTicks(ticks); - } - - /// - /// Initializes a new instance of the structure to a specified number of - /// microseconds. - /// - /// Number of microseconds. - /// Returns a that represents a specified number of microseconds. - /// - /// The parameters specify a value less than or greater than - /// - public static TimeSpan FromMicroseconds(long microseconds) => FromUnits(microseconds, TicksPerMicrosecond, MinMicroseconds, MaxMicroseconds); - public static TimeSpan FromHours(double value) => Interval(value, TicksPerHour); private static TimeSpan Interval(double value, double scale) diff --git a/src/libraries/System.Private.CoreLib/src/System/Type.cs b/src/libraries/System.Private.CoreLib/src/System/Type.cs index 24be352778c841..17e4636296c177 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Type.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Type.cs @@ -135,15 +135,7 @@ public bool IsPrimitive get => IsPrimitiveImpl(); } protected abstract bool IsPrimitiveImpl(); - public bool IsValueType - { -#if NATIVEAOT - // https://github.com/dotnet/runtime/issues/97272 - [MethodImpl(MethodImplOptions.NoOptimization)] -#endif - [Intrinsic] - get => IsValueTypeImpl(); - } + public bool IsValueType { [Intrinsic] get => IsValueTypeImpl(); } protected virtual bool IsValueTypeImpl() => IsSubclassOf(typeof(ValueType)); [Intrinsic] diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Schema/XsdDateTime.cs b/src/libraries/System.Private.Xml/src/System/Xml/Schema/XsdDateTime.cs index 7e75cc55afada2..10b78d608233ee 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Schema/XsdDateTime.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Schema/XsdDateTime.cs @@ -396,7 +396,6 @@ public static implicit operator DateTime(XsdDateTime xdt) { case DateTimeTypeCode.GMonth: case DateTimeTypeCode.GDay: - // codeql[cs/leap-year/unsafe-date-construction-from-two-elements] - The XML specification does not explicitly define this behavior for parsing in a non-leap year. We intentionally throw here. Altering this behavior to be more resilient, producing dates like 2/28 or 3/1, could introduce unintended consequences and may not be desirable for user. result = new DateTime(DateTime.Now.Year, xdt.Month, xdt.Day); break; case DateTimeTypeCode.Time: diff --git a/src/libraries/System.Private.Xml/src/System/Xml/Xsl/QIL/QilXmlWriter.cs b/src/libraries/System.Private.Xml/src/System/Xml/Xsl/QIL/QilXmlWriter.cs index 7480892901b501..574901728e1eb1 100644 --- a/src/libraries/System.Private.Xml/src/System/Xml/Xsl/QIL/QilXmlWriter.cs +++ b/src/libraries/System.Private.Xml/src/System/Xml/Xsl/QIL/QilXmlWriter.cs @@ -206,7 +206,7 @@ protected override QilNode VisitQilExpression(QilExpression qil) foreach (QilNode n in fdecls) { // i.e. - this.writer.WriteStartElement(Enum.GetName(n.NodeType)!); + this.writer.WriteStartElement(Enum.GetName(typeof(QilNodeType), n.NodeType)!); this.writer.WriteAttributeString("id", _ngen.NameOf(n)); WriteXmlType(n); @@ -277,7 +277,7 @@ protected override void BeforeVisit(QilNode node) WriteAnnotations(node.Annotation); // Call WriteStartElement - this.writer.WriteStartElement("", Enum.GetName(node.NodeType)!, ""); + this.writer.WriteStartElement("", Enum.GetName(typeof(QilNodeType), node.NodeType)!, ""); // Write common attributes #if QIL_TRACE_NODE_CREATION diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_AllowXmlAttributes.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_AllowXmlAttributes.cs index 2d5ffe75de7f5a..2bed5cd37c19b5 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_AllowXmlAttributes.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_AllowXmlAttributes.cs @@ -273,7 +273,7 @@ public void v1(string xmlFile, string xsdFile, bool allowXmlAttributes, int expe if (xsdFile != null) xss.Add(null, Path.Combine(testData, xsdFile)); - using XmlReader vr = CreateReader(Path.Combine(testData, xmlFile), xss, allowXmlAttributes); + XmlReader vr = CreateReader(Path.Combine(testData, xmlFile), xss, allowXmlAttributes); while (vr.Read()) ; Assert.Equal(warningCount, expectedWarningCount); diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_EnableUpaCheck.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_EnableUpaCheck.cs index 9576efe8378444..5de2beac167de1 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_EnableUpaCheck.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_EnableUpaCheck.cs @@ -146,7 +146,7 @@ public void v1(object param0, object param1, object param2, int[] expectedErrorL xss.ValidationEventHandler += new ValidationEventHandler(ValidationCallback); xss.Add(null, Path.Combine(testData, xsdFile)); - using XmlReader vr = CreateReader(Path.Combine(testData, xmlFile), xss, false); + XmlReader vr = CreateReader(Path.Combine(testData, xmlFile), xss, false); while (vr.Read()) ; CError.Compare(errorCount, expectedErrorCount, "Error Count mismatch"); diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Misc.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Misc.cs index cd8de8da283594..0cec7918b37879 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Misc.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Misc.cs @@ -89,7 +89,7 @@ public void v2() XmlSchemaValidationFlags.ProcessInlineSchema; settings.ValidationEventHandler += new ValidationEventHandler(ValidationCallback); settings.Schemas.Add(ss); - using XmlReader vr = XmlReader.Create(Path.Combine(TestData._Root, "bug115049.xml"), settings); + XmlReader vr = XmlReader.Create(Path.Combine(TestData._Root, "bug115049.xml"), settings); while (vr.Read()) ; CError.Compare(errorCount, 1, "Error Count mismatch!"); return; @@ -108,7 +108,7 @@ public void v4() XmlSchemaValidationFlags.ProcessSchemaLocation | XmlSchemaValidationFlags.ProcessInlineSchema; settings.ValidationEventHandler += new ValidationEventHandler(ValidationCallback); - using XmlReader vr = XmlReader.Create(new StringReader(xml), settings, (string)null); + XmlReader vr = XmlReader.Create(new StringReader(xml), settings, (string)null); while (vr.Read()) ; CError.Compare(errorCount, 0, "Error Count mismatch!"); CError.Compare(warningCount, 1, "Warning Count mismatch!"); @@ -531,7 +531,7 @@ public void v106() #pragma warning disable 0618 settings.ProhibitDtd = false; #pragma warning restore 0618 - using XmlReader r = XmlReader.Create(Path.Combine(TestData._Root, "XMLSchema.xsd"), settings); + XmlReader r = XmlReader.Create(Path.Combine(TestData._Root, "XMLSchema.xsd"), settings); ss1.Add(null, r); ss1.Compile(); @@ -568,7 +568,7 @@ public void v107() settings.Schemas.Add(schemaSet); settings.ValidationEventHandler += new ValidationEventHandler(ValidationCallback); settings.ValidationType = ValidationType.Schema; - using XmlReader vr = XmlReader.Create(new StringReader(strXml), settings); + XmlReader vr = XmlReader.Create(new StringReader(strXml), settings); while (vr.Read()) ; @@ -742,7 +742,7 @@ public void v112() XmlSchema mainSchema = set.Add(null, Path.Combine(TestData._Root, "bug382035a.xsd")); set.Compile(); - using XmlReader r = XmlReader.Create(Path.Combine(TestData._Root, "bug382035a1.xsd")); + XmlReader r = XmlReader.Create(Path.Combine(TestData._Root, "bug382035a1.xsd")); XmlSchema reParsedInclude = XmlSchema.Read(r, new ValidationEventHandler(ValidationCallback)); ((XmlSchemaExternal)mainSchema.Includes[0]).Schema = reParsedInclude; @@ -766,7 +766,7 @@ public void v113() settings.ValidationFlags |= XmlSchemaValidationFlags.ReportValidationWarnings | XmlSchemaValidationFlags.ProcessSchemaLocation; settings.ValidationEventHandler += new ValidationEventHandler(ValidationCallback); settings.ValidationType = ValidationType.Schema; - using XmlReader vr = XmlReader.Create(new StringReader(strXml), settings); + XmlReader vr = XmlReader.Create(new StringReader(strXml), settings); while (vr.Read()) ; @@ -1056,7 +1056,7 @@ public void Dev10_40509() string xsd = Path.Combine(TestData._Root, "bug511217.xsd"); XmlSchemaSet s = new XmlSchemaSet(); s.XmlResolver = new XmlUrlResolver(); - using XmlReader r = XmlReader.Create(xsd); + XmlReader r = XmlReader.Create(xsd); s.Add(null, r); s.Compile(); XmlReaderSettings rs = new XmlReaderSettings(); diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_NmTokens.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_NmTokens.cs index 86d8c82d1e330d..c9844ef6930638 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_NmTokens.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_NmTokens.cs @@ -30,7 +30,7 @@ public void TestSchemaCompile(string fileName, bool negative) Assert.True(negative, args.Message); numevents++; }; - using XmlReader r = XmlReader.Create(xsd); + XmlReader r = XmlReader.Create(xsd); s.Add(null, r); s.Compile(); Assert.False(negative && numevents != 1); diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_ProhibitDTD.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_ProhibitDTD.cs index 497253ceb83cdc..c1737a3395d078 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_ProhibitDTD.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_ProhibitDTD.cs @@ -149,7 +149,7 @@ public void v2() Initialize(); XmlSchemaSet xss = new XmlSchemaSet(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r = CreateReader(Path.Combine(TestData._Root, "bug356711_a.xsd")); + XmlReader r = CreateReader(Path.Combine(TestData._Root, "bug356711_a.xsd")); try { xss.Add(null, r); @@ -190,7 +190,7 @@ public void v4() XmlSchemaSet xss = new XmlSchemaSet(); xss.XmlResolver = new XmlUrlResolver(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r = CreateReader(Path.Combine(TestData._Root, "bug356711.xsd")); + XmlReader r = CreateReader(Path.Combine(TestData._Root, "bug356711.xsd")); try { xss.Add(null, r); @@ -314,7 +314,7 @@ public void v10(object param0) xss.XmlResolver = new XmlUrlResolver(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r = CreateReader(Path.Combine(TestData._Root, param0.ToString()), false); + XmlReader r = CreateReader(Path.Combine(TestData._Root, param0.ToString()), false); try { xss.Add(null, r); @@ -363,8 +363,8 @@ public void v12(object param0) XmlSchemaSet xss = new XmlSchemaSet(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r = CreateReader(Path.Combine(TestData._Root, param0.ToString()), false); - using XmlReader r2 = CreateReader(r, true); + XmlReader r = CreateReader(Path.Combine(TestData._Root, param0.ToString()), false); + XmlReader r2 = CreateReader(r, true); try { xss.Add(null, r2); @@ -387,8 +387,8 @@ public void v13(object param0) xss.XmlResolver = new XmlUrlResolver(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r = CreateReader(Path.Combine(TestData._Root, param0.ToString()), false); - using XmlReader r2 = CreateReader(r, true); + XmlReader r = CreateReader(Path.Combine(TestData._Root, param0.ToString()), false); + XmlReader r2 = CreateReader(r, true); try { @@ -413,7 +413,7 @@ public void v14() xss.XmlResolver = new XmlUrlResolver(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r = CreateReader(Path.Combine(TestData._Root, "bug356711.xsd"), false); + XmlReader r = CreateReader(Path.Combine(TestData._Root, "bug356711.xsd"), false); try { @@ -437,8 +437,8 @@ public void v15() XmlSchemaSet xss = new XmlSchemaSet(); xss.ValidationEventHandler += ValidationCallback; - using XmlReader r1 = CreateReader(Path.Combine(TestData._Root, "bug356711_a.xsd")); - using XmlReader r2 = CreateReader(Path.Combine(TestData._Root, "bug356711_b.xsd"), false); + XmlReader r1 = CreateReader(Path.Combine(TestData._Root, "bug356711_a.xsd")); + XmlReader r2 = CreateReader(Path.Combine(TestData._Root, "bug356711_b.xsd"), false); try { @@ -482,7 +482,7 @@ public void v20(object param0) try { - using XmlReader reader = CreateReader(Path.Combine(TestData._Root, param0.ToString()), xss, true); + XmlReader reader = CreateReader(Path.Combine(TestData._Root, param0.ToString()), xss, true); while (reader.Read()) ; } catch (XmlException) @@ -539,7 +539,7 @@ public void v22(object param0) try { - using XmlReader reader = CreateReader(Path.Combine(TestData._Root, param0.ToString()), xss, false); + XmlReader reader = CreateReader(Path.Combine(TestData._Root, param0.ToString()), xss, false); while (reader.Read()) ; } catch (XmlException) @@ -561,8 +561,8 @@ public void v23() try { - using XmlReader r1 = CreateReader(Path.Combine(TestData._Root, "bug356711_1.xml"), true); - using XmlReader r2 = CreateReader(r1, xss, false); + XmlReader r1 = CreateReader(Path.Combine(TestData._Root, "bug356711_1.xml"), true); + XmlReader r2 = CreateReader(r1, xss, false); while (r2.Read()) ; } catch (XmlException) diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Reprocess.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Reprocess.cs index 894569f9e3fd59..9aa0e604223791 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Reprocess.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaSet/TC_SchemaSet_Reprocess.cs @@ -560,7 +560,7 @@ public void v51(object param0, object param1, object param2, object param3) settings.ValidationEventHandler += new ValidationEventHandler(ValidationCallback); settings.ValidationType = ValidationType.Schema; settings.Schemas = set; - using XmlReader reader = XmlReader.Create(xmlFile, settings); + XmlReader reader = XmlReader.Create(xmlFile, settings); while (reader.Read()) { } CError.Compare(bWarningCallback, false, "Warning count mismatch"); @@ -581,8 +581,8 @@ public void v51(object param0, object param1, object param2, object param3) bErrorCallback = false; _output.WriteLine("Second validation ***************"); settings.Schemas = set; - using XmlReader reader2 = XmlReader.Create(xmlFile, settings); - while (reader2.Read()) { } + reader = XmlReader.Create(xmlFile, settings); + while (reader.Read()) { } CError.Compare(bWarningCallback, false, "Warning count mismatch"); CError.Compare(bErrorCallback, false, "Error count mismatch"); @@ -606,8 +606,8 @@ public void v51(object param0, object param1, object param2, object param3) _output.WriteLine("Third validation, Expecting errors ***************"); settings.Schemas = set; - using XmlReader reader3 = XmlReader.Create(xmlFile, settings); - while (reader3.Read()) { } + reader = XmlReader.Create(xmlFile, settings); + while (reader.Read()) { } CError.Compare(bWarningCallback, false, "Warning count mismatch"); CError.Compare(bErrorCallback, true, "Error count mismatch"); @@ -623,7 +623,7 @@ public XmlSchema LoadSchema(string path, string baseuri) _output.WriteLine("Correct uri: " + correctUri); using (Stream s = new FileStream(Path.GetFullPath(path), FileMode.Open, FileAccess.Read, FileShare.Read, 1)) { - using XmlReader r = XmlReader.Create(s, new XmlReaderSettings(), includeUri); + XmlReader r = XmlReader.Create(s, new XmlReaderSettings(), includeUri); _output.WriteLine("Reader uri: " + r.BaseURI); using (r) { diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/PropertiesTests.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/PropertiesTests.cs index 0675045b014eed..70fc7f882ce0b6 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/PropertiesTests.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/PropertiesTests.cs @@ -306,7 +306,7 @@ public void XmlReaderAsALineInfoProvider() XmlSchemaInfo info = new XmlSchemaInfo(); XmlSchemaValidator val = CreateValidator(CreateSchemaSetFromXml(xmlSrc)); - using XmlReader r = XmlReader.Create(new StringReader(xmlSrc)); + XmlReader r = XmlReader.Create(new StringReader(xmlSrc)); val.LineInfoProvider = (r as IXmlLineInfo); diff --git a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/ValidateMisc.cs b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/ValidateMisc.cs index 25cb496ed38a46..1e9fb3a25599a9 100644 --- a/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/ValidateMisc.cs +++ b/src/libraries/System.Private.Xml/tests/XmlSchema/XmlSchemaValidatorApi/ValidateMisc.cs @@ -901,7 +901,7 @@ public void XSDValidationGeneratesInvalidError_1() // TempDirectory path must end with a DirectorySeratorChar, otherwise it will throw in the Xml validation. settings.Schemas.Add("mainschema", XmlReader.Create(new StringReader(xsd), null, EnsureTrailingSlash(tempDirectory.Path))); settings.ValidationType = ValidationType.Schema; - using XmlReader reader = XmlReader.Create(new StringReader(xml), settings); + XmlReader reader = XmlReader.Create(new StringReader(xml), settings); XmlDocument doc = new XmlDocument(); doc.Load(reader); @@ -926,7 +926,7 @@ public void XSDValidationGeneratesInvalidError_2() // TempDirectory path must end with a DirectorySeratorChar, otherwise it will throw in the Xml validation. settings.Schemas.Add("mainschema", XmlReader.Create(new StringReader(xsd), null, EnsureTrailingSlash(tempDirectory.Path))); settings.ValidationType = ValidationType.Schema; - using XmlReader reader = XmlReader.Create(new StringReader(xml), settings); + XmlReader reader = XmlReader.Create(new StringReader(xml), settings); XmlDocument doc = new XmlDocument(); doc.Load(reader); diff --git a/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.cs b/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.cs index 3690bf72a4c876..1cd9aa71da7432 100644 --- a/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.cs +++ b/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.cs @@ -24,6 +24,7 @@ protected AssemblyBuilder() { } public static System.Reflection.Emit.AssemblyBuilder DefineDynamicAssembly(System.Reflection.AssemblyName name, System.Reflection.Emit.AssemblyBuilderAccess access) { throw null; } [System.Diagnostics.CodeAnalysis.RequiresDynamicCode("Defining a dynamic assembly requires dynamic code.")] public static System.Reflection.Emit.AssemblyBuilder DefineDynamicAssembly(System.Reflection.AssemblyName name, System.Reflection.Emit.AssemblyBuilderAccess access, System.Collections.Generic.IEnumerable? assemblyAttributes) { throw null; } + public static System.Reflection.Emit.AssemblyBuilder DefinePersistedAssembly(System.Reflection.AssemblyName name, System.Reflection.Assembly coreAssembly, System.Collections.Generic.IEnumerable? assemblyAttributes = null) { throw null; } public System.Reflection.Emit.ModuleBuilder DefineDynamicModule(string name) { throw null; } protected abstract System.Reflection.Emit.ModuleBuilder DefineDynamicModuleCore(string name); public override bool Equals(object? obj) { throw null; } @@ -54,6 +55,9 @@ protected AssemblyBuilder() { } [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("Types might be removed by trimming. If the type name is a string literal, consider using Type.GetType instead.")] public override System.Type? GetType(string name, bool throwOnError, bool ignoreCase) { throw null; } public override bool IsDefined(System.Type attributeType, bool inherit) { throw null; } + public void Save(string assemblyFileName) { throw null; } + public void Save(System.IO.Stream stream) { throw null; } + protected virtual void SaveCore(System.IO.Stream stream) { } public void SetCustomAttribute(System.Reflection.ConstructorInfo con, byte[] binaryAttribute) { } public void SetCustomAttribute(System.Reflection.Emit.CustomAttributeBuilder customBuilder) { } protected abstract void SetCustomAttributeCore(System.Reflection.ConstructorInfo con, System.ReadOnlySpan binaryAttribute); @@ -470,24 +474,6 @@ public void SetCustomAttribute(System.Reflection.ConstructorInfo con, byte[] bin public void SetCustomAttribute(System.Reflection.Emit.CustomAttributeBuilder customBuilder) { } protected abstract void SetCustomAttributeCore(System.Reflection.ConstructorInfo con, System.ReadOnlySpan binaryAttribute); } -#if !BUILDING_CORELIB_REFERENCE - public sealed class PersistedAssemblyBuilder : System.Reflection.Emit.AssemblyBuilder - { - public PersistedAssemblyBuilder(System.Reflection.AssemblyName name, System.Reflection.Assembly coreAssembly, System.Collections.Generic.IEnumerable? assemblyAttributes = null) { } - public override string? FullName { get { throw null; } } - public override bool IsDynamic { get { throw null; } } - public override System.Reflection.Module ManifestModule { get { throw null; } } - [System.Diagnostics.CodeAnalysis.RequiresDynamicCode("Defining a dynamic assembly requires dynamic code.")] - protected override System.Reflection.Emit.ModuleBuilder DefineDynamicModuleCore(string name) { throw null; } - protected override System.Reflection.Emit.ModuleBuilder? GetDynamicModuleCore(string name) { throw null; } - [System.CLSCompliantAttribute(false)] - public System.Reflection.Metadata.Ecma335.MetadataBuilder GenerateMetadata(out System.Reflection.Metadata.BlobBuilder ilStream, out System.Reflection.Metadata.BlobBuilder mappedFieldData) { throw null; } - public override System.Reflection.AssemblyName GetName(bool copiedName) { throw null; } - public void Save(string assemblyFileName) { throw null; } - public void Save(System.IO.Stream stream) { throw null; } - protected override void SetCustomAttributeCore(System.Reflection.ConstructorInfo con, System.ReadOnlySpan binaryAttribute) { throw null; } - } -#endif public abstract partial class PropertyBuilder : System.Reflection.PropertyInfo { protected PropertyBuilder() { } diff --git a/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.csproj b/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.csproj index 44f0feefe67db2..4872ff7f5b6b02 100644 --- a/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.csproj +++ b/src/libraries/System.Reflection.Emit/ref/System.Reflection.Emit.csproj @@ -10,6 +10,5 @@ - \ No newline at end of file diff --git a/src/libraries/System.Reflection.Emit/src/ILLink/ILLink.Descriptors.LibraryBuild.xml b/src/libraries/System.Reflection.Emit/src/ILLink/ILLink.Descriptors.LibraryBuild.xml new file mode 100644 index 00000000000000..8cf11ab4c01f72 --- /dev/null +++ b/src/libraries/System.Reflection.Emit/src/ILLink/ILLink.Descriptors.LibraryBuild.xml @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/src/libraries/System.Reflection.Emit/src/Resources/Strings.resx b/src/libraries/System.Reflection.Emit/src/Resources/Strings.resx index 4ab50a83684c01..0688551573229c 100644 --- a/src/libraries/System.Reflection.Emit/src/Resources/Strings.resx +++ b/src/libraries/System.Reflection.Emit/src/Resources/Strings.resx @@ -117,8 +117,8 @@ System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 - - Cannot populate assembly metadata multiple times. + + Cannot save an assembly multiple times. Cannot set parent to an interface. @@ -249,6 +249,9 @@ Type passed must be an interface. + + Abstract method '{0}' in type '{1}' does not have an implementation. + Method '{0}' cannot have a method body diff --git a/src/libraries/System.Reflection.Emit/src/System.Reflection.Emit.csproj b/src/libraries/System.Reflection.Emit/src/System.Reflection.Emit.csproj index 0c34ef45f44862..3b2f29ec3672f7 100644 --- a/src/libraries/System.Reflection.Emit/src/System.Reflection.Emit.csproj +++ b/src/libraries/System.Reflection.Emit/src/System.Reflection.Emit.csproj @@ -14,6 +14,7 @@ + @@ -23,7 +24,6 @@ - diff --git a/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/PersistedAssemblyBuilder.cs b/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/AssemblyBuilderImpl.cs similarity index 50% rename from src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/PersistedAssemblyBuilder.cs rename to src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/AssemblyBuilderImpl.cs index 12fbf4704b7bac..b52bec9f11c4a8 100644 --- a/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/PersistedAssemblyBuilder.cs +++ b/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/AssemblyBuilderImpl.cs @@ -9,34 +9,18 @@ namespace System.Reflection.Emit { - /// - /// AssemblyBuilder implementation that can persist assembly to a disk or stream. - /// - public sealed class PersistedAssemblyBuilder : AssemblyBuilder + internal sealed class AssemblyBuilderImpl : AssemblyBuilder { private readonly AssemblyName _assemblyName; private readonly Assembly _coreAssembly; private readonly MetadataBuilder _metadataBuilder; private ModuleBuilderImpl? _module; - private bool _isMetadataPopulated; + private bool _previouslySaved; internal List? _customAttributes; - /// - /// Creates a instance that can be saved to a file or stream. - /// - /// The name of the assembly. - /// The assembly that denotes the "system assembly" that houses the well-known types such as - /// A collection that contains the attributes of the assembly. - /// An that can be persisted. - /// The or or is null. - /// Currently the persisted assembly doesn't support running, need to save it and load back to run. - public PersistedAssemblyBuilder(AssemblyName name, Assembly coreAssembly, IEnumerable? assemblyAttributes = null) + internal AssemblyBuilderImpl(AssemblyName name, Assembly coreAssembly, IEnumerable? assemblyAttributes = null) { - ArgumentNullException.ThrowIfNull(name); - ArgumentException.ThrowIfNullOrEmpty(name.Name, "AssemblyName.Name"); - ArgumentNullException.ThrowIfNull(coreAssembly); - _assemblyName = (AssemblyName)name.Clone(); _coreAssembly = coreAssembly; _metadataBuilder = new MetadataBuilder(); @@ -70,70 +54,20 @@ private void WritePEImage(Stream peStream, BlobBuilder ilBuilder, BlobBuilder fi peBlob.WriteContentTo(peStream); } - /// - /// Serializes the assembly to . - /// - /// The to which the assembly serialized. - /// is null. - /// A module not defined for the assembly. - /// The metadata already populated for the assembly before. - public void Save(Stream stream) => SaveInternal(stream); - - /// - /// Saves the assembly to disk. - /// - /// The file name of the assembly. - /// is null. - /// A module not defined for the assembly. - /// The metadata already populated for the assembly before. - public void Save(string assemblyFileName) - { - ArgumentNullException.ThrowIfNull(assemblyFileName); - - using var peStream = new FileStream(assemblyFileName, FileMode.Create, FileAccess.Write); - SaveInternal(peStream); - } - - private void SaveInternal(Stream stream) + protected override void SaveCore(Stream stream) { ArgumentNullException.ThrowIfNull(stream); - PopulateAssemblyMetadata(out BlobBuilder ilStream, out BlobBuilder fieldData); - WritePEImage(stream, ilStream, fieldData); - } - - - /// - /// Generates the metadata for the . - /// - /// Outputs bytes that includes all method's IL (body) emitted. - /// Outputs bytes that includes all field RVA data defined in the assembly. - /// A that includes all members defined in the Assembly. - /// A module not defined for the assembly. - /// The metadata already populated for the assembly before. - [CLSCompliant(false)] - public MetadataBuilder GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder mappedFieldData) - { - PopulateAssemblyMetadata(out ilStream, out mappedFieldData); - - return _metadataBuilder; - } - - private void PopulateAssemblyMetadata(out BlobBuilder ilStream, out BlobBuilder fieldData) - { if (_module == null) { throw new InvalidOperationException(SR.InvalidOperation_AModuleRequired); } - if (_isMetadataPopulated) // Cannot populate assembly metadata multiple times. This is consistent with Save() in .Net Framework. + if (_previouslySaved) // Cannot save an assembly multiple times. This is consistent with Save() in .Net Framework. { - throw new InvalidOperationException(SR.InvalidOperation_CannotPopulateMultipleTimes); + throw new InvalidOperationException(SR.InvalidOperation_CannotSaveMultipleTimes); } - ilStream = new BlobBuilder(); - fieldData = new BlobBuilder(); - // Add assembly metadata AssemblyDefinitionHandle assemblyHandle = _metadataBuilder.AddAssembly( _metadataBuilder.GetOrAddString(value: _assemblyName.Name!), @@ -145,10 +79,15 @@ private void PopulateAssemblyMetadata(out BlobBuilder ilStream, out BlobBuilder hashAlgorithm: (AssemblyHashAlgorithm)_assemblyName.HashAlgorithm #pragma warning restore SYSLIB0037 ); - _module.WriteCustomAttributes(_customAttributes, assemblyHandle); - _module.AppendMetadata(new MethodBodyStreamEncoder(ilStream), fieldData); - _isMetadataPopulated = true; + + var ilBuilder = new BlobBuilder(); + var fieldDataBuilder = new BlobBuilder(); + MethodBodyStreamEncoder methodBodyEncoder = new MethodBodyStreamEncoder(ilBuilder); + _module.AppendMetadata(methodBodyEncoder, fieldDataBuilder); + + WritePEImage(stream, ilBuilder, fieldDataBuilder); + _previouslySaved = true; } private static AssemblyFlags AddContentType(AssemblyFlags flags, AssemblyContentType contentType) diff --git a/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/ModuleBuilderImpl.cs b/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/ModuleBuilderImpl.cs index c79e95962e3db5..b95e74c7846c33 100644 --- a/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/ModuleBuilderImpl.cs +++ b/src/libraries/System.Reflection.Emit/src/System/Reflection/Emit/ModuleBuilderImpl.cs @@ -16,7 +16,7 @@ internal sealed class ModuleBuilderImpl : ModuleBuilder private readonly Assembly _coreAssembly; private readonly string _name; private readonly MetadataBuilder _metadataBuilder; - private readonly PersistedAssemblyBuilder _assemblyBuilder; + private readonly AssemblyBuilderImpl _assemblyBuilder; private readonly TypeBuilderImpl _globalTypeBuilder; private readonly Dictionary _assemblyReferences = new(); private readonly Dictionary _typeReferences = new(); @@ -37,7 +37,7 @@ internal sealed class ModuleBuilderImpl : ModuleBuilder private static readonly Type[] s_coreTypes = { typeof(void), typeof(object), typeof(bool), typeof(char), typeof(sbyte), typeof(byte), typeof(short), typeof(ushort), typeof(int), typeof(uint), typeof(long), typeof(ulong), typeof(float), typeof(double), typeof(string), typeof(nint), typeof(nuint), typeof(TypedReference) }; - internal ModuleBuilderImpl(string name, Assembly coreAssembly, MetadataBuilder builder, PersistedAssemblyBuilder assemblyBuilder) + internal ModuleBuilderImpl(string name, Assembly coreAssembly, MetadataBuilder builder, AssemblyBuilderImpl assemblyBuilder) { _coreAssembly = coreAssembly; _name = name; diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveAssemblyBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveAssemblyBuilder.cs similarity index 85% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveAssemblyBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveAssemblyBuilder.cs index 45f926b1a67bd7..5dbabe683e0734 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveAssemblyBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveAssemblyBuilder.cs @@ -5,19 +5,14 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Reflection.Metadata; -using System.Reflection.Metadata.Ecma335; -using System.Reflection.PortableExecutable; using System.Runtime.InteropServices; -using System.Runtime.Loader; using Xunit; namespace System.Reflection.Emit.Tests { [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotBrowser))] - public class AssemblySaveAssemblyBuilderTests + public class AssemblySaveAssemblyBuilder { - private readonly AssemblyName _assemblyName = new AssemblyName("MyAssembly"); public class Outer { public class Inner @@ -29,82 +24,6 @@ void DoNothing () { } } } - [Fact] - public void PersistedAssemblyBuilder_ConstructorValidations() - { - Assert.Throws("name", () => new PersistedAssemblyBuilder(null, typeof(object).Assembly)); - Assert.Throws("coreAssembly", () => new PersistedAssemblyBuilder(_assemblyName, null)); - Assert.Throws("AssemblyName.Name", () => AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName())); - } - - [Fact] - public void PersistedAssemblyBuilder_SaveValidations() - { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(_assemblyName); - - Assert.Throws("assemblyFileName", () => ab.Save(assemblyFileName: null)); - Assert.Throws("stream", () => ab.Save(stream: null)); - Assert.Throws(() => ab.Save(assemblyFileName: "File")); // no module defined - } - - [Fact] - public void PersistedAssemblyBuilder_GenerateMetadataValidation() - { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(_assemblyName); - Assert.Throws(() => ab.GenerateMetadata(out var _, out var _)); // no module defined - ab.DefineDynamicModule("MyModule"); - MetadataBuilder metadata = ab.GenerateMetadata(out var ilStream, out var mappedFieldData); - Assert.NotNull(metadata); - Assert.NotNull(ilStream); - Assert.NotNull(mappedFieldData); - Assert.Throws(() => ab.GenerateMetadata(out var _, out var _)); // cannot re-generate metadata - } - - [Fact] - public void PersistedAssemblyBuilder_GenerateMetadataWithEntryPoint() - { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(_assemblyName); - TypeBuilder tb = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); - MethodBuilder mb1 = tb.DefineMethod("SumMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int), typeof(int)]); - ILGenerator il = mb1.GetILGenerator(); - il.Emit(OpCodes.Ldarg_0); - il.Emit(OpCodes.Ldarg_1); - il.Emit(OpCodes.Add); - il.Emit(OpCodes.Ret); - MethodBuilder entryPoint = tb.DefineMethod("Main", MethodAttributes.HideBySig | MethodAttributes.Public | MethodAttributes.Static, typeof(int), null); - ILGenerator il2 = entryPoint.GetILGenerator(); - il2.Emit(OpCodes.Ldc_I4_S, 10); - il2.Emit(OpCodes.Ldc_I4_2); - il2.Emit(OpCodes.Call, mb1); - il2.Emit(OpCodes.Ret); - tb.CreateType(); - - MetadataBuilder metadataBuilder = ab.GenerateMetadata(out BlobBuilder ilStream, out BlobBuilder fieldData); - PEHeaderBuilder peHeaderBuilder = new PEHeaderBuilder( - imageCharacteristics: Characteristics.ExecutableImage, - subsystem: Subsystem.WindowsCui); - - ManagedPEBuilder peBuilder = new ManagedPEBuilder( - header: peHeaderBuilder, - metadataRootBuilder: new MetadataRootBuilder(metadataBuilder), - ilStream: ilStream, - mappedFieldData: fieldData, - entryPoint: MetadataTokens.MethodDefinitionHandle(entryPoint.MetadataToken)); - - BlobBuilder peBlob = new BlobBuilder(); - peBuilder.Serialize(peBlob); - - // in case saving to a file: - using var stream = new MemoryStream(); - peBlob.WriteContentTo(stream); - - stream.Seek(0, SeekOrigin.Begin); - Assembly assembly = AssemblyLoadContext.Default.LoadFromStream(stream); - MethodInfo method = assembly.EntryPoint; - Assert.Equal("Main", method.Name); - Assert.Equal(12, method.Invoke(null, null)); - } - [Fact] public void AssemblyWithDifferentTypes() { @@ -115,7 +34,7 @@ public void AssemblyWithDifferentTypes() aName.CultureInfo = new CultureInfo("en"); aName.Flags = AssemblyNameFlags.Retargetable; - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(aName); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(aName); ab.SetCustomAttribute(new CustomAttributeBuilder(typeof(AssemblyDelaySignAttribute).GetConstructor([typeof(bool)]), [true])); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveConstructorBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveConstructorBuilderTests.cs similarity index 94% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveConstructorBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveConstructorBuilderTests.cs index 34fe5c2aec3278..618b01b2a9ae06 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveConstructorBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveConstructorBuilderTests.cs @@ -15,7 +15,7 @@ public void DefineConstructorsTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ConstructorBuilder constructor = type.DefineDefaultConstructor(MethodAttributes.Public); ConstructorBuilder constructor2 = type.DefineConstructor(MethodAttributes.Public, CallingConventions.Standard, [typeof(int)]); constructor2.DefineParameter(1, ParameterAttributes.None, "parameter1"); @@ -51,7 +51,7 @@ public void DefineConstructorsTest() [Fact] public void DefineDefaultConstructor_WithTypeBuilderParent() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); type.CreateType(); TypeBuilder child = ab.GetDynamicModule("MyModule").DefineType("ChildType", TypeAttributes.Public | TypeAttributes.Class); child.SetParent(type); @@ -69,7 +69,7 @@ public void DefineDefaultConstructor_TypesWithGenericParents() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); type.DefineGenericParameters("T"); ConstructorBuilder constructor = type.DefineConstructor(MethodAttributes.Public, CallingConventions.Standard, Type.EmptyTypes); FieldBuilder field = type.DefineField("TestField", typeof(bool), FieldAttributes.Public | FieldAttributes.Static); @@ -112,7 +112,7 @@ public void DefineDefaultConstructor_TypesWithGenericParents() [Fact] public void DefineDefaultConstructor_Interface_ThrowsInvalidOperationException() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); TypeBuilder type = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Interface | TypeAttributes.Abstract); Assert.Throws(() => type.DefineDefaultConstructor(MethodAttributes.Public)); } @@ -120,7 +120,7 @@ public void DefineDefaultConstructor_Interface_ThrowsInvalidOperationException() [Fact] public void DefineDefaultConstructor_ThrowsNotSupportedException_IfParentNotCreated() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); TypeBuilder child = ab.GetDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public); child.SetParent(type); Assert.Throws(() => child.DefineDefaultConstructor(MethodAttributes.Public)); @@ -136,7 +136,7 @@ public void DefineDefaultConstructor_StaticVirtual_ThrowsArgumentException() [Fact] public void DefineDefaultConstructor_ParentNoDefaultConstructor_ThrowsNotSupportedException() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); FieldBuilder field = type.DefineField("TestField", typeof(int), FieldAttributes.Family); ConstructorBuilder constructor = type.DefineConstructor(MethodAttributes.Public, CallingConventions.HasThis, new[] { typeof(int) }); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveCustomAttributeTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveCustomAttributeTests.cs similarity index 98% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveCustomAttributeTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveCustomAttributeTests.cs index 1fd2d53188361a..35f04f3776ccea 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveCustomAttributeTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveCustomAttributeTests.cs @@ -125,7 +125,7 @@ private static void WriteAssemblyToDisk(AssemblyName assemblyName, Type[] types, List? moduleAttributes = null, List? typeAttributes = null, List? methodAttributes = null, List? fieldAttributes = null) { - PersistedAssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(assemblyName, assemblyAttributes); + AssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(assemblyName, assemblyAttributes); ModuleBuilder mb = assemblyBuilder.DefineDynamicModule(assemblyName.Name); PopulateMembersForModule(mb, types, moduleAttributes, typeAttributes, methodAttributes, fieldAttributes); assemblyBuilder.Save(fileLocation); @@ -194,7 +194,7 @@ public void CreateStructWithPseudoCustomAttributesTest() new CustomAttributeBuilder(typeof(SpecialNameAttribute).GetConstructor(Type.EmptyTypes), new object[] { }) }; - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); TypeBuilder tb = ab.DefineDynamicModule("Module").DefineType(type.FullName, type.Attributes, type.BaseType); DefineFieldsAndSetAttributes(fieldAttributes.ToList(), type.GetFields(), tb); typeAttributes.ForEach(tb.SetCustomAttribute); @@ -286,7 +286,7 @@ public void InterfacesWithPseudoCustomAttributes() new CustomAttributeBuilder(marshalAsEnumCtor, new object[] { UnmanagedType.CustomMarshaler }, new FieldInfo[] { typeof(MarshalAsAttribute).GetField("MarshalType")}, new object[] { typeof(EmptyTestClass).AssemblyQualifiedName })}; - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); TypeBuilder tb = ab.DefineDynamicModule("Module").DefineType(type.FullName, type.Attributes); typeAttributes.ForEach(tb.SetCustomAttribute); DefineMethodsAndSetAttributes(methodAttributes, tb, type.GetMethods(), parameterAttributes); @@ -439,7 +439,7 @@ public void MarshalAsPseudoCustomAttributesTest(CustomAttributeBuilder attribute using (TempFile file = TempFile.Create()) { Type type = typeof(StructWithFields); - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); TypeBuilder tb = ab.DefineDynamicModule("Module").DefineType(type.FullName, type.Attributes, type.BaseType); FieldInfo stringField = type.GetFields()[1]; FieldBuilder fb = tb.DefineField(stringField.Name, stringField.FieldType, stringField.Attributes); @@ -477,7 +477,7 @@ public void EnumBuilderSetCustomAttributesTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); EnumBuilder enumBuilder = ab.DefineDynamicModule("Module").DefineEnum("TestEnum", TypeAttributes.Public, typeof(int)); ConstructorInfo attributeConstructor = typeof(BoolAttribute).GetConstructor(new Type[] { typeof(bool) }); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveEnumBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveEnumBuilderTests.cs similarity index 95% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveEnumBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveEnumBuilderTests.cs index 29a94bacd15d84..2711ba48db97f4 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveEnumBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveEnumBuilderTests.cs @@ -64,7 +64,7 @@ public void DefineLiteral(Type underlyingType, object literalValue) { using (TempFile file = TempFile.Create()) { - EnumBuilder enumBuilder = CreateAssemblyAndDefineEnum(out PersistedAssemblyBuilder assemblyBuilder, out TypeBuilder type, underlyingType); + EnumBuilder enumBuilder = CreateAssemblyAndDefineEnum(out AssemblyBuilder assemblyBuilder, out TypeBuilder type, underlyingType); FieldBuilder literal = enumBuilder.DefineLiteral("FieldOne", literalValue); enumBuilder.CreateTypeInfo(); type.CreateTypeInfo(); @@ -97,7 +97,7 @@ public void SaveArrayTypeSignature(int rank, string name) { using (TempFile file = TempFile.Create()) { - EnumBuilder enumBuilder = CreateAssemblyAndDefineEnum(out PersistedAssemblyBuilder ab, out TypeBuilder tb); + EnumBuilder enumBuilder = CreateAssemblyAndDefineEnum(out AssemblyBuilder ab, out TypeBuilder tb); Type arrayType = rank == 0 ? enumBuilder.MakeArrayType() : enumBuilder.MakeArrayType(rank); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); mb.SetReturnType(arrayType); @@ -118,7 +118,7 @@ public void SaveArrayTypeSignature(int rank, string name) } } - private EnumBuilder CreateAssemblyAndDefineEnum(out PersistedAssemblyBuilder assemblyBuilder, + private EnumBuilder CreateAssemblyAndDefineEnum(out AssemblyBuilder assemblyBuilder, out TypeBuilder type, Type? underlyingType = null) { assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); @@ -140,7 +140,7 @@ public void SaveByRefTypeSignature() { using (TempFile file = TempFile.Create()) { - EnumBuilder eb = CreateAssemblyAndDefineEnum(out PersistedAssemblyBuilder assemblyBuilder, out TypeBuilder tb); + EnumBuilder eb = CreateAssemblyAndDefineEnum(out AssemblyBuilder assemblyBuilder, out TypeBuilder tb); Type byrefType = eb.MakeByRefType(); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); mb.SetReturnType(byrefType); @@ -173,7 +173,7 @@ public void SavePointerTypeSignature() { using (TempFile file = TempFile.Create()) { - EnumBuilder eb = CreateAssemblyAndDefineEnum(out PersistedAssemblyBuilder assemblyBuilder, out TypeBuilder tb); + EnumBuilder eb = CreateAssemblyAndDefineEnum(out AssemblyBuilder assemblyBuilder, out TypeBuilder tb); Type pointerType = eb.MakePointerType(); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); mb.SetReturnType(pointerType); @@ -213,7 +213,7 @@ public void EnumTypeField_DefaultValueShouldMatchUnderlyingType() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); + AssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(PopulateAssemblyName()); ModuleBuilder mb = assemblyBuilder.DefineDynamicModule("My Module"); TypeBuilder tb = mb.DefineType("TestType", TypeAttributes.Class | TypeAttributes.Public); EnumBuilder eb = mb.DefineEnum("TestEnum", TypeAttributes.Public, typeof(int)); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveEventBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveEventBuilderTests.cs similarity index 97% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveEventBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveEventBuilderTests.cs index f4692afe194986..65250fb332003e 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveEventBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveEventBuilderTests.cs @@ -17,7 +17,7 @@ public void DefineEventAndItsAccessors() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); EventBuilder eventType = type.DefineEvent("TestEvent", EventAttributes.SpecialName, typeof(int)); MethodBuilder addMethod = type.DefineMethod("AddMethod", MethodAttributes.Public | MethodAttributes.SpecialName); MethodBuilder addMethod2 = type.DefineMethod("AddMethod2", MethodAttributes.Public | MethodAttributes.HideBySig, typeof(int), Type.EmptyTypes); @@ -102,7 +102,7 @@ public void ReferenceEventInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); TypeBuilder delegateType = ab.GetDynamicModule("MyModule").DefineType("OnMissingString", TypeAttributes.Public | TypeAttributes.Sealed, typeof(MulticastDelegate)); delegateType.DefineMethod("Invoke", MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.NewSlot | MethodAttributes.Virtual, typeof(void), [typeof(string)]).SetImplementationFlags(MethodImplAttributes.Runtime); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveILGeneratorTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveILGeneratorTests.cs similarity index 96% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveILGeneratorTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveILGeneratorTests.cs index faded2adaa8b31..2fd95fa874cbed 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveILGeneratorTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveILGeneratorTests.cs @@ -17,7 +17,7 @@ public void MethodWithEmptyBody() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder methodBuilder = type.DefineMethod("EmptyMethod", MethodAttributes.Public, typeof(void), [typeof(Version)]); ILGenerator il = methodBuilder.GetILGenerator(); il.Emit(OpCodes.Ret); @@ -46,7 +46,7 @@ public void MethodReturning_Int(int size) { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method = type.DefineMethod("TestMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), Type.EmptyTypes); ILGenerator ilGenerator = method.GetILGenerator(size); @@ -77,7 +77,7 @@ public void TypeWithTwoMethod_ReferenceMethodArguments(int multiplier) { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder multiplyMethod = type.DefineMethod("MultiplyMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int)]); multiplyMethod.DefineParameter(1, ParameterAttributes.None, "myParam"); MethodBuilder addMethod = type.DefineMethod("AddMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int), typeof(int)]); @@ -123,7 +123,7 @@ public void MultipleTypesWithMultipleMethods() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder multiplyMethod = type.DefineMethod("MultiplyMethod", MethodAttributes.Public, typeof(short), [typeof(short)]); MethodBuilder addMethod = type.DefineMethod("AddMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(double), [typeof(double)]); @@ -187,7 +187,7 @@ public void MultipleTypesWithMultipleMethods() [Fact] public void ILOffset_Test() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method = type.DefineMethod("Method1", MethodAttributes.Public | MethodAttributes.Static, typeof(Type), Type.EmptyTypes); ILGenerator ilGenerator = method.GetILGenerator(); @@ -201,7 +201,7 @@ public void ILMaxStack_Test() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method1 = type.DefineMethod("Method1", MethodAttributes.Public, typeof(long), [typeof(int), typeof(long), typeof(short), typeof(byte)]); ILGenerator il1 = method1.GetILGenerator(); @@ -275,7 +275,7 @@ public void Label_ConditionalBranching() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder methodBuilder = type.DefineMethod("Method1", MethodAttributes.Public, typeof(int), [typeof(int), typeof(int)]); ILGenerator il = methodBuilder.GetILGenerator(); Label failed = il.DefineLabel(); @@ -329,7 +329,7 @@ public void Label_SwitchCase() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder methodBuilder = type.DefineMethod("Method1", MethodAttributes.Public, typeof(string), [typeof(int)]); ILGenerator il = methodBuilder.GetILGenerator(); Label defaultCase = il.DefineLabel(); @@ -397,7 +397,7 @@ public void LocalBuilderMultipleLocalsUsage() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder methodBuilder = type.DefineMethod("Method1", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int), typeof(string)]); ILGenerator il = methodBuilder.GetILGenerator(); LocalBuilder intLocal = il.DeclareLocal(typeof(int)); @@ -481,7 +481,7 @@ public void LocalBuilderMultipleTypesWithMultipleMethodsWithLocals() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder methodBuilder = type.DefineMethod("Method1", MethodAttributes.Public | MethodAttributes.Static, typeof(string), [typeof(int), typeof(string)]); ILGenerator il = methodBuilder.GetILGenerator(); LocalBuilder intLocal = il.DeclareLocal(typeof(int)); @@ -569,7 +569,7 @@ public void LocalBuilderMultipleTypesWithMultipleMethodsWithLocals() [Fact] public void LocalBuilderExceptions() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ILGenerator il = type.DefineMethod("Method1", MethodAttributes.Public).GetILGenerator(); ILGenerator anotherIL = type.DefineMethod("AnotherMethod", MethodAttributes.Public).GetILGenerator(); LocalBuilder stringLocal = il.DeclareLocal(typeof(string)); @@ -585,7 +585,7 @@ public void ReferenceFieldInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder methodBuilder = tb.DefineMethod("Method1", MethodAttributes.Public, typeof(int), [typeof(int)]); FieldBuilder fbNumber = tb.DefineField("_number", typeof(int), FieldAttributes.Private); Assert.Equal(0, fbNumber.MetadataToken); @@ -621,7 +621,7 @@ public void ReferenceFieldAndMethodsInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder methodMain = tb.DefineMethod("Main", MethodAttributes.Public, typeof(void), [typeof(int)]); FieldBuilder field = tb.DefineField("_field", typeof(int), FieldAttributes.Private); MethodInfo writeLineString = typeof(Console).GetMethod("WriteLine", [typeof(string)]); @@ -699,7 +699,7 @@ public void ReferenceConstructedGenericMethod() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ConstructorBuilder ctor = type.DefineDefaultConstructor(MethodAttributes.Public); MethodBuilder genericMethod = type.DefineMethod("GM", MethodAttributes.Public | MethodAttributes.Static); GenericTypeParameterBuilder[] methodParams = genericMethod.DefineGenericParameters("U"); @@ -738,7 +738,7 @@ public void ReferenceConstructedGenericMethodFieldOfConstructedType() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); GenericTypeParameterBuilder[] typeParams = type.DefineGenericParameters(["T"]); ConstructorBuilder ctor = type.DefineDefaultConstructor(MethodAttributes.PrivateScope | MethodAttributes.Public | MethodAttributes.HideBySig | MethodAttributes.SpecialName | MethodAttributes.RTSpecialName); @@ -833,7 +833,7 @@ public void EmitWriteLineMacroTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type1); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type1); MethodBuilder method = type1.DefineMethod("meth", MethodAttributes.Public, typeof(int), Type.EmptyTypes); FieldBuilder field = type1.DefineField("field", typeof(int), FieldAttributes.Public | FieldAttributes.Static); ILGenerator ilGenerator = method.GetILGenerator(); @@ -881,7 +881,7 @@ public void ReferenceStaticFieldAndMethodsInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder methodMain = tb.DefineMethod("Main", MethodAttributes.Public, typeof(int), [typeof(int)]); TypeBuilder anotherType = ab.GetDynamicModule("MyModule").DefineType("AnotherType", TypeAttributes.Public); FieldBuilder field = anotherType.DefineField("StaticField", typeof(int), FieldAttributes.Public | FieldAttributes.Static); @@ -934,7 +934,7 @@ public void ReferenceConstructorInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder methodBuilder = tb.DefineMethod("Method1", MethodAttributes.Public, typeof(Version), [typeof(int), typeof(int)]); ConstructorInfo ctor = typeof(Version).GetConstructor([typeof(int), typeof(int)]); @@ -964,7 +964,7 @@ public void ReferenceAType() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("meth1", MethodAttributes.Public | MethodAttributes.Static, typeof(bool), Type.EmptyTypes); ILGenerator ilGenerator = method.GetILGenerator(); LocalBuilder lb0 = ilGenerator.DeclareLocal(typeof(ValueTuple)); @@ -993,7 +993,7 @@ public void ReferenceAType() [Fact] public void MemberReferenceExceptions() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method = type.DefineMethod("Method1", MethodAttributes.Public); ILGenerator il = method.GetILGenerator(); MethodInfo nullMethod = null; @@ -1026,7 +1026,7 @@ public void SimpleTryCatchBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(float), [typeof(int), typeof(int)]); Type dBZException = typeof(DivideByZeroException); ILGenerator ilGenerator = method.GetILGenerator(); @@ -1080,7 +1080,7 @@ public void TryMultipleCatchBlocks() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(float), [typeof(int), typeof(int)]); Type dBZException = typeof(DivideByZeroException); Type exception = typeof(Exception); @@ -1156,7 +1156,7 @@ public void TryFilterCatchBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(float), [typeof(int), typeof(int)]); Type dBZException = typeof(DivideByZeroException); Type exception = typeof(Exception); @@ -1219,7 +1219,7 @@ public void TryCatchFilterCatchBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(float), [typeof(int), typeof(int)]); Type dBZException = typeof(DivideByZeroException); Type overflowException = typeof(OverflowException); @@ -1294,7 +1294,7 @@ public void TryFinallyBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(float), [typeof(int), typeof(int)]); ILGenerator ilGenerator = method.GetILGenerator(); LocalBuilder local = ilGenerator.DeclareLocal(typeof(float)); @@ -1341,7 +1341,7 @@ public void TryCatchFinallyBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(void), [typeof(int), typeof(int)]); Type exception = typeof(Exception); ILGenerator ilGenerator = method.GetILGenerator(); @@ -1379,7 +1379,7 @@ public void TryFilterCatchFinallyBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int), typeof(int)]); Type overflowEType = typeof(OverflowException); ConstructorInfo myConstructorInfo = overflowEType.GetConstructor([typeof(string)]); @@ -1452,7 +1452,7 @@ public void TryFaultBlock() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(float), [typeof(int), typeof(int)]); ILGenerator ilGenerator = method.GetILGenerator(); Label exBlock = ilGenerator.BeginExceptionBlock(); @@ -1502,7 +1502,7 @@ public void NestedTryCatchBlocks() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(void), [typeof(int), typeof(int)]); Type exception = typeof(Exception); ILGenerator ilGenerator = method.GetILGenerator(); @@ -1572,7 +1572,7 @@ public void DeeperNestedTryCatchFilterFinallyBlocks() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder method = tb.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int), typeof(int)]); Type exception = typeof(Exception); ILGenerator ilGenerator = method.GetILGenerator(); @@ -1678,7 +1678,7 @@ public void EmitCalliBlittable() int a = 1, b = 1, result = 2; using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("EmitCalliBlittable")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("EmitCalliBlittable")); TypeBuilder tb = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); Type returnType = typeof(int); MethodBuilder methodBuilder = tb.DefineMethod("F", MethodAttributes.Public | MethodAttributes.Static, returnType, [typeof(IntPtr), typeof(int), typeof(int)]); @@ -1712,7 +1712,7 @@ public void EmitCalliManagedBlittable() int a = 1, b = 1, result = 2; using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("EmitCalliManagedBlittable")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("EmitCalliManagedBlittable")); TypeBuilder tb = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); Type returnType = typeof(int); MethodBuilder methodBuilder = tb.DefineMethod("F", MethodAttributes.Public | MethodAttributes.Static, returnType, [typeof(IntPtr), typeof(int), typeof(int)]); @@ -1750,7 +1750,7 @@ public void EmitCalliNonBlittable() string input = "Test string!", result = "!gnirts tseT"; using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("EmitCalliNonBlittable")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("EmitCalliNonBlittable")); TypeBuilder tb = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); Type returnType = typeof(string); MethodBuilder methodBuilder = tb.DefineMethod("F", MethodAttributes.Public | MethodAttributes.Static, returnType, [typeof(IntPtr), typeof(string)]); @@ -1783,7 +1783,7 @@ public void EmitCall_VarArgsMethodInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder mb1 = tb.DefineMethod("VarArgMethod", MethodAttributes.Public | MethodAttributes.Static, CallingConventions.VarArgs, null, [typeof(string)]); ILGenerator il1 = mb1.GetILGenerator(); LocalBuilder locAi = il1.DeclareLocal(typeof(ArgIterator)); @@ -1872,7 +1872,7 @@ public void Emit_CallBySignature() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder mb1 = tb.DefineMethod("VarArgMethod", MethodAttributes.Public | MethodAttributes.Static, CallingConventions.VarArgs, null, [typeof(string)]); ILGenerator il1 = mb1.GetILGenerator(); FieldInfo maxStack = GetMaxStackDepthAndCurrentStackDepthField(out FieldInfo currentStack); @@ -2191,7 +2191,7 @@ public void SimpleForLoopTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder mb2 = tb.DefineMethod("SumMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int)]); ILGenerator il = mb2.GetILGenerator(); LocalBuilder sum = il.DeclareLocal(typeof(int)); @@ -2238,7 +2238,7 @@ public void RecursiveSumTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("RecursiveSumTest")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("RecursiveSumTest")); TypeBuilder tb = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); MethodBuilder mb2 = tb.DefineMethod("RecursiveMethod", MethodAttributes.Public | MethodAttributes.Static, typeof(int), [typeof(int)]); ILGenerator il = mb2.GetILGenerator(); @@ -2277,7 +2277,7 @@ public void CallOpenGenericMembersFromConstructedGenericType() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method = type.DefineMethod("M1", MethodAttributes.Public, typeof(string), null); ILGenerator ilGenerator = method.GetILGenerator(); @@ -2312,7 +2312,7 @@ public void ReferenceMethodsOfDictionaryFieldInGenericTypeWorks() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); TypeBuilder tb = ab.GetDynamicModule("MyModule").DefineType("EnumNameCache", TypeAttributes.NotPublic); GenericTypeParameterBuilder[] param = tb.DefineGenericParameters(["TEnum"]); Type fieldType = typeof(Dictionary<,>).MakeGenericType(param[0], typeof(string)); @@ -2368,7 +2368,7 @@ public void ANestedTypeUsedAsGenericArgumentWorks() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); TypeBuilder nested = type.DefineNestedType("Nested", TypeAttributes.NestedPrivate); Type nestedFType = typeof(Dictionary<,>).MakeGenericType(typeof(Type), nested); FieldBuilder nestedField = nested.DefineField("Helpers", nestedFType, FieldAttributes.Static | FieldAttributes.Private); @@ -2425,7 +2425,7 @@ public void ReferenceNestedGenericCollectionsWithTypeBuilderParameterInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); TypeBuilder nestedType = type.DefineNestedType("NestedType", TypeAttributes.NestedPublic); Type returnType = typeof(List<>).MakeGenericType(typeof(Dictionary<,>).MakeGenericType(nestedType, typeof(bool))); @@ -2458,7 +2458,7 @@ public void ReferenceNestedGenericTypeWithConstructedTypeBuilderParameterInIL() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); string[] genParams = new string[] { "T" }; GenericTypeParameterBuilder[] param = type.DefineGenericParameters(genParams); TypeBuilder nestedItem = type.DefineNestedType("ItemInfo", TypeAttributes.NestedPublic); @@ -2509,7 +2509,7 @@ public void ConstructorOfGenericTypeReferencedCorrectly() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); FieldBuilder field = type.DefineField("Field", typeof(int?), FieldAttributes.Public); ConstructorBuilder ctor = type.DefineConstructor(MethodAttributes.Public, CallingConventions.HasThis, Type.EmptyTypes); ILGenerator ctorIL = ctor.GetILGenerator(); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveModuleBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveModuleBuilderTests.cs similarity index 94% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveModuleBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveModuleBuilderTests.cs index 959ea0022d5ce3..7a009e638f589e 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveModuleBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveModuleBuilderTests.cs @@ -15,7 +15,7 @@ public void DefineGlobalMethodAndCreateGlobalFunctionsTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); MethodBuilder method = module.DefineGlobalMethod("TestMethod", MethodAttributes.Static | MethodAttributes.Public, null, null); ILGenerator ilGenerator = method.GetILGenerator(); @@ -63,7 +63,7 @@ public void DefineGlobalMethodAndCreateGlobalFunctionsTest() [Fact] public void DefineGlobalMethodAndCreateGlobalFunctions_Validations() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); Assert.Throws(() => module.DefineGlobalMethod("TestMethod", MethodAttributes.Public, null, null)); // must be static MethodBuilder method = module.DefineGlobalMethod("TestMethod", MethodAttributes.Static | MethodAttributes.Public, null, null); @@ -85,7 +85,7 @@ public static void DefinePInvokeMethodTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); DpmParams p = new DpmParams() { MethodName = "A2", LibName = "Foo2.dll", EntrypointName = "Wha2", ReturnType = typeof(int), ParameterTypes = [typeof(int)], NativeCallConv = CallingConvention.Cdecl }; @@ -117,7 +117,7 @@ public static void DefinePInvokeMethodTest() [InlineData(FieldAttributes.Assembly | FieldAttributes.SpecialName)] public void DefineUninitializedDataTest(FieldAttributes attributes) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); foreach (int size in new int[] { 1, 2, 0x003f0000 - 1 }) { @@ -133,7 +133,7 @@ public void DefineUninitializedDataTest(FieldAttributes attributes) [Fact] public void DefineUninitializedData_Validations() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); AssertExtensions.Throws("name", () => module.DefineUninitializedData(null, 1, FieldAttributes.Family)); @@ -154,7 +154,7 @@ public void DefineUninitializedData_Validations() [InlineData(FieldAttributes.Private)] public void DefineInitializedDataTest(FieldAttributes attributes) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); FieldBuilder field = module.DefineInitializedData("MyField", [01, 00, 01], attributes); @@ -167,7 +167,7 @@ public void DefineInitializedDataTest(FieldAttributes attributes) [Fact] public void DefineInitializedData_Validations() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); AssertExtensions.Throws("name", () => module.DefineInitializedData(null, [1, 0, 1], FieldAttributes.Public)); @@ -190,7 +190,7 @@ public void DefineInitializedData_EnsureAlignmentIsMinimumNeededForUseOfCreateSp { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); TypeBuilder tb = module.DefineType("MyType", TypeAttributes.Public); // Create static field data in a variety of orders that requires the runtime to actively apply alignment @@ -265,7 +265,7 @@ void CheckMethod(string name, int minAlignmentRequired, byte[] dataToVerify) [ActiveIssue("https://github.com/dotnet/runtime/issues/96389", TestRuntimes.Mono)] public void GetABCMetadataToken_Validations() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder module = ab.DefineDynamicModule("MyModule"); TypeBuilder type = module.DefineType("MyType", TypeAttributes.Public); MethodBuilder method = type.DefineMethod("TestMethod", MethodAttributes.Static | MethodAttributes.Public); @@ -294,7 +294,7 @@ public static void GetArrayMethodTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("MyAssembly")); ModuleBuilder mb = ab.DefineDynamicModule("MyModule"); TypeBuilder tb = mb.DefineType("TestClass", TypeAttributes.Public); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySavePropertyBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySavePropertyBuilderTests.cs similarity index 98% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySavePropertyBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySavePropertyBuilderTests.cs index 06caf8dbb6a7de..c18eb017c14284 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySavePropertyBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySavePropertyBuilderTests.cs @@ -18,7 +18,7 @@ public void SetPropertyAccessorsAndOtherValues() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); FieldBuilder field = type.DefineField("TestField", typeof(int), FieldAttributes.Private); PropertyBuilder property = type.DefineProperty("TestProperty", PropertyAttributes.SpecialName | PropertyAttributes.HasDefault, typeof(int), null); MethodBuilder getMethod = type.DefineMethod("GetMethod", MethodAttributes.Public | MethodAttributes.HideBySig, typeof(int), null); @@ -84,7 +84,7 @@ public void SetVariousCustomAttributes_ForProperty() PropertyInfo prop = typeof(CustomAttributeBuilder).GetProperty("Data", BindingFlags.NonPublic | BindingFlags.Instance); byte[] binaryData = (byte[])prop.GetValue(customAttrBuilder, null); - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); PropertyBuilder property = type.DefineProperty("TestProperty", PropertyAttributes.HasDefault, typeof(int), null); property.SetCustomAttribute(con, binaryData); property.SetCustomAttribute(new CustomAttributeBuilder(typeof(SpecialNameAttribute).GetConstructor(Type.EmptyTypes), [])); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTools.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTools.cs similarity index 93% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTools.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTools.cs index 6c2d65fb7ad406..dc083e521815bc 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTools.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTools.cs @@ -31,7 +31,7 @@ internal static class AssemblySaveTools internal static void WriteAssemblyToDisk(AssemblyName assemblyName, Type[] types, string fileLocation) { - PersistedAssemblyBuilder assemblyBuilder = PopulateAssemblyBuilder(assemblyName); + AssemblyBuilder assemblyBuilder = PopulateAssemblyBuilder(assemblyName); ModuleBuilder mb = assemblyBuilder.DefineDynamicModule(assemblyName.Name); PopulateMembersForModule(mb, types); @@ -67,7 +67,7 @@ private static void PopulateMembersForModule(ModuleBuilder mb, Type[] types) internal static void WriteAssemblyToStream(AssemblyName assemblyName, Type[] types, Stream stream) { - PersistedAssemblyBuilder assemblyBuilder = PopulateAssemblyBuilder(assemblyName); + AssemblyBuilder assemblyBuilder = PopulateAssemblyBuilder(assemblyName); ModuleBuilder mb = assemblyBuilder.DefineDynamicModule(assemblyName.Name); PopulateMembersForModule(mb, types); @@ -75,15 +75,15 @@ internal static void WriteAssemblyToStream(AssemblyName assemblyName, Type[] typ assemblyBuilder.Save(stream); } - internal static PersistedAssemblyBuilder PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder typeBuilder) + internal static AssemblyBuilder PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder typeBuilder) { - PersistedAssemblyBuilder ab = PopulateAssemblyBuilder(s_assemblyName, null); + AssemblyBuilder ab = PopulateAssemblyBuilder(s_assemblyName, null); typeBuilder = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); return ab; } - internal static PersistedAssemblyBuilder PopulateAssemblyBuilder(AssemblyName assemblyName, List? assemblyAttributes = null) => - new PersistedAssemblyBuilder(assemblyName, CoreMetadataAssemblyResolver.s_coreAssembly, assemblyAttributes); + internal static AssemblyBuilder PopulateAssemblyBuilder(AssemblyName assemblyName, List? assemblyAttributes = null) => + AssemblyBuilder.DefinePersistedAssembly(assemblyName, CoreMetadataAssemblyResolver.s_coreAssembly, assemblyAttributes); internal static void AssertAssemblyNameAndModule(AssemblyName sourceAName, AssemblyName aNameFromDisk, Module moduleFromDisk) { diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTypeBuilderAPIsTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTypeBuilderAPIsTests.cs similarity index 96% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTypeBuilderAPIsTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTypeBuilderAPIsTests.cs index e533a23446621e..ae84e78cfc24f9 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTypeBuilderAPIsTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTypeBuilderAPIsTests.cs @@ -17,7 +17,7 @@ public void DefineMethodOverride_InterfaceMethod() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method = type.DefineMethod("MImpl", MethodAttributes.Public | MethodAttributes.Virtual, typeof(int), null); ILGenerator ilGenerator = method.GetILGenerator(); ilGenerator.Emit(OpCodes.Ldc_I4, 2); @@ -49,7 +49,7 @@ public void DefineMethodOverride_BaseTypeImplementation() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); type.SetParent(typeof(DefineMethodOverrideClass)); MethodBuilder method = type.DefineMethod("M2", MethodAttributes.Public | MethodAttributes.Virtual, typeof(int), null); ILGenerator ilGenerator = method.GetILGenerator(); @@ -73,7 +73,7 @@ public void DefineMethodOverride_GenericInterface_Succeeds() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); type.AddInterfaceImplementation(typeof(GenericInterface)); MethodBuilder method = type.DefineMethod("Method", MethodAttributes.Public | MethodAttributes.Virtual, typeof(string), Type.EmptyTypes); ILGenerator ilGenerator = method.GetILGenerator(); @@ -219,7 +219,7 @@ public interface InterfaceWithMethod [Fact] public void DefineMethodOverride_StaticVirtualInterfaceMethodWorks() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); TypeBuilder interfaceType = module.DefineType("InterfaceType", TypeAttributes.Public | TypeAttributes.Interface | TypeAttributes.Abstract, parent: null); @@ -249,7 +249,7 @@ public abstract class Impl : InterfaceWithMethod [Fact] public void DefineMethodOverride_InterfaceImplementationWithByRefArrayTypes() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); TypeBuilder interfaceType = module.DefineType("InterfaceType", TypeAttributes.Public | TypeAttributes.Interface | TypeAttributes.Abstract); @@ -279,7 +279,7 @@ public void DefineMethodOverride_InterfaceImplementationWithByRefArrayTypes() [Fact] public void TypeBuilderImplementsGenericInterfaceWithTypeBuilderGenericConstraint() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); TypeBuilder ifaceType = module.DefineType("InterfaceType", TypeAttributes.Public | TypeAttributes.Interface | TypeAttributes.Abstract); TypeBuilder implType = module.DefineType("ImplType", TypeAttributes.Public); @@ -302,7 +302,7 @@ public void TypeBuilderImplementsGenericInterfaceWithTypeBuilderGenericConstrain [Fact] public void TypeBuilderImplementsGenericInterfaceWithTypeBuilderArgument() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); Type constructedGenericInterface = typeof(IComparable<>).MakeGenericType(type); @@ -320,7 +320,7 @@ public void TypeBuilderImplementsGenericInterfaceWithTypeBuilderArgument() [Fact] public void TypeBuilderImplementsGenericInterface() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); TypeBuilder implType = module.DefineType("ImplType", TypeAttributes.Public); @@ -341,7 +341,7 @@ public void TypeBuilderImplementsGenericInterface() [Fact] public void TypeBuilderImplementsConstructedGenericInterface() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); TypeBuilder implType = module.DefineType("ImplType", TypeAttributes.Public, parent: typeof(object), [typeof(IComparable)]); @@ -358,7 +358,7 @@ public void TypeBuilderImplementsConstructedGenericInterface() [Fact] public void GetInterfaceMap_WithImplicitOverride_DefineMethodOverride() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); ModuleBuilder module = ab.GetDynamicModule("MyModule"); TypeBuilder interfaceType = module.DefineType("InterfaceType", TypeAttributes.Public | TypeAttributes.Interface | TypeAttributes.Abstract, parent: null); @@ -448,7 +448,7 @@ public interface IStaticAbstract [Fact] public void CreateType_ValidateMethods() { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder concreteTypeWithAbstractMethod); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder concreteTypeWithAbstractMethod); concreteTypeWithAbstractMethod.DefineMethod("AbstractMethod", MethodAttributes.Public | MethodAttributes.Abstract); Assert.Throws(() => concreteTypeWithAbstractMethod.CreateType()); // Type must be declared abstract if any of its methods are abstract. @@ -514,7 +514,7 @@ public void ReturnTypeAndParameterRequiredOptionalCustomModifiers() Type[] cmodsReq2 = [typeof(uint)]; Type[] cmodsOpt1 = [typeof(int)]; Type[] cmodsOpt2 = [typeof(long), typeof(byte), typeof(bool)]; - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder methodAll = type.DefineMethod("AllModifiers", MethodAttributes.Public | MethodAttributes.Static, CallingConventions.Standard, typeof(string), [typeof(int), typeof(short)], [typeof(Version)], [typeof(int), typeof(long)], [cmodsReq1, cmodsReq2], [cmodsOpt1, cmodsOpt2]); ILGenerator ilGenerator = methodAll.GetILGenerator(); @@ -555,7 +555,7 @@ public static void DefinePInvokeMethodExecution_Windows() using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("DefinePInvokeMethodExecution_Windows")); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilder(new AssemblyName("DefinePInvokeMethodExecution_Windows")); TypeBuilder tb = ab.DefineDynamicModule("MyModule").DefineType("MyType", TypeAttributes.Public | TypeAttributes.Class); MethodBuilder mb = tb.DefinePInvokeMethod( "GetEnvironmentVariableW", @@ -624,7 +624,7 @@ public static void TestDefinePInvokeMethod(DpmParams p) { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); MethodBuilder mb = tb.DefinePInvokeMethod(p.MethodName, p.LibName, p.EntrypointName, p.Attributes, p.ManagedCallConv, p.ReturnType, p.ReturnTypeReqMods, p.ReturnTypeOptMods, p.ParameterTypes, p.ParameterTypeReqMods, p.ParameterTypeOptMods, p.NativeCallConv, p.Charset); mb.SetImplementationFlags(mb.GetMethodImplementationFlags() | MethodImplAttributes.PreserveSig); @@ -756,7 +756,7 @@ public void DefineTypeInitializer() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); FieldBuilder greetingField = tb.DefineField("Greeting", typeof(string), FieldAttributes.Private | FieldAttributes.Static); ConstructorBuilder constructor = tb.DefineTypeInitializer(); ILGenerator constructorIlGenerator = constructor.GetILGenerator(); @@ -780,7 +780,7 @@ public static void DefineUninitializedDataTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder tb); FieldBuilder myFieldBuilder = tb.DefineUninitializedData("MyGreeting", 4, FieldAttributes.Public); var loadAddressMethod = tb.DefineMethod("LoadAddress", MethodAttributes.Public | MethodAttributes.Static, typeof(IntPtr), null); var methodIL = loadAddressMethod.GetILGenerator(); @@ -870,7 +870,7 @@ public void AbstractBaseMethodImplementationReturnsDifferentType() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder ab = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); TypeBuilder baseType = ab.GetDynamicModule("MyModule").DefineType("Base", TypeAttributes.Public | TypeAttributes.Abstract); MethodBuilder getBase = baseType.DefineMethod("Get", MethodAttributes.Public | MethodAttributes.Abstract | MethodAttributes.Virtual, baseType, null); type.SetParent(baseType); diff --git a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTypeBuilderTests.cs b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTypeBuilderTests.cs similarity index 97% rename from src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTypeBuilderTests.cs rename to src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTypeBuilderTests.cs index 1713d91ba45785..eb7976a007866a 100644 --- a/src/libraries/System.Reflection.Emit/tests/PersistedAssemblyBuilder/AssemblySaveTypeBuilderTests.cs +++ b/src/libraries/System.Reflection.Emit/tests/PersistableAssemblyBuilder/AssemblySaveTypeBuilderTests.cs @@ -100,7 +100,7 @@ public void CreateMembersThatUsesTypeLoadedFromCoreAssemblyTest() { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); tb.DefineMethod("TestMethod", MethodAttributes.Public).GetILGenerator().Emit(OpCodes.Ret); tb.CreateType(); assemblyBuilder.Save(file.Path); @@ -124,7 +124,7 @@ public void CreateMembersThatUsesTypeLoadedFromCoreAssemblyTest() } } - private static TypeBuilder CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder) + private static TypeBuilder CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder) { assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(s_assemblyName); return assemblyBuilder.DefineDynamicModule("MyModule") @@ -136,7 +136,7 @@ public void AddInterfaceImplementationTest() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(s_assemblyName); + AssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(s_assemblyName); ModuleBuilder mb = assemblyBuilder.DefineDynamicModule("My Module"); TypeBuilder tb = mb.DefineType("TestInterface", TypeAttributes.Interface | TypeAttributes.Abstract, null, [typeof(IOneMethod)]); tb.AddInterfaceImplementation(typeof(INoMethod)); @@ -176,7 +176,7 @@ public void SaveGenericTypeParametersForAType(string[] typeParamNames) { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); MethodBuilder method = tb.DefineMethod("TestMethod", MethodAttributes.Public); method.GetILGenerator().Emit(OpCodes.Ldarg_0); GenericTypeParameterBuilder[] typeParams = tb.DefineGenericParameters(typeParamNames); @@ -245,7 +245,7 @@ public void SaveGenericTypeParametersForAMethod(string[] typeParamNames) { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); MethodBuilder method = tb.DefineMethod("TestMethod", MethodAttributes.Public); GenericTypeParameterBuilder[] typeParams = method.DefineGenericParameters(typeParamNames); method.GetILGenerator().Emit(OpCodes.Ldarg_0); @@ -282,7 +282,7 @@ public void SaveArrayTypeSignature(int rank, string name) { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); Type arrayType = rank == 0 ? tb.MakeArrayType() : tb.MakeArrayType(rank); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); mb.SetReturnType(arrayType); @@ -320,7 +320,7 @@ public void SaveByRefTypeSignature() { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); Type byrefType = tb.MakeByRefType(); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); mb.SetReturnType(byrefType); @@ -352,7 +352,7 @@ public void SavePointerTypeSignature() { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); Type pointerType = tb.MakePointerType(); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); mb.SetReturnType(pointerType); @@ -423,7 +423,7 @@ public void SaveGenericTypeSignature(string[] genericParams, Type[] typeArgument { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); GenericTypeParameterBuilder[] typeGenParam = tb.DefineGenericParameters(genericParams); Type genericType = tb.MakeGenericType(typeArguments); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); @@ -504,7 +504,7 @@ public void SaveGenericTypeSignatureWithGenericParameter() { using (TempFile file = TempFile.Create()) { - TypeBuilder tb = CreateAssemblyAndDefineType(out PersistedAssemblyBuilder assemblyBuilder); + TypeBuilder tb = CreateAssemblyAndDefineType(out AssemblyBuilder assemblyBuilder); GenericTypeParameterBuilder[] typeParams = tb.DefineGenericParameters(["U", "T", "P"]); MethodBuilder mb = tb.DefineMethod("TestMethod", MethodAttributes.Public); GenericTypeParameterBuilder[] methodParams = mb.DefineGenericParameters(["M", "N"]); @@ -545,7 +545,7 @@ public void SaveMultipleGenericTypeParametersToEnsureSortingWorks() { using (TempFile file = TempFile.Create()) { - PersistedAssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(s_assemblyName); + AssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilder(s_assemblyName); ModuleBuilder mb = assemblyBuilder.DefineDynamicModule("My Module"); TypeBuilder tb = mb.DefineType("TestInterface1", TypeAttributes.Interface | TypeAttributes.Abstract); GenericTypeParameterBuilder[] typeParams = tb.DefineGenericParameters(["U", "T"]); @@ -610,7 +610,7 @@ public void SaveMultipleGenericTypeParametersToEnsureSortingWorks() [Fact] public void MethodBuilderGetParametersReturnParameterTest() { - PersistedAssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); + AssemblyBuilder assemblyBuilder = AssemblySaveTools.PopulateAssemblyBuilderAndTypeBuilder(out TypeBuilder type); MethodBuilder method1 = type.DefineMethod("Method1", MethodAttributes.Public, typeof(long), [typeof(int), typeof(string)]); MethodBuilder method2 = type.DefineMethod("Method2", MethodAttributes.Static); MethodBuilder method3 = type.DefineMethod("Method1", MethodAttributes.Public, typeof(int), [typeof(string)]); diff --git a/src/libraries/System.Reflection.Emit/tests/System.Reflection.Emit.Tests.csproj b/src/libraries/System.Reflection.Emit/tests/System.Reflection.Emit.Tests.csproj index 5317b25145a55d..f70ec0e3d751ed 100644 --- a/src/libraries/System.Reflection.Emit/tests/System.Reflection.Emit.Tests.csproj +++ b/src/libraries/System.Reflection.Emit/tests/System.Reflection.Emit.Tests.csproj @@ -62,17 +62,17 @@ - - - - - - - - - - - + + + + + + + + + + + diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System.Runtime.InteropServices.JavaScript.csproj b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System.Runtime.InteropServices.JavaScript.csproj index 4b637cee5f1743..9b20df8bd80e15 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System.Runtime.InteropServices.JavaScript.csproj +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System.Runtime.InteropServices.JavaScript.csproj @@ -68,7 +68,6 @@ - diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/CancelablePromise.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/CancelablePromise.cs index 361e5febe4ff5a..fc6fe9ce0c6115 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/CancelablePromise.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/CancelablePromise.cs @@ -9,7 +9,7 @@ namespace System.Runtime.InteropServices.JavaScript { public static partial class CancelablePromise { - public static unsafe void CancelPromise(Task promise) + public static void CancelPromise(Task promise) { // this check makes sure that promiseGCHandle is still valid handle if (promise.IsCompleted) @@ -24,6 +24,7 @@ public static unsafe void CancelPromise(Task promise) { return; } + holder.IsCanceling = true; Interop.Runtime.CancelPromise(holder.GCHandle); #else @@ -33,11 +34,7 @@ public static unsafe void CancelPromise(Task promise) { return; } - - if (Interlocked.CompareExchange(ref (*holder.State).IsResolving, 1, 0) != 0) - { - return; - } + holder.IsCanceling = true; if (holder.ProxyContext.IsCurrentThread()) { @@ -45,6 +42,9 @@ public static unsafe void CancelPromise(Task promise) } else { + // FIXME: race condition + // we know that holder.GCHandle is still valid because we hold the ProxyContext lock + // but the message may arrive to the target thread after it was resolved, making GCHandle invalid Interop.Runtime.CancelPromisePost(holder.ProxyContext.JSNativeTID, holder.GCHandle); } } diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Interop/JavaScriptExports.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Interop/JavaScriptExports.cs index 6faf786f3bd539..1cd0730d41f535 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Interop/JavaScriptExports.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Interop/JavaScriptExports.cs @@ -13,7 +13,7 @@ namespace System.Runtime.InteropServices.JavaScript { // this maps to src\mono\browser\runtime\managed-exports.ts // the public methods are protected from trimming by DynamicDependency on JSFunctionBinding.BindJSFunction - // TODO: change all of these to [UnmanagedCallersOnly] and drop the reflection in mono_wasm_invoke_jsexport + // TODO: all the calls here should be running on deputy or TP in MT, not in UI thread internal static unsafe partial class JavaScriptExports { // the marshaled signature is: Task? CallEntrypoint(char* assemblyNamePtr, string[] args) @@ -45,7 +45,7 @@ public static void CallEntrypoint(JSMarshalerArgument* arguments_buffer) } catch (Exception ex) { - Environment.FailFast($"CallEntrypoint: Unexpected synchronous failure (ManagedThreadId {Environment.CurrentManagedThreadId}): " + ex); + Environment.FailFast("CallEntrypoint: Unexpected synchronous failure. " + ex); } } @@ -104,13 +104,13 @@ public static void ReleaseJSOwnedObjectByGCHandle(JSMarshalerArgument* arguments try { - // when we arrive here, we are on the thread which owns the proxies or on IO thread - var ctx = arg_exc.ToManagedContext; + // when we arrive here, we are on the thread which owns the proxies + var ctx = arg_exc.AssertCurrentThreadContext(); ctx.ReleaseJSOwnedObjectByGCHandle(arg_1.slot.GCHandle); } catch (Exception ex) { - Environment.FailFast($"ReleaseJSOwnedObjectByGCHandle: Unexpected synchronous failure (ManagedThreadId {Environment.CurrentManagedThreadId}): " + ex); + Environment.FailFast("ReleaseJSOwnedObjectByGCHandle: Unexpected synchronous failure. " + ex); } } @@ -131,11 +131,6 @@ public static void CallDelegate(JSMarshalerArgument* arguments_buffer) // we may need to consider how to solve blocking of the synchronous call // see also https://github.com/dotnet/runtime/issues/76958#issuecomment-1921418290 arg_exc.AssertCurrentThreadContext(); - - if (JSProxyContext.ThreadBlockingMode == JSHostImplementation.JSThreadBlockingMode.AllowBlockingWaitInAsyncCode) - { - Thread.ThrowOnBlockingWaitOnJSInteropThread = true; - } #endif GCHandle callback_gc_handle = (GCHandle)arg_1.slot.GCHandle; @@ -153,16 +148,8 @@ public static void CallDelegate(JSMarshalerArgument* arguments_buffer) { arg_exc.ToJS(ex); } -#if FEATURE_WASM_MANAGED_THREADS - finally - { - if (JSProxyContext.ThreadBlockingMode == JSHostImplementation.JSThreadBlockingMode.AllowBlockingWaitInAsyncCode) - { - Thread.ThrowOnBlockingWaitOnJSInteropThread = false; - } - } -#endif } + // the marshaled signature is: void CompleteTask(GCHandle holder, Exception? exceptionResult, T? result) public static void CompleteTask(JSMarshalerArgument* arguments_buffer) { @@ -174,8 +161,8 @@ public static void CompleteTask(JSMarshalerArgument* arguments_buffer) try { - // when we arrive here, we are on the thread which owns the proxies or on IO thread - var ctx = arg_exc.ToManagedContext; + // when we arrive here, we are on the thread which owns the proxies + var ctx = arg_exc.AssertCurrentThreadContext(); var holder = ctx.GetPromiseHolder(arg_1.slot.GCHandle); JSHostImplementation.ToManagedCallback callback; @@ -199,7 +186,13 @@ public static void CompleteTask(JSMarshalerArgument* arguments_buffer) lock (ctx) { callback = holder.Callback!; - ctx.ReleasePromiseHolder(arg_1.slot.GCHandle); + // if Interop.Runtime.CancelPromisePost is in flight, we can't free the GCHandle, because it's needed in JS + var isOutOfOrderCancellation = holder.IsCanceling && arg_res.slot.Type != MarshalerType.Discard; + // FIXME: when it happens we are leaking GCHandle + holder + if (!isOutOfOrderCancellation) + { + ctx.ReleasePromiseHolder(arg_1.slot.GCHandle); + } } #else callback = holder.Callback!; @@ -212,7 +205,7 @@ public static void CompleteTask(JSMarshalerArgument* arguments_buffer) } catch (Exception ex) { - Environment.FailFast($"CompleteTask: Unexpected synchronous failure (ManagedThreadId {Environment.CurrentManagedThreadId}): " + ex); + Environment.FailFast("CompleteTask: Unexpected synchronous failure. " + ex); } } @@ -247,21 +240,15 @@ public static void GetManagedStackTrace(JSMarshalerArgument* arguments_buffer) // this is here temporarily, until JSWebWorker becomes public API [DynamicDependency(DynamicallyAccessedMemberTypes.NonPublicMethods, "System.Runtime.InteropServices.JavaScript.JSWebWorker", "System.Runtime.InteropServices.JavaScript")] - // the marshaled signature is: GCHandle InstallMainSynchronizationContext(nint jsNativeTID, JSThreadBlockingMode jsThreadBlockingMode, JSThreadInteropMode jsThreadInteropMode, MainThreadingMode mainThreadingMode) + // the marshaled signature is: GCHandle InstallMainSynchronizationContext(nint jsNativeTID) public static void InstallMainSynchronizationContext(JSMarshalerArgument* arguments_buffer) { ref JSMarshalerArgument arg_exc = ref arguments_buffer[0]; // initialized by caller in alloc_stack_frame() ref JSMarshalerArgument arg_res = ref arguments_buffer[1];// initialized and set by caller ref JSMarshalerArgument arg_1 = ref arguments_buffer[2];// initialized and set by caller - ref JSMarshalerArgument arg_2 = ref arguments_buffer[3];// initialized and set by caller - ref JSMarshalerArgument arg_3 = ref arguments_buffer[4];// initialized and set by caller - ref JSMarshalerArgument arg_4 = ref arguments_buffer[5];// initialized and set by caller try { - JSProxyContext.ThreadBlockingMode = (JSHostImplementation.JSThreadBlockingMode)arg_2.slot.Int32Value; - JSProxyContext.ThreadInteropMode = (JSHostImplementation.JSThreadInteropMode)arg_3.slot.Int32Value; - JSProxyContext.MainThreadingMode = (JSHostImplementation.MainThreadingMode)arg_4.slot.Int32Value; var jsSynchronizationContext = JSSynchronizationContext.InstallWebWorkerInterop(true, CancellationToken.None); jsSynchronizationContext.ProxyContext.JSNativeTID = arg_1.slot.IntPtrValue; arg_res.slot.GCHandle = jsSynchronizationContext.ProxyContext.ContextHandle; @@ -272,50 +259,6 @@ public static void InstallMainSynchronizationContext(JSMarshalerArgument* argume } } -#pragma warning disable CS3016 // Arrays as attribute arguments is not CLS-compliant - [UnmanagedCallersOnly(CallConvs = new[] { typeof(CallConvCdecl) })] -#pragma warning restore CS3016 - // TODO ideally this would be public API callable from generated C# code for JSExport - public static void BeforeSyncJSExport(JSMarshalerArgument* arguments_buffer) - { - ref JSMarshalerArgument arg_exc = ref arguments_buffer[0]; - try - { - var ctx = arg_exc.AssertCurrentThreadContext(); - ctx.IsPendingSynchronousCall = true; - if (JSProxyContext.ThreadBlockingMode == JSHostImplementation.JSThreadBlockingMode.AllowBlockingWaitInAsyncCode) - { - Thread.ThrowOnBlockingWaitOnJSInteropThread = true; - } - } - catch (Exception ex) - { - Environment.FailFast($"BeforeSyncJSExport: Unexpected synchronous failure (ManagedThreadId {Environment.CurrentManagedThreadId}): " + ex); - } - } - -#pragma warning disable CS3016 // Arrays as attribute arguments is not CLS-compliant - [UnmanagedCallersOnly(CallConvs = new[] { typeof(CallConvCdecl) })] -#pragma warning restore CS3016 - // TODO ideally this would be public API callable from generated C# code for JSExport - public static void AfterSyncJSExport(JSMarshalerArgument* arguments_buffer) - { - ref JSMarshalerArgument arg_exc = ref arguments_buffer[0]; - try - { - var ctx = arg_exc.AssertCurrentThreadContext(); - ctx.IsPendingSynchronousCall = false; - if (JSProxyContext.ThreadBlockingMode == JSHostImplementation.JSThreadBlockingMode.AllowBlockingWaitInAsyncCode) - { - Thread.ThrowOnBlockingWaitOnJSInteropThread = false; - } - } - catch (Exception ex) - { - Environment.FailFast($"AfterSyncJSExport: Unexpected synchronous failure (ManagedThreadId {Environment.CurrentManagedThreadId}): " + ex); - } - } - #endif // the marshaled signature is: Task BindAssemblyExports(string assemblyName) @@ -337,7 +280,7 @@ public static void BindAssemblyExports(JSMarshalerArgument* arguments_buffer) } catch (Exception ex) { - Environment.FailFast($"BindAssemblyExports: Unexpected synchronous failure (ManagedThreadId {Environment.CurrentManagedThreadId}): " + ex); + Environment.FailFast("BindAssemblyExports: Unexpected synchronous failure. " + ex); } } diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSAsyncTaskScheduler.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSAsyncTaskScheduler.cs deleted file mode 100644 index 423843af3301fe..00000000000000 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSAsyncTaskScheduler.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Threading.Tasks; - -namespace System.Runtime.InteropServices.JavaScript -{ - // executes all tasks thru queue, never inline - internal sealed class JSAsyncTaskScheduler : TaskScheduler - { - private readonly JSSynchronizationContext m_synchronizationContext; - - internal JSAsyncTaskScheduler(JSSynchronizationContext synchronizationContext) - { - m_synchronizationContext = synchronizationContext; - } - - protected override void QueueTask(Task task) - { - m_synchronizationContext.Post((_) => - { - if (!TryExecuteTask(task)) - { - Environment.FailFast("Unexpected failure in JSAsyncTaskScheduler" + Environment.CurrentManagedThreadId); - } - }, null); - } - - // this is the main difference from the SynchronizationContextTaskScheduler - protected override bool TryExecuteTaskInline(Task task, bool taskWasPreviouslyQueued) - { - return false; - } - - protected override IEnumerable? GetScheduledTasks() - { - return null; - } - - public override int MaximumConcurrencyLevel => 1; - } -} diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSFunctionBinding.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSFunctionBinding.cs index 550956cce3309a..666f2caeb5a4a2 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSFunctionBinding.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSFunctionBinding.cs @@ -151,9 +151,7 @@ internal unsafe JSBindingType this[int position] /// /// Generated metadata about the method signature used for marshaling. /// The intermediate buffer with marshalled arguments. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public static void InvokeJS(JSFunctionBinding signature, Span arguments) { InvokeJSImportImpl(signature, arguments); @@ -206,9 +204,6 @@ internal static unsafe void InvokeJSFunction(JSObject jsFunction, Span arguments) { -#if FEATURE_WASM_MANAGED_THREADS - if (JSProxyContext.ThreadInteropMode == JSHostImplementation.JSThreadInteropMode.NoSyncJSInterop) - { - throw new PlatformNotSupportedException("Cannot call synchronous JS functions."); - } - else if (jsFunction.ProxyContext.IsPendingSynchronousCall) - { - throw new PlatformNotSupportedException("Cannot call synchronous JS function from inside a synchronous call to a C# method."); - } -#endif - var functionHandle = (int)jsFunction.JSHandle; fixed (JSMarshalerArgument* ptr = arguments) { @@ -259,16 +243,6 @@ internal static unsafe void InvokeJSFunctionCurrent(JSObject jsFunction, Span arguments) { -#if FEATURE_WASM_MANAGED_THREADS - if (JSProxyContext.ThreadInteropMode == JSHostImplementation.JSThreadInteropMode.NoSyncJSInterop) - { - throw new PlatformNotSupportedException("Cannot call synchronous JS functions."); - } - else if (jsFunction.ProxyContext.IsPendingSynchronousCall) - { - throw new PlatformNotSupportedException("Cannot call synchronous JS function from inside a synchronous call to a C# method."); - } -#endif var args = (nint)Unsafe.AsPointer(ref arguments[0]); var functionHandle = jsFunction.JSHandle; @@ -293,13 +267,10 @@ internal static unsafe void DispatchJSFunctionSync(JSObject jsFunction, Span arguments) { - ref JSMarshalerArgument exc = ref arguments[0]; - ref JSMarshalerArgument res = ref arguments[1]; #if FEATURE_WASM_MANAGED_THREADS var targetContext = JSProxyContext.SealJSImportCapturing(); - exc.slot.CallerNativeTID = JSProxyContext.GetNativeThreadId(); - exc.slot.ContextHandle = targetContext.ContextHandle; - res.slot.ContextHandle = targetContext.ContextHandle; + arguments[0].slot.ContextHandle = targetContext.ContextHandle; + arguments[1].slot.ContextHandle = targetContext.ContextHandle; #else var targetContext = JSProxyContext.MainThreadContext; #endif @@ -308,25 +279,9 @@ internal static unsafe void InvokeJSImportImpl(JSFunctionBinding signature, Span { // pre-allocate the result handle and Task var holder = targetContext.CreatePromiseHolder(); - res.slot.Type = MarshalerType.TaskPreCreated; - res.slot.GCHandle = holder.GCHandle; -#if FEATURE_WASM_MANAGED_THREADS - res.slot.IntPtrValue = (IntPtr)holder.State; -#endif - } -#if FEATURE_WASM_MANAGED_THREADS - else - { - if (JSProxyContext.ThreadInteropMode == JSHostImplementation.JSThreadInteropMode.NoSyncJSInterop) - { - throw new PlatformNotSupportedException("Cannot call synchronous JS functions."); - } - else if (targetContext.IsPendingSynchronousCall) - { - throw new PlatformNotSupportedException("Cannot call synchronous JS function from inside a synchronous call to a C# method."); - } + arguments[1].slot.Type = MarshalerType.TaskPreCreated; + arguments[1].slot.GCHandle = holder.GCHandle; } -#endif if (signature.IsDiscardNoWait) { @@ -403,8 +358,6 @@ internal static unsafe void DispatchJSImportSyncSend(JSFunctionBinding signature var args = (nint)Unsafe.AsPointer(ref arguments[0]); var sig = (nint)signature.Header; - ref JSMarshalerArgument exc = ref arguments[0]; - // we already know that we are not on the right thread // this will be blocking until resolved by that thread // we don't have to disable ThrowOnBlockingWaitOnJSInteropThread, because this is lock in native code @@ -413,9 +366,10 @@ internal static unsafe void DispatchJSImportSyncSend(JSFunctionBinding signature // see also https://github.com/dotnet/runtime/issues/76958#issuecomment-1921418290 Interop.Runtime.InvokeJSImportSyncSend(targetContext.JSNativeTID, sig, args); - if (exc.slot.Type != MarshalerType.None) + ref JSMarshalerArgument exceptionArg = ref arguments[0]; + if (exceptionArg.slot.Type != MarshalerType.None) { - JSHostImplementation.ThrowException(ref exc); + JSHostImplementation.ThrowException(ref exceptionArg); } } @@ -465,19 +419,17 @@ internal static unsafe JSFunctionBinding BindJSImportImpl(string functionName, s #endif internal static unsafe void ResolveOrRejectPromise(JSProxyContext targetContext, Span arguments) { - ref JSMarshalerArgument exc = ref arguments[0]; #if FEATURE_WASM_MANAGED_THREADS - exc.slot.CallerNativeTID = JSProxyContext.GetNativeThreadId(); - if (targetContext.IsCurrentThread()) #endif { fixed (JSMarshalerArgument* ptr = arguments) { Interop.Runtime.ResolveOrRejectPromise((nint)ptr); - if (exc.slot.Type != MarshalerType.None) + ref JSMarshalerArgument exceptionArg = ref arguments[0]; + if (exceptionArg.slot.Type != MarshalerType.None) { - JSHostImplementation.ThrowException(ref exc); + JSHostImplementation.ThrowException(ref exceptionArg); } } } @@ -485,6 +437,7 @@ internal static unsafe void ResolveOrRejectPromise(JSProxyContext targetContext, else { // meaning JS side needs to dispose it + ref JSMarshalerArgument exc = ref arguments[0]; exc.slot.ReceiverShouldFree = true; // this copy is freed in mono_wasm_resolve_or_reject_promise diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHost.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHost.cs index 20570bbab0fffa..0a685e996882da 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHost.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHost.cs @@ -44,9 +44,7 @@ public static JSObject DotnetInstance /// The location of the module file. /// The token to monitor for cancellation requests. /// A proxy for the JavaScript object that contains the module's exports. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public static Task ImportAsync(string moduleName, string moduleUrl, CancellationToken cancellationToken = default) { return JSHostImplementation.ImportAsync(moduleName, moduleUrl, cancellationToken); diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.Types.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.Types.cs index 1e1e74b711090d..2aa59d1814d45e 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.Types.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.Types.cs @@ -10,46 +10,30 @@ internal static partial class JSHostImplementation { internal unsafe delegate void ToManagedCallback(JSMarshalerArgument* arguments_buffer); - public sealed unsafe class PromiseHolder + public sealed class PromiseHolder { - public bool IsDisposed; public readonly nint GCHandle; // could be also virtual GCVHandle public ToManagedCallback? Callback; public JSProxyContext ProxyContext; + public bool IsDisposed; + public bool IsCanceling; #if FEATURE_WASM_MANAGED_THREADS public ManualResetEventSlim? CallbackReady; - public PromiseHolderState* State; #endif public PromiseHolder(JSProxyContext targetContext) { GCHandle = (IntPtr)InteropServices.GCHandle.Alloc(this, GCHandleType.Normal); ProxyContext = targetContext; -#if FEATURE_WASM_MANAGED_THREADS - State = (PromiseHolderState*)Marshal.AllocHGlobal(sizeof(PromiseHolderState)); - Interlocked.Exchange(ref (*State).IsResolving, 0); -#endif } public PromiseHolder(JSProxyContext targetContext, nint gcvHandle) { GCHandle = gcvHandle; ProxyContext = targetContext; -#if FEATURE_WASM_MANAGED_THREADS - State = (PromiseHolderState*)Marshal.AllocHGlobal(sizeof(PromiseHolderState)); - Interlocked.Exchange(ref (*State).IsResolving, 0); -#endif } } - // NOTE: layout has to match PromiseHolderState in marshal-to-cs.ts - [StructLayout(LayoutKind.Explicit)] - public struct PromiseHolderState - { - [FieldOffset(0)] - public volatile int IsResolving; - } - [StructLayout(LayoutKind.Explicit)] public struct IntPtrAndHandle { @@ -62,42 +46,5 @@ public struct IntPtrAndHandle [FieldOffset(0)] internal RuntimeTypeHandle typeHandle; } - - // keep in sync with types\internal.ts - public enum MainThreadingMode : int - { - // Running the managed main thread on UI thread. - // Managed GC and similar scenarios could be blocking the UI. - // Easy to deadlock. Not recommended for production. - UIThread = 0, - // Running the managed main thread on dedicated WebWorker. Marshaling all JavaScript calls to and from the main thread. - DeputyThread = 1, - // TODO comments - DeputyAndIOThreads = 2, - } - - // keep in sync with types\internal.ts - public enum JSThreadBlockingMode : int - { - // throw PlatformNotSupportedException if blocking .Wait is called on threads with JS interop, like JSWebWorker and Main thread. - // Avoids deadlocks (typically with pending JS promises on the same thread) by throwing exceptions. - NoBlockingWait = 0, - // TODO comments - AllowBlockingWaitInAsyncCode = 1, - // allow .Wait on all threads. - // Could cause deadlocks with blocking .Wait on a pending JS Task/Promise on the same thread or similar Task/Promise chain. - AllowBlockingWait = 100, - } - - // keep in sync with types\internal.ts - public enum JSThreadInteropMode : int - { - // throw PlatformNotSupportedException if synchronous JSImport/JSExport is called on threads with JS interop, like JSWebWorker and Main thread. - // calling synchronous JSImport on thread pool or new threads is allowed. - NoSyncJSInterop = 0, - // allow non-re-entrant synchronous blocking calls to and from JS on JSWebWorker on threads with JS interop, like JSWebWorker and Main thread. - // calling synchronous JSImport on thread pool or new threads is allowed. - SimpleSynchronousJSInterop = 1, - } } } diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.cs index abadf7f1d01913..736f2de5a134d1 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSHostImplementation.cs @@ -64,9 +64,7 @@ public static MethodInfo GetTaskResultMethodInfo(Type taskType) throw new InvalidOperationException(); } -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public static void ThrowException(ref JSMarshalerArgument arg) { arg.ToManaged(out Exception? ex); @@ -87,9 +85,7 @@ public static async Task ImportAsync(string moduleName, string moduleU ConfigureAwaitOptions.ForceYielding); // this helps to finish the import before we bind the module in [JSImport] } -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public static async Task CancellationHelper(Task jsTask, CancellationToken cancellationToken) { if (jsTask.IsCompletedSuccessfully) @@ -101,7 +97,7 @@ public static async Task CancellationHelper(Task jsTask, Can CancelablePromise.CancelPromise((Task)s!); }, jsTask)) { - return await jsTask.ConfigureAwait(false); + return await jsTask.ConfigureAwait(true); } } @@ -297,7 +293,6 @@ public static unsafe JSFunctionBinding BindManagedFunction(string fullyQualified var signature = GetMethodSignature(signatures, null, null); - // this will hit JS side possibly on another thread, depending on JSProxyContext.CurrentThreadContext JavaScriptImports.BindCSFunction(monoMethod, assemblyName, nameSpace, shortClassName, methodName, signatureHash, (IntPtr)signature.Header); FreeMethodSignatureBuffer(signature); @@ -315,9 +310,7 @@ public static void SetHasExternalEventLoop(Thread thread) } #endif -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public static RuntimeMethodHandle GetMethodHandleFromIntPtr(IntPtr ptr) { var temp = new IntPtrAndHandle { ptr = ptr }; diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSMarshalerArgument.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSMarshalerArgument.cs index afd0325800dc4b..cdb4b0bc627756 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSMarshalerArgument.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSMarshalerArgument.cs @@ -65,18 +65,13 @@ internal struct JSMarshalerArgumentImpl [FieldOffset(20)] internal bool ReceiverShouldFree; - - [FieldOffset(24)] - internal IntPtr CallerNativeTID; #endif } /// /// This API supports JSImport infrastructure and is not intended to be used directly from your code. /// -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void Initialize() { slot.Type = MarshalerType.None; @@ -90,9 +85,7 @@ public unsafe void Initialize() } #if FEATURE_WASM_MANAGED_THREADS -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif internal unsafe void InitializeWithContext(JSProxyContext knownProxyContext) { slot.Type = MarshalerType.None; @@ -118,11 +111,11 @@ internal JSProxyContext ToManagedContext // during JSExport, this is marshaling parameters and it would be set by: // - alloc_stack_frame // - set_js_handle/set_gc_handle - if (slot.ContextHandle == IntPtr.Zero) + var proxyContextGCHandle = (GCHandle)slot.ContextHandle; + if (proxyContextGCHandle == default) { - Environment.FailFast($"ContextHandle not set (ManagedThreadId {Environment.CurrentManagedThreadId}): {Environment.NewLine} {Environment.StackTrace}"); + Environment.FailFast($"ContextHandle not set, ManagedThreadId: {Environment.CurrentManagedThreadId}. {Environment.NewLine} {Environment.StackTrace}"); } - var proxyContextGCHandle = (GCHandle)slot.ContextHandle; var argumentContext = (JSProxyContext)proxyContextGCHandle.Target!; return argumentContext; #endif diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSObject.References.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSObject.References.cs index 48c6cbc32862c2..35424deea557bd 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSObject.References.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSObject.References.cs @@ -42,9 +42,7 @@ internal JSObject(IntPtr jsHandle, JSProxyContext ctx) /// public override string ToString() => $"(js-obj js '{JSHandle}')"; -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif internal void AssertNotDisposed() { lock (ProxyContext) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSProxyContext.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSProxyContext.cs index 02618e778bc823..e03f924de110ce 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSProxyContext.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSProxyContext.cs @@ -40,19 +40,11 @@ private JSProxyContext() public int ManagedTID; // current managed thread id public bool IsMainThread; public JSSynchronizationContext SynchronizationContext; - public JSAsyncTaskScheduler? AsyncTaskScheduler; - public static MainThreadingMode MainThreadingMode = MainThreadingMode.DeputyThread; - public static JSThreadBlockingMode ThreadBlockingMode = JSThreadBlockingMode.AllowBlockingWaitInAsyncCode; - public static JSThreadInteropMode ThreadInteropMode = JSThreadInteropMode.SimpleSynchronousJSInterop; - public bool IsPendingSynchronousCall; - -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public bool IsCurrentThread() { - return ManagedTID == Environment.CurrentManagedThreadId && (!IsMainThread || MainThreadingMode == MainThreadingMode.UIThread); + return ManagedTID == Environment.CurrentManagedThreadId; } [UnsafeAccessor(UnsafeAccessorKind.Field, Name = "thread_id")] @@ -240,9 +232,7 @@ public static JSProxyContext CurrentOperationContext #endif -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public static JSProxyContext AssertIsInteropThread() { #if FEATURE_WASM_MANAGED_THREADS @@ -281,14 +271,16 @@ public nint AllocJSVHandle() { lock (this) { - ObjectDisposedException.ThrowIf(_isDisposed, this); - if (JSVHandleFreeList.Count > 0) { var jsvHandle = JSVHandleFreeList[JSVHandleFreeList.Count - 1]; JSVHandleFreeList.RemoveAt(JSVHandleFreeList.Count - 1); return jsvHandle; } + if (NextJSVHandle == IntPtr.Zero) + { + NextJSVHandle = -2; + } return NextJSVHandle--; } } @@ -385,10 +377,6 @@ public unsafe void ReleasePromiseHolder(nint holderGCHandle) holder.IsDisposed = true; handle.Free(); } -#if FEATURE_WASM_MANAGED_THREADS - Marshal.FreeHGlobal((IntPtr)holder.State); - holder.State = null; -#endif } } @@ -426,10 +414,6 @@ public unsafe void ReleaseJSOwnedObjectByGCHandle(nint gcHandle) { holderCallback = holder.Callback; holder.IsDisposed = true; -#if FEATURE_WASM_MANAGED_THREADS - Marshal.FreeHGlobal((IntPtr)holder.State); - holder.State = null; -#endif } } holderCallback?.Invoke(null); @@ -484,7 +468,7 @@ public static void ReleaseCSOwnedObject(JSObject jso, bool skipJS) { if (IsJSVHandle(jsHandle)) { - Environment.FailFast($"TODO implement blocking ReleaseCSOwnedObjectSend to make sure the order of FreeJSVHandle is correct, ManagedThreadId: {Environment.CurrentManagedThreadId}. {Environment.NewLine} {Environment.StackTrace}"); + Environment.FailFast("TODO implement blocking ReleaseCSOwnedObjectSend to make sure the order of FreeJSVHandle is correct."); } // this is async message, we need to call this as the last thing @@ -502,7 +486,7 @@ public static void ReleaseCSOwnedObject(JSObject jso, bool skipJS) } } - #endregion +#endregion #region Dispose @@ -517,6 +501,7 @@ private void Dispose(bool disposing) { Environment.FailFast($"JSProxyContext must be disposed on the thread which owns it, ManagedThreadId: {Environment.CurrentManagedThreadId}. {Environment.NewLine} {Environment.StackTrace}"); } + ((GCHandle)ContextHandle).Free(); #endif List> copy = new(ThreadCsOwnedObjects.Values); @@ -530,7 +515,6 @@ private void Dispose(bool disposing) #if FEATURE_WASM_MANAGED_THREADS Interop.Runtime.UninstallWebWorkerInterop(); - ((GCHandle)ContextHandle).Free(); #endif foreach (var gch in ThreadJsOwnedObjects.Values) @@ -544,7 +528,6 @@ private void Dispose(bool disposing) { holder.Callback!.Invoke(null); } - ((GCHandle)holder.GCHandle).Free(); } ThreadCsOwnedObjects.Clear(); diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSSynchronizationContext.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSSynchronizationContext.cs index f967540d450793..b9c151ac79d08f 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSSynchronizationContext.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/JSSynchronizationContext.cs @@ -44,19 +44,30 @@ public WorkItem(SendOrPostCallback callback, object? data, ManualResetEventSlim? } // this need to be called from JSWebWorker or UI thread - public static unsafe JSSynchronizationContext InstallWebWorkerInterop(bool isMainThread, CancellationToken cancellationToken) + public static JSSynchronizationContext InstallWebWorkerInterop(bool isMainThread, CancellationToken cancellationToken) { var ctx = new JSSynchronizationContext(isMainThread, cancellationToken); ctx.previousSynchronizationContext = SynchronizationContext.Current; SynchronizationContext.SetSynchronizationContext(ctx); - if (JSProxyContext.ThreadBlockingMode == JSHostImplementation.JSThreadBlockingMode.NoBlockingWait) - { - Thread.ThrowOnBlockingWaitOnJSInteropThread = true; - } + // FIXME: make this configurable + // we could have 3 different modes of this + // 1) throwing on UI + JSWebWorker + // 2) throwing only on UI - small risk, more convenient. + // 3) not throwing at all - quite risky + // deadlock scenarios are: + // - .Wait for more than 5000ms and deadlock the GC suspend + // - .Wait on the Task from HTTP client, on the same thread as the HTTP client needs to resolve the Task/Promise. This could be also be a chain of promises. + // - try to create new pthread when UI thread is blocked and we run out of posix/emscripten pool of loaded workers. + // Things which lead to it are + // - Task.Wait, Signal.Wait etc + // - Monitor.Enter etc, if the lock is held by another thread for long time + // - synchronous [JSExport] into managed code, which would block + // - synchronous [JSImport] to another thread, which would block + // see also https://github.com/dotnet/runtime/issues/76958#issuecomment-1921418290 + Thread.ThrowOnBlockingWaitOnJSInteropThread = true; var proxyContext = ctx.ProxyContext; - proxyContext.AsyncTaskScheduler = new JSAsyncTaskScheduler(ctx); JSProxyContext.CurrentThreadContext = proxyContext; JSProxyContext.ExecutionContext = proxyContext; if (isMainThread) @@ -66,9 +77,7 @@ public static unsafe JSSynchronizationContext InstallWebWorkerInterop(bool isMai ctx.AwaitNewData(); - Interop.Runtime.InstallWebWorkerInterop(proxyContext.ContextHandle, - (delegate* unmanaged[Cdecl])&JavaScriptExports.BeforeSyncJSExport, - (delegate* unmanaged[Cdecl])&JavaScriptExports.AfterSyncJSExport); + Interop.Runtime.InstallWebWorkerInterop(proxyContext.ContextHandle); return ctx; } @@ -257,10 +266,6 @@ private void Pump() } try { - if (SynchronizationContext.Current == null) - { - SetSynchronizationContext(this); - } while (Queue.Reader.TryRead(out var item)) { try diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.BigInt64.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.BigInt64.cs index 87eaa76966039f..a2bce80d13b89c 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.BigInt64.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.BigInt64.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManagedBig(out long value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManagedBig(out long value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJSBig(long value) { slot.Type = MarshalerType.BigInt64; @@ -44,9 +40,7 @@ public void ToJSBig(long value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManagedBig(out long? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManagedBig(out long? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJSBig(long? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Bool.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Bool.cs index 51ae3b6aed043a..375e4b97f4a5a4 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Bool.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Bool.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out bool value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out bool value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(bool value) { slot.Type = MarshalerType.Boolean; @@ -44,9 +40,7 @@ public void ToJS(bool value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out bool? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out bool? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(bool? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Byte.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Byte.cs index 113d4f1a069084..5392fca48fae8b 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Byte.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Byte.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out byte value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out byte value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(byte value) { slot.Type = MarshalerType.Byte; @@ -44,9 +40,7 @@ public void ToJS(byte value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out byte? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out byte? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(byte? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Char.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Char.cs index d31a8d3cd35c89..7daddfb0fd4448 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Char.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Char.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out char value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out char value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(char value) { slot.Type = MarshalerType.Char; @@ -44,9 +40,7 @@ public void ToJS(char value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out char? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out char? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(char? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.DateTime.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.DateTime.cs index 7f63e5034c3b81..6521ac0c54b9a5 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.DateTime.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.DateTime.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out DateTimeOffset value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out DateTimeOffset value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(DateTimeOffset value) { slot.Type = MarshalerType.DateTimeOffset; @@ -44,9 +40,7 @@ public void ToJS(DateTimeOffset value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out DateTimeOffset? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out DateTimeOffset? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(DateTimeOffset? value) { if (value.HasValue) @@ -83,9 +75,7 @@ public void ToJS(DateTimeOffset? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out DateTime value) { if (slot.Type == MarshalerType.None) @@ -101,9 +91,7 @@ public unsafe void ToManaged(out DateTime value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(DateTime value) { slot.Type = MarshalerType.DateTime; @@ -115,9 +103,7 @@ public void ToJS(DateTime value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out DateTime? value) { if (slot.Type == MarshalerType.None) @@ -133,9 +119,7 @@ public unsafe void ToManaged(out DateTime? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(DateTime? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Double.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Double.cs index c83930bda82545..9b7f48ed4b3acd 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Double.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Double.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out double value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out double value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(double value) { slot.Type = MarshalerType.Double; @@ -44,9 +40,7 @@ public void ToJS(double value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out double? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out double? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(double? value) { if (value.HasValue) @@ -83,9 +75,7 @@ public void ToJS(double? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out double[]? value) { if (slot.Type == MarshalerType.None) @@ -103,9 +93,7 @@ public unsafe void ToManaged(out double[]? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(double[] value) { if (value == null) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Exception.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Exception.cs index 86a57b48b345aa..e526fe4b52c891 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Exception.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Exception.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out Exception? value) { if (slot.Type == MarshalerType.None) @@ -50,9 +48,7 @@ public unsafe void ToManaged(out Exception? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToJS(Exception? value) { if (value == null) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int16.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int16.cs index 54ef3ee53c242c..6a2fec5e0f2fa8 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int16.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int16.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out short value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out short value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(short value) { slot.Type = MarshalerType.Int16; @@ -44,9 +40,7 @@ public void ToJS(short value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out short? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out short? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(short? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int32.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int32.cs index a6990113c8ff4e..501484af3ab4fa 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int32.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int32.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out int value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out int value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(int value) { slot.Type = MarshalerType.Int32; @@ -44,9 +40,7 @@ public void ToJS(int value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out int? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out int? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(int? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int52.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int52.cs index 6b639ae77ef87e..4893f32f5f20b2 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int52.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Int52.cs @@ -15,9 +15,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out long value) { if (slot.Type == MarshalerType.None) @@ -33,9 +31,7 @@ public unsafe void ToManaged(out long value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(long value) { if (value < I52_MIN_VALUE || value > I52_MAX_VALUE) @@ -52,9 +48,7 @@ public void ToJS(long value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out long? value) { if (slot.Type == MarshalerType.None) @@ -70,9 +64,7 @@ public unsafe void ToManaged(out long? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(long? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.IntPtr.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.IntPtr.cs index 2737005542df48..251db16215122f 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.IntPtr.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.IntPtr.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out IntPtr value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out IntPtr value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(IntPtr value) { slot.Type = MarshalerType.IntPtr; @@ -44,9 +40,7 @@ public void ToJS(IntPtr value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out IntPtr? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out IntPtr? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(IntPtr? value) { if (value.HasValue) @@ -83,9 +75,7 @@ public void ToJS(IntPtr? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out void* value) { if (slot.Type == MarshalerType.None) @@ -101,9 +91,7 @@ public unsafe void ToManaged(out void* value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToJS(void* value) { slot.Type = MarshalerType.IntPtr; diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.JSObject.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.JSObject.cs index bdaf228f6a7200..7eb4440c565d6f 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.JSObject.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.JSObject.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out JSObject? value) { if (slot.Type == MarshalerType.None) @@ -31,9 +29,7 @@ public unsafe void ToManaged(out JSObject? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(JSObject? value) { if (value == null) @@ -68,9 +64,7 @@ public void ToJS(JSObject? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out JSObject?[]? value) { if (slot.Type == MarshalerType.None) @@ -96,9 +90,7 @@ public unsafe void ToManaged(out JSObject?[]? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToJS(JSObject?[] value) { if (value == null) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Object.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Object.cs index 8a5105cbce828a..3ce627aeff21c5 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Object.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Object.cs @@ -18,9 +18,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out object? value) { if (slot.Type == MarshalerType.None) @@ -112,9 +110,7 @@ public unsafe void ToManaged(out object? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(object? value) { if (value == null) @@ -331,9 +327,7 @@ public void ToJS(object? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out object?[]? value) { if (slot.Type == MarshalerType.None) @@ -362,9 +356,7 @@ public unsafe void ToManaged(out object?[]? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToJS(object?[] value) { if (value == null) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Single.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Single.cs index 696cc9a60089b3..c22d26c86520a5 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Single.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Single.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out float value) { if (slot.Type == MarshalerType.None) @@ -30,9 +28,7 @@ public unsafe void ToManaged(out float value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(float value) { slot.Type = MarshalerType.Single; @@ -44,9 +40,7 @@ public void ToJS(float value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out float? value) { if (slot.Type == MarshalerType.None) @@ -62,9 +56,7 @@ public unsafe void ToManaged(out float? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public void ToJS(float? value) { if (value.HasValue) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.String.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.String.cs index 247aad1ec613a6..efe764cd837fb7 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.String.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.String.cs @@ -12,9 +12,7 @@ public partial struct JSMarshalerArgument /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out string? value) { if (slot.Type == MarshalerType.None) @@ -38,9 +36,7 @@ public unsafe void ToManaged(out string? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToJS(string? value) { if (value == null) @@ -73,9 +69,7 @@ public unsafe void ToJS(string? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToManaged(out string?[]? value) { if (slot.Type == MarshalerType.None) @@ -104,9 +98,7 @@ public unsafe void ToManaged(out string?[]? value) /// It's used by JSImport code generator and should not be used by developers in source code. /// /// The value to be marshaled. -#if !DEBUG [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif public unsafe void ToJS(string?[] value) { if (value == null) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Task.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Task.cs index cf636727ce623e..eee6b5f3037cf9 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Task.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/src/System/Runtime/InteropServices/JavaScript/Marshaling/JSMarshalerArgument.Task.cs @@ -7,7 +7,6 @@ using System.ComponentModel; using System.Threading; using static System.Runtime.InteropServices.JavaScript.JSHostImplementation; -using System.Runtime.CompilerServices; namespace System.Runtime.InteropServices.JavaScript { @@ -50,15 +49,14 @@ public unsafe void ToManaged(out Task? value) lock (ctx) { PromiseHolder holder = ctx.GetPromiseHolder(slot.GCHandle); - TaskCompletionSource tcs = new TaskCompletionSource(holder, TaskCreationOptions.RunContinuationsAsynchronously); + // we want to run the continuations on the original thread which called the JSImport, so RunContinuationsAsynchronously, rather than ExecuteSynchronously + // TODO TaskCreationOptions.RunContinuationsAsynchronously + TaskCompletionSource tcs = new TaskCompletionSource(holder); ToManagedCallback callback = (JSMarshalerArgument* arguments_buffer) => { if (arguments_buffer == null) { - if (!tcs.TrySetException(new TaskCanceledException("WebWorker which is origin of the Promise is being terminated."))) - { - Environment.FailFast("Failed to set exception to TaskCompletionSource (arguments buffer is null)"); - } + tcs.TrySetException(new TaskCanceledException("WebWorker which is origin of the Promise is being terminated.")); return; } ref JSMarshalerArgument arg_2 = ref arguments_buffer[3]; // set by caller when this is SetException call @@ -66,17 +64,11 @@ public unsafe void ToManaged(out Task? value) if (arg_2.slot.Type != MarshalerType.None) { arg_2.ToManaged(out Exception? fail); - if (!tcs.TrySetException(fail!)) - { - Environment.FailFast("Failed to set exception to TaskCompletionSource (exception raised)"); - } + tcs.TrySetException(fail!); } else { - if (!tcs.TrySetResult()) - { - Environment.FailFast("Failed to set result to TaskCompletionSource (marshaler type is none)"); - } + tcs.TrySetResult(); } // eventual exception is handled by caller }; @@ -108,15 +100,14 @@ public unsafe void ToManaged(out Task? value, ArgumentToManagedCallback lock (ctx) { var holder = ctx.GetPromiseHolder(slot.GCHandle); - TaskCompletionSource tcs = new TaskCompletionSource(holder, TaskCreationOptions.RunContinuationsAsynchronously); + // we want to run the continuations on the original thread which called the JSImport, so RunContinuationsAsynchronously, rather than ExecuteSynchronously + // TODO TaskCreationOptions.RunContinuationsAsynchronously + TaskCompletionSource tcs = new TaskCompletionSource(holder); ToManagedCallback callback = (JSMarshalerArgument* arguments_buffer) => { if (arguments_buffer == null) { - if (!tcs.TrySetException(new TaskCanceledException("WebWorker which is origin of the Promise is being terminated."))) - { - Environment.FailFast("Failed to set exception to TaskCompletionSource (arguments buffer is null)"); - } + tcs.TrySetException(new TaskCanceledException("WebWorker which is origin of the Promise is being terminated.")); return; } @@ -126,18 +117,12 @@ public unsafe void ToManaged(out Task? value, ArgumentToManagedCallback { arg_2.ToManaged(out Exception? fail); if (fail == null) throw new InvalidOperationException(SR.FailedToMarshalException); - if (!tcs.TrySetException(fail)) - { - Environment.FailFast("Failed to set exception to TaskCompletionSource (exception raised)"); - } + tcs.TrySetException(fail); } else { marshaler(ref arg_3, out T result); - if(!tcs.TrySetResult(result)) - { - Environment.FailFast("Failed to set result to TaskCompletionSource (marshaler type is none)"); - } + tcs.TrySetResult(result); } // eventual exception is handled by caller }; @@ -155,20 +140,13 @@ internal void ToJSDynamic(Task? value) { Task? task = value; - var ctx = ToJSContext; - var canMarshalTaskResultOnSameCall = CanMarshalTaskResultOnSameCall(ctx); - if (task == null) { - if (!canMarshalTaskResultOnSameCall) - { - Environment.FailFast("Marshalling null return Task to JS is not supported in MT"); - } slot.Type = MarshalerType.None; return; } - if (canMarshalTaskResultOnSameCall && task.IsCompleted) + if (task.IsCompleted) { if (task.Exception != null) { @@ -194,6 +172,7 @@ internal void ToJSDynamic(Task? value) } } + var ctx = ToJSContext; if (slot.Type != MarshalerType.TaskPreCreated) { @@ -210,9 +189,7 @@ internal void ToJSDynamic(Task? value) var taskHolder = ctx.CreateCSOwnedProxy(slot.JSHandle); #if FEATURE_WASM_MANAGED_THREADS - // AsyncTaskScheduler will make sure that the resolve message is always sent after this call is completed - // that is: synchronous marshaling and eventually message to the target thread, which need to arrive before the resolve message - task.ContinueWith(Complete, taskHolder, ctx.AsyncTaskScheduler!); + task.ContinueWith(Complete, taskHolder, TaskScheduler.FromCurrentSynchronizationContext()); #else task.ContinueWith(Complete, taskHolder, TaskScheduler.Current); #endif @@ -252,18 +229,18 @@ public void ToJS(Task? value) { Task? task = value; var ctx = ToJSContext; - var canMarshalTaskResultOnSameCall = CanMarshalTaskResultOnSameCall(ctx); + var isCurrentThread = ctx.IsCurrentThread(); if (task == null) { - if (!canMarshalTaskResultOnSameCall) + if (!isCurrentThread) { - Environment.FailFast("Marshalling null return Task to JS is not supported in MT"); + Environment.FailFast("Marshalling null task to JS is not supported in MT"); } slot.Type = MarshalerType.None; return; } - if (canMarshalTaskResultOnSameCall && task.IsCompleted) + if (isCurrentThread && task.IsCompleted) { if (task.Exception != null) { @@ -296,9 +273,7 @@ public void ToJS(Task? value) var taskHolder = ctx.CreateCSOwnedProxy(slot.JSHandle); #if FEATURE_WASM_MANAGED_THREADS - // AsyncTaskScheduler will make sure that the resolve message is always sent after this call is completed - // that is: synchronous marshaling and eventually message to the target thread, which need to arrive before the resolve message - task.ContinueWith(Complete, taskHolder, ctx.AsyncTaskScheduler!); + task.ContinueWith(Complete, taskHolder, TaskScheduler.FromCurrentSynchronizationContext()); #else task.ContinueWith(Complete, taskHolder, TaskScheduler.Current); #endif @@ -328,19 +303,19 @@ public void ToJS(Task? value, ArgumentToJSCallback marshaler) { Task? task = value; var ctx = ToJSContext; - var canMarshalTaskResultOnSameCall = CanMarshalTaskResultOnSameCall(ctx); + var isCurrentThread = ctx.IsCurrentThread(); if (task == null) { - if (!canMarshalTaskResultOnSameCall) + if (!isCurrentThread) { - Environment.FailFast("Marshalling null return Task to JS is not supported in MT"); + Environment.FailFast("NULL not supported in MT"); } slot.Type = MarshalerType.None; return; } - if (canMarshalTaskResultOnSameCall && task.IsCompleted) + if (isCurrentThread && task.IsCompleted) { if (task.Exception != null) { @@ -375,9 +350,7 @@ public void ToJS(Task? value, ArgumentToJSCallback marshaler) var taskHolder = ctx.CreateCSOwnedProxy(slot.JSHandle); #if FEATURE_WASM_MANAGED_THREADS - // AsyncTaskScheduler will make sure that the resolve message is always sent after this call is completed - // that is: synchronous marshaling and eventually message to the target thread, which need to arrive before the resolve message - task.ContinueWith(Complete, new HolderAndMarshaler(taskHolder, marshaler), ctx.AsyncTaskScheduler!); + task.ContinueWith(Complete, new HolderAndMarshaler(taskHolder, marshaler), TaskScheduler.FromCurrentSynchronizationContext()); #else task.ContinueWith(Complete, new HolderAndMarshaler(taskHolder, marshaler), TaskScheduler.Current); #endif @@ -397,44 +370,6 @@ static void Complete(Task task, object? thm) } } -#if !DEBUG - [MethodImpl(MethodImplOptions.AggressiveInlining)] -#endif -#if FEATURE_WASM_MANAGED_THREADS - // We can't marshal resolved/rejected/null Task.Result directly into current argument when this is marshaling return of JSExport across threads - private bool CanMarshalTaskResultOnSameCall(JSProxyContext ctx) - { - if (slot.Type != MarshalerType.TaskPreCreated) - { - // this means that we are not in the return value of JSExport - // we are marshaling parameter of JSImport - return true; - } - - if (ctx.IsCurrentThread()) - { - // If the JS and Managed is running on the same thread we can use the args buffer, - // because the call is synchronous and the buffer will be processed. - // In that case the pre-allocated Promise would be discarded as necessary - // and the result will be marshaled by `try_marshal_sync_task_to_js` - return true; - } - - // Otherwise this is JSExport return value and we can't use the args buffer, because the args buffer arrived in async message and nobody is reading after this. - // In such case the JS side already pre-created the Promise and we have to use it, to resolve it in separate call via `mono_wasm_resolve_or_reject_promise_post` - // there is JSVHandle in this arg - return false; - } -#else -#pragma warning disable CA1822 // Mark members as static - private bool CanMarshalTaskResultOnSameCall(JSProxyContext _) - { - // in ST build this is always synchronous and we can marshal the result directly - return true; - } -#pragma warning restore CA1822 // Mark members as static -#endif - private sealed record HolderAndMarshaler(JSObject TaskHolder, ArgumentToJSCallback Marshaler); private static void RejectPromise(JSObject holder, Exception ex) diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj index ef19eaa15fc203..c3881268f2a6cd 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj @@ -20,17 +20,17 @@ false - - - - - - + + + + + + - - - + + + @@ -41,19 +41,19 @@ - - - + + + - - - - - - - - - + + + + + + + + + diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSExportTest.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSExportTest.cs index 170f393deb7c60..bcc2c85e9e7857 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSExportTest.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSExportTest.cs @@ -14,13 +14,6 @@ namespace System.Runtime.InteropServices.JavaScript.Tests { public class JSExportAsyncTest : JSInteropTestBase, IAsyncLifetime { - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupportedNotBrowserBackgroundExec))] - public void SyncJsImportJsExportThrows() - { - var ex = Assert.Throws(()=>JavaScriptTestHelper.invoke1_Boolean(true, nameof(JavaScriptTestHelper.EchoBoolean))); - Assert.Contains("Cannot call synchronous C# method", ex.Message); - } - [Theory] [MemberData(nameof(MarshalBooleanCases))] public async Task JsExportBooleanAsync(bool value) @@ -37,7 +30,7 @@ public async Task JsExportInt32DiscardNoWait(int value) { JavaScriptTestHelper.optimizedReached=0; JavaScriptTestHelper.invoke1O(value); - await JavaScriptTestHelper.Delay(50); + await JavaScriptTestHelper.Delay(0); Assert.Equal(value, JavaScriptTestHelper.optimizedReached); } @@ -50,7 +43,7 @@ public async Task JsExportInt32DiscardNoWait(int value) } } - [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsWasmBackgroundExecOrSingleThread))] + //TODO [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public class JSExportTest : JSInteropTestBase, IAsyncLifetime { [Theory] @@ -157,7 +150,6 @@ public void JsExportIntPtr(IntPtr value) [MemberData(nameof(MarshalIntPtrCases))] public unsafe void JsExportVoidPtr(IntPtr xvalue) { - JavaScriptTestHelper.AssertWasmBackgroundExec(); void* value = (void*)xvalue; void* res = JavaScriptTestHelper.invoke1_VoidPtr(value, nameof(JavaScriptTestHelper.EchoVoidPtr)); Assert.True(value == res); @@ -188,7 +180,6 @@ public void JsExportDateTimeOffset(DateTimeOffset value) [MemberData(nameof(MarshalNullableBooleanCases))] public void JsExportNullableBoolean(bool? value) { - JavaScriptTestHelper.AssertWasmBackgroundExec(); JsExportTest(value, JavaScriptTestHelper.invoke1_NullableBoolean, nameof(JavaScriptTestHelper.EchoNullableBoolean), @@ -392,7 +383,7 @@ public async Task JsExportTaskOfInt(int value) //GC.Collect(); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] public void JsExportCallback_FunctionIntInt() { int called = -1; @@ -408,7 +399,7 @@ public void JsExportCallback_FunctionIntInt() Assert.Equal(42, called); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] public void JsExportCallback_FunctionIntIntThrow() { int called = -1; diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSImportTest.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSImportTest.cs index f3ce777e824565..96b89ffa9e4f4c 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSImportTest.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JSImportTest.cs @@ -113,10 +113,9 @@ public unsafe void OutOfRange() Assert.Contains("Overflow: value 9007199254740991 is out of -2147483648 2147483647 range", ex.Message); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsWasmBackgroundExecOrSingleThread))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public unsafe void OptimizedPaths() { - JavaScriptTestHelper.AssertWasmBackgroundExec(); JavaScriptTestHelper.optimizedReached = 0; JavaScriptTestHelper.invoke0V(); Assert.Equal(1, JavaScriptTestHelper.optimizedReached); @@ -923,7 +922,7 @@ public async Task JsImportSleep() await JavaScriptTestHelper.sleep(100); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // slow + [Fact] public async Task JsImportTaskTypes() { for (int i = 0; i < 100; i++) @@ -1118,7 +1117,7 @@ public async Task JsImportTaskAwait() #region Action - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_EchoAction() { bool called = false; @@ -1133,6 +1132,7 @@ public void JsImportCallback_EchoAction() Assert.True(called); } + /* TODO deputy [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public void JsImportCallback_EchoActionThrows_MT() { @@ -1148,6 +1148,7 @@ public void JsImportCallback_EchoActionThrows_MT() Assert.Throws(()=>actual()); Assert.False(called); } + */ [Fact] public async Task JsImportCallback_Async() @@ -1165,7 +1166,7 @@ public async Task JsImportCallback_Async() } - [Fact] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy [OuterLoop] public async Task JsImportCallback_EchoActionMany() { @@ -1186,7 +1187,7 @@ public async Task JsImportCallback_EchoActionMany() } } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_Action() { bool called = false; @@ -1197,7 +1198,7 @@ public void JsImportCallback_Action() Assert.True(called); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportEcho_ActionAction() { bool called = false; @@ -1210,7 +1211,7 @@ public void JsImportEcho_ActionAction() Assert.True(called); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportEcho_ActionIntActionInt() { int calledA = -1; @@ -1223,7 +1224,7 @@ public void JsImportEcho_ActionIntActionInt() Assert.Equal(42, calledA); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_ActionInt() { int called = -1; @@ -1234,7 +1235,7 @@ public void JsImportCallback_ActionInt() Assert.Equal(42, called); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_FunctionIntInt() { int called = -1; @@ -1247,7 +1248,7 @@ public void JsImportCallback_FunctionIntInt() Assert.Equal(42, res); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportBackCallback_FunctionIntInt() { int called = -1; @@ -1262,7 +1263,7 @@ public void JsImportBackCallback_FunctionIntInt() Assert.Equal(84, called); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportBackCallback_FunctionIntIntIntInt() { int calledA = -1; @@ -1281,7 +1282,7 @@ public void JsImportBackCallback_FunctionIntIntIntInt() Assert.Equal(84, calledB); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_ActionIntInt() { int calledA = -1; @@ -1295,7 +1296,7 @@ public void JsImportCallback_ActionIntInt() Assert.Equal(43, calledB); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_ActionLongLong() { long calledA = -1; @@ -1309,7 +1310,7 @@ public void JsImportCallback_ActionLongLong() Assert.Equal(43, calledB); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_ActionIntLong() { int calledA = -1; @@ -1323,7 +1324,7 @@ public void JsImportCallback_ActionIntLong() Assert.Equal(43, calledB); } - [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] + [Fact] //TODO [Fact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWasmThreadingSupported))] // this test doesn't make sense with deputy public void JsImportCallback_ActionIntThrow() { int called = -1; diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JavaScriptTestHelper.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JavaScriptTestHelper.cs index 232397defef1dc..e5ea3a887e24b9 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JavaScriptTestHelper.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/JavaScriptTestHelper.cs @@ -1020,24 +1020,14 @@ public static JSObject EchoIJSObject([JSMarshalAs] JSObject arg1) [JSImport("INTERNAL.forceDisposeProxies")] internal static partial void ForceDisposeProxies(bool disposeMethods, bool verbose); - public static void AssertWasmBackgroundExec() - { - if (PlatformDetection.IsWasmBackgroundExec && Environment.CurrentManagedThreadId == 1) - { - throw new Exception("With WasmBackgroundExec we are expecting to run tests on the thread pool"); - } - } - static JSObject _module; public static async Task InitializeAsync() { - AssertWasmBackgroundExec(); if (_module == null) { _module = await JSHost.ImportAsync("JavaScriptTestHelper", "../JavaScriptTestHelper.mjs"); ; await Setup(); } - AssertWasmBackgroundExec(); #if FEATURE_WASM_MANAGED_THREADS // are we in the UI thread ? @@ -1047,7 +1037,6 @@ public static async Task InitializeAsync() // this gives browser chance to serve UI thread event loop before every test await Task.Yield(); } - AssertWasmBackgroundExec(); } public static Task DisposeAsync() diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.Http.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.Http.cs index 53d45b99678e90..8939cca759b19d 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.Http.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.Http.cs @@ -81,23 +81,9 @@ await HttpClient_ActionInDifferentThread(url, executor1, executor2, async (HttpR await Assert.ThrowsAsync(async () => { CancellationTokenSource cts = new CancellationTokenSource(); - try - { - var promise = response.Content.ReadAsStringAsync(cts.Token); - Console.WriteLine("HttpClient_CancelInDifferentThread: ManagedThreadId: " + Environment.CurrentManagedThreadId + " NativeThreadId: " + WebWorkerTestHelper.NativeThreadId); - cts.Cancel(); - await promise; - } - catch (TaskCanceledException ex) - { - Console.WriteLine("HttpClient_CancelInDifferentThread: TaskCanceledException is thrown with message: " + ex.ToString()); - throw; - } - catch (OperationCanceledException ex) - { - Console.WriteLine("HttpClient_CancelInDifferentThread: OperationCanceledException is thrown with message: " + ex.ToString()); - throw; - } + var promise = response.Content.ReadAsStringAsync(cts.Token); + cts.Cancel(); + await promise; }); }); } diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.cs index c67bac997294b9..514741a6b8753b 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTest.cs @@ -68,34 +68,6 @@ public async Task JSDelay_Cancellation(Executor executor) await Assert.ThrowsAnyAsync(() => canceledTask); } - [Theory, MemberData(nameof(GetBlockingFriendlyTargetThreads))] - public async Task JSDelay_Blocking_Wait(Executor executor) - { - var cts = new CancellationTokenSource(); - var blockedTask = executor.Execute(async () => - { - await executor.StickyAwait(WebWorkerTestHelper.CreateDelay(), cts.Token); - var promise = WebWorkerTestHelper.JSDelay(100); - promise.Wait(); - }, cts.Token); - - await blockedTask; - } - - [Theory, MemberData(nameof(GetBlockingFriendlyTargetThreads))] - public async Task JSDelay_Blocking_GetResult(Executor executor) - { - var cts = new CancellationTokenSource(); - var blockedTask = executor.Execute(async () => - { - await executor.StickyAwait(WebWorkerTestHelper.CreateDelay(), cts.Token); - var promise = WebWorkerTestHelper.JSDelay(100); - promise.GetAwaiter().GetResult(); - }, cts.Token); - - await blockedTask; - } - [Fact] public async Task JSSynchronizationContext_Send_Post_Items_Cancellation() { @@ -371,7 +343,7 @@ await executor.Execute(async () => var jsTid = WebWorkerTestHelper.GetTid(); var csTid = WebWorkerTestHelper.NativeThreadId; - if (executor.Type == ExecutorType.JSWebWorker) + if (executor.Type == ExecutorType.Main || executor.Type == ExecutorType.JSWebWorker) { Assert.Equal(jsTid, csTid); } @@ -392,25 +364,23 @@ public async Task ThreadingTimer(Executor executor) await executor.Execute(async () => { TaskCompletionSource tcs = new TaskCompletionSource(); - Console.WriteLine("ThreadingTimer: Start Time: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff") + " ManagedThreadId: " + Environment.CurrentManagedThreadId + " NativeThreadId: " + WebWorkerTestHelper.NativeThreadId); using var timer = new Timer(_ => { Assert.NotEqual(1, Environment.CurrentManagedThreadId); Assert.True(Thread.CurrentThread.IsThreadPoolThread); - hit = true; tcs.SetResult(); + hit = true; }, null, 100, Timeout.Infinite); await tcs.Task; }, cts.Token); - Console.WriteLine("ThreadingTimer: End Time: " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff") + " ManagedThreadId: " + Environment.CurrentManagedThreadId + " NativeThreadId: " + WebWorkerTestHelper.NativeThreadId); Assert.True(hit); } [Theory, MemberData(nameof(GetTargetThreads))] - public async Task JSDelay_ContinueWith_Async(Executor executor) + public async Task JSDelay_ContinueWith(Executor executor) { using var cts = CreateTestCaseTimeoutSource(); await executor.Execute(async () => @@ -419,23 +389,9 @@ await executor.Execute(async () => await WebWorkerTestHelper.JSDelay(10).ContinueWith(_ => { - Assert.True(Thread.CurrentThread.IsThreadPoolThread); - }, TaskContinuationOptions.RunContinuationsAsynchronously); - }, cts.Token); - } - - [Theory, MemberData(nameof(GetTargetThreads))] - public async Task JSDelay_ContinueWith_Sync(Executor executor) - { - using var cts = CreateTestCaseTimeoutSource(); - await executor.Execute(async () => - { - await executor.StickyAwait(WebWorkerTestHelper.CreateDelay(), cts.Token); - - await WebWorkerTestHelper.JSDelay(10).ContinueWith(_ => - { - Assert.True(Thread.CurrentThread.IsThreadPoolThread); - }, TaskContinuationOptions.ExecuteSynchronously); // ExecuteSynchronously is ignored + // continue on the context of the target JS interop + executor.AssertInteropThread(); + }, TaskContinuationOptions.ExecuteSynchronously); }, cts.Token); } @@ -453,21 +409,6 @@ await executor.Execute(async () => }, cts.Token); } - [Theory, MemberData(nameof(GetTargetThreads))] - public async Task JSDelay_ConfigureAwait_False(Executor executor) - { - using var cts = CreateTestCaseTimeoutSource(); - await executor.Execute(async () => - { - await executor.StickyAwait(WebWorkerTestHelper.CreateDelay(), cts.Token); - - await WebWorkerTestHelper.JSDelay(10).ConfigureAwait(false); - - // resolve/reject on I/O thread -> thread pool - Assert.True(Thread.CurrentThread.IsThreadPoolThread); - }, cts.Token); - } - [Theory, MemberData(nameof(GetTargetThreads))] public async Task ManagedDelay_ContinueWith(Executor executor) { @@ -496,13 +437,11 @@ await executor.Execute(async () => } [Theory, MemberData(nameof(GetTargetThreadsAndBlockingCalls))] - public async Task WaitDoesNotAssertInAsyncCode(Executor executor, NamedCall method) + public async Task WaitAssertsOnJSInteropThreads(Executor executor, NamedCall method) { using var cts = CreateTestCaseTimeoutSource(); - await executor.Execute(async () => + await executor.Execute(Task () => { - await executor.StickyAwait(WebWorkerTestHelper.InitializeAsync(), cts.Token); - Exception? exception = null; try { @@ -513,54 +452,9 @@ await executor.Execute(async () => exception = ex; } - Assert.Null(exception); - }, cts.Token); - } - - [Theory, MemberData(nameof(GetTargetThreadsAndBlockingCalls))] - public async Task WaitAssertsOnSyncCallback(Executor executor, NamedCall method) - { - using var cts = CreateTestCaseTimeoutSource(); - await executor.Execute(async () => - { - await executor.StickyAwait(WebWorkerTestHelper.InitializeAsync(), cts.Token); - - Exception? exception = null; - // the callback will hit Main or JSWebWorker, not the original executor thread - await WebWorkerTestHelper.CallMeBackSync(() => { - // when we are inside of synchronous callback, all blocking .Wait is forbidden - try - { - method.Call(cts.Token); - } - catch (Exception ex) - { - exception = ex; - } - }); - - Console.WriteLine("WaitAssertsOnJSInteropThreads: ExecuterType: " + executor.Type + " ManagedThreadId: " + Environment.CurrentManagedThreadId + " NativeThreadId: " + WebWorkerTestHelper.NativeThreadId); - Assert.NotNull(exception); - Assert.IsType(exception); - }, cts.Token); - } - - [Theory, MemberData(nameof(GetTargetThreadsAndBlockingCalls))] - public async Task WaitAssertsOnSyncJSExport(Executor executor, NamedCall method) - { - using var cts = CreateTestCaseTimeoutSource(); - await executor.Execute(async () => - { - await executor.StickyAwait(WebWorkerTestHelper.InitializeAsync(), cts.Token); + executor.AssertBlockingWait(exception); - WebWorkerTestHelper.CurrentCallback = method; - WebWorkerTestHelper.CurrentCancellationToken = cts.Token; - // the callback will hit Main or JSWebWorker, not the original executor thread - await WebWorkerTestHelper.CallExportBackSync(nameof(WebWorkerTestHelper.CallCurrentCallback)); - - Console.WriteLine("WaitAssertsOnJSInteropThreads: ExecuterType: " + executor.Type + " ManagedThreadId: " + Environment.CurrentManagedThreadId + " NativeThreadId: " + WebWorkerTestHelper.NativeThreadId); - Assert.NotNull(WebWorkerTestHelper.LastException); - Assert.IsType(WebWorkerTestHelper.LastException); + return Task.CompletedTask; }, cts.Token); } diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestBase.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestBase.cs index 3fb53d484def06..87f88745377b02 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestBase.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestBase.cs @@ -45,17 +45,9 @@ public static IEnumerable GetTargetThreads() return Enum.GetValues().Select(type => new object[] { new Executor(type) }); } - public static IEnumerable GetBlockingFriendlyTargetThreads() + public static IEnumerable GetSpecificTargetThreads() { - yield return new object[] { new Executor(ExecutorType.Main) }; - yield return new object[] { new Executor(ExecutorType.NewThread) }; - yield return new object[] { new Executor(ExecutorType.ThreadPool) }; - // JSWebWorker is missing here because JS can't resolve promises while blocked - } - - public static IEnumerable GetSpecificTargetThreads2x() - { - yield return new object[] { new Executor(ExecutorType.Main), new Executor(ExecutorType.Main) }; + yield return new object[] { new Executor(ExecutorType.JSWebWorker), new Executor(ExecutorType.Main) }; yield break; } @@ -145,6 +137,15 @@ async Task ActionsInDifferentThreads2() } } + public class NamedCall + { + public string Name { get; set; } + public delegate void Method(CancellationToken ct); + public Method Call { get; set; } + + override public string ToString() => Name; + } + public static IEnumerable BlockingCalls = new List { new NamedCall { Name = "Task.Wait", Call = delegate (CancellationToken ct) { Task.Delay(10, ct).Wait(ct); }}, diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.cs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.cs index 1fc94b83adba99..96312c7aba35c8 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.cs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.cs @@ -18,7 +18,6 @@ public partial class WebWorkerTestHelper public static readonly string LocalWsEcho = "ws://" + Environment.GetEnvironmentVariable("DOTNET_TEST_WEBSOCKETHOST") + "/WebSocket/EchoWebSocket.ashx"; [JSImport("globalThis.console.log")] - [return: JSMarshalAs] public static partial void Log(string message); [JSImport("delay", "InlineTestHelper")] @@ -39,30 +38,6 @@ public partial class WebWorkerTestHelper [JSImport("promiseValidateState", "WebWorkerTestHelper")] public static partial Task PromiseValidateState(JSObject state); - [JSImport("callMeBackSync", "WebWorkerTestHelper")] - public static partial Task CallMeBackSync([JSMarshalAs] Action syncCallback); - - [JSImport("callExportBackSync", "WebWorkerTestHelper")] - public static partial Task CallExportBackSync(string syncExportName); - - public static NamedCall CurrentCallback; - public static CancellationToken CurrentCancellationToken = CancellationToken.None; - public static Exception? LastException = null; - - [JSExport] - public static void CallCurrentCallback() - { - LastException = null; - try - { - CurrentCallback.Call(CurrentCancellationToken); - } - catch (Exception ex) - { - LastException = ex; - } - } - public static string GetOriginUrl() { using var globalThis = JSHost.GlobalThis; @@ -146,6 +121,7 @@ public enum ExecutorType public class Executor { public int ExecutorTID; + public SynchronizationContext ExecutorSynchronizationContext; private static SynchronizationContext _mainSynchronizationContext; public static SynchronizationContext MainSynchronizationContext { @@ -180,6 +156,7 @@ public Task Execute(Func job, CancellationToken cancellationToken) Task wrapExecute() { ExecutorTID = Environment.CurrentManagedThreadId; + ExecutorSynchronizationContext = SynchronizationContext.Current ?? MainSynchronizationContext; AssertTargetThread(); return job(); } @@ -217,15 +194,6 @@ public void AssertTargetThread() { Assert.False(Thread.CurrentThread.IsThreadPoolThread, "IsThreadPoolThread:" + Thread.CurrentThread.IsThreadPoolThread + " Type " + Type); } - if (Type == ExecutorType.Main || Type == ExecutorType.JSWebWorker) - { - Assert.NotNull(SynchronizationContext.Current); - Assert.Equal("System.Runtime.InteropServices.JavaScript.JSSynchronizationContext", SynchronizationContext.Current.GetType().FullName); - } - else - { - Assert.Null(SynchronizationContext.Current); - } } public void AssertAwaitCapturedContext() @@ -262,6 +230,51 @@ public void AssertAwaitCapturedContext() } } + public void AssertBlockingWait(Exception? exception) + { + switch (Type) + { + case ExecutorType.Main: + case ExecutorType.JSWebWorker: + Assert.NotNull(exception); + Assert.IsType(exception); + break; + case ExecutorType.NewThread: + case ExecutorType.ThreadPool: + Assert.Null(exception); + break; + } + } + + public void AssertInteropThread() + { + switch (Type) + { + case ExecutorType.Main: + Assert.Equal(1, Environment.CurrentManagedThreadId); + Assert.Equal(ExecutorTID, Environment.CurrentManagedThreadId); + Assert.False(Thread.CurrentThread.IsThreadPoolThread); + break; + case ExecutorType.JSWebWorker: + Assert.NotEqual(1, Environment.CurrentManagedThreadId); + Assert.Equal(ExecutorTID, Environment.CurrentManagedThreadId); + Assert.False(Thread.CurrentThread.IsThreadPoolThread); + break; + case ExecutorType.NewThread: + // it will synchronously continue on the UI thread + Assert.Equal(1, Environment.CurrentManagedThreadId); + Assert.NotEqual(ExecutorTID, Environment.CurrentManagedThreadId); + Assert.False(Thread.CurrentThread.IsThreadPoolThread); + break; + case ExecutorType.ThreadPool: + // it will synchronously continue on the UI thread + Assert.Equal(1, Environment.CurrentManagedThreadId); + Assert.NotEqual(ExecutorTID, Environment.CurrentManagedThreadId); + Assert.False(Thread.CurrentThread.IsThreadPoolThread); + break; + } + } + public override string ToString() => Type.ToString(); // make sure we stay on the executor @@ -312,6 +325,10 @@ public static Task RunOnThreadPool(Func job, CancellationToken cancellatio public static Task RunOnNewThread(Func job, CancellationToken cancellationToken) { + if( Environment.CurrentManagedThreadId == 1) + { + throw new Exception("This unit test should be executed with -backgroundExec otherwise it's prone to consume all threads too quickly"); + } TaskCompletionSource tcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var thread = new Thread(() => { @@ -381,14 +398,4 @@ public static Task RunOnTargetAsync(SynchronizationContext ctx, Func job, } #endregion - - public class NamedCall - { - public string Name { get; set; } - public delegate void Method(CancellationToken ct); - public Method Call { get; set; } - - override public string ToString() => Name; - } - } diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.mjs b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.mjs index e2a8cfadfaea7e..558fb181b47d67 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.mjs +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System/Runtime/InteropServices/JavaScript/WebWorkerTestHelper.mjs @@ -73,13 +73,3 @@ export function delay(ms) { export function getRndInteger(min, max) { return Math.floor(Math.random() * (max - min)) + min; } - -export async function callMeBackSync(syncCallback) { - syncCallback(); -} - -export async function callExportBackSync(syncExportName) { - const WebWorkerTestHelper = dllExports.System.Runtime.InteropServices.JavaScript.Tests.WebWorkerTestHelper; - const method = WebWorkerTestHelper[syncExportName] - method(); -} \ No newline at end of file diff --git a/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/Analyzers/ConvertComImportToGeneratedComInterfaceAnalyzer.cs b/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/Analyzers/ConvertComImportToGeneratedComInterfaceAnalyzer.cs index f9b96eb7e30184..3035552c59dd8c 100644 --- a/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/Analyzers/ConvertComImportToGeneratedComInterfaceAnalyzer.cs +++ b/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/Analyzers/ConvertComImportToGeneratedComInterfaceAnalyzer.cs @@ -50,7 +50,7 @@ public override void Initialize(AnalysisContext context) INamedTypeSymbol type = (INamedTypeSymbol)context.Symbol; AttributeData? interfaceTypeAttributeData = type.GetAttributes().FirstOrDefault(a => a.AttributeClass.Equals(interfaceTypeAttribute, SymbolEqualityComparer.Default)); if (type is not { TypeKind: TypeKind.Interface, IsComImport: true } - || interfaceTypeAttributeData is not { ConstructorArguments: [{ Value: (int)ComInterfaceType.InterfaceIsIUnknown }] }) + || interfaceTypeAttributeData.ConstructorArguments.Length == 1 && (int)interfaceTypeAttributeData.ConstructorArguments[0].Value != (int)ComInterfaceType.InterfaceIsIUnknown) { return; } diff --git a/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/ComInterfaceGenerator.cs b/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/ComInterfaceGenerator.cs index d3c5c631bb0b5f..dd04872d97b4a6 100644 --- a/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/ComInterfaceGenerator.cs +++ b/src/libraries/System.Runtime.InteropServices/gen/ComInterfaceGenerator/ComInterfaceGenerator.cs @@ -637,15 +637,24 @@ private static ClassDeclarationSyntax GenerateInterfaceInformation(ComInterfaceI static ExpressionSyntax CreateEmbeddedDataBlobCreationStatement(ReadOnlySpan bytes) { - var literals = new CollectionElementSyntax[bytes.Length]; + var literals = new LiteralExpressionSyntax[bytes.Length]; for (int i = 0; i < bytes.Length; i++) { - literals[i] = ExpressionElement(LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(bytes[i]))); + literals[i] = LiteralExpression(SyntaxKind.NumericLiteralExpression, Literal(bytes[i])); } - // [ ] - return CollectionExpression(SeparatedList(literals)); + // new System.ReadOnlySpan(new[] { } ) + return ObjectCreationExpression( + GenericName(TypeNames.System_ReadOnlySpan) + .AddTypeArgumentListArguments(PredefinedType(Token(SyntaxKind.ByteKeyword)))) + .AddArgumentListArguments( + Argument( + ArrayCreationExpression( + ArrayType(PredefinedType(Token(SyntaxKind.ByteKeyword)), SingletonList(ArrayRankSpecifier())), + InitializerExpression( + SyntaxKind.ArrayInitializerExpression, + SeparatedList(literals))))); } } } diff --git a/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/ConvertToGeneratedComInterfaceTests.cs b/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/ConvertToGeneratedComInterfaceTests.cs index 7260a4d4d392c7..308cc66f48b9f8 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/ConvertToGeneratedComInterfaceTests.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/ComInterfaceGenerator.Unit.Tests/ConvertToGeneratedComInterfaceTests.cs @@ -370,43 +370,5 @@ public struct HResult await VerifyCS.VerifyCodeFixAsync(source, fixedSource); } - - [Fact] - public async Task UnsupportedInterfaceTypes_DoesNotReportDiagnostic() - { - // This also tests the case where InterfaceType is missing (defaulting to ComInterfaceType.InterfaceIsDual). - string source = """ - using System.Runtime.InteropServices; - - [ComImport] - [Guid("73EB4AF8-BE9C-4b49-B3A4-24F4FF657B26")] - public interface IInterfaceIsDualMissingAttribute - { - } - - [ComImport] - [Guid("5DA39CDF-DCAD-447A-836E-EA80DB34D81B")] - [InterfaceType(ComInterfaceType.InterfaceIsDual)] - public interface IInterfaceIsDual - { - } - - [ComImport] - [Guid("F59AB2FE-523D-4B28-911C-21363808C51E")] - [InterfaceType(ComInterfaceType.InterfaceIsIDispatch)] - public interface IInterfaceIsIDispatch - { - } - - [ComImport] - [Guid("DC1C5A9C-E88A-4dde-A5A1-60F82A20AEF7")] - [InterfaceType(ComInterfaceType.InterfaceIsIInspectable)] - public interface IInterfaceIsIInspectable - { - } - """; - - await VerifyCS.VerifyCodeFixAsync(source, source); - } } } diff --git a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/GetDelegateForFunctionPointerTests.cs b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/GetDelegateForFunctionPointerTests.cs index f0745272af906c..abcf514822786d 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/GetDelegateForFunctionPointerTests.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/GetDelegateForFunctionPointerTests.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Reflection; using System.Reflection.Emit; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices.Tests.Common; using Xunit; @@ -154,66 +153,6 @@ public void GetDelegateForFunctionPointer_CantCast_ThrowsInvalidCastException() GC.KeepAlive(d); } - [Fact] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99478", TestRuntimes.Mono)] - public void GetDelegateForFunctionPointer_Resurrection() - { - GCHandle handle = Alloc(); - - if (PlatformDetection.IsPreciseGcSupported) - { - while (!IsNullTarget(handle)) - { - GC.Collect(); - GC.WaitForPendingFinalizers(); - } - } - - handle.Free(); - - [MethodImpl(MethodImplOptions.NoInlining)] - static GCHandle Alloc() - { - GCHandle gcHandle = default; - gcHandle = GCHandle.Alloc(new FreachableObject(), GCHandleType.WeakTrackResurrection); - return gcHandle; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - static bool IsNullTarget(GCHandle handle) - { - return handle.Target is null; - } - } - - private class FreachableObject - { - private readonly Action _del; - private readonly IntPtr _fnptr; - private int _count; - - internal FreachableObject() - { - _del = new Action(SomeFunction); - _fnptr = Marshal.GetFunctionPointerForDelegate(_del); - } - - // Note: This method cannot be replaced by a lambda for the test to trigger the delegate resurrection - private void SomeFunction() - { - } - - ~FreachableObject() - { - Assert.Same(Marshal.GetDelegateForFunctionPointer(_fnptr), _del); - - if (_count++ < 3) - { - GC.ReRegisterForFinalize(this); - } - } - } - public delegate void GenericDelegate(T t); public delegate void NonGenericDelegate(string t); public delegate void OtherNonGenericDelegate(string t); diff --git a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToCoTaskMemAnsiTests.cs b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToCoTaskMemAnsiTests.cs index 6beaf2d3a67a38..7bc52efa15e0c7 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToCoTaskMemAnsiTests.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToCoTaskMemAnsiTests.cs @@ -33,9 +33,9 @@ public void SecureStringToCoTaskMemAnsi_InvokePtrToStringAnsi_Roundtrips(string { Assert.NotEqual(IntPtr.Zero, ptr); - // The check is incorrect for UTF8 encoding of non-Ansi chars. Detect UTF8 encoding via SystemMaxDBCSCharSize. + // Unix is incorrect with unicode chars. bool containsNonAnsiChars = s.Any(c => c > 0xFF); - if (!containsNonAnsiChars || Marshal.SystemMaxDBCSCharSize < 3) + if (!containsNonAnsiChars || PlatformDetection.IsWindows) { // Make sure the native memory is correctly laid out. for (int i = 0; i < s.Length; i++) diff --git a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToGlobalAllocAnsiTests.cs b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToGlobalAllocAnsiTests.cs index 5c1ca653c75769..a97ea54fb3b162 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToGlobalAllocAnsiTests.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/SecureStringToGlobalAllocAnsiTests.cs @@ -33,9 +33,9 @@ public void SecureStringToGlobalAllocAnsi_InvokePtrToStringAnsi_Roundtrips(strin { Assert.NotEqual(IntPtr.Zero, ptr); - // The check is incorrect for UTF8 encoding of non-Ansi chars. Detect UTF8 encoding via SystemMaxDBCSCharSize. + // Unix uses UTF8 for Ansi marshalling. bool containsNonAnsiChars = s.Any(c => c > 0xFF); - if (!containsNonAnsiChars || Marshal.SystemMaxDBCSCharSize < 3) + if (!containsNonAnsiChars || PlatformDetection.IsWindows) { // Make sure the native memory is correctly laid out. for (int i = 0; i < s.Length; i++) diff --git a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToCoTaskMemAnsiTests.cs b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToCoTaskMemAnsiTests.cs index 9375b6095e36e7..abb9666544c7eb 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToCoTaskMemAnsiTests.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToCoTaskMemAnsiTests.cs @@ -30,9 +30,9 @@ public void StringToCoTaskMemAnsi_InvokePtrToStringAnsi_Roundtrips(string s) { Assert.NotEqual(IntPtr.Zero, ptr); - // The check is incorrect for UTF8 encoding of non-Ansi chars. Detect UTF8 encoding via SystemMaxDBCSCharSize. + // Unix uses UTF8 for Ansi marshalling. bool containsNonAnsiChars = s.Any(c => c > 0xFF); - if (!containsNonAnsiChars || Marshal.SystemMaxDBCSCharSize < 3) + if (!containsNonAnsiChars || PlatformDetection.IsWindows) { // Make sure the native memory is correctly laid out. for (int i = 0; i < s.Length; i++) diff --git a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToHGlobalAnsiTests.cs b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToHGlobalAnsiTests.cs index b089a5406788c8..7802a4b99f2d55 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToHGlobalAnsiTests.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/System.Runtime.InteropServices.UnitTests/System/Runtime/InteropServices/Marshal/StringToHGlobalAnsiTests.cs @@ -30,9 +30,9 @@ public void StringToHGlobalAnsi_InvokePtrToStringAnsi_Roundtrips(string s) { Assert.NotEqual(IntPtr.Zero, ptr); - // The check is incorrect for UTF8 encoding of non-Ansi chars. Detect UTF8 encoding via SystemMaxDBCSCharSize. + // Unix uses UTF8 for Ansi marshalling. bool containsNonAnsiChars = s.Any(c => c > 0xFF); - if (!containsNonAnsiChars || Marshal.SystemMaxDBCSCharSize < 3) + if (!containsNonAnsiChars || PlatformDetection.IsWindows) { // Make sure the native memory is correctly laid out. for (int i = 0; i < s.Length; i++) diff --git a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatefulPinnedMarshalling.cs b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatefulPinnedMarshalling.cs index 1347a2858e3725..eaac2608d8184a 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatefulPinnedMarshalling.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatefulPinnedMarshalling.cs @@ -86,7 +86,6 @@ public ref StatefulPinnedNative GetPinnableReference() _canFree = true; if (_isPinned) { - // Unsafe.AsPointer is safe, because the result from GetPinnableReference is pinned _refNativeStruct = new StatefulPinnedNative() { I = _managed.I }; return (StatefulPinnedNative*)Unsafe.AsPointer(ref _refNativeStruct); } diff --git a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatelessCallerAllocateBufferMarshalling.cs b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatelessCallerAllocateBufferMarshalling.cs index 6388ab2990113b..f8b1174799b136 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatelessCallerAllocateBufferMarshalling.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/ComInterfaces/IStatelessCallerAllocateBufferMarshalling.cs @@ -94,7 +94,6 @@ public static class ManagedToUnmanagedIn { var unmanaged = new StatelessCallerAllocatedBufferNative() { I = managed.I }; MemoryMarshal.Write(buffer, in unmanaged); - // Unsafe.AsPointer is safe since buffer is pinned return (StatelessCallerAllocatedBufferNative*)Unsafe.AsPointer(ref MemoryMarshal.AsRef(buffer)); } diff --git a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/NonBlittable.cs b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/NonBlittable.cs index aa4df06e002f4f..88e594d9f6a2d2 100644 --- a/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/NonBlittable.cs +++ b/src/libraries/System.Runtime.InteropServices/tests/TestAssets/SharedTypes/NonBlittable.cs @@ -84,7 +84,6 @@ public static unsafe class DoubleToBytesBigEndianMarshaller public static byte* ConvertToUnmanaged(double managed, Span buffer) { - // Unsafe.AsPointer is safe since buffer must be pinned BinaryPrimitives.WriteDoubleBigEndian(buffer, managed); return (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(buffer)); } @@ -306,7 +305,6 @@ public struct StatefulGetPinnableReference private IntWrapperWithoutGetPinnableReference _managed; public void FromManaged(IntWrapperWithoutGetPinnableReference managed) => _managed = managed; - // Unsafe.AsPointer is safe since buffer must be pinned public int* ToUnmanaged() => (int*)Unsafe.AsPointer(ref _managed.i); public ref int GetPinnableReference() => ref _managed.i; @@ -465,7 +463,6 @@ public unsafe static class ListMarshallerWithBuffer where if (spaceRequired > buffer.Length) throw new InvalidOperationException(); - // Unsafe.AsPointer is safe since buffer must be pinned return (byte*)Unsafe.AsPointer(ref MemoryMarshal.GetReference(buffer)); } @@ -523,7 +520,6 @@ public void FromManaged(List managed, Span buffer) public ref TUnmanagedElement GetPinnableReference() => ref MemoryMarshal.GetReference(_span); - // Unsafe.AsPointer is safe since buffer must be pinned public byte* ToUnmanaged() => (byte*)Unsafe.AsPointer(ref GetPinnableReference()); public Span GetManagedValuesDestination(int length) diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 77fe06ddc5c02c..82e179628b11f4 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -2904,6 +2904,8 @@ public static unsafe void StoreSelectedScalar(ushort* address, System.Runtime.In public static unsafe void StoreSelectedScalar(uint* address, System.Runtime.Intrinsics.Vector128 value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(3))] byte index) { } public static unsafe void StoreSelectedScalar(uint* address, System.Runtime.Intrinsics.Vector64 value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { } public static unsafe void StoreSelectedScalar(ulong* address, System.Runtime.Intrinsics.Vector128 value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { } +#if false + // Should be disabled until Mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 public static unsafe void StoreSelectedScalar(byte* address, (System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(7))] byte index) { throw null; } public static unsafe void StoreSelectedScalar(sbyte* address, (System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(7))] byte index) { throw null; } public static unsafe void StoreSelectedScalar(short* address, (System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(3))] byte index) { throw null; } @@ -2925,6 +2927,7 @@ public static unsafe void StoreSelectedScalar(ulong* address, System.Runtime.Int public static unsafe void StoreSelectedScalar(int* address, (System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2, System.Runtime.Intrinsics.Vector64 value3, System.Runtime.Intrinsics.Vector64 value4) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { throw new PlatformNotSupportedException(); } public static unsafe void StoreSelectedScalar(uint* address, (System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2, System.Runtime.Intrinsics.Vector64 value3, System.Runtime.Intrinsics.Vector64 value4) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { throw new PlatformNotSupportedException(); } public static unsafe void StoreSelectedScalar(float* address, (System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2, System.Runtime.Intrinsics.Vector64 value3, System.Runtime.Intrinsics.Vector64 value4) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { throw new PlatformNotSupportedException(); } +#endif public unsafe static void StoreVector64x2AndZip(byte* address, (System.Runtime.Intrinsics.Vector64 Value1, System.Runtime.Intrinsics.Vector64 Value2) value) { throw null; } public unsafe static void StoreVector64x2AndZip(sbyte* address, (System.Runtime.Intrinsics.Vector64 Value1, System.Runtime.Intrinsics.Vector64 Value2) value) { throw null; } public unsafe static void StoreVector64x2AndZip(short* address, (System.Runtime.Intrinsics.Vector64 Value1, System.Runtime.Intrinsics.Vector64 Value2) value) { throw null; } @@ -3745,6 +3748,8 @@ public static unsafe void StorePairScalar(uint* address, System.Runtime.Intrinsi public static unsafe void StorePairScalarNonTemporal(int* address, System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2) { } public static unsafe void StorePairScalarNonTemporal(float* address, System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2) { } public static unsafe void StorePairScalarNonTemporal(uint* address, System.Runtime.Intrinsics.Vector64 value1, System.Runtime.Intrinsics.Vector64 value2) { } +#if false + // Should be disabled until Mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 public static unsafe void StoreSelectedScalar(byte* address, (System.Runtime.Intrinsics.Vector128 value1, System.Runtime.Intrinsics.Vector128 value2) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(15))] byte index) { throw null; } public static unsafe void StoreSelectedScalar(sbyte* address, (System.Runtime.Intrinsics.Vector128 value1, System.Runtime.Intrinsics.Vector128 value2) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(15))] byte index) { throw null; } public static unsafe void StoreSelectedScalar(short* address, (System.Runtime.Intrinsics.Vector128 value1, System.Runtime.Intrinsics.Vector128 value2) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(7))] byte index) { throw null; } @@ -3775,6 +3780,7 @@ public static unsafe void StorePairScalarNonTemporal(uint* address, System.Runti public static unsafe void StoreSelectedScalar(ulong* address, (System.Runtime.Intrinsics.Vector128 value1, System.Runtime.Intrinsics.Vector128 value2, System.Runtime.Intrinsics.Vector128 value3, System.Runtime.Intrinsics.Vector128 value4) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { throw null; } public static unsafe void StoreSelectedScalar(float* address, (System.Runtime.Intrinsics.Vector128 value1, System.Runtime.Intrinsics.Vector128 value2, System.Runtime.Intrinsics.Vector128 value3, System.Runtime.Intrinsics.Vector128 value4) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(3))] byte index) { throw null; } public static unsafe void StoreSelectedScalar(double* address, (System.Runtime.Intrinsics.Vector128 value1, System.Runtime.Intrinsics.Vector128 value2, System.Runtime.Intrinsics.Vector128 value3, System.Runtime.Intrinsics.Vector128 value4) value, [System.Diagnostics.CodeAnalysis.ConstantExpectedAttribute(Max = (byte)(1))] byte index) { throw null; } +#endif public unsafe static void StoreVector128x2AndZip(byte* address, (System.Runtime.Intrinsics.Vector128 Value1, System.Runtime.Intrinsics.Vector128 Value2) value) { throw null; } public unsafe static void StoreVector128x2AndZip(sbyte* address, (System.Runtime.Intrinsics.Vector128 Value1, System.Runtime.Intrinsics.Vector128 Value2) value) { throw null; } public unsafe static void StoreVector128x2AndZip(short* address, (System.Runtime.Intrinsics.Vector128 Value1, System.Runtime.Intrinsics.Vector128 Value2) value) { throw null; } @@ -4138,51 +4144,7 @@ internal Sve() { } internal Arm64() { } public static new bool IsSupported { get { throw null; } } } - - public static System.Numerics.Vector CreateTrueMaskByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskDouble([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskSByte([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskSingle([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskUInt16([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskUInt32([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - public static System.Numerics.Vector CreateTrueMaskUInt64([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; } - - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, sbyte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, short* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, int* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, long* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, byte* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, ushort* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, uint* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, ulong* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, float* address) { throw null; } - public static unsafe System.Numerics.Vector LoadVector(System.Numerics.Vector mask, double* address) { throw null; } - } - - public enum SveMaskPattern : byte - { - LargestPowerOf2 = 0, // The largest power of 2. - VectorCount1 = 1, // 1 element. - VectorCount2 = 2, // 2 elements. - VectorCount3 = 3, // 3 elements. - VectorCount4 = 4, // 4 elements. - VectorCount5 = 5, // 5 elements. - VectorCount6 = 6, // 6 elements. - VectorCount7 = 7, // 7 elements. - VectorCount8 = 8, // 8 elements. - VectorCount16 = 9, // 16 elements. - VectorCount32 = 10, // 32 elements. - VectorCount64 = 11, // 64 elements. - VectorCount128 = 12, // 128 elements. - VectorCount256 = 13, // 256 elements. - LargestMultipleOf4 = 29, // The largest multiple of 4. - LargestMultipleOf3 = 30, // The largest multiple of 3. - All = 31 // All available (implicitly a multiple of two). - }; } namespace System.Runtime.Intrinsics.X86 { diff --git a/src/libraries/System.Runtime.Numerics/src/System/Number.BigInteger.cs b/src/libraries/System.Runtime.Numerics/src/System/Number.BigInteger.cs index 01db0ef3777e87..a5fc0e6f1f1d53 100644 --- a/src/libraries/System.Runtime.Numerics/src/System/Number.BigInteger.cs +++ b/src/libraries/System.Runtime.Numerics/src/System/Number.BigInteger.cs @@ -445,7 +445,7 @@ internal static ParsingStatus TryParseBigIntegerHexOrBinaryNumberStyle public bool IsEmpty { get { throw null; } } public ref readonly T this[int index] { get { throw null; } } public int Length { get { throw null; } } - public static System.ReadOnlySpan CastUp(System.ReadOnlySpan items) where TDerived : class?, T { throw null; } public void CopyTo(System.Span destination) { } [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.ObsoleteAttribute("Equals() on ReadOnlySpan will always throw an exception. Use the equality operator instead.")] @@ -5614,16 +5613,6 @@ protected TimeProvider() { } public override bool Equals([System.Diagnostics.CodeAnalysis.NotNullWhenAttribute(true)] object? value) { throw null; } public bool Equals(System.TimeSpan obj) { throw null; } public static bool Equals(System.TimeSpan t1, System.TimeSpan t2) { throw null; } - public static System.TimeSpan FromDays(int days) { throw null; } - public static System.TimeSpan FromDays(int days, int hours = 0, long minutes = 0, long seconds = 0, long milliseconds = 0, long microseconds = 0) { throw null; } - public static System.TimeSpan FromHours(int hours) { throw null; } - public static System.TimeSpan FromHours(int hours, long minutes = 0, long seconds = 0, long milliseconds = 0, long microseconds = 0) { throw null; } - public static System.TimeSpan FromMinutes(long minutes) { throw null; } - public static System.TimeSpan FromMinutes(long minutes, long seconds = 0, long milliseconds = 0, long microseconds = 0) { throw null; } - public static System.TimeSpan FromSeconds(long seconds) { throw null; } - public static System.TimeSpan FromSeconds(long seconds, long milliseconds = 0, long microseconds = 0) { throw null; } - public static System.TimeSpan FromMilliseconds(long milliseconds, long microseconds = 0) { throw null; } - public static System.TimeSpan FromMicroseconds(long microseconds) { throw null; } public static System.TimeSpan FromDays(double value) { throw null; } public static System.TimeSpan FromHours(double value) { throw null; } public static System.TimeSpan FromMicroseconds(double value) { throw null; } @@ -8428,18 +8417,6 @@ public ExperimentalAttribute(string diagnosticId) { } public string DiagnosticId { get { throw null; } } public string? UrlFormat { get { throw null; } set { } } } - [System.AttributeUsageAttribute(System.AttributeTargets.Property, Inherited = false, AllowMultiple = true)] - public sealed class FeatureGuardAttribute : System.Attribute - { - public FeatureGuardAttribute(System.Type featureType) { } - public System.Type FeatureType { get { throw null; } } - } - [System.AttributeUsage(System.AttributeTargets.Property, Inherited = false)] - public sealed class FeatureSwitchDefinitionAttribute : Attribute - { - public FeatureSwitchDefinitionAttribute(string switchName) { } - public string SwitchName { get { throw null; } } - } [System.AttributeUsageAttribute(System.AttributeTargets.Field | System.AttributeTargets.Parameter | System.AttributeTargets.Property | System.AttributeTargets.ReturnValue, Inherited=false)] public sealed partial class MaybeNullAttribute : System.Attribute { @@ -11611,7 +11588,6 @@ public enum GenericParameterAttributes NotNullableValueTypeConstraint = 8, DefaultConstructorConstraint = 16, SpecialConstraintMask = 28, - AllowByRefLike = 32, } public partial interface ICustomAttributeProvider { @@ -13059,11 +13035,6 @@ public sealed partial class NullablePublicOnlyAttribute : System.Attribute public readonly bool IncludesInternals; public NullablePublicOnlyAttribute(bool value) { } } - [System.AttributeUsageAttribute(System.AttributeTargets.Parameter, Inherited = true, AllowMultiple = false)] - public sealed partial class ParamCollectionAttribute : System.Attribute - { - public ParamCollectionAttribute() { } - } public partial struct PoolingAsyncValueTaskMethodBuilder { private object _dummy; @@ -13131,16 +13102,13 @@ public RuntimeCompatibilityAttribute() { } public static partial class RuntimeFeature { public const string ByRefFields = "ByRefFields"; - public const string ByRefLikeGenerics = "ByRefLikeGenerics"; public const string CovariantReturnsOfClasses = "CovariantReturnsOfClasses"; public const string DefaultImplementationsOfInterfaces = "DefaultImplementationsOfInterfaces"; public const string NumericIntPtr = "NumericIntPtr"; public const string PortablePdb = "PortablePdb"; public const string UnmanagedSignatureCallingConvention = "UnmanagedSignatureCallingConvention"; public const string VirtualStaticsInInterfaces = "VirtualStaticsInInterfaces"; - [System.Diagnostics.CodeAnalysis.FeatureGuard(typeof(System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute))] public static bool IsDynamicCodeCompiled { get { throw null; } } - [System.Diagnostics.CodeAnalysis.FeatureSwitchDefinition("System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeSupported")] public static bool IsDynamicCodeSupported { get { throw null; } } public static bool IsSupported(string feature) { throw null; } } diff --git a/src/libraries/System.Runtime/tests/System.Buffers.Tests/ArrayPool/UnitTests.cs b/src/libraries/System.Runtime/tests/System.Buffers.Tests/ArrayPool/UnitTests.cs index d58a8838d90a97..cc704c68fac131 100644 --- a/src/libraries/System.Runtime/tests/System.Buffers.Tests/ArrayPool/UnitTests.cs +++ b/src/libraries/System.Runtime/tests/System.Buffers.Tests/ArrayPool/UnitTests.cs @@ -272,7 +272,7 @@ public static void RentingReturningThenRentingABufferShouldNotAllocate() Assert.Equal(id, bt.GetHashCode()); } - [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupportedOrBrowserBackgroundExec))] + [Theory] [MemberData(nameof(BytePoolInstances))] public static void CanRentManySizedBuffers(ArrayPool pool) { diff --git a/src/libraries/System.Runtime/tests/System.Reflection.Tests/GetTypeTests.cs b/src/libraries/System.Runtime/tests/System.Reflection.Tests/GetTypeTests.cs index ba1cb8a203c4c5..efa17feb0d2df7 100644 --- a/src/libraries/System.Runtime/tests/System.Reflection.Tests/GetTypeTests.cs +++ b/src/libraries/System.Runtime/tests/System.Reflection.Tests/GetTypeTests.cs @@ -3,8 +3,6 @@ using System.Collections.Generic; using System.IO; -using System.Reflection.Emit; -using System.Runtime.CompilerServices; using Xunit; namespace System.Reflection.Tests @@ -299,142 +297,6 @@ public void TestTypeIdentifierAttribute() args = new object[1] { Activator.CreateInstance(otherEquivalentValueType) }; Assert.Equal(42, mi.Invoke(null, args)); } - - [Fact] - public void IgnoreLeadingDotForTypeNamesWithoutNamespace() - { - Type typeWithNoNamespace = typeof(NoNamespace); - - Assert.Equal(typeWithNoNamespace, Type.GetType($".{typeWithNoNamespace.AssemblyQualifiedName}")); - Assert.Equal(typeWithNoNamespace, Type.GetType(typeWithNoNamespace.AssemblyQualifiedName)); - - Assert.Equal(typeWithNoNamespace, typeWithNoNamespace.Assembly.GetType($".{typeWithNoNamespace.FullName}")); - Assert.Equal(typeWithNoNamespace, typeWithNoNamespace.Assembly.GetType(typeWithNoNamespace.FullName)); - - Assert.Equal(typeof(List), Type.GetType($"{typeof(List<>).FullName}[[{typeWithNoNamespace.AssemblyQualifiedName}]]")); - Assert.Equal(typeof(List), Type.GetType($"{typeof(List<>).FullName}[[.{typeWithNoNamespace.AssemblyQualifiedName}]]")); - - Type typeWithNamespace = typeof(int); - - Assert.Equal(typeWithNamespace, Type.GetType(typeWithNamespace.AssemblyQualifiedName)); - Assert.Null(Type.GetType($".{typeWithNamespace.AssemblyQualifiedName}")); - } - - public static IEnumerable GetTypesThatRequireEscaping() - { - if (RuntimeFeature.IsDynamicCodeSupported) - { - AssemblyBuilder assembly = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName("TypeNamesThatRequireEscaping"), AssemblyBuilderAccess.Run); - ModuleBuilder module = assembly.DefineDynamicModule("TypeNamesThatRequireEscapingModule"); - - yield return new object[] { module.DefineType("TypeNameWith+ThatIsNotNestedType").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith\\TheEscapingCharacter").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith&Ampersand").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith*Asterisk").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith[OpeningSquareBracket").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith]ClosingSquareBracket").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith[]BothSquareBrackets").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith[[]]NestedSquareBrackets").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith,Comma").CreateType(), assembly }; - yield return new object[] { module.DefineType("TypeNameWith\\[]+*&,AllSpecialCharacters").CreateType(), assembly }; - - TypeBuilder containingType = module.DefineType("ContainingTypeWithA+Plus"); - _ = containingType.CreateType(); // containing type must exist! - yield return new object[] { containingType.DefineNestedType("NoSpecialCharacters").CreateType(), assembly }; - yield return new object[] { containingType.DefineNestedType("Contains+Plus").CreateType(), assembly }; - } - } - - [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsReflectionEmitSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/45033", typeof(PlatformDetection), nameof(PlatformDetection.IsMonoRuntime))] - [MemberData(nameof(GetTypesThatRequireEscaping))] - public void TypeNamesThatRequireEscaping(Type type, Assembly assembly) - { - Assert.Contains('\\', type.FullName); - - Assert.Equal(type, assembly.GetType(type.FullName)); - Assert.Equal(type, assembly.GetType(type.FullName.ToLower(), throwOnError: true, ignoreCase: true)); - Assert.Equal(type, assembly.GetType(type.FullName.ToUpper(), throwOnError: true, ignoreCase: true)); - } - - [Fact] - [ActiveIssue("https://github.com/dotnet/runtime/issues/45033", typeof(PlatformDetection), nameof(PlatformDetection.IsMonoRuntime))] - public void EscapingCharacterThatDoesNotRequireEscapingIsTreatedAsError() - { - for (char character = (char)0; character <= 255; character++) - { - Func testCode = () => Type.GetType($"System.\\{character}", throwOnError: true); - - if (character is '\\' or '[' or ']' or '+' or '*' or '&' or ',') - { - Assert.Throws(testCode); // such type does not exist - } - else - { - Assert.Throws(testCode); // such name is invalid - } - - Assert.Null(Type.GetType($"System.\\{character}", throwOnError: false)); - } - } - - public static IEnumerable AllWhitespacesArguments() - { - // leading whitespaces are allowed for type names: - yield return new object[] - { - " \t\r\nSystem.Int32", - typeof(int) - }; - yield return new object[] - { - $"System.Collections.Generic.List`1[\r\n\t [\t\r\n {typeof(int).AssemblyQualifiedName}]], {typeof(List<>).Assembly.FullName}", - typeof(List) - }; - yield return new object[] - { - $"System.Collections.Generic.List`1[\r\n\t{typeof(int).FullName}]", - typeof(List) - }; - // leading whitespaces are NOT allowed for modifiers: - yield return new object[] - { - "System.Int32\t\r\n []", - null - }; - yield return new object[] - { - "System.Int32\r\n\t [,]", - null - }; - yield return new object[] - { - "System.Int32 \r\n\t [*]", - null - }; - yield return new object[] - { - "System.Int32 *", - null - }; - yield return new object[] - { - "System.Int32\t&", - null - }; - // trailing whitespaces are NOT allowed: - yield return new object[] - { - $"System.Int32 \t\r\n", - null - }; - } - - [Theory] - [ActiveIssue("https://github.com/dotnet/runtime/issues/45033", typeof(PlatformDetection), nameof(PlatformDetection.IsMonoRuntime))] - [MemberData(nameof(AllWhitespacesArguments))] - public void AllWhitespaces(string input, Type? expectedType) - => Assert.Equal(expectedType, Type.GetType(input)); } namespace MyNamespace1 @@ -490,8 +352,3 @@ public class MyClass1 { } public class GenericClass { } } - -public class NoNamespace -{ - -} diff --git a/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs index 955bc189b43314..13488161781df5 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.CompilerServices.Unsafe.Tests/UnsafeTests.cs @@ -14,7 +14,7 @@ public class UnsafeTests public static unsafe void ReadInt32() { int expected = 10; - void* address = Unsafe.AsPointer(ref expected); // Unsafe.AsPointer is safe since expected is on stack + void* address = Unsafe.AsPointer(ref expected); int ret = Unsafe.Read(address); Assert.Equal(expected, ret); } @@ -23,7 +23,7 @@ public static unsafe void ReadInt32() public static unsafe void WriteInt32() { int value = 10; - int* address = (int*)Unsafe.AsPointer(ref value); // Unsafe.AsPointer is safe since value is on stack + int* address = (int*)Unsafe.AsPointer(ref value); int expected = 20; Unsafe.Write(address, expected); @@ -36,7 +36,7 @@ public static unsafe void WriteInt32() public static unsafe void WriteBytesIntoInt32() { int value = 20; - int* intAddress = (int*)Unsafe.AsPointer(ref value); // Unsafe.AsPointer is safe since value is on stack + int* intAddress = (int*)Unsafe.AsPointer(ref value); byte* byteAddress = (byte*)intAddress; for (int i = 0; i < 4; i++) { @@ -70,7 +70,7 @@ public static unsafe void WriteBytesIntoInt32() public static unsafe void LongIntoCompoundStruct() { long value = 1234567891011121314L; - long* longAddress = (long*)Unsafe.AsPointer(ref value); // Unsafe.AsPointer is safe since value is on stack + long* longAddress = (long*)Unsafe.AsPointer(ref value); Byte4Short2 b4s2 = Unsafe.Read(longAddress); if (BitConverter.IsLittleEndian) { @@ -117,7 +117,7 @@ public static unsafe void ReadWriteDoublePointer() { int value1 = 10; int value2 = 20; - int* valueAddress = (int*)Unsafe.AsPointer(ref value1); // Unsafe.AsPointer is safe since value1 is on stack + int* valueAddress = (int*)Unsafe.AsPointer(ref value1); int** valueAddressPtr = &valueAddress; Unsafe.Write(valueAddressPtr, new IntPtr(&value2)); @@ -132,7 +132,7 @@ public static unsafe void CopyToRef() { int value = 10; int destination = -1; - Unsafe.Copy(ref destination, Unsafe.AsPointer(ref value)); // Unsafe.AsPointer is safe since value is on stack + Unsafe.Copy(ref destination, Unsafe.AsPointer(ref value)); Assert.Equal(10, destination); Assert.Equal(10, value); @@ -147,7 +147,7 @@ public static unsafe void CopyToVoidPtr() { int value = 10; int destination = -1; - Unsafe.Copy(Unsafe.AsPointer(ref destination), ref value); // Unsafe.AsPointer is safe since destination is on stack + Unsafe.Copy(Unsafe.AsPointer(ref destination), ref value); Assert.Equal(10, destination); Assert.Equal(10, value); @@ -163,7 +163,7 @@ public static unsafe void CopyToRefGenericStruct() Int32Generic destination = default; Int32Generic value = new() { Int32 = 5, Value = "a" }; - Unsafe.Copy(ref destination, Unsafe.AsPointer(ref value)); // Unsafe.AsPointer is safe since value is on stack + Unsafe.Copy(ref destination, Unsafe.AsPointer(ref value)); Assert.Equal(5, destination.Int32); Assert.Equal("a", destination.Value); @@ -175,7 +175,7 @@ public static unsafe void CopyToVoidPtrGenericStruct() Int32Generic destination = default; Int32Generic value = new() { Int32 = 5, Value = "a" }; - Unsafe.Copy(Unsafe.AsPointer(ref destination), ref value); // Unsafe.AsPointer is safe since destination is on stack + Unsafe.Copy(Unsafe.AsPointer(ref destination), ref value); Assert.Equal(5, destination.Int32); Assert.Equal("a", destination.Value); @@ -462,26 +462,6 @@ public static void ByteOffsetStackByte4() Assert.Equal(new IntPtr(-3), Unsafe.ByteOffset(ref byte4.B3, ref byte4.B0)); } - private static unsafe class StaticReadonlyHolder - { - public static readonly void* Pointer = (void*)RuntimeHelpers.AllocateTypeAssociatedMemory(typeof(StaticReadonlyHolder), 1); - } - - [Fact] - public static unsafe void ByteOffsetConstantRef() - { - // https://github.com/dotnet/runtime/pull/99019 - [MethodImpl(MethodImplOptions.NoInlining)] - static nint NullTest(ref byte origin) => Unsafe.ByteOffset(ref origin, ref Unsafe.NullRef()); - Assert.Equal(0, NullTest(ref Unsafe.NullRef())); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - static ref byte GetStatic(ref byte x) => ref x; - [MethodImpl(MethodImplOptions.NoInlining)] - static nint StaticReadonlyTest(ref byte x) => Unsafe.ByteOffset(ref GetStatic(ref Unsafe.AsRef(StaticReadonlyHolder.Pointer)), ref x); - Assert.Equal(0, StaticReadonlyTest(ref Unsafe.AsRef(StaticReadonlyHolder.Pointer))); - } - [Fact] public static unsafe void AsRef() { @@ -617,7 +597,7 @@ public static void RefAddNuintByteOffset() } [Fact] - public static unsafe void RefSubtract() + public static void RefSubtract() { string[] a = new string[] { "abc", "def", "ghi", "jkl" }; @@ -629,11 +609,6 @@ public static unsafe void RefSubtract() ref string r3 = ref Unsafe.Subtract(ref r2, 3); Assert.Equal("abc", r3); - - // https://github.com/dotnet/runtime/pull/99019 - [MethodImpl(MethodImplOptions.NoInlining)] - static ref byte NullTest(nuint offset) => ref Unsafe.Subtract(ref Unsafe.NullRef(), offset); - Assert.True(Unsafe.IsNullRef(ref NullTest(0))); } [Fact] diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System.Runtime.Tests.csproj b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System.Runtime.Tests.csproj index af0f02ec7bc37c..6bcf41fe129bd3 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System.Runtime.Tests.csproj +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System.Runtime.Tests.csproj @@ -196,8 +196,6 @@ - - @@ -268,7 +266,6 @@ - diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Diagnostics/CodeAnalysis/FeatureGuardAttributeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Diagnostics/CodeAnalysis/FeatureGuardAttributeTests.cs deleted file mode 100644 index 3a3b88ff339d30..00000000000000 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Diagnostics/CodeAnalysis/FeatureGuardAttributeTests.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics.CodeAnalysis; -using Xunit; - -namespace System.Diagnostics.CodeAnalysis.Tests -{ - public class FeatureGuardAttributeTests - { - [Fact] - public void TestConstructor() - { - var attr = new FeatureGuardAttribute(typeof(RequiresUnreferencedCodeAttribute)); - Assert.Equal(typeof(RequiresUnreferencedCodeAttribute), attr.FeatureType); - } - } -} diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Diagnostics/CodeAnalysis/FeatureSwitchDefinitionAttributeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Diagnostics/CodeAnalysis/FeatureSwitchDefinitionAttributeTests.cs deleted file mode 100644 index 9d67be3ec1abb3..00000000000000 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Diagnostics/CodeAnalysis/FeatureSwitchDefinitionAttributeTests.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Xunit; - -namespace System.Diagnostics.CodeAnalysis.Tests -{ - public class FeatureSwitchDefinitionAttributeTests - { - [Fact] - public void TestConstructor() - { - var attr = new FeatureSwitchDefinitionAttribute("SwitchName"); - Assert.Equal("SwitchName", attr.SwitchName); - } - } -} diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/GCTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/GCTests.cs index e4aad71c5ac357..59cedd9a5cea69 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/GCTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/GCTests.cs @@ -1096,7 +1096,7 @@ private unsafe static void AllocateArrayPinned_ManagedValueType_CanRoundtripThro var rng = new Random(0xAF); EmbeddedValueType[] array = uninitialized ? GC.AllocateUninitializedArray>(length, pinned: true) : GC.AllocateArray>(length, pinned: true); - byte* pointer = (byte*)Unsafe.AsPointer(ref array[0]); // Unsafe.AsPointer is safe since array is pinned + byte* pointer = (byte*)Unsafe.AsPointer(ref array[0]); var size = Unsafe.SizeOf>(); for(int i = 0; i < length; ++i) diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs index 77c6917a138b29..600326b39d85f7 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/HalfTests.cs @@ -522,16 +522,12 @@ public static IEnumerable ExplicitConversion_FromSingle_TestData() BitConverter.UInt16BitsToHalf(0b0_11001_0000000100)), // 1027.5 rounds to even (BitConverter.Int32BitsToSingle(0b0_10001001_00000000110111111111111), BitConverter.UInt16BitsToHalf(0b0_11001_0000000011)), // 1027.5-ULP rounds down - (BitConverter.Int32BitsToSingle(0b0_10001001_00000000101000000000000), - BitConverter.UInt16BitsToHalf(0b0_11001_0000000010)), // 1026.5 rounds to even (BitConverter.Int32BitsToSingle(unchecked((int)0b1_10001001_00000000110111111111111)), BitConverter.UInt16BitsToHalf(0b1_11001_0000000011)), // -1027.5+ULP rounds towards zero (BitConverter.Int32BitsToSingle(unchecked((int)0b1_10001001_00000000111000000000000)), BitConverter.UInt16BitsToHalf(0b1_11001_0000000100)), // -1027.5 rounds to even (BitConverter.Int32BitsToSingle(unchecked((int)0b1_10001001_00000000111000000000001)), BitConverter.UInt16BitsToHalf(0b1_11001_0000000100)), // -1027.5-ULP rounds away from zero - (BitConverter.Int32BitsToSingle(unchecked((int)0b1_10001001_00000000101000000000000)), - BitConverter.UInt16BitsToHalf(0b1_11001_0000000010)), // -1026.5 rounds to even (BitConverter.Int32BitsToSingle(0b0_01110000_00000001110000000000001), BitConverter.UInt16BitsToHalf(0b0_00000_1000000100)), // subnormal + ULP rounds up (BitConverter.Int32BitsToSingle(0b0_01110000_00000001110000000000000), @@ -542,8 +538,8 @@ public static IEnumerable ExplicitConversion_FromSingle_TestData() BitConverter.UInt16BitsToHalf(0b1_00000_1000000011)), // neg subnormal + ULP rounds higher (BitConverter.Int32BitsToSingle(unchecked((int)0b1_01110000_00000001110000000000000)), BitConverter.UInt16BitsToHalf(0b1_00000_1000000100)), // neg subnormal rounds to even - (BitConverter.Int32BitsToSingle(unchecked((int)0b1_01110000_00000001110000000000001)), - BitConverter.UInt16BitsToHalf(0b1_00000_1000000100)), // neg subnormal - ULP rounds lower, + (BitConverter.Int32BitsToSingle(unchecked((int)0b1_01110000_00000001101111111111111)), + BitConverter.UInt16BitsToHalf(0b1_00000_1000000011)), // neg subnormal - ULP rounds lower, (BitConverter.Int32BitsToSingle(0x33000000), BitConverter.UInt16BitsToHalf(0b0_00000_000000000)), // (half-precision minimum subnormal / 2) should underflow to zero }; @@ -620,16 +616,12 @@ public static IEnumerable ExplicitConversion_FromDouble_TestData() BitConverter.UInt16BitsToHalf(0b0_11001_0000000100)), // 1027.5 rounds to even (BitConverter.Int64BitsToDouble(0x40900DFFFFFFFFFF), BitConverter.UInt16BitsToHalf(0b0_11001_0000000011)), // 1027.5-ULP rounds down - (BitConverter.Int64BitsToDouble(0x40900A0000000000), - BitConverter.UInt16BitsToHalf(0b0_11001_0000000010)), // 1026.5 rounds to even (BitConverter.Int64BitsToDouble(unchecked((long)0xC0900DFFFFFFFFFF)), BitConverter.UInt16BitsToHalf(0b1_11001_0000000011)), // -1027.5+ULP rounds towards zero (BitConverter.Int64BitsToDouble(unchecked((long)0xC0900E0000000000)), BitConverter.UInt16BitsToHalf(0b1_11001_0000000100)), // -1027.5 rounds to even (BitConverter.Int64BitsToDouble(unchecked((long)0xC0900E0000000001)), BitConverter.UInt16BitsToHalf(0b1_11001_0000000100)), // -1027.5-ULP rounds away from zero - (BitConverter.Int64BitsToDouble(unchecked((long)0xC0900A0000000000)), - BitConverter.UInt16BitsToHalf(0b1_11001_0000000010)), // -1026.5 rounds to even (BitConverter.Int64BitsToDouble(0x3F001C0000000001), BitConverter.UInt16BitsToHalf(0b0_00000_1000000100)), // subnormal + ULP rounds up (BitConverter.Int64BitsToDouble(0x3F001C0000000001), diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/MulticastDelegateTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/MulticastDelegateTests.cs index 9994fa02f10fac..354f997ed5c7b3 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/MulticastDelegateTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/MulticastDelegateTests.cs @@ -66,21 +66,6 @@ public static void EqualsTest() Assert.Equal(d1.GetHashCode(), d1.GetHashCode()); } - [Fact] - public static void ArrayDelegates() - { - // Delegate implementation may use Delegate[] arrays as sentinels. Validate that - // the sentinels are not confused with user provided targets. - - Action da = new Delegate[5].MyExtension; - Assert.True(da.HasSingleTarget); - Assert.Equal(1, da.GetInvocationList().Length); - - Func dd = new Delegate[10].GetLength; - Assert.True(dd.HasSingleTarget); - Assert.Equal(1, dd.GetInvocationList().Length); - } - [Fact] public static void CombineReturn() { @@ -214,7 +199,7 @@ private static void CheckInvokeList(D[] expected, D combo, Tracker target) { CheckIsSingletonDelegate((D)(expected[i]), (D)(invokeList[i]), target); } - Assert.Same(combo.Target, expected[^1].Target); + Assert.Same(combo.Target, expected[expected.Length - 1].Target); Assert.Same(combo.Target, target); Assert.Equal(combo.HasSingleTarget, invokeList.Length == 1); int count = 0; @@ -224,7 +209,6 @@ private static void CheckInvokeList(D[] expected, D combo, Tracker target) count++; } Assert.Equal(count, invokeList.Length); - Assert.Equal(combo.Method, invokeList[^1].Method); } private static void CheckIsSingletonDelegate(D expected, D actual, Tracker target) @@ -299,11 +283,4 @@ private class C public string Goo(int x) => new string('A', x); } } - - static class MulticastDelegateTestsExtensions - { - public static void MyExtension(this Delegate[] delegates) - { - } - } } diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/ParamCollectionAttributeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/ParamCollectionAttributeTests.cs deleted file mode 100644 index 25d2f5e87b4e54..00000000000000 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/ParamCollectionAttributeTests.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Xunit; - -namespace System.Runtime.CompilerServices.Tests -{ - public static class ParamCollectionAttributeTests - { - [Fact] - public static void Ctor() - { - var attribute = new ParamCollectionAttribute(); - } - } -} diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/RuntimeHelpersTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/RuntimeHelpersTests.cs index 5148ca0c488d80..7359e11283f740 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/RuntimeHelpersTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Runtime/CompilerServices/RuntimeHelpersTests.cs @@ -68,26 +68,6 @@ public static unsafe void GetObjectValue() Assert.Equal(i, (int)iOV); } - [Fact] - public static void EqualsTest() - { - // Boolean RuntimeHelpers.Equals(Object, Object) - - Assert.True(RuntimeHelpers.Equals(Guid.Empty, Guid.Empty)); - Assert.False(RuntimeHelpers.Equals(Guid.Empty, Guid.NewGuid())); - - // Reference equal - object o = new object(); - Assert.True(RuntimeHelpers.Equals(o, o)); - - // Type mismatch - Assert.False(RuntimeHelpers.Equals(Guid.Empty, string.Empty)); - - // Non value types - Assert.False(RuntimeHelpers.Equals(new object(), new object())); - Assert.False(RuntimeHelpers.Equals(new int[] { 1, 2, 3 }, new int[] { 1, 2, 3 })); - } - [Fact] public static void InitializeArray() { @@ -394,6 +374,7 @@ public static void ArrayGetSubArrayCoVarianceTest() } [Fact] + [SkipOnMono("Not presently implemented on Mono")] public static void AllocateTypeAssociatedMemoryInvalidArguments() { Assert.Throws(() => { RuntimeHelpers.AllocateTypeAssociatedMemory(null, 10); }); @@ -401,6 +382,7 @@ public static void AllocateTypeAssociatedMemoryInvalidArguments() } [Fact] + [SkipOnMono("Not presently implemented on Mono")] public static unsafe void AllocateTypeAssociatedMemoryValidArguments() { IntPtr memory = RuntimeHelpers.AllocateTypeAssociatedMemory(typeof(RuntimeHelpersTests), 32); diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/StringTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/StringTests.cs index 9f59853b136dad..dbd6bf7cbee5b9 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/StringTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/StringTests.cs @@ -999,7 +999,6 @@ public static unsafe void GetPinnableReference_ReturnsSameAsGCHandleAndLegacyFix GCHandle gcHandle = GCHandle.Alloc(input, GCHandleType.Pinned); try { - // Unsafe.AsPointer is safe since it's pinned by the gc handle Assert.Equal((IntPtr)Unsafe.AsPointer(ref Unsafe.AsRef(in rChar)), gcHandle.AddrOfPinnedObject()); } finally diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/TimeSpanTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/TimeSpanTests.cs index 9085d957fa7e63..de434b059fad92 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/TimeSpanTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/TimeSpanTests.cs @@ -347,412 +347,6 @@ public static void EqualsTest(TimeSpan timeSpan1, object obj, bool expected) Assert.Equal(expected, timeSpan1.Equals(obj)); } -#region FromX_int_overloads - [Fact] - public static void FromDays_Int_Positive() - { - var expectedaaa = new TimeSpan(1, 2, 3, 4, 5, 6); - var actual = TimeSpan.FromDays(1, 2, 3, 4, 5, 6); - Assert.Equal(expectedaaa, actual); - } - [Fact] - public static void FromDays_Int_Negative() - { - var expectedaaa = new TimeSpan(-1, -2, -3, -4, -5, -6); - var actual = TimeSpan.FromDays(-1, -2, -3, -4, -5, -6); - Assert.Equal(expectedaaa, actual); - } - [Fact] - public static void FromDays_Int_Zero() - { - var expectedaaa = new TimeSpan(0, 0, 0, 0, 0, 0); - var actual = TimeSpan.FromDays(0, 0, 0, 0, 0, 0); - Assert.Equal(expectedaaa, actual); - } - - [Fact] - public static void FromSeconds_Int_ShouldGiveResultWithPrecision() - { - // Given example of problem with double in: https://github.com/dotnet/runtime/issues/93890#issue-1957706751 - Assert.Equal(new TimeSpan(0, 0, 0, 101, 832), TimeSpan.FromSeconds(101, 832)); - } - - [Fact] - public static void FromDays_Int_ShouldOverflow_WhenIntermediateCalculationCouldOverflowBackIntoValidRange() - { - // Given example of problematic day count in comment in abandoned pr https://github.com/dotnet/runtime/pull/95779/files#r1439772903 - Assert.Throws(() => TimeSpan.FromDays(1067519900)); - } - - [Fact] - public static void FromDays_Int_ShouldConstructMaxValueAproximation() - { - var expected = TimeSpan.MaxValue; - var actual = TimeSpan.FromDays(expected.Days, expected.Hours, expected.Minutes, expected.Seconds, expected.Milliseconds, expected.Microseconds); - // Should be within TicksPerMicrosecond (10) ticks of expected - var diffTicks = (expected - actual).Ticks; - Assert.True(Math.Abs(diffTicks) < 10, $"Diff ticks was {diffTicks}"); - } - [Fact] - public static void FromDays_Int_ShouldConstructMinValueAproximation() - { - var expected = TimeSpan.MinValue; - var actual = TimeSpan.FromDays(expected.Days, expected.Hours, expected.Minutes, expected.Seconds, expected.Milliseconds, expected.Microseconds); - // Should be within TicksPerMicrosecond (10) ticks of expected - var diffTicks = (actual - expected).Ticks; - Assert.True(Math.Abs(diffTicks) < 10, $"Diff ticks was {diffTicks}"); - } - - // Consts copied from internal const in TimeSpan - // Max and Min are symmetrical - private const long maxMicroseconds = 922_337_203_685_477_580; - private const long maxMilliseconds = 922_337_203_685_477; - private const long maxSeconds = 922_337_203_685; - private const long maxMinutes = 15_372_286_728; - private const int maxHours = 256_204_778; - private const int maxDays = 10_675_199; - public static IEnumerable FromDays_Int_ShouldOverflowOrUnderflow_Data() - { - long[] individualMaxValues = [ maxDays, maxHours, maxMinutes, maxSeconds, maxMilliseconds, maxMicroseconds ]; - // Each possibility for individual property to overflow or underflow - for (var i = 0; i < individualMaxValues.Length; i++) - { - var iVal = individualMaxValues[i] + 1; - object[] resultPos = [ 0, 0, 0, 0, 0, 0 ]; - resultPos[i] = iVal; - yield return resultPos; - object[] resultNeg = [ 0, 0, 0, 0, 0, 0 ]; - resultNeg[i] = -iVal; - yield return resultNeg; - } - // Each possibility for 2 properties to overflow or underflow - // while neither of them individually overflow or underflow - for (var i = 0; i < individualMaxValues.Length; i++) - { - for (var j = i + 1; j < individualMaxValues.Length; j++) - { - var iVal = individualMaxValues[i]; - var jVal = individualMaxValues[j]; - object[] resultPos = [ 0, 0, 0, 0, 0, 0 ]; - resultPos[i] = iVal; - resultPos[j] = jVal; - yield return resultPos; - object[] resultNeg = [ 0, 0, 0, 0, 0, 0 ]; - resultNeg[i] = -iVal; - resultNeg[j] = -jVal; - yield return resultNeg; - } - } - } - [Theory] - [MemberData(nameof(FromDays_Int_ShouldOverflowOrUnderflow_Data))] - public static void FromDays_Int_ShouldOverflowOrUnderflow(int days, int hours, long minutes, long seconds, long milliseconds, long microseconds) - { - Assert.Throws(() => TimeSpan.FromDays(days, hours, minutes, seconds, milliseconds, microseconds)); - } - - public static IEnumerable FromDays_Int_ShouldNotOverflow_WhenOverflowingParamIsCounteredByOppositeSignParam_Data() - { - long[] individualMaxValues = [ maxDays, maxHours, maxMinutes, maxSeconds, maxMilliseconds, maxMicroseconds ]; - for (var i = 0; i < individualMaxValues.Length; i++) - { - for (var j = 0; j < individualMaxValues.Length; j++) - { - if (i == j) - { - continue; - } - var iVal = individualMaxValues[i] + 1; - var jVal = individualMaxValues[j] + 1; - object[] result = [ 0, 0, 0, 0, 0, 0 ]; - result[i] = iVal; - result[j] = -jVal; - yield return result; - } - } - } - [Theory] - [MemberData(nameof(FromDays_Int_ShouldNotOverflow_WhenOverflowingParamIsCounteredByOppositeSignParam_Data))] - public static void FromDays_Int_ShouldNotOverflow_WhenOverflowingParamIsCounteredByOppositeSignParam(int days, int hours, long minutes, long seconds, long milliseconds, long microseconds) - { - var actual = TimeSpan.FromDays(days, hours, minutes, seconds, milliseconds, microseconds); - // 2 individually overflowing or underflowing params with opposite sign should end up close to TimeSpan.FromDays(0) - // This is an implementation detail of the chosen test data, but a nice sanity check of expected result - Assert.True(actual > TimeSpan.FromDays(-1)); - Assert.True(actual < TimeSpan.FromDays(1)); - } - [Theory] - [InlineData(maxDays, maxHours, maxMinutes, maxSeconds, maxMilliseconds, maxMicroseconds)] - [InlineData(-maxDays, -maxHours, -maxMinutes, -maxSeconds, -maxMilliseconds, -maxMicroseconds)] - [InlineData(int.MaxValue, int.MaxValue, long.MaxValue, long.MaxValue, long.MaxValue, long.MaxValue)] - [InlineData(int.MinValue, int.MinValue, long.MinValue, long.MinValue, long.MinValue, long.MinValue)] - [InlineData(int.MaxValue, 0, 0, 0, 0, 0)] - [InlineData(int.MinValue, 0, 0, 0, 0, 0)] - [InlineData(0, int.MaxValue, 0, 0, 0, 0)] - [InlineData(0, int.MinValue, 0, 0, 0, 0)] - [InlineData(0, 0, long.MaxValue, 0, 0, 0)] - [InlineData(0, 0, long.MinValue, 0, 0, 0)] - [InlineData(0, 0, 0, long.MaxValue, 0, 0)] - [InlineData(0, 0, 0, long.MinValue, 0, 0)] - [InlineData(0, 0, 0, 0, long.MaxValue, 0)] - [InlineData(0, 0, 0, 0, long.MinValue, 0)] - [InlineData(0, 0, 0, 0, 0, long.MaxValue)] - [InlineData(0, 0, 0, 0, 0, long.MinValue)] - public static void FromDays_Int_ShouldOverflow(int days, int hours, long minutes, long seconds, long milliseconds, long microseconds) - { - Assert.Throws(() => TimeSpan.FromDays(days, hours, minutes, seconds, milliseconds, microseconds)); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - [InlineData(-1)] - [InlineData(maxDays)] - [InlineData(-maxDays)] - public static void FromDays_Int_Single_ShouldCreate(int days) - { - Assert.Equal(new TimeSpan(days, 0, 0, 0), TimeSpan.FromDays(days)); - } - [Theory] - [InlineData(maxDays + 1)] - [InlineData(-(maxDays + 1))] - [InlineData(int.MaxValue)] - [InlineData(int.MinValue)] - public static void FromDays_Int_Single_ShouldOverflow(int days) - { - Assert.Throws(() => TimeSpan.FromDays(days)); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - [InlineData(-1)] - [InlineData(maxHours)] - [InlineData(-maxHours)] - public static void FromHours_Int_Single_ShouldCreate(int hours) - { - Assert.Equal(new TimeSpan(0, hours, 0, 0), TimeSpan.FromHours(hours)); - } - [Theory] - [InlineData(maxHours + 1)] - [InlineData(-(maxHours + 1))] - [InlineData(int.MaxValue)] - [InlineData(int.MinValue)] - public static void FromHours_Int_Single_ShouldOverflow(int hours) - { - Assert.Throws(() => TimeSpan.FromHours(hours)); - } - - [Theory] - [InlineData(0, 0, 0, 0, 0)] - [InlineData(1, 1, 1, 1, 1)] - [InlineData(-1, -1, -1, -1, -1)] - [InlineData(maxHours, 0, 0, 0, 0)] - [InlineData(-maxHours, 0, 0, 0, 0)] - [InlineData(0, maxMinutes, 0, 0, 0)] - [InlineData(0, -maxMinutes, 0, 0, 0)] - [InlineData(0, 0, maxSeconds, 0, 0)] - [InlineData(0, 0, -maxSeconds, 0, 0)] - [InlineData(0, 0, 0, maxMilliseconds, 0)] - [InlineData(0, 0, 0, -maxMilliseconds, 0)] - [InlineData(0, 0, 0, 0, maxMicroseconds)] - [InlineData(0, 0, 0, 0, -maxMicroseconds)] - public static void FromHours_Int_ShouldCreate(int hours, long minutes, long seconds, long milliseconds, long microseconds) - { - var ticksFromHours = hours * TimeSpan.TicksPerHour; - var ticksFromMinutes = minutes * TimeSpan.TicksPerMinute; - var ticksFromSeconds = seconds * TimeSpan.TicksPerSecond; - var ticksFromMilliseconds = milliseconds * TimeSpan.TicksPerMillisecond; - var ticksFromMicroseconds = microseconds * TimeSpan.TicksPerMicrosecond; - var expected = TimeSpan.FromTicks(ticksFromHours + ticksFromMinutes + ticksFromSeconds + ticksFromMilliseconds + ticksFromMicroseconds); - Assert.Equal(expected, TimeSpan.FromHours(hours, minutes, seconds, milliseconds, microseconds)); - } - [Theory] - [InlineData(maxHours + 1, 0, 0, 0, 0)] - [InlineData(-(maxHours + 1), 0, 0, 0, 0)] - [InlineData(0, maxMinutes + 1, 0, 0, 0)] - [InlineData(0, -(maxMinutes + 1), 0, 0, 0)] - [InlineData(0, 0, maxSeconds + 1, 0, 0)] - [InlineData(0, 0, -(maxSeconds + 1), 0, 0)] - [InlineData(0, 0, 0, maxMilliseconds + 1, 0)] - [InlineData(0, 0, 0, -(maxMilliseconds + 1), 0)] - [InlineData(0, 0, 0, 0, maxMicroseconds + 1)] - [InlineData(0, 0, 0, 0, -(maxMicroseconds + 1))] - public static void FromHours_Int_ShouldOverflow(int hours, long minutes, long seconds, long milliseconds, long microseconds) - { - Assert.Throws(() => TimeSpan.FromHours(hours, minutes, seconds, milliseconds, microseconds)); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - [InlineData(-1)] - [InlineData(maxMinutes)] - [InlineData(-maxMinutes)] - public static void FromMinutes_Int_Single_ShouldCreate(long minutes) - { - Assert.Equal(TimeSpan.FromDays(0, minutes: minutes), TimeSpan.FromMinutes(minutes)); - } - [Theory] - [InlineData(maxMinutes + 1)] - [InlineData(-(maxMinutes + 1))] - [InlineData(long.MaxValue)] - [InlineData(long.MinValue)] - public static void FromMinutes_Int_Single_ShouldOverflow(long minutes) - { - Assert.Throws(() => TimeSpan.FromMinutes(minutes)); - } - - [Theory] - [InlineData(0, 0, 0, 0)] - [InlineData(1, 1, 1, 1)] - [InlineData(-1, -1, -1, -1)] - [InlineData(maxMinutes, 0, 0, 0)] - [InlineData(-maxMinutes, 0, 0, 0)] - [InlineData(0, maxSeconds, 0, 0)] - [InlineData(0, -maxSeconds, 0, 0)] - [InlineData(0, 0, maxMilliseconds, 0)] - [InlineData(0, 0, -maxMilliseconds, 0)] - [InlineData(0, 0, 0, maxMicroseconds)] - [InlineData(0, 0, 0, -maxMicroseconds)] - public static void FromMinutes_Int_ShouldCreate(long minutes, long seconds, long milliseconds, long microseconds) - { - var ticksFromMinutes = minutes * TimeSpan.TicksPerMinute; - var ticksFromSeconds = seconds * TimeSpan.TicksPerSecond; - var ticksFromMilliseconds = milliseconds * TimeSpan.TicksPerMillisecond; - var ticksFromMicroseconds = microseconds * TimeSpan.TicksPerMicrosecond; - var expected = TimeSpan.FromTicks(ticksFromMinutes + ticksFromSeconds + ticksFromMilliseconds + ticksFromMicroseconds); - Assert.Equal(expected, TimeSpan.FromMinutes(minutes, seconds, milliseconds, microseconds)); - } - [Theory] - [InlineData(maxMinutes + 1, 0, 0, 0)] - [InlineData(-(maxMinutes + 1), 0, 0, 0)] - [InlineData(0, maxSeconds + 1, 0, 0)] - [InlineData(0, -(maxSeconds + 1), 0, 0)] - [InlineData(0, 0, maxMilliseconds + 1, 0)] - [InlineData(0, 0, -(maxMilliseconds + 1), 0)] - [InlineData(0, 0, 0, maxMicroseconds + 1)] - [InlineData(0, 0, 0, -(maxMicroseconds + 1))] - public static void FromMinutes_Int_ShouldOverflow(long minutes, long seconds, long milliseconds, long microseconds) - { - Assert.Throws(() => TimeSpan.FromMinutes(minutes, seconds, milliseconds, microseconds)); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - [InlineData(-1)] - [InlineData(maxSeconds)] - [InlineData(-maxSeconds)] - public static void FromSeconds_Int_Single_ShouldCreate(long seconds) - { - Assert.Equal(TimeSpan.FromDays(0, seconds: seconds), TimeSpan.FromSeconds(seconds)); - } - [Theory] - [InlineData(maxSeconds + 1)] - [InlineData(-(maxSeconds + 1))] - [InlineData(long.MaxValue)] - [InlineData(long.MinValue)] - public static void FromSeconds_Int_Single_ShouldOverflow(long seconds) - { - Assert.Throws(() => TimeSpan.FromSeconds(seconds)); - } - - [Theory] - [InlineData(0, 0, 0)] - [InlineData(1, 1, 1)] - [InlineData(-1, -1, -1)] - [InlineData(maxSeconds, 0, 0)] - [InlineData(-maxSeconds, 0, 0)] - [InlineData(0, maxMilliseconds, 0)] - [InlineData(0, -maxMilliseconds, 0)] - [InlineData(0, 0, maxMicroseconds)] - [InlineData(0, 0, -maxMicroseconds)] - public static void FromSeconds_Int_ShouldCreate(long seconds, long milliseconds, long microseconds) - { - var ticksFromSeconds = seconds * TimeSpan.TicksPerSecond; - var ticksFromMilliseconds = milliseconds * TimeSpan.TicksPerMillisecond; - var ticksFromMicroseconds = microseconds * TimeSpan.TicksPerMicrosecond; - var expected = TimeSpan.FromTicks(ticksFromSeconds + ticksFromMilliseconds + ticksFromMicroseconds); - Assert.Equal(expected, TimeSpan.FromSeconds(seconds, milliseconds, microseconds)); - } - [Theory] - [InlineData(maxSeconds + 1, 0, 0)] - [InlineData(-(maxSeconds + 1), 0, 0)] - [InlineData(0, maxMilliseconds + 1, 0)] - [InlineData(0, -(maxMilliseconds + 1), 0)] - [InlineData(0, 0, maxMicroseconds + 1)] - [InlineData(0, 0, -(maxMicroseconds + 1))] - [InlineData(long.MaxValue, 0, 0)] - [InlineData(long.MinValue, 0, 0)] - [InlineData(0, long.MaxValue, 0)] - [InlineData(0, long.MinValue, 0)] - [InlineData(0, 0, long.MaxValue)] - [InlineData(0, 0, long.MinValue)] - [InlineData(maxSeconds, maxMilliseconds, 0)] - [InlineData(0, maxMilliseconds, maxMicroseconds)] - [InlineData(maxSeconds, 0, maxMicroseconds)] - public static void FromSeconds_Int_ShouldOverflow(long seconds, long milliseconds, long microseconds) - { - Assert.Throws(() => TimeSpan.FromSeconds(seconds, milliseconds, microseconds)); - } - - [Theory] - [InlineData(0, 0)] - [InlineData(1, 0)] - [InlineData(0, 1)] - [InlineData(-1, 0)] - [InlineData(0, -1)] - [InlineData(maxMilliseconds, 0)] - [InlineData(-maxMilliseconds, 0)] - [InlineData(0, maxMicroseconds)] - [InlineData(0, -maxMicroseconds)] - public static void FromMilliseconds_Int_ShouldCreate(long milliseconds, long microseconds) - { - long ticksFromMilliseconds = milliseconds * TimeSpan.TicksPerMillisecond; - long ticksFromMicroseconds = microseconds * TimeSpan.TicksPerMicrosecond; - var expected = TimeSpan.FromTicks(ticksFromMilliseconds + ticksFromMicroseconds); - Assert.Equal(expected, TimeSpan.FromMilliseconds(milliseconds, microseconds)); - } - [Theory] - [InlineData(maxMilliseconds + 1, 0)] - [InlineData(-(maxMilliseconds + 1), 0)] - [InlineData(long.MaxValue, 0)] - [InlineData(long.MinValue, 0)] - [InlineData(0, maxMicroseconds + 1)] - [InlineData(0, -(maxMicroseconds + 1))] - [InlineData(0, long.MaxValue)] - [InlineData(0, long.MinValue)] - [InlineData(maxMilliseconds, 1000)] - [InlineData(-maxMilliseconds, -1000)] - [InlineData(1, maxMicroseconds)] - [InlineData(-1, -maxMicroseconds)] - public static void FromMilliseconds_Int_ShouldOverflow(long milliseconds, long microseconds) - { - Assert.Throws(() => TimeSpan.FromMilliseconds(milliseconds, microseconds)); - } - - [Theory] - [InlineData(0)] - [InlineData(1)] - [InlineData(-1)] - [InlineData(maxMicroseconds)] - [InlineData(-maxMicroseconds)] - public static void FromMicroseconds_Int_Single_ShouldCreate(long microseconds) - { - Assert.Equal(TimeSpan.FromDays(0, microseconds: microseconds), TimeSpan.FromMicroseconds(microseconds)); - } - [Theory] - [InlineData(maxMicroseconds + 1)] - [InlineData(-(maxMicroseconds + 1))] - [InlineData(long.MaxValue)] - [InlineData(long.MinValue)] - public static void FromMicroseconds_Int_Single_ShouldOverflow(long microseconds) - { - Assert.Throws(() => TimeSpan.FromMicroseconds(microseconds)); - } -#endregion - public static IEnumerable FromDays_TestData() { yield return new object[] { 100.5, new TimeSpan(100, 12, 0, 0) }; diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.cs index ab1d6c66c12796..d604b588621ed9 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/Type/TypeTests.cs @@ -513,26 +513,6 @@ public void GetTypeByName_ValidType_ReturnsExpected(string typeName, Type expect Assert.Equal(expectedType, Type.GetType(typeName.ToLower(), throwOnError: false, ignoreCase: true)); } - public static IEnumerable GetTypeByName_InvalidElementType() - { - Type expectedException = PlatformDetection.IsMonoRuntime - ? typeof(ArgumentException) // https://github.com/dotnet/runtime/issues/45033 - : typeof(TypeLoadException); - - yield return new object[] { "System.Int32&&", expectedException, true }; - yield return new object[] { "System.Int32&*", expectedException, true }; - yield return new object[] { "System.Int32&[]", expectedException, true }; - yield return new object[] { "System.Int32&[*]", expectedException, true }; - yield return new object[] { "System.Int32&[,]", expectedException, true }; - - // https://github.com/dotnet/runtime/issues/45033 - if (!PlatformDetection.IsMonoRuntime) - { - yield return new object[] { "System.Void[]", expectedException, true }; - yield return new object[] { "System.TypedReference[]", expectedException, true }; - } - } - [Theory] [InlineData("system.nullable`1[system.int32]", typeof(TypeLoadException), false)] [InlineData("System.NonExistingType", typeof(TypeLoadException), false)] @@ -542,7 +522,6 @@ public static IEnumerable GetTypeByName_InvalidElementType() [InlineData("Outside`1[System.Boolean, System.Int32]", typeof(ArgumentException), true)] [InlineData(".System.Int32", typeof(TypeLoadException), false)] [InlineData("..Outside`1", typeof(TypeLoadException), false)] - [MemberData(nameof(GetTypeByName_InvalidElementType))] public void GetTypeByName_Invalid(string typeName, Type expectedException, bool alwaysThrowsException) { if (!alwaysThrowsException) @@ -1188,6 +1167,9 @@ public static IEnumerable GetInterfaceMap_TestData() } else { + // [ActiveIssue("https://github.com/dotnet/runtime/issues/90863")] + if (classType.Type == typeof(SIMs.C2Implicit) && interfaceType.Type == typeof(SIMs.I1)) continue; + // It's implemented implicitly by the level 2 interface MTarget = interfaceType.Level2InterfaceType.GetMethod(interfaceType.MethodNamePrefix + "M", bindingFlags); GTarget = interfaceType.Level2InterfaceType.GetMethod(interfaceType.MethodNamePrefix + "G", bindingFlags); @@ -1312,7 +1294,7 @@ public void Method(string arg) { } static class DIMs { - + internal interface I1 { void M() { throw new Exception("e"); } diff --git a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ValueTypeTests.cs b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ValueTypeTests.cs index 92c7000ed414d5..422f71e11c04f0 100644 --- a/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ValueTypeTests.cs +++ b/src/libraries/System.Runtime/tests/System.Runtime.Tests/System/ValueTypeTests.cs @@ -299,36 +299,6 @@ public static void StructContainsPointerCompareTest() Assert.True(obj1.Equals(obj2)); Assert.Equal(obj1.GetHashCode(), obj2.GetHashCode()); } - - [Fact] - public static void StructContainsPointerNestedCompareTest() - { - StructContainsPointerNested obj1 = new StructContainsPointerNested(); - obj1.o = null; - obj1.value.value = 1; - - StructContainsPointerNested obj2 = new StructContainsPointerNested(); - obj2.o = null; - obj2.value.value = 1; - - Assert.True(obj1.Equals(obj2)); - Assert.Equal(obj1.GetHashCode(), obj2.GetHashCode()); - } - - [Fact] - public static void StructWithNestedOverriddenNotBitwiseComparableTest() - { - StructWithNestedOverriddenNotBitwiseComparable obj1 = new StructWithNestedOverriddenNotBitwiseComparable(); - obj1.value1.value = 1; - obj1.value2.value = 0; - - StructWithNestedOverriddenNotBitwiseComparable obj2 = new StructWithNestedOverriddenNotBitwiseComparable(); - obj2.value1.value = -1; - obj2.value2.value = 0; - - Assert.True(obj1.Equals(obj2)); - Assert.Equal(obj1.GetHashCode(), obj2.GetHashCode()); - } public struct S { @@ -422,26 +392,5 @@ public struct StructContainsPointer public double value1; public double value2; } - - public struct StructContainsPointerNested - { - public object o; - public StructNonOverriddenEqualsOrGetHasCode value; - } - - public struct StructOverriddenNotBitwiseComparable - { - public int value; - - public override bool Equals(object obj) => obj is StructOverriddenNotBitwiseComparable other && (value == other.value || value == -other.value); - - public override int GetHashCode() => value < 0 ? -value : value; - } - - public struct StructWithNestedOverriddenNotBitwiseComparable - { - public StructOverriddenNotBitwiseComparable value1; - public StructOverriddenNotBitwiseComparable value2; - } } } diff --git a/src/libraries/System.Runtime/tests/System.Text.Encoding.Tests/NegativeEncodingTests.cs b/src/libraries/System.Runtime/tests/System.Text.Encoding.Tests/NegativeEncodingTests.cs index 8ae05a1faa40a9..3eb90f87286fad 100644 --- a/src/libraries/System.Runtime/tests/System.Text.Encoding.Tests/NegativeEncodingTests.cs +++ b/src/libraries/System.Runtime/tests/System.Text.Encoding.Tests/NegativeEncodingTests.cs @@ -623,9 +623,6 @@ void VerifyOutParams() // Chars does not have enough space AssertExtensions.Throws("chars", () => decoder.Convert(new byte[4], 0, 4, new char[0], 0, 0, flush, out charsUsed, out bytesUsed, out completed)); VerifyOutParams(); - - AssertExtensions.Throws("chars", () => decoder.Convert(encoding.GetBytes("\uD800\uDC00".ToCharArray()).AsSpan(), new char[0].AsSpan(), flush, out charsUsed, out bytesUsed, out completed)); - VerifyOutParams(); } [Theory] diff --git a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/CancellationTokenTests.cs b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/CancellationTokenTests.cs index ba75f617bf0b5d..72df26fa4c9886 100644 --- a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/CancellationTokenTests.cs +++ b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/CancellationTokenTests.cs @@ -874,7 +874,6 @@ static void FinalizeHelper(DisposeTracker disposeTracker) // Several tests for deriving custom user types from CancellationTokenSource [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void DerivedCancellationTokenSource() { // Verify that a derived CTS is functional diff --git a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/MethodCoverage.cs b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/MethodCoverage.cs index eca5c0c92e203d..3509d10843bcbd 100644 --- a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/MethodCoverage.cs +++ b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/MethodCoverage.cs @@ -279,7 +279,6 @@ public static async Task Task_WhenAny_TwoTasks_WakesOnFirstCompletion() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99500", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void CancellationTokenRegitration() { ManualResetEvent mre = new ManualResetEvent(false); @@ -297,7 +296,6 @@ public static void CancellationTokenRegitration() /// verify that the taskawaiter.UnsafeOnCompleted is invoked /// [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void TaskAwaiter() { ManualResetEvent mre = new ManualResetEvent(false); diff --git a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj index 8f7a8cb6b51cf3..19521db08c3331 100644 --- a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj +++ b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj @@ -3,8 +3,6 @@ true true $(NetCoreAppCurrent) - - true diff --git a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/AsyncEnumerableToBlockingEnumerableTests.cs b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/AsyncEnumerableToBlockingEnumerableTests.cs index 786734ad8391e3..0692aedb514f9e 100644 --- a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/AsyncEnumerableToBlockingEnumerableTests.cs +++ b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/AsyncEnumerableToBlockingEnumerableTests.cs @@ -70,7 +70,6 @@ static async IAsyncEnumerable CreateSourceEnumerable() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void AsyncEnumerableWithDelays() { var source = new InstrumentedAsyncEnumerable(CreateSourceEnumerable()); @@ -105,7 +104,6 @@ static async IAsyncEnumerable CreateSourceEnumerable() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void AsyncEnumerableWithException() { var source = new InstrumentedAsyncEnumerable(CreateSourceEnumerable()); @@ -134,7 +132,6 @@ static async IAsyncEnumerable CreateSourceEnumerable() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void AsyncEnumerableWithCancellation() { var source = new InstrumentedAsyncEnumerable(CreateSourceEnumerable()); diff --git a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/TaskContinueWithTests.cs b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/TaskContinueWithTests.cs index 76803afb0195ad..2800a5e9e24c46 100644 --- a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/TaskContinueWithTests.cs +++ b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/Task/TaskContinueWithTests.cs @@ -1076,7 +1076,6 @@ public static void RunContinuationCancelTest_State() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void TestNoDeadlockOnContinueWith() { Debug.WriteLine("TestNoDeadlockOnContinueWith: shouldn't deadlock if it passes."); @@ -1256,7 +1255,6 @@ public static void LongContinuationChain_Unwrap_DoesNotStackOverflow() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99519", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void LongContinuationChain_Await_DoesNotStackOverflow() { const int DiveDepth = 12_000; diff --git a/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs b/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs index ec6e6cabc2cdfa..001aaf26a21aee 100644 --- a/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs +++ b/src/libraries/System.Security.Cryptography.Xml/src/System/Security/Cryptography/Xml/Reference.cs @@ -266,31 +266,18 @@ public void LoadXml(XmlElement value) // let the transform read the children of the transformElement for data transform.LoadInnerXml(transformElement.ChildNodes); // Hack! this is done to get around the lack of here() function support in XPath - if (transform is XmlDsigEnvelopedSignatureTransform) + if (transform is XmlDsigEnvelopedSignatureTransform + && _uri != null && (_uri.Length == 0 || _uri[0] == '#')) { // Walk back to the Signature tag. Find the nearest signature ancestor // Signature-->SignedInfo-->Reference-->Transforms-->Transform XmlNode? signatureTag = transformElement.SelectSingleNode("ancestor::ds:Signature[1]", nsm); // Resolve the reference to get starting point for position calculation. - // This needs to match the way CalculateSignature resolves URI references. - XmlNode? referenceTarget = null; - if (_uri == null || _uri.Length == 0) - { - referenceTarget = transformElement.OwnerDocument; - } - else if (_uri[0] == '#') - { - string idref = Utils.ExtractIdFromLocalUri(_uri); - if (idref == "xpointer(/)") - { - referenceTarget = transformElement.OwnerDocument; - } - else - { - referenceTarget = SignedXml!.GetIdElement(transformElement.OwnerDocument, idref); - } - } + XmlNode? referenceTarget = + _uri.Length == 0 + ? transformElement.OwnerDocument + : SignedXml!.GetIdElement(transformElement.OwnerDocument, Utils.GetIdFromLocalUri(_uri, out bool _)); XmlNodeList? signatureList = referenceTarget?.SelectNodes(".//ds:Signature", nsm); if (signatureList != null) diff --git a/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs b/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs index bd22c5b835eed8..3db8c44aed8270 100644 --- a/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs +++ b/src/libraries/System.Security.Cryptography.Xml/tests/SignedXmlTest.cs @@ -9,7 +9,6 @@ // (C) 2002, 2003 Motus Technologies Inc. (http://www.motus.com) // Copyright (C) 2004-2005, 2008 Novell, Inc (http://www.novell.com) -using System.Collections.Generic; using System.Globalization; using System.IO; using System.Net; @@ -1994,45 +1993,5 @@ public void CheckSignatureHandlesIncorrectOrTamperedReferenceWithMultipleEnvelop Assert.False(subject.CheckSignature()); } - - public static object[][] EnvelopedSignatureWithRootXpointerReference = new object[][] - { - new object[] { true, """HiSVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=dqcBmS1ZvDJNhmCEgobpAb+A2XaiuB69dfGIhisZvqoxaWqAqv/0w49jp38+usJ5t3wcq3aMC631QE8iln+lHWrarojDMDWLa00isv3oE3q9UgOIV9e6MUSoRTTvQkmlK/LSYV9T/SKx6h03vLLcIkUMXaTkC/n2kthlJTGkLbU=t6qV1iTlkCPoaIeOTvnDczQv5pytUxMoyNXws5vaMQYxfJMKos47dvmiLtfWUDLYXFX3Yf/JMC14plJw2JA5jLrlHLnZj/vCjRtXckmWW/wGYewXUqrgR1CytStUeQKj9mNsi76erukua10UhzIrWG+H6YQ/qS4AMMJZU6jBvO0=AQAB""" }, - new object[] { false, """Tempered worldSVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=dqcBmS1ZvDJNhmCEgobpAb+A2XaiuB69dfGIhisZvqoxaWqAqv/0w49jp38+usJ5t3wcq3aMC631QE8iln+lHWrarojDMDWLa00isv3oE3q9UgOIV9e6MUSoRTTvQkmlK/LSYV9T/SKx6h03vLLcIkUMXaTkC/n2kthlJTGkLbU=t6qV1iTlkCPoaIeOTvnDczQv5pytUxMoyNXws5vaMQYxfJMKos47dvmiLtfWUDLYXFX3Yf/JMC14plJw2JA5jLrlHLnZj/vCjRtXckmWW/wGYewXUqrgR1CytStUeQKj9mNsi76erukua10UhzIrWG+H6YQ/qS4AMMJZU6jBvO0=AQAB""" }, - }; - - [Theory] - [MemberData(nameof(EnvelopedSignatureWithRootXpointerReference))] - public void CheckSignatureHandlesEnvelopedSignatureWithRootXpointerReference(bool isValid, string xml) - { - XmlDocument xmlDoc = new (); - xmlDoc.LoadXml(xml); - SignedXml signedXml = new (xmlDoc); - signedXml.LoadXml(xmlDoc.GetElementsByTagName("Signature", SignedXml.XmlDsigNamespaceUrl)[0] as XmlElement); - - Assert.Equal(isValid, signedXml.CheckSignature()); - } - - - public static object[][] EnvelopedSignatureWithEmptyReference = new object[][] - { - new object[] { true, """HiSVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=CiB9jgIS7+Wq+lpyzCGsBZQcQ2BxqQuEU9VCvb3Li5jMtjwRV1bMO+4Wfnb4VWhEtEUq6NdiVGXhC1xvtVLnnLDX7CD/jG6NvM1Yd0/rf0UUceBhzYLFE9HLsopsBmmm3t8FO6ZtRr1QqKM0XDaQleGK9vYd2m2Jq8OR3r/w4OY=vcM1wQVmLB9DwdnAym8l8nw63/HlTVzgTDhIwNzWPhsPE/qr2wlK4TEQ3rjU+RAdNytfFNCnuuh75ZVMjAWCV9h6VDlp0DOvBhb6GenhymtTAdJJKzBXKJP6mNPga9cPOP31IZ36Ui00G3fjBBPrHa7nStludgL9Wi0dBU28DjU=AQAB""" }, - new object[] { false, """HISVaCE5w9iLXTVYTKP1t/yjjmPXvWovMYpgljGgpgz2Y=CiB9jgIS7+Wq+lpyzCGsBZQcQ2BxqQuEU9VCvb3Li5jMtjwRV1bMO+4Wfnb4VWhEtEUq6NdiVGXhC1xvtVLnnLDX7CD/jG6NvM1Yd0/rf0UUceBhzYLFE9HLsopsBmmm3t8FO6ZtRr1QqKM0XDaQleGK9vYd2m2Jq8OR3r/w4OY=vcM1wQVmLB9DwdnAym8l8nw63/HlTVzgTDhIwNzWPhsPE/qr2wlK4TEQ3rjU+RAdNytfFNCnuuh75ZVMjAWCV9h6VDlp0DOvBhb6GenhymtTAdJJKzBXKJP6mNPga9cPOP31IZ36Ui00G3fjBBPrHa7nStludgL9Wi0dBU28DjU=AQAB""" }, - }; - - [Theory] - [MemberData(nameof(EnvelopedSignatureWithEmptyReference))] - public void CheckSignatureHandlesEnvelopedSignatureWithEmptyReference(bool isValid, string xml) - { - XmlDocument xmlDoc = new (); - xmlDoc.LoadXml(xml); - SignedXml signedXml = new (xmlDoc); - signedXml.LoadXml(xmlDoc.GetElementsByTagName("Signature", SignedXml.XmlDsigNamespaceUrl)[0] as XmlElement); - - // without this, CheckSignature throws - ((Reference)signedXml.SignedInfo.References[0]).TransformChain[0].LoadInput(xmlDoc); - - Assert.Equal(isValid, signedXml.CheckSignature()); - } } } diff --git a/src/libraries/System.Security.Cryptography/src/Microsoft/Win32/SafeHandles/SafePasswordHandle.cs b/src/libraries/System.Security.Cryptography/src/Microsoft/Win32/SafeHandles/SafePasswordHandle.cs index ec3c1d6e995f73..f53e582af6e295 100644 --- a/src/libraries/System.Security.Cryptography/src/Microsoft/Win32/SafeHandles/SafePasswordHandle.cs +++ b/src/libraries/System.Security.Cryptography/src/Microsoft/Win32/SafeHandles/SafePasswordHandle.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Security; @@ -36,8 +35,8 @@ public SafePasswordHandle(string? password, bool passwordProvided) public SafePasswordHandle(ReadOnlySpan password, bool passwordProvided) : base(ownsHandle: true) { - // "".AsSpan() does not contain a null ref, so this is compat for "null tries NULL first". - if (!Unsafe.IsNullRef(ref MemoryMarshal.GetReference(password))) + // "".AsSpan() is not default, so this is compat for "null tries NULL first". + if (password != default) { int spanLen; diff --git a/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj b/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj index 7e24c20a7fc418..06de21c43ea96c 100644 --- a/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj +++ b/src/libraries/System.Security.Cryptography/src/System.Security.Cryptography.csproj @@ -403,7 +403,6 @@ - diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.Android.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.Android.cs index 0f7297af6aef58..15fd39ef995da1 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.Android.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.Android.cs @@ -9,16 +9,16 @@ namespace System.Security.Cryptography { public sealed partial class AesCcm { - private FixedMemoryKeyBox _keyBox; + private byte[]? _key; public static bool IsSupported => true; - [MemberNotNull(nameof(_keyBox))] + [MemberNotNull(nameof(_key))] private void ImportKey(ReadOnlySpan key) { - // We should only be calling this in the constructor, so there shouldn't be a previous key. - Debug.Assert(_keyBox is null); - _keyBox = new FixedMemoryKeyBox(key); + // Pin the array on the POH so that the GC doesn't move it around to allow zeroing to be more effective. + _key = GC.AllocateArray(key.Length, pinned: true); + key.CopyTo(_key); } private void EncryptCore( @@ -28,93 +28,80 @@ private void EncryptCore( Span tag, ReadOnlySpan associatedData = default) { - bool acquired = false; + CheckDisposed(); - try + // Convert key length to bits. + using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(_key.Length * 8))) { - _keyBox.DangerousAddRef(ref acquired); - ReadOnlySpan key = _keyBox.DangerousKeySpan; + if (ctx.IsInvalid) + { + throw new CryptographicException(); + } + + if (!Interop.Crypto.CipherSetTagLength(ctx, tag.Length)) + { + throw new CryptographicException(); + } + + Interop.Crypto.CipherSetNonceLength(ctx, nonce.Length); + Interop.Crypto.EvpCipherSetKeyAndIV(ctx, _key, nonce, Interop.Crypto.EvpCipherDirection.Encrypt); + + if (associatedData.Length != 0) + { + Interop.Crypto.CipherUpdateAAD(ctx, associatedData); + } - // Convert key length to bits. - using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(key.Length * 8))) + byte[]? rented = null; + try { - if (ctx.IsInvalid) + scoped Span ciphertextAndTag; + + // Arbitrary limit. + const int StackAllocMax = 128; + if (checked(ciphertext.Length + tag.Length) <= StackAllocMax) { - throw new CryptographicException(); + ciphertextAndTag = stackalloc byte[ciphertext.Length + tag.Length]; } - - if (!Interop.Crypto.CipherSetTagLength(ctx, tag.Length)) + else { - throw new CryptographicException(); + rented = CryptoPool.Rent(ciphertext.Length + tag.Length); + ciphertextAndTag = new Span(rented, 0, ciphertext.Length + tag.Length); } - Interop.Crypto.CipherSetNonceLength(ctx, nonce.Length); - Interop.Crypto.EvpCipherSetKeyAndIV(ctx, key, nonce, Interop.Crypto.EvpCipherDirection.Encrypt); - - if (associatedData.Length != 0) + if (!Interop.Crypto.EvpCipherUpdate(ctx, ciphertextAndTag, out int ciphertextBytesWritten, plaintext)) { - Interop.Crypto.CipherUpdateAAD(ctx, associatedData); + throw new CryptographicException(); } - byte[]? rented = null; - try + if (!Interop.Crypto.EvpAeadCipherFinalEx( + ctx, + ciphertextAndTag.Slice(ciphertextBytesWritten), + out int bytesWritten, + out bool authTagMismatch)) { - scoped Span ciphertextAndTag; - - // Arbitrary limit. - const int StackAllocMax = 128; - if (checked(ciphertext.Length + tag.Length) <= StackAllocMax) - { - ciphertextAndTag = stackalloc byte[ciphertext.Length + tag.Length]; - } - else - { - rented = CryptoPool.Rent(ciphertext.Length + tag.Length); - ciphertextAndTag = new Span(rented, 0, ciphertext.Length + tag.Length); - } - - if (!Interop.Crypto.EvpCipherUpdate(ctx, ciphertextAndTag, out int ciphertextBytesWritten, plaintext)) - { - throw new CryptographicException(); - } - - if (!Interop.Crypto.EvpAeadCipherFinalEx( - ctx, - ciphertextAndTag.Slice(ciphertextBytesWritten), - out int bytesWritten, - out bool authTagMismatch)) - { - Debug.Assert(!authTagMismatch); - throw new CryptographicException(); - } - - ciphertextBytesWritten += bytesWritten; - - // NOTE: Android appends tag to the end of the ciphertext in case of CCM/GCM and "encryption" mode - - if (ciphertextBytesWritten != ciphertextAndTag.Length) - { - Debug.Fail($"CCM encrypt wrote {ciphertextBytesWritten} of {ciphertextAndTag.Length} bytes."); - throw new CryptographicException(); - } - - ciphertextAndTag[..ciphertext.Length].CopyTo(ciphertext); - ciphertextAndTag[ciphertext.Length..].CopyTo(tag); + Debug.Assert(!authTagMismatch); + throw new CryptographicException(); } - finally + + ciphertextBytesWritten += bytesWritten; + + // NOTE: Android appends tag to the end of the ciphertext in case of CCM/GCM and "encryption" mode + + if (ciphertextBytesWritten != ciphertextAndTag.Length) { - if (rented != null) - { - CryptoPool.Return(rented, ciphertext.Length + tag.Length); - } + Debug.Fail($"CCM encrypt wrote {ciphertextBytesWritten} of {ciphertextAndTag.Length} bytes."); + throw new CryptographicException(); } + + ciphertextAndTag[..ciphertext.Length].CopyTo(ciphertext); + ciphertextAndTag[ciphertext.Length..].CopyTo(tag); } - } - finally - { - if (acquired) + finally { - _keyBox.DangerousRelease(); + if (rented != null) + { + CryptoPool.Return(rented, ciphertext.Length + tag.Length); + } } } } @@ -126,77 +113,64 @@ private void DecryptCore( Span plaintext, ReadOnlySpan associatedData) { - bool acquired = false; + CheckDisposed(); - try + using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(_key.Length * 8))) { - _keyBox.DangerousAddRef(ref acquired); - ReadOnlySpan key = _keyBox.DangerousKeySpan; + if (ctx.IsInvalid) + { + throw new CryptographicException(); + } + Interop.Crypto.CipherSetNonceLength(ctx, nonce.Length); - using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(key.Length * 8))) + if (!Interop.Crypto.CipherSetTagLength(ctx, tag.Length)) { - if (ctx.IsInvalid) - { - throw new CryptographicException(); - } - Interop.Crypto.CipherSetNonceLength(ctx, nonce.Length); + throw new CryptographicException(); + } - if (!Interop.Crypto.CipherSetTagLength(ctx, tag.Length)) - { - throw new CryptographicException(); - } + Interop.Crypto.EvpCipherSetKeyAndIV(ctx, _key, nonce, Interop.Crypto.EvpCipherDirection.Decrypt); - Interop.Crypto.EvpCipherSetKeyAndIV(ctx, key, nonce, Interop.Crypto.EvpCipherDirection.Decrypt); + if (associatedData.Length != 0) + { + Interop.Crypto.CipherUpdateAAD(ctx, associatedData); + } - if (associatedData.Length != 0) - { - Interop.Crypto.CipherUpdateAAD(ctx, associatedData); - } + if (!Interop.Crypto.EvpCipherUpdate(ctx, plaintext, out int plaintextBytesWritten, ciphertext)) + { + CryptographicOperations.ZeroMemory(plaintext); + throw new CryptographicException(); + } - if (!Interop.Crypto.EvpCipherUpdate(ctx, plaintext, out int plaintextBytesWritten, ciphertext)) - { - CryptographicOperations.ZeroMemory(plaintext); - throw new CryptographicException(); - } + if (!Interop.Crypto.EvpCipherUpdate(ctx, plaintext.Slice(plaintextBytesWritten), out int bytesWritten, tag)) + { + CryptographicOperations.ZeroMemory(plaintext); + throw new CryptographicException(); + } - if (!Interop.Crypto.EvpCipherUpdate(ctx, plaintext.Slice(plaintextBytesWritten), out int bytesWritten, tag)) - { - CryptographicOperations.ZeroMemory(plaintext); - throw new CryptographicException(); - } + plaintextBytesWritten += bytesWritten; - plaintextBytesWritten += bytesWritten; + if (!Interop.Crypto.EvpAeadCipherFinalEx( + ctx, + plaintext.Slice(plaintextBytesWritten), + out bytesWritten, + out bool authTagMismatch)) + { + CryptographicOperations.ZeroMemory(plaintext); - if (!Interop.Crypto.EvpAeadCipherFinalEx( - ctx, - plaintext.Slice(plaintextBytesWritten), - out bytesWritten, - out bool authTagMismatch)) + if (authTagMismatch) { - CryptographicOperations.ZeroMemory(plaintext); - - if (authTagMismatch) - { - throw new AuthenticationTagMismatchException(); - } - - throw new CryptographicException(SR.Arg_CryptographyException); + throw new AuthenticationTagMismatchException(); } - plaintextBytesWritten += bytesWritten; - - if (plaintextBytesWritten != plaintext.Length) - { - Debug.Fail($"CCM decrypt wrote {plaintextBytesWritten} of {plaintext.Length} bytes."); - throw new CryptographicException(); - } + throw new CryptographicException(SR.Arg_CryptographyException); } - } - finally - { - if (acquired) + + plaintextBytesWritten += bytesWritten; + + if (plaintextBytesWritten != plaintext.Length) { - _keyBox.DangerousRelease(); + Debug.Fail($"CCM decrypt wrote {plaintextBytesWritten} of {plaintext.Length} bytes."); + throw new CryptographicException(); } } } @@ -212,6 +186,16 @@ private static IntPtr GetCipher(int keySizeInBits) }; } - public void Dispose() => _keyBox.Dispose(); + [MemberNotNull(nameof(_key))] + private void CheckDisposed() + { + ObjectDisposedException.ThrowIf(_key is null, this); + } + + public void Dispose() + { + CryptographicOperations.ZeroMemory(_key); + _key = null; + } } } diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.OpenSsl.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.OpenSsl.cs index 554d759acf407e..8f96682ca3258e 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.OpenSsl.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesCcm.OpenSsl.cs @@ -9,18 +9,18 @@ namespace System.Security.Cryptography { public sealed partial class AesCcm { - private FixedMemoryKeyBox _keyBox; + private byte[]? _key; public static bool IsSupported { get; } = Interop.OpenSslNoInit.OpenSslIsAvailable; - [MemberNotNull(nameof(_keyBox))] + [MemberNotNull(nameof(_key))] private void ImportKey(ReadOnlySpan key) { // OpenSSL does not allow setting nonce length after setting the key // we need to store it as bytes instead - // We should only be calling this in the constructor, so there shouldn't be a previous key. - Debug.Assert(_keyBox is null); - _keyBox = new FixedMemoryKeyBox(key); + // Pin the array on the POH so that the GC doesn't move it around to allow zeroing to be more effective. + _key = GC.AllocateArray(key.Length, pinned: true); + key.CopyTo(_key); } private void EncryptCore( @@ -30,65 +30,52 @@ private void EncryptCore( Span tag, ReadOnlySpan associatedData = default) { - bool acquired = false; + CheckDisposed(); - try + using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(_key.Length * 8))) { - _keyBox.DangerousAddRef(ref acquired); - ReadOnlySpan key = _keyBox.DangerousKeySpan; + Interop.Crypto.CheckValidOpenSslHandle(ctx); - using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(key.Length * 8))) - { - Interop.Crypto.CheckValidOpenSslHandle(ctx); - - // We need to set mode to encryption before setting the tag and nonce length - // otherwise older versions of OpenSSL (i.e. 1.0.1f which can be found on Ubuntu 14.04) will fail - Interop.Crypto.EvpCipherSetKeyAndIV(ctx, Span.Empty, Span.Empty, Interop.Crypto.EvpCipherDirection.Encrypt); - Interop.Crypto.EvpCipherSetCcmTagLength(ctx, tag.Length); - Interop.Crypto.EvpCipherSetCcmNonceLength(ctx, nonce.Length); - Interop.Crypto.EvpCipherSetKeyAndIV(ctx, key, nonce, Interop.Crypto.EvpCipherDirection.NoChange); - - if (associatedData.Length != 0) - { - // length needs to be known ahead of time in CCM mode - Interop.Crypto.EvpCipherSetInputLength(ctx, plaintext.Length); + // We need to set mode to encryption before setting the tag and nonce length + // otherwise older versions of OpenSSL (i.e. 1.0.1f which can be found on Ubuntu 14.04) will fail + Interop.Crypto.EvpCipherSetKeyAndIV(ctx, Span.Empty, Span.Empty, Interop.Crypto.EvpCipherDirection.Encrypt); + Interop.Crypto.EvpCipherSetCcmTagLength(ctx, tag.Length); + Interop.Crypto.EvpCipherSetCcmNonceLength(ctx, nonce.Length); + Interop.Crypto.EvpCipherSetKeyAndIV(ctx, _key, nonce, Interop.Crypto.EvpCipherDirection.NoChange); - if (!Interop.Crypto.EvpCipherUpdate(ctx, Span.Empty, out _, associatedData)) - { - throw Interop.Crypto.CreateOpenSslCryptographicException(); - } - } + if (associatedData.Length != 0) + { + // length needs to be known ahead of time in CCM mode + Interop.Crypto.EvpCipherSetInputLength(ctx, plaintext.Length); - if (!Interop.Crypto.EvpCipherUpdate(ctx, ciphertext, out int ciphertextBytesWritten, plaintext)) + if (!Interop.Crypto.EvpCipherUpdate(ctx, Span.Empty, out _, associatedData)) { throw Interop.Crypto.CreateOpenSslCryptographicException(); } + } - if (!Interop.Crypto.EvpCipherFinalEx( - ctx, - ciphertext.Slice(ciphertextBytesWritten), - out int bytesWritten)) - { - throw Interop.Crypto.CreateOpenSslCryptographicException(); - } + if (!Interop.Crypto.EvpCipherUpdate(ctx, ciphertext, out int ciphertextBytesWritten, plaintext)) + { + throw Interop.Crypto.CreateOpenSslCryptographicException(); + } - ciphertextBytesWritten += bytesWritten; + if (!Interop.Crypto.EvpCipherFinalEx( + ctx, + ciphertext.Slice(ciphertextBytesWritten), + out int bytesWritten)) + { + throw Interop.Crypto.CreateOpenSslCryptographicException(); + } - if (ciphertextBytesWritten != ciphertext.Length) - { - Debug.Fail($"CCM encrypt wrote {ciphertextBytesWritten} of {ciphertext.Length} bytes."); - throw new CryptographicException(); - } + ciphertextBytesWritten += bytesWritten; - Interop.Crypto.EvpCipherGetCcmTag(ctx, tag); - } - } - finally - { - if (acquired) + if (ciphertextBytesWritten != ciphertext.Length) { - _keyBox.DangerousRelease(); + Debug.Fail($"CCM encrypt wrote {ciphertextBytesWritten} of {ciphertext.Length} bytes."); + throw new CryptographicException(); } + + Interop.Crypto.EvpCipherGetCcmTag(ctx, tag); } } @@ -99,54 +86,41 @@ private void DecryptCore( Span plaintext, ReadOnlySpan associatedData) { - bool acquired = false; + CheckDisposed(); - try + using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(_key.Length * 8))) { - _keyBox.DangerousAddRef(ref acquired); - ReadOnlySpan key = _keyBox.DangerousKeySpan; + Interop.Crypto.CheckValidOpenSslHandle(ctx); + Interop.Crypto.EvpCipherSetCcmNonceLength(ctx, nonce.Length); + Interop.Crypto.EvpCipherSetCcmTag(ctx, tag); - using (SafeEvpCipherCtxHandle ctx = Interop.Crypto.EvpCipherCreatePartial(GetCipher(key.Length * 8))) - { - Interop.Crypto.CheckValidOpenSslHandle(ctx); - Interop.Crypto.EvpCipherSetCcmNonceLength(ctx, nonce.Length); - Interop.Crypto.EvpCipherSetCcmTag(ctx, tag); + Interop.Crypto.EvpCipherSetKeyAndIV(ctx, _key, nonce, Interop.Crypto.EvpCipherDirection.Decrypt); - Interop.Crypto.EvpCipherSetKeyAndIV(ctx, key, nonce, Interop.Crypto.EvpCipherDirection.Decrypt); - - if (associatedData.Length != 0) - { - // length needs to be known ahead of time in CCM mode - Interop.Crypto.EvpCipherSetInputLength(ctx, ciphertext.Length); - - if (!Interop.Crypto.EvpCipherUpdate(ctx, Span.Empty, out _, associatedData)) - { - throw Interop.Crypto.CreateOpenSslCryptographicException(); - } - } - - if (!Interop.Crypto.EvpCipherUpdate(ctx, plaintext, out int plaintextBytesWritten, ciphertext)) - { - plaintext.Clear(); - throw new AuthenticationTagMismatchException(); - } + if (associatedData.Length != 0) + { + // length needs to be known ahead of time in CCM mode + Interop.Crypto.EvpCipherSetInputLength(ctx, ciphertext.Length); - if (plaintextBytesWritten != plaintext.Length) + if (!Interop.Crypto.EvpCipherUpdate(ctx, Span.Empty, out _, associatedData)) { - Debug.Fail($"CCM decrypt wrote {plaintextBytesWritten} of {plaintext.Length} bytes."); - throw new CryptographicException(); + throw Interop.Crypto.CreateOpenSslCryptographicException(); } + } - // The OpenSSL documentation says not to call EvpCipherFinalEx for CCM decryption, and calling it will report failure. - // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Decryption_using_CCM_mode + if (!Interop.Crypto.EvpCipherUpdate(ctx, plaintext, out int plaintextBytesWritten, ciphertext)) + { + plaintext.Clear(); + throw new AuthenticationTagMismatchException(); } - } - finally - { - if (acquired) + + if (plaintextBytesWritten != plaintext.Length) { - _keyBox.DangerousRelease(); + Debug.Fail($"CCM decrypt wrote {plaintextBytesWritten} of {plaintext.Length} bytes."); + throw new CryptographicException(); } + + // The OpenSSL documentation says not to call EvpCipherFinalEx for CCM decryption, and calling it will report failure. + // https://wiki.openssl.org/index.php/EVP_Authenticated_Encryption_and_Decryption#Authenticated_Decryption_using_CCM_mode } } @@ -163,6 +137,16 @@ private static IntPtr GetCipher(int keySizeInBits) } } - public void Dispose() => _keyBox.Dispose(); + public void Dispose() + { + CryptographicOperations.ZeroMemory(_key); + _key = null; + } + + [MemberNotNull(nameof(_key))] + private void CheckDisposed() + { + ObjectDisposedException.ThrowIf(_key is null, this); + } } } diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesGcm.macOS.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesGcm.macOS.cs index 3b528f496f4d91..5b5b2638164f7a 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesGcm.macOS.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/AesGcm.macOS.cs @@ -9,7 +9,7 @@ namespace System.Security.Cryptography { public sealed partial class AesGcm { - private FixedMemoryKeyBox _keyBox; + private byte[]? _key; // CryptoKit added AES.GCM in macOS 10.15, which is our minimum target for macOS. public static bool IsSupported => true; @@ -17,12 +17,15 @@ public sealed partial class AesGcm // CryptoKit only supports 16 byte tags. public static KeySizes TagByteSizes { get; } = new KeySizes(16, 16, 1); - [MemberNotNull(nameof(_keyBox))] + [MemberNotNull(nameof(_key))] private void ImportKey(ReadOnlySpan key) { // We should only be calling this in the constructor, so there shouldn't be a previous key. - Debug.Assert(_keyBox is null); - _keyBox = new FixedMemoryKeyBox(key); + Debug.Assert(_key is null); + + // Pin the array on the POH so that the GC doesn't move it around to allow zeroing to be more effective. + _key = GC.AllocateArray(key.Length, pinned: true); + key.CopyTo(_key); } private void EncryptCore( @@ -32,26 +35,14 @@ private void EncryptCore( Span tag, ReadOnlySpan associatedData) { - bool acquired = false; - - try - { - _keyBox.DangerousAddRef(ref acquired); - Interop.AppleCrypto.AesGcmEncrypt( - _keyBox.DangerousKeySpan, - nonce, - plaintext, - ciphertext, - tag, - associatedData); - } - finally - { - if (acquired) - { - _keyBox.DangerousRelease(); - } - } + CheckDisposed(); + Interop.AppleCrypto.AesGcmEncrypt( + _key, + nonce, + plaintext, + ciphertext, + tag, + associatedData); } private void DecryptCore( @@ -61,28 +52,26 @@ private void DecryptCore( Span plaintext, ReadOnlySpan associatedData) { - bool acquired = false; + CheckDisposed(); + Interop.AppleCrypto.AesGcmDecrypt( + _key, + nonce, + ciphertext, + tag, + plaintext, + associatedData); + } - try - { - _keyBox.DangerousAddRef(ref acquired); - Interop.AppleCrypto.AesGcmDecrypt( - _keyBox.DangerousKeySpan, - nonce, - ciphertext, - tag, - plaintext, - associatedData); - } - finally - { - if (acquired) - { - _keyBox.DangerousRelease(); - } - } + public void Dispose() + { + CryptographicOperations.ZeroMemory(_key); + _key = null; } - public void Dispose() => _keyBox.Dispose(); + [MemberNotNull(nameof(_key))] + private void CheckDisposed() + { + ObjectDisposedException.ThrowIf(_key is null, this); + } } } diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ChaCha20Poly1305.macOS.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ChaCha20Poly1305.macOS.cs index aa2cba5f234ba7..82f1633c1882e1 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ChaCha20Poly1305.macOS.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/ChaCha20Poly1305.macOS.cs @@ -10,14 +10,17 @@ public sealed partial class ChaCha20Poly1305 { // CryptoKit added ChaCha20Poly1305 in macOS 10.15, which is our minimum target for macOS. public static bool IsSupported => true; - private FixedMemoryKeyBox _keyBox; + private byte[]? _key; - [MemberNotNull(nameof(_keyBox))] + [MemberNotNull(nameof(_key))] private void ImportKey(ReadOnlySpan key) { // We should only be calling this in the constructor, so there shouldn't be a previous key. - Debug.Assert(_keyBox is null); - _keyBox = new FixedMemoryKeyBox(key); + Debug.Assert(_key is null); + + // Pin the array on the POH so that the GC doesn't move it around to allow zeroing to be more effective. + _key = GC.AllocateArray(key.Length, pinned: true); + key.CopyTo(_key); } private void EncryptCore( @@ -27,26 +30,14 @@ private void EncryptCore( Span tag, ReadOnlySpan associatedData = default) { - bool acquired = false; - - try - { - _keyBox.DangerousAddRef(ref acquired); - Interop.AppleCrypto.ChaCha20Poly1305Encrypt( - _keyBox.DangerousKeySpan, - nonce, - plaintext, - ciphertext, - tag, - associatedData); - } - finally - { - if (acquired) - { - _keyBox.DangerousRelease(); - } - } + CheckDisposed(); + Interop.AppleCrypto.ChaCha20Poly1305Encrypt( + _key, + nonce, + plaintext, + ciphertext, + tag, + associatedData); } private void DecryptCore( @@ -56,28 +47,26 @@ private void DecryptCore( Span plaintext, ReadOnlySpan associatedData = default) { - bool acquired = false; + CheckDisposed(); + Interop.AppleCrypto.ChaCha20Poly1305Decrypt( + _key, + nonce, + ciphertext, + tag, + plaintext, + associatedData); + } - try - { - _keyBox.DangerousAddRef(ref acquired); - Interop.AppleCrypto.ChaCha20Poly1305Decrypt( - _keyBox.DangerousKeySpan, - nonce, - ciphertext, - tag, - plaintext, - associatedData); - } - finally - { - if (acquired) - { - _keyBox.DangerousRelease(); - } - } + public void Dispose() + { + CryptographicOperations.ZeroMemory(_key); + _key = null; } - public void Dispose() => _keyBox.Dispose(); + [MemberNotNull(nameof(_key))] + private void CheckDisposed() + { + ObjectDisposedException.ThrowIf(_key is null, this); + } } } diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/CngKey.StandardProperties.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/CngKey.StandardProperties.cs index ceb080af407a07..51584566f560a2 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/CngKey.StandardProperties.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/CngKey.StandardProperties.cs @@ -15,14 +15,12 @@ namespace System.Security.Cryptography /// public sealed partial class CngKey : IDisposable { - private const int CachedKeySizeUninitializedSentinel = -1; - private volatile int _cachedKeySize = CachedKeySizeUninitializedSentinel; + // + // Key properties + // - private volatile CngAlgorithm? _cachedAlgorithm; - private volatile bool _hasCachedAlgorithmGroup; - private volatile CngAlgorithmGroup? _cachedAlgorithmGroup; - private volatile bool _hasCachedProvider; - private volatile CngProvider? _cachedProvider; + private const int CachedKeySizeUninitializedSentinel = -1; + private int _cachedKeySize = CachedKeySizeUninitializedSentinel; /// /// Algorithm group this key can be used with @@ -31,38 +29,25 @@ public CngAlgorithm Algorithm { get { - if (_cachedAlgorithm is null || _keyHandle.IsClosed) - { - string algorithm = _keyHandle.GetPropertyAsString(KeyPropertyName.Algorithm, CngPropertyOptions.None)!; - - // .NET Framework compat: Don't check for null. Just let CngAlgorithm handle it. - _cachedAlgorithm = new CngAlgorithm(algorithm); - } - - return _cachedAlgorithm; + string algorithm = _keyHandle.GetPropertyAsString(KeyPropertyName.Algorithm, CngPropertyOptions.None)!; + // .NET Framework compat: Don't check for null. Just let CngAlgorithm handle it. + return new CngAlgorithm(algorithm); } + } /// /// Name of the algorithm this key can be used with /// public CngAlgorithmGroup? AlgorithmGroup + { get { - if (!_hasCachedAlgorithmGroup || _keyHandle.IsClosed) - { - string? algorithmGroup = _keyHandle.GetPropertyAsString(KeyPropertyName.AlgorithmGroup, CngPropertyOptions.None); - - if (algorithmGroup is not null) - { - _cachedAlgorithmGroup = new CngAlgorithmGroup(algorithmGroup); - } - - _hasCachedAlgorithmGroup = true; - } - - return _cachedAlgorithmGroup; + string? algorithmGroup = _keyHandle.GetPropertyAsString(KeyPropertyName.AlgorithmGroup, CngPropertyOptions.None); + if (algorithmGroup == null) + return null; + return new CngAlgorithmGroup(algorithmGroup); } } @@ -257,6 +242,7 @@ int ComputeKeySize() /// Usage restrictions on the key /// public CngKeyUsages KeyUsage + { get { @@ -293,19 +279,10 @@ public CngProvider? Provider { get { - if (!_hasCachedProvider || _providerHandle.IsClosed) - { - string? provider = _providerHandle.GetPropertyAsString(ProviderPropertyName.Name, CngPropertyOptions.None); - - if (provider is not null) - { - _cachedProvider = new CngProvider(provider); - } - - _hasCachedProvider = true; - } - - return _cachedProvider; + string? provider = _providerHandle.GetPropertyAsString(ProviderPropertyName.Name, CngPropertyOptions.None); + if (provider == null) + return null; + return new CngProvider(provider); } } diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/FixedMemoryKeyBox.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/FixedMemoryKeyBox.cs deleted file mode 100644 index 7f6f1fce35dc8b..00000000000000 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/FixedMemoryKeyBox.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.InteropServices; - -namespace System.Security.Cryptography -{ - internal sealed unsafe class FixedMemoryKeyBox : SafeHandle - { - private readonly int _length; - - internal FixedMemoryKeyBox(ReadOnlySpan key) : base(IntPtr.Zero, ownsHandle: true) - { - void* memory = NativeMemory.Alloc((nuint)key.Length); - key.CopyTo(new Span(memory, key.Length)); - SetHandle((IntPtr)memory); - _length = key.Length; - } - - internal ReadOnlySpan DangerousKeySpan => new ReadOnlySpan((void*)handle, _length); - - protected override bool ReleaseHandle() - { - CryptographicOperations.ZeroMemory(new Span((void*)handle, _length)); - NativeMemory.Free((void*)handle); - return true; - } - - public override bool IsInvalid => handle == IntPtr.Zero; - } -} diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderCng.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderCng.cs index 36f4767e9939e6..831b846a74447e 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderCng.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/HashProviderCng.cs @@ -39,7 +39,7 @@ internal HashProviderCng(string hashAlgId, ReadOnlySpan key, bool isHmac) // So keep hHash trapped in this scope to prevent (mis-)use of it. { SafeBCryptHashHandle hHash; - NTSTATUS ntStatus = Interop.BCrypt.BCryptCreateHash(_hAlgorithm, out hHash, IntPtr.Zero, 0, key, key.Length, BCryptCreateHashFlags.BCRYPT_HASH_REUSABLE_FLAG); + NTSTATUS ntStatus = Interop.BCrypt.BCryptCreateHash(_hAlgorithm, out hHash, IntPtr.Zero, 0, key, key == null ? 0 : key.Length, BCryptCreateHashFlags.BCRYPT_HASH_REUSABLE_FLAG); if (ntStatus == NTSTATUS.STATUS_INVALID_PARAMETER) { hHash.Dispose(); diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificatePal.Windows.PrivateKey.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificatePal.Windows.PrivateKey.cs index 947d7bbee5e9f9..e80641afc1818b 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificatePal.Windows.PrivateKey.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificatePal.Windows.PrivateKey.cs @@ -62,21 +62,9 @@ public bool HasPrivateKey public ECDiffieHellman? GetECDiffieHellmanPrivateKey() { - static ECDiffieHellmanCng? FromCngKey(CngKey cngKey) - { - if (cngKey.AlgorithmGroup == CngAlgorithmGroup.ECDiffieHellman) - { - return new ECDiffieHellmanCng(cngKey, transferOwnership: true); - } - - // We might be getting an ECDSA key here. CNG allows ECDH to be either ECDH or ECDSA, however if - // the AlgorithmGroup is ECDSA, then it cannot be used for ECDH, even though both of them are ECC keys. - return null; - } - return GetPrivateKey( csp => throw new NotSupportedException(SR.NotSupported_ECDiffieHellman_Csp), - FromCngKey + cngKey => new ECDiffieHellmanCng(cngKey, transferOwnership: true) ); } @@ -211,7 +199,7 @@ public ICertificatePal CopyWithPrivateKey(RSA rsa) } } - private T? GetPrivateKey(Func createCsp, Func createCng) where T : AsymmetricAlgorithm + private T? GetPrivateKey(Func createCsp, Func createCng) where T : AsymmetricAlgorithm { using (SafeCertContextHandle certContext = GetCertContext()) { @@ -219,15 +207,7 @@ public ICertificatePal CopyWithPrivateKey(RSA rsa) if (ncryptKey != null) { CngKey cngKey = CngKey.OpenNoDuplicate(ncryptKey, cngHandleOptions); - T? result = createCng(cngKey); - - // Dispose of cngKey if its ownership did not transfer to the underlying algorithm. - if (result is null) - { - cngKey.Dispose(); - } - - return result; + return createCng(cngKey); } } diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificateRequest.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificateRequest.cs index 405d37188e3729..c3084c964d7dc2 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificateRequest.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/CertificateRequest.cs @@ -812,7 +812,7 @@ public X509Certificate2 Create( if (notAfter < notBefore) throw new ArgumentException(SR.Cryptography_CertReq_DatesReversed); - if (serialNumber.Length < 1) + if (serialNumber == null || serialNumber.Length < 1) throw new ArgumentException(SR.Arg_EmptyOrNullArray, nameof(serialNumber)); byte[] signatureAlgorithm = generator.GetSignatureAlgorithmIdentifier(HashAlgorithm); diff --git a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X500NameEncoder.cs b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X500NameEncoder.cs index 865b18e0951888..d2509eca3c3da1 100644 --- a/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X500NameEncoder.cs +++ b/src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/X509Certificates/X500NameEncoder.cs @@ -444,6 +444,7 @@ private static List ParseDistinguishedName( // then some whitespace. case ParseState.MaybeEndQuote: case ParseState.SeekComma: + Debug.Assert(tagOid != null); Debug.Assert(valueStart != -1); Debug.Assert(valueEnd != -1); diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs index a2479d92b71876..c97b8040d34a16 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/CertTests.cs @@ -27,108 +27,6 @@ public CertTests(ITestOutputHelper output) _log = output; } - [Fact] - public static void PrivateKey_FromCertificate_CanExportPrivate_ECDsa() - { - using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) - { - CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256); - - using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) - using (ECDsa certKey = cert.GetECDsaPrivateKey()) - { - ECParameters certParameters = certKey.ExportParameters(true); - ECParameters originalParameters = ca.ExportParameters(true); - AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); - } - } - } - - [Fact] - public static void PrivateKey_FromCertificate_CanExportPrivate_RSA() - { - using (RSA ca = RSA.Create(2048)) - { - CertificateRequest req = new("CN=potatos", ca, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); - - using (X509Certificate2 cert = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3))) - using (RSA certKey = cert.GetRSAPrivateKey()) - { - RSAParameters certParameters = certKey.ExportParameters(true); - RSAParameters originalParameters = ca.ExportParameters(true); - AssertExtensions.SequenceEqual(originalParameters.P, certParameters.P); - AssertExtensions.SequenceEqual(originalParameters.Q, certParameters.Q); - } - } - } - - [Fact] - [SkipOnPlatform(PlatformSupport.MobileAppleCrypto, "DSA is not available")] - public static void PrivateKey_FromCertificate_CanExportPrivate_DSA() - { - DSAParameters originalParameters = DSATestData.GetDSA1024Params(); - - using (DSA ca = DSA.Create()) - { - ca.ImportParameters(originalParameters); - DSAX509SignatureGenerator gen = new DSAX509SignatureGenerator(ca); - X500DistinguishedName dn = new X500DistinguishedName("CN=potatos"); - - CertificateRequest req = new CertificateRequest( - dn, - gen.PublicKey, - HashAlgorithmName.SHA1); - - using (X509Certificate2 cert = req.Create(dn, gen, DateTimeOffset.Now, DateTimeOffset.Now.AddDays(3), [1, 2, 3])) - using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ca)) - using (DSA certKey = certWithKey.GetDSAPrivateKey()) - { - DSAParameters certParameters = certKey.ExportParameters(true); - AssertExtensions.SequenceEqual(originalParameters.X, certParameters.X); - } - } - } - - [Fact] - public static void PrivateKey_FromCertificate_CanExportPrivate_ECDiffieHellman() - { - using (ECDsa ca = ECDsa.Create(ECCurve.NamedCurves.nistP256)) - using (ECDiffieHellman ecdh = ECDiffieHellman.Create(ECCurve.NamedCurves.nistP256)) - { - CertificateRequest issuerRequest = new CertificateRequest( - new X500DistinguishedName("CN=root"), - ca, - HashAlgorithmName.SHA256); - - issuerRequest.CertificateExtensions.Add( - new X509BasicConstraintsExtension(true, false, 0, true)); - - CertificateRequest request = new CertificateRequest( - new X500DistinguishedName("CN=potato"), - new PublicKey(ecdh), - HashAlgorithmName.SHA256); - - request.CertificateExtensions.Add( - new X509BasicConstraintsExtension(false, false, 0, true)); - request.CertificateExtensions.Add( - new X509KeyUsageExtension(X509KeyUsageFlags.KeyAgreement, true)); - - DateTimeOffset notBefore = DateTimeOffset.UtcNow; - DateTimeOffset notAfter = notBefore.AddDays(30); - byte[] serial = [1, 2, 3, 4, 5, 6, 7, 8]; - - using (X509Certificate2 issuer = issuerRequest.CreateSelfSigned(notBefore, notAfter)) - using (X509Certificate2 cert = request.Create(issuer, notBefore, notAfter, serial)) - using (X509Certificate2 certWithKey = cert.CopyWithPrivateKey(ecdh)) - using (ECDiffieHellman certKey = certWithKey.GetECDiffieHellmanPrivateKey()) - { - ECParameters certParameters = certKey.ExportParameters(true); - ECParameters originalParameters = ecdh.ExportParameters(true); - AssertExtensions.SequenceEqual(originalParameters.D, certParameters.D); - } - } - } - [Fact] public static void PublicPrivateKey_IndependentLifetimes_ECDsa() { diff --git a/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxTests.cs b/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxTests.cs index c70260c9dc0db7..a8454f1cf3c3e0 100644 --- a/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxTests.cs +++ b/src/libraries/System.Security.Cryptography/tests/X509Certificates/PfxTests.cs @@ -250,19 +250,6 @@ public static void ECDHPrivateKeyProperty_WindowsPfx() } } - [Fact] - [PlatformSpecific(TestPlatforms.Windows)] // Only windows cares about the key usage attribute in the PKCS12 - public static void ECDHPrivateKey_PfxKeyIsEcdsaConstrained() - { - // [SuppressMessage("Microsoft.Security", "CSCAN0220.DefaultPasswordContexts", Justification="Legacy Test Data")] - using (X509Certificate2 cert = new X509Certificate2(TestData.ECDsaP256_DigitalSignature_Pfx_Windows, "Test")) - { - Assert.Null(cert.GetECDiffieHellmanPrivateKey()); - Assert.NotNull(cert.GetECDiffieHellmanPublicKey()); - Assert.NotNull(cert.GetECDsaPrivateKey()); - } - } - [Fact] [SkipOnPlatform(PlatformSupport.MobileAppleCrypto, "DSA is not available")] public static void DsaPrivateKeyProperty() diff --git a/src/libraries/System.Text.Encoding.CodePages/src/System/Text/EncodingCharBuffer.cs b/src/libraries/System.Text.Encoding.CodePages/src/System/Text/EncodingCharBuffer.cs index 1ba281dc68afbd..c65756dc34719c 100644 --- a/src/libraries/System.Text.Encoding.CodePages/src/System/Text/EncodingCharBuffer.cs +++ b/src/libraries/System.Text.Encoding.CodePages/src/System/Text/EncodingCharBuffer.cs @@ -53,7 +53,7 @@ internal unsafe bool AddChar(char ch, int numBytes) { // Throw maybe _bytes -= numBytes; // Didn't encode these bytes - _enc.ThrowCharsOverflow(_decoder, _chars == _charStart); // Throw? + _enc.ThrowCharsOverflow(_decoder, _bytes <= _byteStart); // Throw? return false; // No throw, but no store either } @@ -72,11 +72,11 @@ internal unsafe bool AddChar(char ch) internal unsafe bool AddChar(char ch1, char ch2, int numBytes) { // Need room for 2 chars - if (_charEnd - _chars < 2) + if (_chars >= _charEnd - 1) { // Throw maybe _bytes -= numBytes; // Didn't encode these bytes - _enc.ThrowCharsOverflow(_decoder, _chars == _charStart); // Throw? + _enc.ThrowCharsOverflow(_decoder, _bytes <= _byteStart); // Throw? return false; // No throw, but no store either } return AddChar(ch1, numBytes) && AddChar(ch2, numBytes); diff --git a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs index 7dbc4de3a57f59..c41234d54b7b2e 100644 --- a/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs +++ b/src/libraries/System.Text.Json/gen/JsonSourceGenerator.Emitter.cs @@ -6,7 +6,7 @@ using System.Linq; using System.Reflection; using System.Text.Json.Serialization; -using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.Text; using SourceGenerators; @@ -638,16 +638,16 @@ private void GeneratePropMetadataInitFunc(SourceWriter writer, string propInitMe writer.WriteLine($$""" var {{InfoVarName}}{{i}} = new {{JsonPropertyInfoValuesTypeRef}}<{{propertyTypeFQN}}> { - IsProperty = {{FormatBoolLiteral(property.IsProperty)}}, - IsPublic = {{FormatBoolLiteral(property.IsPublic)}}, - IsVirtual = {{FormatBoolLiteral(property.IsVirtual)}}, + IsProperty = {{FormatBool(property.IsProperty)}}, + IsPublic = {{FormatBool(property.IsPublic)}}, + IsVirtual = {{FormatBool(property.IsVirtual)}}, DeclaringType = typeof({{property.DeclaringType.FullyQualifiedName}}), Converter = {{converterInstantiationExpr ?? "null"}}, Getter = {{getterValue}}, Setter = {{setterValue}}, IgnoreCondition = {{ignoreConditionNamedArg}}, - HasJsonInclude = {{FormatBoolLiteral(property.HasJsonInclude)}}, - IsExtensionData = {{FormatBoolLiteral(property.IsExtensionData)}}, + HasJsonInclude = {{FormatBool(property.HasJsonInclude)}}, + IsExtensionData = {{FormatBool(property.IsExtensionData)}}, NumberHandling = {{FormatNumberHandling(property.NumberHandling)}}, PropertyName = {{FormatStringLiteral(property.MemberName)}}, JsonPropertyName = {{FormatStringLiteral(property.JsonPropertyName)}} @@ -701,10 +701,10 @@ private static void GenerateCtorParamMetadataInitFunc(SourceWriter writer, strin writer.WriteLine($$""" {{parametersVarName}}[{{spec.ParameterIndex}}] = new() { - Name = {{FormatStringLiteral(spec.Name)}}, + Name = "{{spec.Name}}", ParameterType = typeof({{spec.ParameterType.FullyQualifiedName}}), Position = {{spec.ParameterIndex}}, - HasDefaultValue = {{FormatBoolLiteral(spec.HasDefaultValue)}}, + HasDefaultValue = {{FormatBool(spec.HasDefaultValue)}}, DefaultValue = {{CSharpSyntaxUtilities.FormatLiteral(spec.DefaultValue, spec.ParameterType)}} }; @@ -721,7 +721,7 @@ private static void GenerateCtorParamMetadataInitFunc(SourceWriter writer, strin writer.WriteLine($$""" {{parametersVarName}}[{{spec.ParameterIndex}}] = new() { - Name = {{FormatStringLiteral(spec.Name)}}, + Name = "{{spec.Name}}", ParameterType = typeof({{spec.ParameterType.FullyQualifiedName}}), Position = {{spec.ParameterIndex}}, }; @@ -1106,10 +1106,10 @@ private static void GetLogicForDefaultSerializerOptionsInit(SourceGenerationOpti writer.Indentation++; if (optionsSpec.AllowOutOfOrderMetadataProperties is bool allowOutOfOrderMetadataProperties) - writer.WriteLine($"AllowOutOfOrderMetadataProperties = {FormatBoolLiteral(allowOutOfOrderMetadataProperties)},"); + writer.WriteLine($"AllowOutOfOrderMetadataProperties = {FormatBool(allowOutOfOrderMetadataProperties)},"); if (optionsSpec.AllowTrailingCommas is bool allowTrailingCommas) - writer.WriteLine($"AllowTrailingCommas = {FormatBoolLiteral(allowTrailingCommas)},"); + writer.WriteLine($"AllowTrailingCommas = {FormatBool(allowTrailingCommas)},"); if (optionsSpec.Converters is { Count: > 0 } converters) { @@ -1136,13 +1136,13 @@ private static void GetLogicForDefaultSerializerOptionsInit(SourceGenerationOpti writer.WriteLine($"DictionaryKeyPolicy = {FormatNamingPolicy(dictionaryKeyPolicy)},"); if (optionsSpec.IgnoreReadOnlyFields is bool ignoreReadOnlyFields) - writer.WriteLine($"IgnoreReadOnlyFields = {FormatBoolLiteral(ignoreReadOnlyFields)},"); + writer.WriteLine($"IgnoreReadOnlyFields = {FormatBool(ignoreReadOnlyFields)},"); if (optionsSpec.IgnoreReadOnlyProperties is bool ignoreReadOnlyProperties) - writer.WriteLine($"IgnoreReadOnlyProperties = {FormatBoolLiteral(ignoreReadOnlyProperties)},"); + writer.WriteLine($"IgnoreReadOnlyProperties = {FormatBool(ignoreReadOnlyProperties)},"); if (optionsSpec.IncludeFields is bool includeFields) - writer.WriteLine($"IncludeFields = {FormatBoolLiteral(includeFields)},"); + writer.WriteLine($"IncludeFields = {FormatBool(includeFields)},"); if (optionsSpec.MaxDepth is int maxDepth) writer.WriteLine($"MaxDepth = {maxDepth},"); @@ -1154,7 +1154,7 @@ private static void GetLogicForDefaultSerializerOptionsInit(SourceGenerationOpti writer.WriteLine($"PreferredObjectCreationHandling = {FormatObjectCreationHandling(preferredObjectCreationHandling)},"); if (optionsSpec.PropertyNameCaseInsensitive is bool propertyNameCaseInsensitive) - writer.WriteLine($"PropertyNameCaseInsensitive = {FormatBoolLiteral(propertyNameCaseInsensitive)},"); + writer.WriteLine($"PropertyNameCaseInsensitive = {FormatBool(propertyNameCaseInsensitive)},"); if (optionsSpec.PropertyNamingPolicy is JsonKnownNamingPolicy propertyNamingPolicy) writer.WriteLine($"PropertyNamingPolicy = {FormatNamingPolicy(propertyNamingPolicy)},"); @@ -1169,10 +1169,10 @@ private static void GetLogicForDefaultSerializerOptionsInit(SourceGenerationOpti writer.WriteLine($"UnmappedMemberHandling = {FormatUnmappedMemberHandling(unmappedMemberHandling)},"); if (optionsSpec.WriteIndented is bool writeIndented) - writer.WriteLine($"WriteIndented = {FormatBoolLiteral(writeIndented)},"); + writer.WriteLine($"WriteIndented = {FormatBool(writeIndented)},"); if (optionsSpec.IndentCharacter is char indentCharacter) - writer.WriteLine($"IndentCharacter = {FormatCharLiteral(indentCharacter)},"); + writer.WriteLine($"IndentCharacter = {FormatIndentChar(indentCharacter)},"); if (optionsSpec.IndentSize is int indentSize) writer.WriteLine($"IndentSize = {indentSize},"); @@ -1238,7 +1238,7 @@ private static void GenerateConverterHelpers(SourceWriter writer, bool emitGetCo { throw new {{InvalidOperationExceptionTypeRef}}(string.Format("{{ExceptionMessages.IncompatibleConverterType}}", converter.GetType(), type)); } - + if (converter is {{JsonConverterFactoryTypeRef}} factory) { converter = factory.CreateConverter(type, options); @@ -1247,7 +1247,7 @@ private static void GenerateConverterHelpers(SourceWriter writer, bool emitGetCo throw new {{InvalidOperationExceptionTypeRef}}(string.Format("{{ExceptionMessages.InvalidJsonConverterFactoryOutput}}", factory.GetType())); } } - + return converter; } """); @@ -1263,7 +1263,7 @@ private static void GenerateConverterHelpers(SourceWriter writer, bool emitGetCo { return ({{JsonConverterTypeRef}}<{{TypeParameter}}?>){{ExpandConverterMethodName}}(typeof({{TypeParameter}}?), converter, options, validateCanConvert: false); } - + converter = {{ExpandConverterMethodName}}(typeof({{TypeParameter}}), converter, options); {{JsonTypeInfoTypeRef}}<{{TypeParameter}}> typeInfo = {{JsonMetadataServicesTypeRef}}.{{CreateValueInfoMethodName}}<{{TypeParameter}}>(options, converter); return {{JsonMetadataServicesTypeRef}}.GetNullableConverter<{{TypeParameter}}>(typeInfo); @@ -1320,7 +1320,7 @@ private SourceText GetPropertyNameInitialization(ContextGenerationSpec contextSp foreach (KeyValuePair name_varName_pair in _propertyNames) { - writer.WriteLine($$"""private static readonly {{JsonEncodedTextTypeRef}} {{name_varName_pair.Value}} = {{JsonEncodedTextTypeRef}}.Encode({{FormatStringLiteral(name_varName_pair.Key)}});"""); + writer.WriteLine($$"""private static readonly {{JsonEncodedTextTypeRef}} {{name_varName_pair.Value}} = {{JsonEncodedTextTypeRef}}.Encode("{{name_varName_pair.Key}}");"""); } return CompleteSourceFileAndReturnText(writer); @@ -1351,9 +1351,9 @@ private static string FormatJsonSerializerDefaults(JsonSerializerDefaults defaul private static string GetCreateValueInfoMethodRef(string typeCompilableName) => $"{CreateValueInfoMethodName}<{typeCompilableName}>"; - private static string FormatBoolLiteral(bool value) => value ? "true" : "false"; - private static string FormatStringLiteral(string? value) => value is null ? "null" : SymbolDisplay.FormatLiteral(value, quote: true); - private static string FormatCharLiteral(char value) => SymbolDisplay.FormatLiteral(value, quote: true); + private static string FormatBool(bool value) => value ? "true" : "false"; + private static string FormatStringLiteral(string? value) => value is null ? "null" : $"\"{value}\""; + private static string FormatIndentChar(char value) => value is '\t' ? "'\\t'" : $"'{value}'"; /// /// Method used to generate JsonTypeInfo given options instance diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.Parse.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.Parse.cs index f9b1fffd4243b2..e5719888d80a79 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.Parse.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.Parse.cs @@ -165,6 +165,8 @@ internal static JsonDocument ParseValue(Stream utf8Json, JsonDocumentOptions opt internal static JsonDocument ParseValue(ReadOnlySpan utf8Json, JsonDocumentOptions options) { + Debug.Assert(utf8Json != null); + byte[] owned = new byte[utf8Json.Length]; utf8Json.CopyTo(owned); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs index 1ca7fff9f7e333..c5d687892335aa 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs @@ -5,7 +5,6 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; namespace System.Text.Json { @@ -1244,7 +1243,7 @@ public bool ValueEquals(ReadOnlySpan utf8Text) if (TokenType == JsonTokenType.Null) { // This is different than Length == 0, in that it tests true for null, but false for "" - return Unsafe.IsNullRef(ref MemoryMarshal.GetReference(utf8Text)); + return utf8Text == default; } return TextEqualsHelper(utf8Text, isPropertyName: false, shouldUnescape: true); @@ -1272,7 +1271,7 @@ public bool ValueEquals(ReadOnlySpan text) if (TokenType == JsonTokenType.Null) { // This is different than Length == 0, in that it tests true for null, but false for "" - return Unsafe.IsNullRef(ref MemoryMarshal.GetReference(text)); + return text == default; } return TextEqualsHelper(text, isPropertyName: false); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptionsUpdateHandler.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptionsUpdateHandler.cs index a2beec2eae3b76..9e1d5e4c0d95bd 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptionsUpdateHandler.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializerOptionsUpdateHandler.cs @@ -28,7 +28,9 @@ public static void ClearCache(Type[]? types) if (RuntimeFeature.IsDynamicCodeSupported) { // Flush the dynamic method cache +#pragma warning disable IL3050 // The analyzer doesn't understand runtime feature conditions: https://github.com/dotnet/linker/issues/2715 ReflectionEmitCachingMemberAccessor.Clear(); +#pragma warning restore IL3050 } } } diff --git a/src/libraries/System.Text.Json/tests/Common/PropertyNameTests.cs b/src/libraries/System.Text.Json/tests/Common/PropertyNameTests.cs index 9e6066fc308caa..021481ae5a1362 100644 --- a/src/libraries/System.Text.Json/tests/Common/PropertyNameTests.cs +++ b/src/libraries/System.Text.Json/tests/Common/PropertyNameTests.cs @@ -33,7 +33,7 @@ public async Task BuiltInPolicyDeserializeMatch() await DeserializeAndAssert(JsonNamingPolicy.SnakeCaseLower, @"{""my_int16"":1}", 1); await DeserializeAndAssert(JsonNamingPolicy.SnakeCaseUpper, @"{""MY_INT16"":1}", 1); await DeserializeAndAssert(JsonNamingPolicy.KebabCaseLower, @"{""my-int16"":1}", 1); - await DeserializeAndAssert(JsonNamingPolicy.KebabCaseUpper, @"{""MY-INT16"":1}", 1); + await DeserializeAndAssert(JsonNamingPolicy.KebabCaseUpper, @"{""MY-INT16"":1}", 1); } private async Task DeserializeAndAssert(JsonNamingPolicy policy, string json, short expected) @@ -449,8 +449,7 @@ public async Task SpecialCharacters() SmtpId = 3, Emojies = 4, \uA000 = 5, - YiIt_2 = 6, - PropertyNameWithWhitespace = 7, + YiIt_2 = 6 }; string json = await Serializer.SerializeWrapper(obj); @@ -460,8 +459,7 @@ public async Task SpecialCharacters() "\"smtp-id\":3," + "\"\\uD83D\\uDE00\\uD83D\\uDE01\":4," + "\"\\uA000\":5," + - "\"\\uA000_2\":6," + - "\"\\u0022ab \\n\\r\\t\\f\\bc\\u0022\":7}", json); + "\"\\uA000_2\":6}", json); obj = await Serializer.DeserializeWrapper(json); Assert.Equal(1, obj.Baseline); @@ -495,10 +493,6 @@ public class ClassWithSpecialCharacters [JsonPropertyOrder(6)] [JsonPropertyName("\uA000_2")] // Valid C# property name: \uA000_2 public int YiIt_2 { get; set; } - - [JsonPropertyOrder(7)] - [JsonPropertyName("\"ab \n\r\t\f\bc\"")] // Regression test for https://github.com/dotnet/runtime/issues/98638 - public int PropertyNameWithWhitespace { get; set; } } [Theory] diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs index 5e5d83de2c09f9..a554d2681d43d1 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorDiagnosticsTests.cs @@ -11,8 +11,6 @@ namespace System.Text.Json.SourceGeneration.UnitTests { [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/71962", ~RuntimeConfiguration.Release)] - [SkipOnMono("https://github.com/dotnet/runtime/issues/92467")] - [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotX86Process))] // https://github.com/dotnet/runtime/issues/71962 public class JsonSourceGeneratorDiagnosticsTests { /// diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorIncrementalTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorIncrementalTests.cs index 5bcb01a94bde9a..daa6498cbc9b2d 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorIncrementalTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorIncrementalTests.cs @@ -13,8 +13,6 @@ namespace System.Text.Json.SourceGeneration.UnitTests { [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/71962", ~RuntimeConfiguration.Release)] - [SkipOnMono("https://github.com/dotnet/runtime/issues/92467")] - [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotX86Process))] // https://github.com/dotnet/runtime/issues/71962 public static class JsonSourceGeneratorIncrementalTests { [Theory] diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs index e2f08b988441c0..eb6d0991585c02 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.SourceGeneration.Unit.Tests/JsonSourceGeneratorTests.cs @@ -10,8 +10,6 @@ namespace System.Text.Json.SourceGeneration.UnitTests { [ActiveIssue("https://github.com/dotnet/runtime/issues/58226", TestPlatforms.Browser)] [SkipOnCoreClr("https://github.com/dotnet/runtime/issues/71962", ~RuntimeConfiguration.Release)] - [SkipOnMono("https://github.com/dotnet/runtime/issues/92467")] - [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsNotX86Process))] // https://github.com/dotnet/runtime/issues/71962 public class GeneratorTests { [Fact] diff --git a/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs b/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs index c7c587c3a5bdc5..7e7fed6cab65f2 100644 --- a/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs +++ b/src/libraries/System.Text.RegularExpressions/gen/RegexGenerator.Emitter.cs @@ -10,7 +10,6 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Threading; using Microsoft.CodeAnalysis; @@ -733,11 +732,6 @@ private static void EmitTryFindNextPossibleStartingPosition(IndentedTextWriter w EmitIndexOfString_RightToLeft(); break; - case FindNextStartingPositionMode.LeadingStrings_LeftToRight: - case FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight: - EmitIndexOfStrings_LeftToRight(); - break; - case FindNextStartingPositionMode.LeadingSet_LeftToRight: case FindNextStartingPositionMode.FixedDistanceSets_LeftToRight: EmitFixedSet_LeftToRight(); @@ -1047,37 +1041,6 @@ UnicodeCategory.NonSpacingMark or } } - // Emits a case-sensitive left-to-right search for any one of multiple leading prefixes. - void EmitIndexOfStrings_LeftToRight() - { - RegexFindOptimizations opts = regexTree.FindOptimizations; - Debug.Assert(opts.FindMode is FindNextStartingPositionMode.LeadingStrings_LeftToRight or FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight); - - string prefixes = string.Join(", ", opts.LeadingPrefixes.Select(prefix => Literal(prefix))); - StringComparison stringComparison = opts.FindMode is FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight ? - StringComparison.OrdinalIgnoreCase : - StringComparison.Ordinal; - string fieldName = GetSHA256FieldName($"s_indexOfAnyStrings_{stringComparison}_", prefixes); - - if (!requiredHelpers.ContainsKey(fieldName)) - { - requiredHelpers.Add(fieldName, - [ - $"/// Supports searching for the specified strings.", - $"internal static readonly SearchValues {fieldName} = SearchValues.Create([{prefixes}], StringComparison.{stringComparison});", // explicitly using an array in case prefixes is large - ]); - } - - writer.WriteLine($"// The pattern has multiple strings that could begin the match. Search for any of them."); - writer.WriteLine($"// If none can be found, there's no match."); - writer.WriteLine($"int i = inputSpan.Slice(pos).IndexOfAny({HelpersTypeName}.{fieldName});"); - using (EmitBlock(writer, "if (i >= 0)")) - { - writer.WriteLine("base.runtextpos = pos + i;"); - writer.WriteLine("return true;"); - } - } - // Emits a case-sensitive right-to-left search for a substring. void EmitIndexOfString_RightToLeft() { @@ -1453,16 +1416,6 @@ private static void EmitTryMatchAtCurrentPosition(IndentedTextWriter writer, Reg HashSet additionalDeclarations = new(); Dictionary additionalLocalFunctions = new(); - // In debug builds, additional code is emitted to validate that the backtracking stack is being maintained appropriately. - // When state is pushed onto the backtracking stack, an additional known value is pushed, and when it's popped, it's - // the popped value is checked against that known value, throwing an exception if they don't match. This validation code - // is currently not part of RegexCompiler, though it could be added there in the future if desired. -#if DEBUG -#pragma warning disable RS1035 // Random isn't always deterministic, but this is only for debug builds, and we've seeded the Random with a constant - Random stackCookieGenerator = new(12345); // seed for deterministic behavior -#pragma warning restore RS1035 -#endif - // Declare some locals. string sliceSpan = "slice"; writer.WriteLine("int pos = base.runtextpos;"); @@ -1665,7 +1618,7 @@ void EmitAlternation(RegexNode node) } // Detect whether every branch begins with one or more unique characters. - const int SetCharsSize = 64; // arbitrary limit; we want it to be large enough to handle ignore-case of common sets, like hex, the latin alphabet, etc. + const int SetCharsSize = 5; // arbitrary limit (for IgnoreCase, we want this to be at least 3 to handle the vast majority of values) Span setChars = stackalloc char[SetCharsSize]; if (useSwitchedBranches) { @@ -1675,10 +1628,8 @@ void EmitAlternation(RegexNode node) var seenChars = new HashSet(); for (int i = 0; i < childCount && useSwitchedBranches; i++) { - // Look for the guaranteed starting node that's a one, multi, set, - // or loop of one of those with at least one minimum iteration. We need to exclude notones. - if (node.Child(i).FindStartingLiteralNode(allowZeroWidth: false) is not RegexNode startingLiteralNode || - startingLiteralNode.IsNotoneFamily) + // If it's not a One, Multi, or Set, we can't apply this optimization. + if (node.Child(i).FindBranchOneMultiOrSetStart() is not RegexNode oneMultiOrSet) { useSwitchedBranches = false; break; @@ -1686,9 +1637,9 @@ void EmitAlternation(RegexNode node) // If it's a One or a Multi, get the first character and add it to the set. // If it was already in the set, we can't apply this optimization. - if (startingLiteralNode.IsOneFamily || startingLiteralNode.Kind is RegexNodeKind.Multi) + if (oneMultiOrSet.Kind is RegexNodeKind.One or RegexNodeKind.Multi) { - if (!seenChars.Add(startingLiteralNode.FirstCharOfOneOrMulti())) + if (!seenChars.Add(oneMultiOrSet.FirstCharOfOneOrMulti())) { useSwitchedBranches = false; break; @@ -1698,10 +1649,10 @@ void EmitAlternation(RegexNode node) { // The branch begins with a set. Make sure it's a set of only a few characters // and get them. If we can't, we can't apply this optimization. - Debug.Assert(startingLiteralNode.IsSetFamily); + Debug.Assert(oneMultiOrSet.Kind is RegexNodeKind.Set); int numChars; - if (RegexCharClass.IsNegated(startingLiteralNode.Str!) || - (numChars = RegexCharClass.GetSetChars(startingLiteralNode.Str!, setChars)) == 0) + if (RegexCharClass.IsNegated(oneMultiOrSet.Str!) || + (numChars = RegexCharClass.GetSetChars(oneMultiOrSet.Str!, setChars)) == 0) { useSwitchedBranches = false; break; @@ -1743,7 +1694,7 @@ void EmitSwitchedBranches() writer.WriteLine(); // Emit a switch statement on the first char of each branch. - using (EmitBlock(writer, $"switch ({sliceSpan}[{sliceStaticPos}])")) + using (EmitBlock(writer, $"switch ({sliceSpan}[{sliceStaticPos++}])")) { Span setChars = stackalloc char[SetCharsSize]; // needs to be same size as detection check in caller int startingSliceStaticPos = sliceStaticPos; @@ -1753,74 +1704,56 @@ void EmitSwitchedBranches() { sliceStaticPos = startingSliceStaticPos; - // We know we're only in this code if every branch has a valid starting literal node. Get it. - // We also get the immediate child. Ideally they're the same, in which case we might be able to - // use the switch as the processing of that node, e.g. if the node is a One, then by matching the - // literal via the switch, we've fully processed it. But there may be other cases in which it's not - // sufficient, e.g. if that one was wrapped in a Capture, we still want to emit the capture code, - // and for simplicity, we still end up emitting the re-evaluation of that character. It's still much - // cheaper to do this than to emit the full alternation code. - RegexNode child = node.Child(i); - RegexNode? startingLiteralNode = child.FindStartingLiteralNode(allowZeroWidth: false); - Debug.Assert(startingLiteralNode is not null, "Unexpectedly couldn't find the branch starting node."); + Debug.Assert(child.Kind is RegexNodeKind.One or RegexNodeKind.Multi or RegexNodeKind.Set or RegexNodeKind.Concatenate, DescribeNode(child, rm)); + Debug.Assert(child.Kind is not RegexNodeKind.Concatenate || (child.ChildCount() >= 2 && child.Child(0).Kind is RegexNodeKind.One or RegexNodeKind.Multi or RegexNodeKind.Set)); - // Emit the case for this branch to match on the first character. - if (startingLiteralNode.IsSetFamily) + RegexNode? childStart = child.FindBranchOneMultiOrSetStart(); + Debug.Assert(childStart is not null, "Unexpectedly couldn't find the branch starting node."); + + if (childStart.Kind is RegexNodeKind.Set) { - int numChars = RegexCharClass.GetSetChars(startingLiteralNode.Str!, setChars); + int numChars = RegexCharClass.GetSetChars(childStart.Str!, setChars); Debug.Assert(numChars != 0); writer.WriteLine($"case {string.Join(" or ", setChars.Slice(0, numChars).ToArray().Select(Literal))}:"); } else { - writer.WriteLine($"case {Literal(startingLiteralNode.FirstCharOfOneOrMulti())}:"); + writer.WriteLine($"case {Literal(childStart.FirstCharOfOneOrMulti())}:"); } writer.Indent++; // Emit the code for the branch, without the first character that was already matched in the switch. - RegexNode? remainder = null; - HandleChild: switch (child.Kind) { - case RegexNodeKind.One: - case RegexNodeKind.Set: - // The character was handled entirely by the switch. No additional matching is needed. - sliceStaticPos++; - break; - case RegexNodeKind.Multi: - // First character was handled by the switch. Emit matching code for the remainder of the multi string. - sliceStaticPos++; - EmitNode(child.Str!.Length == 2 ? - new RegexNode(RegexNodeKind.One, child.Options, child.Str![1]) : - new RegexNode(RegexNodeKind.Multi, child.Options, child.Str!.Substring(1))); + EmitNode(CloneMultiWithoutFirstChar(child)); writer.WriteLine(); break; - case RegexNodeKind.Concatenate when child.Child(0) == startingLiteralNode && (startingLiteralNode.Kind is RegexNodeKind.One or RegexNodeKind.Set or RegexNodeKind.Multi): - // This is a concatenation where its first node is the starting literal we found and that starting literal - // is one of the nodes above that we know how to handle completely. This is a common - // enough case that we want to special-case it to avoid duplicating the processing for that character - // unnecessarily. So, we'll shave off that first node from the concatenation and then handle the remainder. - // Note that it's critical startingLiteralNode is something we can fully handle above: if it's not, - // we'll end up losing some of the pattern due to overwriting `remainder`. - remainder = child; - child = child.Child(0); - remainder.ReplaceChild(0, new RegexNode(RegexNodeKind.Empty, remainder.Options)); - goto HandleChild; // reprocess just the first node that was saved; the remainder will then be processed below - - default: - Debug.Assert(remainder is null); - remainder = child; + case RegexNodeKind.Concatenate: + var newConcat = new RegexNode(RegexNodeKind.Concatenate, child.Options); + if (childStart.Kind == RegexNodeKind.Multi) + { + newConcat.AddChild(CloneMultiWithoutFirstChar(childStart)); + } + int concatChildCount = child.ChildCount(); + for (int j = 1; j < concatChildCount; j++) + { + newConcat.AddChild(child.Child(j)); + } + EmitNode(newConcat.Reduce()); + writer.WriteLine(); break; - } - if (remainder is not null) - { - // Emit a full match for whatever part of the child we haven't yet handled. - EmitNode(remainder); - writer.WriteLine(); + static RegexNode CloneMultiWithoutFirstChar(RegexNode node) + { + Debug.Assert(node.Kind is RegexNodeKind.Multi); + Debug.Assert(node.Str!.Length >= 2); + return node.Str!.Length == 2 ? + new RegexNode(RegexNodeKind.One, node.Options, node.Str![1]) : + new RegexNode(RegexNodeKind.Multi, node.Options, node.Str!.Substring(1)); + } } // This is only ever used for atomic alternations, so we can simply reset the doneLabel @@ -1924,7 +1857,6 @@ void EmitAllBranches() additionalDeclarations.Add($"int {currentBranch} = 0;"); } - int stackCookie = CreateStackCookie(); for (int i = 0; i < childCount; i++) { // If the alternation isn't atomic, backtracking may require our jump table jumping back @@ -1964,7 +1896,7 @@ void EmitAllBranches() // the relevant state is stored in our locals. if (currentBranch is null) { - EmitStackPush(stackCookie + i, startingCapturePos is not null ? + EmitStackPush(startingCapturePos is not null ? [i.ToString(), startingPos, startingCapturePos] : [i.ToString(), startingPos]); } @@ -2034,12 +1966,11 @@ void EmitAllBranches() string switchClause; if (currentBranch is null) { - // We're in a loop, so we use the backtracking stack to persist our state. - // Pop it off and validate the stack position. - EmitStackPop(0, startingCapturePos is not null ? + // We're in a loop, so we use the backtracking stack to persist our state. Pop it off. + EmitStackPop(startingCapturePos is not null ? [startingCapturePos, startingPos] : [startingPos]); - switchClause = ValidateStackCookieWithAdditionAndReturnPoppedStack(stackCookie); + switchClause = StackPop(); } else { @@ -2139,7 +2070,6 @@ void EmitBackreferenceConditional(RegexNode node) // We're branching in a complicated fashion. Make sure sliceStaticPos is 0. TransferSliceStaticPosToPos(); - int stackCookie = CreateStackCookie(); // Get the capture number to test. int capnum = RegexParser.MapCaptureNumber(node.M, rm.Tree.CaptureNumberSparseMapping); @@ -2271,7 +2201,7 @@ void EmitBackreferenceConditional(RegexNode node) // the local. if (isInLoop) { - EmitStackPop(stackCookie, resumeAt); + EmitStackPop(resumeAt); } using (EmitBlock(writer, $"switch ({resumeAt})")) { @@ -2300,7 +2230,7 @@ void EmitBackreferenceConditional(RegexNode node) // so finish outputting our backtracking logic, which involves pushing onto the stack which // branch to backtrack into. If we're not in a loop, though, nothing else can overwrite this local // in the interim, so we can avoid pushing it. - EmitStackPush(stackCookie, resumeAt); + EmitStackPush(resumeAt); } } @@ -2368,19 +2298,10 @@ void EmitExpressionConditional(RegexNode node) writer.WriteLine(); int startingSliceStaticPos = sliceStaticPos; - // Emit the condition. The condition expression is a zero-width assertion, which is atomic, + // Emit the child. The condition expression is a zero-width assertion, which is atomic, // so prevent backtracking into it. writer.WriteLine("// Condition:"); - if (rm.Analysis.MayBacktrack(condition)) - { - // Condition expressions are treated like positive lookarounds and thus are implicitly atomic, - // so we need to emit the node as atomic if it might backtrack. - EmitAtomic(node, null); - } - else - { - EmitNode(condition); - } + EmitNode(condition); writer.WriteLine(); doneLabel = originalDoneLabel; @@ -2459,13 +2380,11 @@ void EmitExpressionConditional(RegexNode node) doneLabel = backtrack; MarkLabel(backtrack, emitSemicolon: false); - int stackCookie = CreateStackCookie(); - if (isInLoop) { // If we're not in a loop, the local will maintain its value until backtracking occurs. // If we are in a loop, multiple iterations need their own value, so we need to use the stack. - EmitStackPop(stackCookie, resumeAt); + EmitStackPop(resumeAt); } using (EmitBlock(writer, $"switch ({resumeAt})")) @@ -2486,7 +2405,7 @@ void EmitExpressionConditional(RegexNode node) MarkLabel(endConditional, emitSemicolon: !isInLoop); if (isInLoop) { - EmitStackPush(stackCookie, resumeAt); + EmitStackPush(resumeAt); } } } @@ -2558,13 +2477,12 @@ void EmitCapture(RegexNode node, RegexNode? subsequent = null) // pushes/pops the starting position before falling through. writer.WriteLine(); - int stackCookie = CreateStackCookie(); if (isInLoop) { // If we're in a loop, different iterations of the loop need their own // starting position, so push it on to the stack. If we're not in a loop, // the local will maintain its value and will suffice. - EmitStackPush(stackCookie, startingPos); + EmitStackPush(startingPos); } // Skip past the backtracking section @@ -2577,7 +2495,7 @@ void EmitCapture(RegexNode node, RegexNode? subsequent = null) MarkLabel(backtrack, emitSemicolon: false); if (isInLoop) { - EmitStackPop(stackCookie, startingPos); + EmitStackPop(startingPos); } Goto(doneLabel); writer.WriteLine(); @@ -2671,7 +2589,6 @@ void EmitNegativeLookaroundAssertion(RegexNode node) RegexNode child = node.Child(0); // Ensure we're able to uncapture anything captured by the child. - int stackCookie = CreateStackCookie(); bool isInLoop = false; string? capturePos = null; bool hasCaptures = rm.Analysis.MayContainCapture(child); @@ -2682,7 +2599,7 @@ void EmitNegativeLookaroundAssertion(RegexNode node) isInLoop = rm.Analysis.IsInLoop(node); if (isInLoop) { - EmitStackPush(stackCookie, "base.Crawlpos()"); + EmitStackPush("base.Crawlpos()"); } else { @@ -2706,12 +2623,6 @@ void EmitNegativeLookaroundAssertion(RegexNode node) // If the generated code ends up here, it matched the lookaround, which actually // means failure for a _negative_ lookaround, so we need to jump to the original done. writer.WriteLine(); - if (hasCaptures && isInLoop) - { - // Pop the crawl position from the stack. - writer.WriteLine("stackpos--;"); - EmitStackCookieValidate(stackCookie); - } Goto(originalDoneLabel); writer.WriteLine(); @@ -2726,15 +2637,7 @@ void EmitNegativeLookaroundAssertion(RegexNode node) // And uncapture anything if necessary. Negative lookaround captures don't persist beyond the lookaround. if (hasCaptures) { - if (isInLoop) - { - EmitUncaptureUntil(StackPop()); - EmitStackCookieValidate(stackCookie); - } - else - { - EmitUncaptureUntil(capturePos!); - } + EmitUncaptureUntil(isInLoop ? StackPop() : capturePos!); } doneLabel = originalDoneLabel; @@ -2914,8 +2817,8 @@ void EmitNode(RegexNode node, RegexNode? subsequent = null, bool emitLengthCheck // Emits the node for an atomic. void EmitAtomic(RegexNode node, RegexNode? subsequent) { - Debug.Assert(node.Kind is RegexNodeKind.Atomic or RegexNodeKind.PositiveLookaround or RegexNodeKind.NegativeLookaround or RegexNodeKind.ExpressionConditional, $"Unexpected type: {node.Kind}"); - Debug.Assert(node.Kind is RegexNodeKind.ExpressionConditional ? node.ChildCount() >= 1 : node.ChildCount() == 1, $"Unexpected number of children: {node.ChildCount()}"); + Debug.Assert(node.Kind is RegexNodeKind.Atomic or RegexNodeKind.PositiveLookaround or RegexNodeKind.NegativeLookaround, $"Unexpected type: {node.Kind}"); + Debug.Assert(node.ChildCount() == 1, $"Expected 1 child, found {node.ChildCount()}"); Debug.Assert(rm.Analysis.MayBacktrack(node.Child(0)), "Expected child to potentially backtrack"); // Grab the current done label and the current backtracking position. The purpose of the atomic node @@ -3324,7 +3227,6 @@ void EmitSingleCharLoop(RegexNode node, RegexNode? subsequent = null, bool emitL // point we decrement the matched count as long as it's above the minimum // required, and try again by flowing to everything that comes after this. MarkLabel(backtrackingLabel, emitSemicolon: false); - int stackCookie = CreateStackCookie(); string? capturePos = null; if (isInLoop) { @@ -3337,7 +3239,7 @@ void EmitSingleCharLoop(RegexNode node, RegexNode? subsequent = null, bool emitL { EmitUncaptureUntil(StackPop()); } - EmitStackPop(stackCookie, endingPos, startingPos); + EmitStackPop(endingPos, startingPos); } else if (expressionHasCaptures) { @@ -3392,7 +3294,7 @@ void EmitSingleCharLoop(RegexNode node, RegexNode? subsequent = null, bool emitL // We're in a loop and thus can't rely on locals correctly holding the state we // need (the locals could be overwritten by a subsequent iteration). Push the state // on to the backtracking stack. - EmitStackPush(stackCookie, expressionHasCaptures ? + EmitStackPush(expressionHasCaptures ? [startingPos, endingPos, "base.Crawlpos()"] : [startingPos, endingPos]); } @@ -3633,10 +3535,9 @@ node.Kind is RegexNodeKind.Setlazy && if (isInLoop) { writer.WriteLine(); - int stackCookie = CreateStackCookie(); // Store the loop's state. - EmitStackPush(stackCookie, + EmitStackPush( capturePos is not null && iterationCount is not null ? [startingPos, capturePos, iterationCount] : capturePos is not null ? [startingPos, capturePos] : iterationCount is not null ? [startingPos, iterationCount] : @@ -3652,7 +3553,7 @@ node.Kind is RegexNodeKind.Setlazy && MarkLabel(backtrack, emitSemicolon: false); // Restore the loop's state. - EmitStackPop(stackCookie, + EmitStackPop( capturePos is not null && iterationCount is not null ? [iterationCount, capturePos, startingPos] : capturePos is not null ? [capturePos, startingPos] : iterationCount is not null ? [iterationCount, startingPos] : @@ -3739,13 +3640,8 @@ void EmitLazy(RegexNode node) // iterations, this state needs to be stored on to the backtracking stack. if (!isAtomic) { - int stackCookie = CreateStackCookie(); - int entriesPerIteration = - 1/*pos*/ + - (iterationMayBeEmpty ? 2/*startingPos+sawEmpty*/ : 0) + - (expressionHasCaptures ? 1/*Crawlpos*/ : 0) + - (stackCookie != 0 ? 1 : 0); - EmitStackPush(stackCookie, + int entriesPerIteration = 1/*pos*/ + (iterationMayBeEmpty ? 2/*startingPos+sawEmpty*/ : 0) + (expressionHasCaptures ? 1/*Crawlpos*/ : 0); + EmitStackPush( expressionHasCaptures && iterationMayBeEmpty ? ["pos", startingPos!, sawEmpty!, "base.Crawlpos()"] : iterationMayBeEmpty ? ["pos", startingPos!, sawEmpty!] : expressionHasCaptures ? ["pos", "base.Crawlpos()"] : @@ -3825,7 +3721,7 @@ void EmitLazy(RegexNode node) { EmitUncaptureUntil(StackPop()); } - EmitStackPop(stackCookie, iterationMayBeEmpty ? + EmitStackPop(iterationMayBeEmpty ? [sawEmpty!, startingPos!, "pos"] : ["pos"]); SliceInputSpan(); @@ -3882,8 +3778,7 @@ void EmitLazy(RegexNode node) // of another loop, then any number of iterations might have such state that needs to be stored, // and thus it needs to be pushed on to the backtracking stack. bool isInLoop = rm.Analysis.IsInLoop(node); - stackCookie = CreateStackCookie(); - EmitStackPush(stackCookie, + EmitStackPush( !isInLoop ? (expressionHasCaptures ? ["pos", "base.Crawlpos()"] : ["pos"]) : iterationMayBeEmpty ? (expressionHasCaptures ? ["pos", iterationCount, startingPos!, sawEmpty!, "base.Crawlpos()"] : ["pos", iterationCount, startingPos!, sawEmpty!]) : expressionHasCaptures ? ["pos", iterationCount, "base.Crawlpos()"] : @@ -3905,7 +3800,7 @@ void EmitLazy(RegexNode node) { EmitUncaptureUntil(StackPop()); } - EmitStackPop(stackCookie, + EmitStackPop( !isInLoop ? ["pos"] : iterationMayBeEmpty ? [sawEmpty!, startingPos!, iterationCount, "pos"] : [iterationCount, "pos"]); @@ -4288,7 +4183,6 @@ void EmitLoop(RegexNode node) int minIterations = node.M; int maxIterations = node.N; - int stackCookie = CreateStackCookie(); // Special-case some repeaters. if (minIterations == maxIterations) @@ -4367,7 +4261,7 @@ void EmitLoop(RegexNode node) // need to know where each iteration began so when backtracking we can jump back to that location. This is // true even if the loop is atomic, as we might need to backtrack within the loop in order to match the // minimum iteration count. - EmitStackPush(stackCookie, + EmitStackPush( expressionHasCaptures && iterationMayBeEmpty ? ["base.Crawlpos()", startingPos!, "pos"] : expressionHasCaptures ? ["base.Crawlpos()", "pos"] : iterationMayBeEmpty ? [startingPos!, "pos"] : @@ -4477,14 +4371,13 @@ void EmitLoop(RegexNode node) writer.WriteLine("// Unable to match the remainder of the expression after exhausting the loop."); Goto(originalDoneLabel); } - EmitStackPop(0, iterationMayBeEmpty ? // stack cookie handled is explicitly 0 to handle it below + EmitStackPop(iterationMayBeEmpty ? ["pos", startingPos!] : ["pos"]); if (expressionHasCaptures) { EmitUncaptureUntil(StackPop()); } - EmitStackCookieValidate(stackCookie); SliceInputSpan(); // If there's a required minimum iteration count, validate now that we've processed enough iterations. @@ -4594,8 +4487,7 @@ void EmitLoop(RegexNode node) writer.WriteLine(); // Store the loop's state - stackCookie = CreateStackCookie(); - EmitStackPush(stackCookie, + EmitStackPush( startingPos is not null && startingStackpos is not null ? [startingPos, startingStackpos, iterationCount] : startingPos is not null ? [startingPos, iterationCount] : startingStackpos is not null ? [startingStackpos, iterationCount] : @@ -4609,7 +4501,7 @@ void EmitLoop(RegexNode node) // Emit a backtracking section that restores the loop's state and then jumps to the previous done label string backtrack = ReserveName("LoopBacktrack"); MarkLabel(backtrack, emitSemicolon: false); - EmitStackPop(stackCookie, + EmitStackPop( startingPos is not null && startingStackpos is not null ? [iterationCount, startingStackpos, startingPos] : startingPos is not null ? [iterationCount, startingPos] : startingStackpos is not null ? [iterationCount, startingStackpos] : @@ -4660,7 +4552,7 @@ void EmitUncaptureUntil(string capturepos) } /// Pushes values on to the backtracking stack. - void EmitStackPush(int stackCookie, params string[] args) + void EmitStackPush(params string[] args) { Debug.Assert(args.Length is >= 1); @@ -4704,134 +4596,41 @@ void EmitStackPush(int stackCookie, params string[] args) requiredHelpers.Add(key, lines); } - if (stackCookie != 0) - { - EmitStackCookie(stackCookie); - } writer.WriteLine($"{HelpersTypeName}.{MethodName}(ref base.runstack!, ref stackpos, {string.Join(", ", args)});"); } /// Pops values from the backtracking stack into the specified locations. - void EmitStackPop(int stackCookie, params string[] args) + void EmitStackPop(params string[] args) { Debug.Assert(args.Length is >= 1); if (args.Length == 1) { writer.WriteLine($"{args[0]} = {StackPop()};"); + return; } - else - { - const string MethodName = "StackPop"; - string key = $"{MethodName}{args.Length}"; - - if (!requiredHelpers.ContainsKey(key)) - { - var lines = new string[5 + args.Length]; - lines[0] = $"/// Pops {args.Length} value{(args.Length == 1 ? "" : "s")} from the backtracking stack."; - lines[1] = $"[MethodImpl(MethodImplOptions.AggressiveInlining)]"; - lines[2] = $"internal static void {MethodName}(int[] stack, ref int pos{FormatN(", out int arg{0}", args.Length)})"; - lines[3] = $"{{"; - for (int i = 0; i < args.Length; i++) - { - lines[4 + i] = $" arg{i} = stack[--pos];"; - } - lines[4 + args.Length] = $"}}"; - - requiredHelpers.Add(key, lines); - } - - writer.WriteLine($"{HelpersTypeName}.{MethodName}(base.runstack!, ref stackpos, out {string.Join(", out ", args)});"); - } - - if (stackCookie != 0) - { - EmitStackCookieValidate(stackCookie); - } - } - /// Initializes a debug stack cookie for a new backtracking stack push. - int CreateStackCookie() => -#if DEBUG -#pragma warning disable RS1035 // Random is banned from generators due to non-determinism, but this Random is seeded with a constant and it's only for debug builds - stackCookieGenerator.Next() + 1; -#pragma warning restore RS1035 -#else - 0; -#endif - - /// Emits a debug stack cookie for a new backtracking stack push. - void EmitStackCookie(int stackCookie) - { -#if DEBUG - EmitStackPush(0, stackCookie.ToString()); -#endif - } - - /// Emits validation for a debug stack cookie. - void EmitStackCookieValidate(int stackCookie) - { -#if DEBUG - writer.WriteLine($"{StackCookieValidate(stackCookie)};"); -#endif - } + const string MethodName = "StackPop"; + string key = $"{MethodName}{args.Length}"; - /// - /// Returns an expression that: - /// In debug, pops item 1 from the backtracking stack, pops item 2 and validates it against the cookie, then evaluates to item1. - /// In release, pops and evaluates to an item from the backtracking stack. - /// - string ValidateStackCookieWithAdditionAndReturnPoppedStack(int stackCookie) - { -#if DEBUG - const string MethodName = "ValidateStackCookieWithAdditionAndReturnPoppedStack"; - if (!requiredHelpers.ContainsKey(MethodName)) + if (!requiredHelpers.ContainsKey(key)) { - requiredHelpers.Add(MethodName, - [ - $"/// Validates that a stack cookie popped off the backtracking stack holds the expected value. Debug only.", - $"internal static int {MethodName}(int poppedStack, int expectedCookie, int actualCookie)", - $"{{", - $" expectedCookie += poppedStack;", - $" if (expectedCookie != actualCookie)", - $" {{", - $" throw new Exception($\"Backtracking stack imbalance detected. Expected {{expectedCookie}}. Actual {{actualCookie}}.\");", - $" }}", - $" return poppedStack;", - $"}}", - ]); - } - - return $"{HelpersTypeName}.{MethodName}({StackPop()}, {stackCookie}, {StackPop()})"; -#else - return StackPop(); -#endif - } + var lines = new string[5 + args.Length]; + lines[0] = $"/// Pops {args.Length} value{(args.Length == 1 ? "" : "s")} from the backtracking stack."; + lines[1] = $"[MethodImpl(MethodImplOptions.AggressiveInlining)]"; + lines[2] = $"internal static void {MethodName}(int[] stack, ref int pos{FormatN(", out int arg{0}", args.Length)})"; + lines[3] = $"{{"; + for (int i = 0; i < args.Length; i++) + { + lines[4 + i] = $" arg{i} = stack[--pos];"; + } + lines[4 + args.Length] = $"}}"; -#if DEBUG - /// Returns an expression that validates and returns a debug stack cookie. - string StackCookieValidate(int stackCookie) - { - const string MethodName = "ValidateStackCookie"; - if (!requiredHelpers.ContainsKey(MethodName)) - { - requiredHelpers.Add(MethodName, - [ - $"/// Validates that a stack cookie popped off the backtracking stack holds the expected value. Debug only.", - $"internal static int {MethodName}(int expected, int actual)", - $"{{", - $" if (expected != actual)", - $" {{", - $" throw new Exception($\"Backtracking stack imbalance detected. Expected {{expected}}. Actual {{actual}}.\");", - $" }}", - $" return actual;", - $"}}", - ]); + requiredHelpers.Add(key, lines); } - return $"{HelpersTypeName}.{MethodName}({stackCookie}, {StackPop()})"; + writer.WriteLine($"{HelpersTypeName}.{MethodName}(base.runstack!, ref stackpos, out {string.Join(", out ", args)});"); } -#endif /// Expression for popping the next item from the backtracking stack. string StackPop() => "base.runstack![--stackpos]"; diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexAssemblyCompiler.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexAssemblyCompiler.cs index 662d8322e17575..2e5dc74f733dce 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexAssemblyCompiler.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexAssemblyCompiler.cs @@ -25,7 +25,7 @@ internal sealed class RegexAssemblyCompiler : RegexCompiler /// Type count used to augment generated type names to create unique names. private static int s_typeCount; - private readonly PersistedAssemblyBuilder _assembly; + private readonly AssemblyBuilder _assembly; private readonly ModuleBuilder _module; internal RegexAssemblyCompiler(AssemblyName an, CustomAttributeBuilder[]? attribs, string? resourceFile) @@ -36,7 +36,7 @@ internal RegexAssemblyCompiler(AssemblyName an, CustomAttributeBuilder[]? attrib throw new PlatformNotSupportedException(); } - _assembly = new PersistedAssemblyBuilder(an, typeof(object).Assembly, attribs is not null ? new List(attribs) : null) ?? + _assembly = AssemblyBuilder.DefinePersistedAssembly(an, typeof(object).Assembly, attribs is not null ? new List(attribs) : null) ?? throw new InvalidOperationException("DefinePersistedAssembly returned null"); _module = _assembly.DefineDynamicModule(an.Name + ".dll"); diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCaseEquivalences.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCaseEquivalences.cs index 70587b19a6c009..4367da61026dca 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCaseEquivalences.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCaseEquivalences.cs @@ -52,7 +52,7 @@ public static bool TryFindCaseEquivalencesForCharWithIBehavior(char c, CultureIn // Default _ => default }; - return !equivalences.IsEmpty; + return equivalences != default; } else { diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs index ed67df6819023d..c56ad4b5b6e054 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCharClass.cs @@ -1054,21 +1054,6 @@ public static bool IsAscii(ReadOnlySpan s) #endif } - /// Gets whether the set description string is for two ASCII letters that case to each other under OrdinalIgnoreCase rules. - public static bool SetContainsAsciiOrdinalIgnoreCaseCharacter(string set, Span twoChars) - { - Debug.Assert(twoChars.Length >= 2); - return - !IsNegated(set) && - GetSetChars(set, twoChars) == 2 && - twoChars[0] < 128 && - twoChars[1] < 128 && - twoChars[0] != twoChars[1] && - char.IsLetter(twoChars[0]) && - char.IsLetter(twoChars[1]) && - (twoChars[0] | 0x20) == (twoChars[1] | 0x20); - } - /// Gets whether we can iterate through the set list pairs in order to completely enumerate the set's contents. /// This may enumerate negated characters if the set is negated. This will return false if the set has subtraction. private static bool CanEasilyEnumerateSetContents(string set) => diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs index dd2357183d4b3a..bac950c6db2f97 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs @@ -460,8 +460,6 @@ protected void EmitTryFindNextPossibleStartingPosition() { case FindNextStartingPositionMode.LeadingString_LeftToRight: case FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight: - case FindNextStartingPositionMode.LeadingStrings_LeftToRight: - case FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight: case FindNextStartingPositionMode.FixedDistanceString_LeftToRight: EmitIndexOfString_LeftToRight(); break; @@ -747,19 +745,15 @@ bool EmitAnchors() return false; } - // Emits a case-sensitive left-to-right search for a substring or substrings. + // Emits a case-sensitive left-to-right search for a substring. void EmitIndexOfString_LeftToRight() { RegexFindOptimizations opts = _regexTree.FindOptimizations; - Debug.Assert(opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or - FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight or - FindNextStartingPositionMode.FixedDistanceString_LeftToRight or - FindNextStartingPositionMode.LeadingStrings_LeftToRight or - FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight); + Debug.Assert(opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight or FindNextStartingPositionMode.FixedDistanceString_LeftToRight); using RentedLocalBuilder i = RentInt32Local(); - // int i = inputSpan.Slice(pos)... + // int i = inputSpan.Slice(pos).IndexOf(prefix); Ldloca(inputSpan); Ldloc(pos); if (opts.FindMode is FindNextStartingPositionMode.FixedDistanceString_LeftToRight && @@ -769,21 +763,11 @@ FindNextStartingPositionMode.LeadingStrings_LeftToRight or Add(); } Call(s_spanSliceIntMethod); - - // ...IndexOf(prefix); - if (opts.FindMode is FindNextStartingPositionMode.LeadingStrings_LeftToRight or FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight) - { - LoadSearchValues(opts.LeadingPrefixes, opts.FindMode is FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); - Call(s_spanIndexOfAnySearchValuesString); - } - else - { - string literalString = opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight ? - opts.LeadingPrefix : - opts.FixedDistanceLiteral.String!; - LoadSearchValues([literalString], opts.FindMode is FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); - Call(s_spanIndexOfAnySearchValuesString); - } + string literalString = opts.FindMode is FindNextStartingPositionMode.LeadingString_LeftToRight or FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight ? + opts.LeadingPrefix : + opts.FixedDistanceLiteral.String!; + LoadSearchValues([literalString], opts.FindMode is FindNextStartingPositionMode.LeadingString_OrdinalIgnoreCase_LeftToRight ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal); + Call(s_spanIndexOfAnySearchValuesString); Stloc(i); // if (i < 0) goto ReturnFalse; @@ -2239,18 +2223,9 @@ void EmitExpressionConditional(RegexNode node) Stloc(startingPos); int startingSliceStaticPos = sliceStaticPos; - // Emit the condition. The condition expression is a zero-width assertion, which is atomic, + // Emit the child. The condition expression is a zero-width assertion, which is atomic, // so prevent backtracking into it. - if (analysis.MayBacktrack(condition)) - { - // Condition expressions are treated like positive lookarounds and thus are implicitly atomic, - // so we need to emit the node as atomic if it might backtrack. - EmitAtomic(node, null); - } - else - { - EmitNode(condition); - } + EmitNode(condition); doneLabel = originalDoneLabel; // After the condition completes successfully, reset the text positions. @@ -2640,15 +2615,6 @@ void EmitNegativeLookaroundAssertion(RegexNode node) // If the generated code ends up here, it matched the lookaround, which actually // means failure for a _negative_ lookaround, so we need to jump to the original done. // goto originalDoneLabel; - if (capturePos is not null && isInLoop) - { - // Pop the crawl position from the stack. - // stackpos--; - Ldloc(stackpos); - Ldc(1); - Sub(); - Stloc(stackpos); - } BrFar(originalDoneLabel); // Failures (success for a negative lookaround) jump here. @@ -2827,8 +2793,8 @@ void EmitNode(RegexNode node, RegexNode? subsequent = null, bool emitLengthCheck // Emits the node for an atomic. void EmitAtomic(RegexNode node, RegexNode? subsequent) { - Debug.Assert(node.Kind is RegexNodeKind.Atomic or RegexNodeKind.PositiveLookaround or RegexNodeKind.NegativeLookaround or RegexNodeKind.ExpressionConditional, $"Unexpected type: {node.Kind}"); - Debug.Assert(node.Kind is RegexNodeKind.ExpressionConditional ? node.ChildCount() >= 1 : node.ChildCount() == 1, $"Unexpected number of children: {node.ChildCount()}"); + Debug.Assert(node.Kind is RegexNodeKind.Atomic or RegexNodeKind.PositiveLookaround or RegexNodeKind.NegativeLookaround, $"Unexpected type: {node.Kind}"); + Debug.Assert(node.ChildCount() == 1, $"Expected 1 child, found {node.ChildCount()}"); RegexNode child = node.Child(0); diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs index a8dc9f4fd0e581..f40f48e35a6d98 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexFindOptimizations.cs @@ -137,28 +137,7 @@ public RegexFindOptimizations(RegexNode root, RegexOptions options) return; } - // We're now left-to-right only and looking for multiple prefixes and/or sets. - - // If there are multiple leading strings, we can search for any of them. - if (compiled) - { - if (RegexPrefixAnalyzer.FindPrefixes(root, ignoreCase: true) is { Length: > 1 } caseInsensitivePrefixes) - { - LeadingPrefixes = caseInsensitivePrefixes; - FindMode = FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight; - return; - } - - // TODO: While some benchmarks benefit from this significantly, others regressed a bit (in particular those with few - // matches). Before enabling this, we need to investigate the performance impact on real-world scenarios, - // and see if there are ways to reduce the impact. - //if (RegexPrefixAnalyzer.FindPrefixes(root, ignoreCase: false) is { Length: > 1 } caseSensitivePrefixes) - //{ - // LeadingPrefixes = caseSensitivePrefixes; - // FindMode = FindNextStartingPositionMode.LeadingStrings_LeftToRight; - // return; - //} - } + // We're now left-to-right only and looking for sets. // Build up a list of all of the sets that are a fixed distance from the start of the expression. List? fixedDistanceSets = RegexPrefixAnalyzer.FindFixedDistanceSets(root, thorough: !interpreter); @@ -265,9 +244,6 @@ public RegexFindOptimizations(RegexNode root, RegexOptions options) /// Gets the leading prefix. May be an empty string. public string LeadingPrefix { get; } = string.Empty; - /// Gets the leading prefixes. May be an empty array. - public string[] LeadingPrefixes { get; } = Array.Empty(); - /// When in fixed distance literal mode, gets the literal and how far it is from the start of the pattern. public (char Char, string? String, int Distance) FixedDistanceLiteral { get; } @@ -791,16 +767,10 @@ public bool TryFindNextStartingPositionLeftToRight(ReadOnlySpan textSpan, return false; } - // Not supported in the interpreter, but we could end up here for patterns so complex the compiler gave up on them. - - case FindNextStartingPositionMode.LeadingStrings_LeftToRight: - case FindNextStartingPositionMode.LeadingStrings_OrdinalIgnoreCase_LeftToRight: - return true; - // Nothing special to look for. Just return true indicating this is a valid position to try to match. default: - Debug.Assert(FindMode == FindNextStartingPositionMode.NoSearch, $"Unexpected FindMode {FindMode}"); + Debug.Assert(FindMode == FindNextStartingPositionMode.NoSearch); return true; } } @@ -840,11 +810,6 @@ internal enum FindNextStartingPositionMode /// A multi-character ordinal case-insensitive substring at the beginning of the pattern. LeadingString_OrdinalIgnoreCase_LeftToRight, - /// Multiple leading prefix strings - LeadingStrings_LeftToRight, - /// Multiple leading ordinal case-insensitive prefix strings - LeadingStrings_OrdinalIgnoreCase_LeftToRight, - /// A set starting the pattern. LeadingSet_LeftToRight, /// A set starting the right-to-left pattern. diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs index 42cb10d85ed0f3..5445f696423e4e 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs @@ -83,24 +83,6 @@ public RegexNode(RegexNodeKind kind, RegexOptions options, int m, int n) N = n; } - /// Creates a new node from an existing one/notone/setone {lazy/atomic} loop with one less iteration. - public RegexNode CloneCharLoopWithOneLessIteration() - { - Debug.Assert(Kind is RegexNodeKind.Onelazy or RegexNodeKind.Oneloop or RegexNodeKind.Oneloopatomic or - RegexNodeKind.Notonelazy or RegexNodeKind.Notoneloop or RegexNodeKind.Notoneloopatomic or - RegexNodeKind.Setlazy or RegexNodeKind.Setloop or RegexNodeKind.Setloopatomic); - Debug.Assert(M > 0); - - RegexNode newNode = IsSetFamily ? - new RegexNode(Kind, Options, Str!) : - new RegexNode(Kind, Options, Ch); - - newNode.M = M - 1; - newNode.N = N == int.MaxValue ? int.MaxValue : N - 1; - - return newNode; - } - /// Creates a RegexNode representing a single character. /// The character. /// The node's options. @@ -1379,16 +1361,27 @@ static void ProcessOneOrMulti(RegexNode node, ReadOnlySpan startingSpan) return branch.Kind is RegexNodeKind.One or RegexNodeKind.Multi ? branch : null; } + /// Same as but also for Sets. + public RegexNode? FindBranchOneMultiOrSetStart() + { + RegexNode branch = Kind == RegexNodeKind.Concatenate ? Child(0) : this; + return branch.Kind is RegexNodeKind.One or RegexNodeKind.Multi or RegexNodeKind.Set ? branch : null; + } + /// Gets the character that begins a One or Multi. public char FirstCharOfOneOrMulti() { - Debug.Assert(Kind is RegexNodeKind.One or RegexNodeKind.Multi || (IsOneFamily && M > 0)); + Debug.Assert(Kind is RegexNodeKind.One or RegexNodeKind.Multi); Debug.Assert((Options & RegexOptions.RightToLeft) == 0); - return IsOneFamily ? Ch : Str![0]; + return Kind == RegexNodeKind.One ? Ch : Str![0]; } /// Finds the guaranteed beginning literal(s) of the node, or null if none exists. - public RegexNode? FindStartingLiteralNode(bool allowZeroWidth = true) + /// + /// A tuple of data about the literal: only one of the Char/String/SetChars fields is relevant. + /// The Negated value indicates whether the Char/SetChars should be considered exclusionary. + /// + public RegexNode? FindStartingLiteralNode() { RegexNode? node = this; while (true) @@ -1411,7 +1404,7 @@ public char FirstCharOfOneOrMulti() case RegexNodeKind.Capture: case RegexNodeKind.Group: case RegexNodeKind.Loop or RegexNodeKind.Lazyloop when node.M > 0: - case RegexNodeKind.PositiveLookaround when allowZeroWidth: + case RegexNodeKind.PositiveLookaround: node = node.Child(0); continue; } @@ -2568,7 +2561,14 @@ public bool TryGetOrdinalCaseInsensitiveString(int childIndex, int exclusiveChil { // In particular we want to look for sets that contain only the upper and lowercase variant // of the same ASCII letter. - if (!RegexCharClass.SetContainsAsciiOrdinalIgnoreCaseCharacter(child.Str!, twoChars)) + if (RegexCharClass.IsNegated(child.Str!) || + RegexCharClass.GetSetChars(child.Str!, twoChars) != 2 || + twoChars[0] >= 128 || + twoChars[1] >= 128 || + twoChars[0] == twoChars[1] || + !char.IsLetter(twoChars[0]) || + !char.IsLetter(twoChars[1]) || + ((twoChars[0] | 0x20) != (twoChars[1] | 0x20))) { break; } diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs index 97aba89b9804c5..1658e5bcdf2ad1 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexPrefixAnalyzer.cs @@ -11,316 +11,6 @@ namespace System.Text.RegularExpressions /// Detects various forms of prefixes in the regular expression that can help FindFirstChars optimize its search. internal static class RegexPrefixAnalyzer { - /// Finds an array of multiple prefixes that a node can begin with. - /// The node to search. - /// true to find ordinal ignore-case prefixes; false for case-sensitive. - /// - /// If a fixed set of prefixes is found, such that a match for this node is guaranteed to begin - /// with one of those prefixes, an array of those prefixes is returned. Otherwise, null. - /// - public static string[]? FindPrefixes(RegexNode node, bool ignoreCase) - { - // Minimum string length for prefixes to be useful. If any prefix has length 1, - // then we're generally better off just using IndexOfAny with chars. - const int MinPrefixLength = 2; - - // Arbitrary string length limit (with some wiggle room) to avoid creating strings that are longer than is useful and consuming too much memory. - const int MaxPrefixLength = 8; - - // Arbitrary limit on the number of prefixes to find. If we find more than this, we're likely to be spending too much time finding prefixes that won't be useful. - const int MaxPrefixes = 16; - - // Analyze the node to find prefixes. - List results = [new StringBuilder()]; - FindPrefixesCore(node, results, ignoreCase); - - // If we found too many prefixes or if any found is too short, fail. - if (results.Count > MaxPrefixes || !results.TrueForAll(sb => sb.Length >= MinPrefixLength)) - { - return null; - } - - // Return the prefixes. - string[] resultStrings = new string[results.Count]; - for (int i = 0; i < results.Count; i++) - { - resultStrings[i] = results[i].ToString(); - } - return resultStrings; - - // - // Updates the results list with found prefixes. All existing strings in the list are treated as existing - // discovered prefixes prior to the node being processed. The method returns true if subsequent nodes after - // this one should be examined, or returns false if they shouldn't be because the node wasn't guaranteed - // to be fully processed. - // - static bool FindPrefixesCore(RegexNode node, List results, bool ignoreCase) - { - // If we're too deep to analyze further, we can't trust what we've already computed, so stop iterating. - // Also bail if any of our results is already hitting the threshold, or if this node is RTL, which is - // not worth the complexity of handling. - if (!StackHelper.TryEnsureSufficientExecutionStack() || - !results.TrueForAll(sb => sb.Length < MaxPrefixLength) || - (node.Options & RegexOptions.RightToLeft) != 0) - { - return false; - } - - // These limits are approximations. We'll stop trying to make strings longer once we exceed the max length, - // and if we exceed the max number of prefixes by a non-trivial amount, we'll fail the operation. - Span setChars = stackalloc char[MaxPrefixes]; // limit how many chars we get from a set based on the max prefixes we care about - - // Loop down the left side of the tree, looking for a starting node we can handle. We only loop through - // atomic and capture nodes, as the child is guaranteed to execute once, as well as loops with a positive - // minimum and thus at least one guaranteed iteration. - while (true) - { - switch (node.Kind) - { - // These nodes are all guaranteed to execute at least once, so we can just - // skip through them to their child. - case RegexNodeKind.Atomic: - case RegexNodeKind.Capture: - node = node.Child(0); - continue; - - // Zero-width anchors and assertions don't impact a prefix and may be skipped over. - case RegexNodeKind.Bol: - case RegexNodeKind.Eol: - case RegexNodeKind.Boundary: - case RegexNodeKind.ECMABoundary: - case RegexNodeKind.NonBoundary: - case RegexNodeKind.NonECMABoundary: - case RegexNodeKind.Beginning: - case RegexNodeKind.Start: - case RegexNodeKind.EndZ: - case RegexNodeKind.End: - case RegexNodeKind.Empty: - case RegexNodeKind.UpdateBumpalong: - case RegexNodeKind.PositiveLookaround: - case RegexNodeKind.NegativeLookaround: - return true; - - // If we hit a single character, we can just return that character. - // This is only relevant for case-sensitive searches, as for case-insensitive we'd have sets for anything - // that produces a different result when case-folded, or for strings composed entirely of characters that - // don't participate in case conversion. Single character loops are handled the same as single characters - // up to the min iteration limit. We can continue processing after them as well if they're repeaters such - // that their min and max are the same. - case RegexNodeKind.One or RegexNodeKind.Oneloop or RegexNodeKind.Onelazy or RegexNodeKind.Oneloopatomic when !ignoreCase || !RegexCharClass.ParticipatesInCaseConversion(node.Ch): - { - int reps = node.Kind is RegexNodeKind.One ? 1 : Math.Min(node.M, MaxPrefixLength); - foreach (StringBuilder sb in results) - { - sb.Append(node.Ch, reps); - } - return node.Kind is RegexNodeKind.One || reps == node.N; - } - - // If we hit a string, we can just return that string. - // As with One above, this is only relevant for case-sensitive searches. - case RegexNodeKind.Multi: - if (!ignoreCase) - { - foreach (StringBuilder sb in results) - { - sb.Append(node.Str); - } - } - else - { - // If we're ignoring case, then only append up through characters that don't participate in case conversion. - // If there are any beyond that, we can't go further and need to stop with what we have. - foreach (char c in node.Str!) - { - if (RegexCharClass.ParticipatesInCaseConversion(c)) - { - return false; - } - - foreach (StringBuilder sb in results) - { - sb.Append(c); - } - } - } - return true; - - // For case-sensitive, try to extract the characters that comprise it, and if there are - // any and there aren't more than the max number of prefixes, we can return - // them each as a prefix. Effectively, this is an alternation of the characters - // that comprise the set. For case-insensitive, we need the set to be two ASCII letters that case fold to the same thing. - // As with One and loops, set loops are handled the same as sets up to the min iteration limit. - case RegexNodeKind.Set or RegexNodeKind.Setloop or RegexNodeKind.Setlazy or RegexNodeKind.Setloopatomic when !RegexCharClass.IsNegated(node.Str!): // negated sets are too complex to analyze - { - int charCount = RegexCharClass.GetSetChars(node.Str!, setChars); - if (charCount == 0) - { - return false; - } - - int reps = node.Kind is RegexNodeKind.Set ? 1 : Math.Min(node.M, MaxPrefixLength); - if (!ignoreCase) - { - int existingCount = results.Count; - - // Duplicate all of the existing strings for all of the new suffixes, other than the first. - foreach (char suffix in setChars.Slice(1, charCount - 1)) - { - for (int existing = 0; existing < existingCount; existing++) - { - StringBuilder newSb = new StringBuilder().Append(results[existing]); - newSb.Append(suffix, reps); - results.Add(newSb); - } - } - - // Then append the first suffix to all of the existing strings. - for (int existing = 0; existing < existingCount; existing++) - { - results[existing].Append(setChars[0], reps); - } - } - else - { - // For ignore-case, we currently only handle the simple (but common) case of a single - // ASCII character that case folds to the same char. - if (!RegexCharClass.SetContainsAsciiOrdinalIgnoreCaseCharacter(node.Str!, setChars)) - { - return false; - } - - // Append it to each. - foreach (StringBuilder sb in results) - { - sb.Append(setChars[1], reps); - } - } - - return node.Kind is RegexNodeKind.Set || reps == node.N; - } - - case RegexNodeKind.Concatenate: - { - int childCount = node.ChildCount(); - for (int i = 0; i < childCount; i++) - { - if (!FindPrefixesCore(node.Child(i), results, ignoreCase)) - { - return false; - } - } - } - return true; - - // We can append any guaranteed iterations as if they were a concatenation. - case RegexNodeKind.Loop or RegexNodeKind.Lazyloop when node.M > 0: - { - int limit = Math.Min(node.M, MaxPrefixLength); // MaxPrefixLength here is somewhat arbitrary, as a single loop iteration could yield multiple chars - for (int i = 0; i < limit; i++) - { - if (!FindPrefixesCore(node.Child(0), results, ignoreCase)) - { - return false; - } - } - return limit == node.N; - } - - // For alternations, we need to find a prefix for every branch; if we can't compute a - // prefix for any one branch, we can't trust the results and need to give up, since we don't - // know if our set of prefixes is complete. - case RegexNodeKind.Alternate: - { - // If there are more children than our maximum, just give up immediately, as we - // won't be able to get a prefix for every branch and have it be within our max. - int childCount = node.ChildCount(); - Debug.Assert(childCount >= 2); // otherwise it would have been optimized out - if (childCount > MaxPrefixes) - { - return false; - } - - // Build up the list of all prefixes across all branches. - List? allBranchResults = null; - List? alternateBranchResults = [new StringBuilder()]; - for (int i = 0; i < childCount; i++) - { - _ = FindPrefixesCore(node.Child(i), alternateBranchResults, ignoreCase); - - Debug.Assert(alternateBranchResults.Count > 0); - foreach (StringBuilder sb in alternateBranchResults) - { - // If a branch yields an empty prefix, then none of the other branches - // matter, e.g. if the pattern is abc(def|ghi|), then this would result - // in prefixes abcdef, abcghi, and abc, and since abc is a prefix of both - // abcdef and abcghi, the former two would never be used. - if (sb.Length == 0) - { - return false; - } - } - - if (allBranchResults is null) - { - allBranchResults = alternateBranchResults; - alternateBranchResults = [new StringBuilder()]; - } - else - { - allBranchResults.AddRange(alternateBranchResults); - alternateBranchResults.Clear(); - alternateBranchResults.Add(new StringBuilder()); - } - } - - // At this point, we know we can successfully incorporate the alternation's results - // into the main results. - - // If the results are currently empty (meaning a single empty StringBuilder), we can remove - // that builder and just replace the results with the alternation's results. We would otherwise - // be creating a dot product of every builder in the results with every branch's result, which - // is logically the same thing. - if (results.Count == 1 && results[0].Length == 0) - { - results.Clear(); - results.AddRange(allBranchResults!); - } - else - { - // Duplicate all of the existing strings for all of the new suffixes, other than the first. - int existingCount = results.Count; - for (int i = 1; i < allBranchResults!.Count; i++) - { - StringBuilder suffix = allBranchResults[i]; - for (int existing = 0; existing < existingCount; existing++) - { - StringBuilder newSb = new StringBuilder().Append(results[existing]); - newSb.Append(suffix); - results.Add(newSb); - } - } - - // Then append the first suffix to all of the existing strings. - for (int existing = 0; existing < existingCount; existing++) - { - results[existing].Append(allBranchResults[0]); - } - } - } - - // We don't know that we fully processed every branch, so we can't iterate through what comes after this node. - // The results were successfully updated, but return false to indicate that nothing after this node should be examined. - return false; - - // Something else we don't recognize, so stop iterating. - default: - return false; - } - } - } - } - /// Computes the leading substring in ; may be empty. public static string FindPrefix(RegexNode node) { diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexWriter.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexWriter.cs index 5284c09339bd7b..c0b743d5f1b99f 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexWriter.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexWriter.cs @@ -28,8 +28,8 @@ internal ref struct RegexWriter #if DEBUG static RegexWriter() { - Debug.Assert(!Enum.IsDefined(BeforeChild)); - Debug.Assert(!Enum.IsDefined(AfterChild)); + Debug.Assert(!Enum.IsDefined(typeof(RegexNodeKind), BeforeChild)); + Debug.Assert(!Enum.IsDefined(typeof(RegexNodeKind), AfterChild)); } #endif diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs index 2c7b9b0f185e43..c003c03bba3a4d 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/Regex.Match.Tests.cs @@ -92,7 +92,6 @@ public static IEnumerable Match_MemberData() yield return (@"(?:(?!(b)b)\1a)+", "babababa", RegexOptions.None, 0, 8, false, string.Empty); yield return (@"(?:(?!(b)b)\1a)*", "babababa", RegexOptions.None, 0, 8, true, string.Empty); yield return (@"(.*?)a(?!(a+)b\2c)\2(.*)", "baaabaac", RegexOptions.None, 0, 8, false, string.Empty); - yield return (@"(?!(abc))+\w\w\w", "abcdef", RegexOptions.None, 0, 6, true, "bcd"); // Zero-width positive lookbehind assertion yield return (@"(\w){6}(?<=XXX)def", "abcXXXdef", RegexOptions.None, 0, 9, true, "abcXXXdef"); @@ -661,15 +660,8 @@ public static IEnumerable Match_MemberData() yield return (@$"^{aOptional}{{1,2}}?b", "aaab", RegexOptions.None, 0, 4, false, ""); yield return (@$"^{aOptional}{{2}}b", "aab", RegexOptions.None, 0, 3, true, "aab"); } - yield return (@"(a+.|b+.)", "aaac", RegexOptions.None, 0, 4, true, "aaac"); - yield return (@"(a+?.|b+?.)", "aaac", RegexOptions.None, 0, 4, true, "aa"); - yield return (@"((a+?).|(b+?).)", "aaac", RegexOptions.None, 0, 4, true, "aa"); - yield return (@"((a+?)+.|(b+?)+.)", "aaac", RegexOptions.None, 0, 4, true, "aaac"); - yield return (@"((a+?)+?.|(b+?)+?.)", "aaac", RegexOptions.None, 0, 4, true, "aa"); if (!RegexHelpers.IsNonBacktracking(engine)) { - yield return (@"((?>a+).|(?>b+).)", "aaac", RegexOptions.None, 0, 4, true, "aaac"); - yield return ("(?(dog2))", "dog2", RegexOptions.None, 0, 4, true, string.Empty); yield return ("(?(a:b))", "a", RegexOptions.None, 0, 1, true, string.Empty); yield return ("(?(a:))", "a", RegexOptions.None, 0, 1, true, string.Empty); diff --git a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexGeneratorOutputTests.cs b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexGeneratorOutputTests.cs index 57d4232ee1ed18..181c978a376661 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexGeneratorOutputTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/FunctionalTests/RegexGeneratorOutputTests.cs @@ -2,24 +2,15 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.Diagnostics; using System.IO; -using System.Linq; -using System.Text.RegularExpressions.Generator; using System.Threading.Tasks; using Xunit; namespace System.Text.RegularExpressions.Tests { - [ConditionalClass(typeof(RegexGeneratorOutputTests), nameof(GeneratorOutputTestsSupported))] + [ConditionalClass(typeof(PlatformDetection), nameof(PlatformDetection.IsReflectionEmitSupported), nameof(PlatformDetection.IsNotMobile), nameof(PlatformDetection.IsNotBrowser))] public partial class RegexGeneratorOutputTests { - public static bool GeneratorOutputTestsSupported => - PlatformDetection.IsReflectionEmitSupported && - PlatformDetection.IsNotMobile && - PlatformDetection.IsNotBrowser && - typeof(RegexGenerator).Assembly.GetCustomAttributes(false).OfType().Any(da => da.IsJITTrackingEnabled); // output differs between debug and release - // This exists to ensure we're aware of any egregious breaks to formatting / readability. // Any updates that impact the generated code in these baselines will need to be updated // as changes are made to the code emitted by the generator. @@ -267,7 +258,6 @@ private bool TryMatchAtCurrentPosition(ReadOnlySpan inputSpan) loop_iteration = 0; LoopBody: - Utilities.StackPush(ref base.runstack!, ref stackpos, 143337952); Utilities.StackPush(ref base.runstack!, ref stackpos, base.Crawlpos(), pos); loop_iteration++; @@ -321,7 +311,6 @@ private bool TryMatchAtCurrentPosition(ReadOnlySpan inputSpan) } pos = base.runstack![--stackpos]; UncaptureUntil(base.runstack![--stackpos]); - Utilities.ValidateStackCookie(143337952, base.runstack![--stackpos]); slice = inputSpan.Slice(pos); LoopEnd:; //} @@ -392,32 +381,6 @@ internal static bool IsWordChar(char ch) (WordCategoriesMask & (1 << (int)CharUnicodeInfo.GetUnicodeCategory(ch))) != 0; } - /// Pushes 1 value onto the backtracking stack. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static void StackPush(ref int[] stack, ref int pos, int arg0) - { - // If there's space available for the value, store it. - int[] s = stack; - int p = pos; - if ((uint)p < (uint)s.Length) - { - s[p] = arg0; - pos++; - return; - } - - // Otherwise, resize the stack to make room and try again. - WithResize(ref stack, ref pos, arg0); - - // Resize the backtracking stack array and push 1 value onto the stack. - [MethodImpl(MethodImplOptions.NoInlining)] - static void WithResize(ref int[] stack, ref int pos, int arg0) - { - Array.Resize(ref stack, (pos + 0) * 2); - StackPush(ref stack, ref pos, arg0); - } - } - /// Pushes 2 values onto the backtracking stack. [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void StackPush(ref int[] stack, ref int pos, int arg0, int arg1) @@ -444,16 +407,6 @@ static void WithResize(ref int[] stack, ref int pos, int arg0, int arg1) StackPush(ref stack, ref pos, arg0, arg1); } } - - /// Validates that a stack cookie popped off the backtracking stack holds the expected value. Debug only. - internal static int ValidateStackCookie(int expected, int actual) - { - if (expected != actual) - { - throw new Exception($"Backtracking stack imbalance detected. Expected {expected}. Actual {actual}."); - } - return actual; - } } } """ diff --git a/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs b/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs index 783b45e9d3c980..9c592d7c57f60e 100644 --- a/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs +++ b/src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexPrefixAnalyzerTests.cs @@ -70,76 +70,6 @@ public void FindFirstCharClass_StressDeep() FindFirstCharClass(string.Concat(Enumerable.Repeat($"(a?", nesting).Concat(Enumerable.Repeat(")*", nesting))), 0, null); } - [Theory] - // case-sensitive - [InlineData("abc", new[] { "abc" }, false)] - [InlineData("(abc+|bcd+)", new[] { "abc", "bcd" }, false)] - [InlineData("(ab+c|bcd+)", new[] { "ab", "bcd" }, false)] - [InlineData("(ab+c|bcd+)*", null, false)] - [InlineData("(ab+c|bcd+)+", new[] { "ab", "bcd" }, false)] - [InlineData("(ab+c|bcd+){3,5}", new[] { "ab", "bcd" }, false)] - [InlineData("abc|def", new[] { "abc", "def" }, false)] - [InlineData("ab{4}c|def{5}|g{2,4}h", new[] { "abbbbc", "defffff", "gg" }, false)] - [InlineData("abc|def|(ghi|jklm)", new[] { "abc", "def", "ghi", "jklm" }, false)] - [InlineData("abc[def]ghi", new[] { "abcdghi", "abceghi", "abcfghi" }, false)] - [InlineData("abc[def]ghi|[jkl]m", new[] { "abcdghi", "abceghi", "abcfghi", "jm", "km", "lm" }, false)] - [InlineData("agggtaaa|tttaccct", new[] { "agggtaaa", "tttaccct" }, false)] - [InlineData("[cgt]gggtaaa|tttaccc[acg]", new[] { "cgggtaaa", "ggggtaaa", "tgggtaaa", "tttaccca", "tttacccc", "tttacccg" }, false)] - [InlineData("a[act]ggtaaa|tttacc[agt]t", new[] { "aaggtaaa", "acggtaaa", "atggtaaa", "tttaccat", "tttaccgt", "tttacctt" }, false)] - [InlineData("ag[act]gtaaa|tttac[agt]ct", new[] { "agagtaaa", "agcgtaaa", "agtgtaaa", "tttacact", "tttacgct", "tttactct" }, false)] - [InlineData("agg[act]taaa|ttta[agt]cct", new[] { "aggataaa", "aggctaaa", "aggttaaa", "tttaacct", "tttagcct", "tttatcct" }, false)] - [InlineData(@"\b(abc|def)\b", new[] { "abc", "def" }, false)] - [InlineData("^(abc|def)$", new[] { "abc", "def" }, false)] - [InlineData("abcdefg|h", null, false)] - [InlineData("abc[def]ghi|[jkl]", null, false)] - [InlineData("[12][45][789]", new[] { "147", "148", "149", "157", "158", "159", "247", "248", "249", "257", "258", "259" }, false)] - [InlineData("[12]a[45]b[789]c", new[] { "1a4b7c", "1a4b8c", "1a4b9c", "1a5b7c", "1a5b8c", "1a5b9c", "2a4b7c", "2a4b8c", "2a4b9c", "2a5b7c", "2a5b8c", "2a5b9c" }, false)] - [InlineData("(abc){3}|(def){3}", new[] { "abcabcabc", "defdefdef" }, false)] - [InlineData("(abc){4,8}|(def){2,3}", new[] { "abcabcabc", "defdef" }, false)] - [InlineData("(abc){4,8}|(de+f){2,3}", new[] { "abcabcabc", "de" }, false)] - [InlineData("(ab{2}c){4,8}|(de+f){2,3}", new[] { "abbcabbc", "de" }, false)] - // case-insensitive - [InlineData("[Aa][Bb][Cc]", new[] { "abc" }, true)] - [InlineData("[Aa][Bbc][Cc]", null, true)] - [InlineData(":[Aa]![Bb]@", new[] { ":a!b@" }, true)] - [InlineData("(?i)abc", new[] { "abc" }, true)] - [InlineData("(?i)(abc+|bcd+)", new[] { "abc", "bcd" }, true)] - [InlineData("(?i)(ab+c|bcd+)", new[] { "ab", "bcd" }, true)] - [InlineData("(?i)(ab+c|bcd+)*", null, true)] - [InlineData("(?i)(ab+c|bcd+)+", new[] { "ab", "bcd" }, true)] - [InlineData("(?i)(ab+c|bcd+){3,5}", new[] { "ab", "bcd" }, true)] - [InlineData("(?i)abc|def", new[] { "abc", "def" }, true)] - [InlineData("(?i)ab{4}c|def{5}|g{2,4}h", new[] { "abbbbc", "defffff", "gg" }, true)] - [InlineData("(?i)(((?>abc)|(?>def)))", new[] { "abc", "def" }, true)] - [InlineData("(?i)(abc|def|(ghi|jklm))", null, true)] - [InlineData("(?i)(abc|def|(ghi|jlmn))", new[] { "abc", "def", "ghi", "jlmn" }, true)] - [InlineData("abc", null, true)] - [InlineData("abc|def", null, true)] - [InlineData("abc|def|(ghi|jklm)", null, true)] - [InlineData("://[Aa][Bb]|[Cc]@!", new[] { "://ab", "c@!" }, true)] - [InlineData("(?i)((abc){4,8}|(def){2,3})", new[] { "abcabcab", "defdef" }, true)] - [InlineData("(?i)((abc){4,8}|(de+f){2,3})", new[] { "abcabcab", "de" }, true)] - [InlineData("(?i)((ab{2}c){4,8}|(de+f){2,3})", new[] { "abbcabbc", "de" }, true)] - public void FindPrefixes(string pattern, string[] expectedSet, bool ignoreCase) - { - RegexTree tree = RegexParser.Parse(pattern, RegexOptions.None, CultureInfo.InvariantCulture); - string[] actual = RegexPrefixAnalyzer.FindPrefixes(tree.Root, ignoreCase); - - if (expectedSet is null) - { - Assert.Null(actual); - } - else - { - Assert.NotNull(actual); - - Array.Sort(actual, StringComparer.Ordinal); - Array.Sort(expectedSet, StringComparer.Ordinal); - - Assert.Equal(expectedSet, actual); - } - } - private static string FormatSet(string set) { if (set is null) diff --git a/src/libraries/System.Threading.Thread/ref/System.Threading.Thread.cs b/src/libraries/System.Threading.Thread/ref/System.Threading.Thread.cs index 200fe545e56381..c4c038036ce341 100644 --- a/src/libraries/System.Threading.Thread/ref/System.Threading.Thread.cs +++ b/src/libraries/System.Threading.Thread/ref/System.Threading.Thread.cs @@ -110,93 +110,41 @@ public void UnsafeStart() { } [System.Runtime.Versioning.UnsupportedOSPlatformAttribute("browser")] #endif public void UnsafeStart(object? parameter) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static byte VolatileRead(ref byte address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static double VolatileRead(ref double address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static short VolatileRead(ref short address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static int VolatileRead(ref int address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static long VolatileRead(ref long address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static System.IntPtr VolatileRead(ref System.IntPtr address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [return: System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute("address")] public static object? VolatileRead([System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute("address")] ref object? address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static sbyte VolatileRead(ref sbyte address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static float VolatileRead(ref float address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static ushort VolatileRead(ref ushort address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static uint VolatileRead(ref uint address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static ulong VolatileRead(ref ulong address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static System.UIntPtr VolatileRead(ref System.UIntPtr address) { throw null; } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref byte address, byte value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref double address, double value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref short address, short value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref int address, int value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref long address, long value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref System.IntPtr address, System.IntPtr value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite([System.Diagnostics.CodeAnalysis.NotNullIfNotNullAttribute("value")] ref object? address, object? value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static void VolatileWrite(ref sbyte address, sbyte value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] public static void VolatileWrite(ref float address, float value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static void VolatileWrite(ref ushort address, ushort value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static void VolatileWrite(ref uint address, uint value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static void VolatileWrite(ref ulong address, ulong value) { } - [System.ObsoleteAttribute("Thread.VolatileRead and Thread.VolatileWrite are obsolete. Use Volatile.Read or Volatile.Write respectively instead.", DiagnosticId = "SYSLIB0054", UrlFormat = "https://aka.ms/dotnet-warnings/{0}")] - [System.ComponentModel.EditorBrowsableAttribute(System.ComponentModel.EditorBrowsableState.Never)] [System.CLSCompliantAttribute(false)] public static void VolatileWrite(ref System.UIntPtr address, System.UIntPtr value) { } public static bool Yield() { throw null; } diff --git a/src/libraries/System.Threading.ThreadPool/tests/ThreadPoolTests.cs b/src/libraries/System.Threading.ThreadPool/tests/ThreadPoolTests.cs index c96ad22b47b531..0b51d5e07a6ec8 100644 --- a/src/libraries/System.Threading.ThreadPool/tests/ThreadPoolTests.cs +++ b/src/libraries/System.Threading.ThreadPool/tests/ThreadPoolTests.cs @@ -6,8 +6,6 @@ using System.Diagnostics.Tracing; using System.IO; using System.Linq; -using System.Net.Sockets; -using System.Net; using System.Reflection; using System.Threading.Tasks; using System.Threading.Tests; @@ -1162,95 +1160,6 @@ public void ThreadPoolMinMaxThreadsEventTest() }).Dispose(); } - private sealed class RuntimeEventListener : EventListener - { - private const string ClrProviderName = "Microsoft-Windows-DotNETRuntime"; - private const EventKeywords ThreadingKeyword = (EventKeywords)0x10000; - - public volatile int tpIOEnqueue = 0; - public volatile int tpIODequeue = 0; - public ManualResetEvent tpWaitIOEnqueueEvent = new ManualResetEvent(false); - public ManualResetEvent tpWaitIODequeueEvent = new ManualResetEvent(false); - - protected override void OnEventSourceCreated(EventSource eventSource) - { - if (eventSource.Name.Equals(ClrProviderName)) - { - EnableEvents(eventSource, EventLevel.Verbose, ThreadingKeyword); - } - - base.OnEventSourceCreated(eventSource); - } - - protected override void OnEventWritten(EventWrittenEventArgs eventData) - { - if (eventData.EventName.Equals("ThreadPoolIOEnqueue")) - { - Interlocked.Increment(ref tpIOEnqueue); - tpWaitIOEnqueueEvent.Set(); - } - else if (eventData.EventName.Equals("ThreadPoolIODequeue")) - { - Interlocked.Increment(ref tpIODequeue); - tpWaitIODequeueEvent.Set(); - } - } - } - - [ConditionalFact(nameof(IsThreadingAndRemoteExecutorSupported), nameof(UseWindowsThreadPool))] - public void ReadWriteAsyncTest() - { - RemoteExecutor.Invoke(async () => - { - using (RuntimeEventListener eventListener = new RuntimeEventListener()) - { - TaskCompletionSource portTcs = new TaskCompletionSource(); - TaskCompletionSource readAsyncReadyTcs = new TaskCompletionSource(); - - async Task StartListenerAsync() - { - using TcpListener listener = new TcpListener(IPAddress.Loopback, 0); - listener.Start(); - int port = ((IPEndPoint)listener.LocalEndpoint).Port; - portTcs.SetResult(port); - using TcpClient client = await listener.AcceptTcpClientAsync(); - using (NetworkStream stream = client.GetStream()) - { - byte[] buffer = new byte[1]; - Task readAsyncTask = stream.ReadAsync(buffer, 0, buffer.Length); - readAsyncReadyTcs.SetResult(true); - await readAsyncTask; - } - listener.Stop(); - } - - async Task StartClientAsync() - { - int port = await portTcs.Task; - using (TcpClient client = new TcpClient(new IPEndPoint(IPAddress.Loopback, 0))) - { - await client.ConnectAsync(IPAddress.Loopback, port); - using (NetworkStream stream = client.GetStream()) - { - bool readAsyncReady = await readAsyncReadyTcs.Task; - byte[] data = new byte[1]; - await stream.WriteAsync(data, 0, data.Length); - } - } - } - - Task listenerTask = StartListenerAsync(); - Task clientTask = StartClientAsync(); - await Task.WhenAll(listenerTask, clientTask); - ManualResetEvent[] waitEvents = [eventListener.tpWaitIOEnqueueEvent, eventListener.tpWaitIODequeueEvent]; - - Assert.True(WaitHandle.WaitAll(waitEvents, TimeSpan.FromSeconds(15))); // Assert that there wasn't a timeout - Assert.True(eventListener.tpIOEnqueue > 0); - Assert.True(eventListener.tpIODequeue > 0); - } - }).Dispose(); - } - public static bool IsThreadingAndRemoteExecutorSupported => PlatformDetection.IsThreadingSupported && RemoteExecutor.IsSupported; @@ -1260,7 +1169,6 @@ private static bool GetUseWindowsThreadPool() return useWindowsThreadPool; } - private static bool UseWindowsThreadPool { get; } = GetUseWindowsThreadPool(); - private static bool UsePortableThreadPool { get; } = !UseWindowsThreadPool; + private static bool UsePortableThreadPool { get; } = !GetUseWindowsThreadPool(); } } diff --git a/src/libraries/System.Threading/tests/SemaphoreSlimTests.cs b/src/libraries/System.Threading/tests/SemaphoreSlimTests.cs index dfc75fc2a89e69..7aabd01c39f1e2 100644 --- a/src/libraries/System.Threading/tests/SemaphoreSlimTests.cs +++ b/src/libraries/System.Threading/tests/SemaphoreSlimTests.cs @@ -90,7 +90,6 @@ public static void RunSemaphoreSlimTest1_WaitAsync() } [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] - [ActiveIssue("https://github.com/dotnet/runtime/issues/99501", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] public static void RunSemaphoreSlimTest1_WaitAsync_NegativeCases() { // Invalid timeout diff --git a/src/libraries/System.Threading/tests/System.Threading.Tests.csproj b/src/libraries/System.Threading/tests/System.Threading.Tests.csproj index 54261d3a1fe665..e938db9863da3d 100644 --- a/src/libraries/System.Threading/tests/System.Threading.Tests.csproj +++ b/src/libraries/System.Threading/tests/System.Threading.Tests.csproj @@ -7,9 +7,6 @@ true - - true - diff --git a/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml b/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml index b908b6cbab0714..2f7dfb841c805b 100644 --- a/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml +++ b/src/libraries/apicompat/ApiCompatBaseline.NetCoreAppLatestStable.xml @@ -1,436 +1,4 @@  - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/netstandard.dll - net9.0/netstandard.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.Primitives.dll - net9.0/System.ComponentModel.Primitives.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.ComponentModel.TypeConverter.dll - net9.0/System.ComponentModel.TypeConverter.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.DesignerAttribute.#ctor(System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.String)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.String,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$0:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - M:System.ComponentModel.EditorAttribute.#ctor(System.Type,System.Type)$1:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - P:System.ComponentModel.DesignerAttribute.DesignerTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorBaseTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - - - CP0014 - P:System.ComponentModel.EditorAttribute.EditorTypeName:[T:System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembersAttribute] - net8.0/System.dll - net9.0/System.dll - \ No newline at end of file diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj index 7c668b03471249..910cf44926098d 100644 --- a/src/libraries/sendtohelixhelp.proj +++ b/src/libraries/sendtohelixhelp.proj @@ -31,9 +31,9 @@ '$(Scenario)' == 'gcstress0x3' or '$(Scenario)' == 'gcstress0xc' or '$(Scenario)' == 'heapverify1' or - '$(Scenario)' == 'gcstress0xc_disabler2r' or - '$(Scenario)' == 'gcstress0xc_disabler2r_jitstress2' or - '$(Scenario)' == 'gcstress0xc_disabler2r_heapverify1' or + '$(Scenario)' == 'gcstress0xc_zapdisable' or + '$(Scenario)' == 'gcstress0xc_zapdisable_jitstress2' or + '$(Scenario)' == 'gcstress0xc_zapdisable_heapverify1' or '$(Scenario)' == 'gcstress0xc_jitstress1' or '$(Scenario)' == 'gcstress0xc_jitstress2' or '$(Scenario)' == 'gcstress0xc_jitminopts_heapverify1'">06:00:00 @@ -240,20 +240,9 @@ - - - sos - https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/flat2/dotnet-sos/$(DotnetSosVersion)/dotnet-sos.$(DotnetSosVersion).nupkg - - - - - set _NT_SYMBOL_PATH=%25HELIX_CORRELATION_PAYLOAD%25%3B%25HELIX_CORRELATION_PAYLOAD%25\PDB%3B%25HELIX_CORRELATION_PAYLOAD%25\shared\$(MicrosoftNetCoreAppFrameworkName)\$(ProductVersion) - %25HELIX_CORRELATION_PAYLOAD%25\dotnet %25HELIX_CORRELATION_PAYLOAD%25\sos\tools\net$(DotnetSosTargetFrameworkVersion)\any\dotnet-sos.dll install --architecture $(TargetArchitecture) - $(HelixPreCommands);$(NtSymbolPathEnvVar);$(ExecuteDotNetSos) - + + + @@ -395,6 +397,8 @@ + + @@ -441,10 +445,6 @@ - - - @@ -756,7 +756,7 @@ BuildInParallel="$(Samples_BuildInParallel)" /> - + diff --git a/src/mono/CMakeLists.txt b/src/mono/CMakeLists.txt index 1d17921a277d7b..00f883aca07e4b 100644 --- a/src/mono/CMakeLists.txt +++ b/src/mono/CMakeLists.txt @@ -288,7 +288,6 @@ elseif(CLR_CMAKE_HOST_OS STREQUAL "windows") add_compile_options($<$:/Zi>) # enable debugging information add_link_options(/LTCG) # link-time code generation add_link_options(/DEBUG) # enable debugging information - add_link_options(/DEBUGTYPE:CV,FIXUP) # enable fixup debug information add_link_options(/OPT:REF) # optimize: remove unreferenced functions & data add_link_options(/OPT:ICF) # optimize: enable COMDAT folding # the combination of /Zi compiler flag and /DEBUG /OPT:REF /OPT:ICF diff --git a/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj index c273e6ba4172f7..1c085ed36e446e 100644 --- a/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -246,6 +246,7 @@ + @@ -263,6 +264,7 @@ Condition="'$(FeatureObjCMarshal)' == 'true'"/> + diff --git a/src/mono/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.iOS.xml b/src/mono/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.iOS.xml index ed9ce57b2481c6..d0c319ec93aae2 100644 --- a/src/mono/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.iOS.xml +++ b/src/mono/System.Private.CoreLib/src/ILLink/ILLink.Substitutions.iOS.xml @@ -3,5 +3,8 @@ + + + diff --git a/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs index bcb9b6b38f2e39..8f45f602e6fb11 100644 --- a/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Buffer.Mono.cs @@ -23,7 +23,7 @@ internal static unsafe void Memmove(ref T destination, ref T source, nuint el { #pragma warning disable 8500 // sizeof of managed types // Blittable memmove - SpanHelpers.Memmove( + Memmove( ref Unsafe.As(ref destination), ref Unsafe.As(ref source), elementCount * (nuint)sizeof(T)); diff --git a/src/mono/System.Private.CoreLib/src/System/Delegate.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Delegate.Mono.cs index 98ade01313be25..9f71ff470f3e4a 100644 --- a/src/mono/System.Private.CoreLib/src/System/Delegate.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Delegate.Mono.cs @@ -556,7 +556,7 @@ private DelegateData CreateDelegateData() return delegate_data; } - internal static bool InternalEqualTypes(object source, object value) + private static bool InternalEqualTypes(object source, object value) { return source.GetType() == value.GetType(); } diff --git a/src/mono/System.Private.CoreLib/src/System/MulticastDelegate.Mono.cs b/src/mono/System.Private.CoreLib/src/System/MulticastDelegate.Mono.cs index 73aacd9ca0e0c6..e634afd23448d0 100644 --- a/src/mono/System.Private.CoreLib/src/System/MulticastDelegate.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/MulticastDelegate.Mono.cs @@ -131,10 +131,6 @@ protected sealed override Delegate CombineImpl(Delegate? follow) if (follow == null) return this; - // Verify that the types are the same... - if (!InternalEqualTypes(this, follow)) - throw new ArgumentException(SR.Arg_DlgtTypeMis); - MulticastDelegate other = (MulticastDelegate)follow; MulticastDelegate ret = AllocDelegateLike_internal(this); diff --git a/src/mono/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.Mono.cs new file mode 100644 index 00000000000000..54871091da0f4f --- /dev/null +++ b/src/mono/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.Mono.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Reflection; + +namespace System.Resources +{ + internal partial class ManifestBasedResourceGroveler + { + private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, CultureInfo culture, Version? version) + { + return (RuntimeAssembly.InternalGetSatelliteAssembly(mainAssembly, culture, version, throwOnFileNotFound: false)); + } + } +} diff --git a/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs index ca9ecdbe9f829c..391bfaec6a2a46 100644 --- a/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs @@ -4,7 +4,6 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Tracing; -using System.Runtime.InteropServices; using System.Runtime.Serialization; namespace System.Runtime.CompilerServices @@ -138,15 +137,9 @@ public static void RunModuleConstructor(ModuleHandle module) RunModuleConstructor(module.Value); } - public static unsafe IntPtr AllocateTypeAssociatedMemory(Type type, int size) + public static IntPtr AllocateTypeAssociatedMemory(Type type, int size) { - if (type is not RuntimeType) - throw new ArgumentException(SR.Arg_MustBeType, nameof(type)); - - ArgumentOutOfRangeException.ThrowIfNegative(size); - - // We don't support unloading; the memory will never be freed. - return (IntPtr)NativeMemory.AllocZeroed((uint)size); + throw new PlatformNotSupportedException(); } [Intrinsic] diff --git a/src/mono/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs b/src/mono/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs new file mode 100644 index 00000000000000..e3dae854517e29 --- /dev/null +++ b/src/mono/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Security +{ + // DynamicSecurityMethodAttribute: + // All methods that use StackCrawlMark should be marked with this attribute. This attribute + // disables inlining of the calling method to allow stackwalking to find the exact caller. + // + // This attribute used to indicate that the target method requires space for a security object + // to be allocated on the callers stack. It is not used for this purpose anymore because of security + // stackwalks are not ever done in CoreCLR. + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Constructor, AllowMultiple = true, Inherited = false)] + internal sealed class DynamicSecurityMethodAttribute : Attribute + { + public DynamicSecurityMethodAttribute() { } + } +} diff --git a/src/mono/System.Private.CoreLib/src/System/String.Mono.cs b/src/mono/System.Private.CoreLib/src/System/String.Mono.cs index 7dedf5a6e536d0..7314504aff9a22 100644 --- a/src/mono/System.Private.CoreLib/src/System/String.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/String.Mono.cs @@ -116,7 +116,7 @@ private static unsafe void memset(byte* dest, int val, int len) private static unsafe void memcpy(byte* dest, byte* src, int size) { - SpanHelpers.Memmove(ref *dest, ref *src, (nuint)size); + Buffer.Memmove(ref *dest, ref *src, (nuint)size); } /* Used by the runtime */ diff --git a/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.Browser.Threads.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.Browser.Threads.Mono.cs index cc3f606fe6273d..632b0c934ee4c0 100644 --- a/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.Browser.Threads.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.Browser.Threads.Mono.cs @@ -7,7 +7,7 @@ internal sealed partial class PortableThreadPool { private static partial class WorkerThread { - private static bool IsIOPending => false; + private static bool IsIOPending => WebWorkerEventLoop.HasJavaScriptInteropDependents; } private struct CpuUtilizationReader diff --git a/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs index 82645c987d913c..b45dee7fa2fd6f 100644 --- a/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.Browser.Threads.Mono.cs @@ -37,7 +37,7 @@ private static partial class WorkerThread private static readonly ThreadStart s_workerThreadStart = WorkerThreadStart; - private sealed record SemaphoreWaitState(PortableThreadPool ThreadPoolInstance, LowLevelLock ThreadAdjustmentLock) + private sealed record SemaphoreWaitState(PortableThreadPool ThreadPoolInstance, LowLevelLock ThreadAdjustmentLock, WebWorkerEventLoop.KeepaliveToken KeepaliveToken) { public bool SpinWait = true; @@ -59,7 +59,8 @@ private static void WorkerThreadStart() } LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock; - SemaphoreWaitState state = new(threadPoolInstance, threadAdjustmentLock) { SpinWait = true }; + var keepaliveToken = WebWorkerEventLoop.KeepalivePush(); + SemaphoreWaitState state = new(threadPoolInstance, threadAdjustmentLock, keepaliveToken) { SpinWait = true }; // set up the callbacks for semaphore waits, tell // emscripten to keep the thread alive, and return to // the JS event loop. @@ -73,6 +74,8 @@ private static void WorkerThreadStart() private static void WaitForWorkLoop(LowLevelLifoAsyncWaitSemaphore semaphore, SemaphoreWaitState state) { semaphore.PrepareAsyncWait(ThreadPoolThreadTimeoutMs, s_WorkLoopSemaphoreSuccess, s_WorkLoopSemaphoreTimedOut, state); + // thread should still be kept alive + Debug.Assert(state.KeepaliveToken.Valid); } private static void WorkLoopSemaphoreSuccess(LowLevelLifoAsyncWaitSemaphore semaphore, object? stateObject) @@ -88,6 +91,11 @@ private static void WorkLoopSemaphoreTimedOut(LowLevelLifoAsyncWaitSemaphore sem SemaphoreWaitState state = (SemaphoreWaitState)stateObject!; if (ShouldExitWorker(state.ThreadPoolInstance, state.ThreadAdjustmentLock)) { // we're done, kill the thread. + + // we're wrapped in an emscripten eventloop handler which will consult the + // keepalive count, destroy the thread and run the TLS dtor which will + // unregister the thread from Mono + state.KeepaliveToken.Pop(); return; } else { // more work showed up while we were shutting down, go around one more time diff --git a/src/mono/System.Private.CoreLib/src/System/Threading/WebWorkerEventLoop.Browser.Threads.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Threading/WebWorkerEventLoop.Browser.Threads.Mono.cs index 367b1df00ee64c..73c2959293d525 100644 --- a/src/mono/System.Private.CoreLib/src/System/Threading/WebWorkerEventLoop.Browser.Threads.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Threading/WebWorkerEventLoop.Browser.Threads.Mono.cs @@ -12,6 +12,49 @@ namespace System.Threading; /// internal static class WebWorkerEventLoop { + // FIXME: these keepalive calls could be qcalls with a SuppressGCTransitionAttribute + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern void KeepalivePushInternal(); + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern void KeepalivePopInternal(); + + /// + /// A keepalive token prevents a thread from shutting down even if it returns to the JS event + /// loop. A thread may want a keepalive token if it needs to allow JS code to run to settle JS + /// promises or execute JS timeout callbacks. + /// + internal sealed class KeepaliveToken + { + public bool Valid {get; private set; } + + private KeepaliveToken() { Valid = true; } + + /// + /// Decrement the Emscripten keepalive count. A thread with a zero keepalive count will + /// terminate when it returns from its start function or from an async invocation from the + /// JS event loop. + /// + internal void Pop() { + if (!Valid) + throw new InvalidOperationException(); + Valid = false; + KeepalivePopInternal(); + } + + internal static KeepaliveToken Create() + { + KeepalivePushInternal(); + return new KeepaliveToken(); + } + } + + /// + /// Increment the Emscripten keepalive count. A thread with a positive keepalive can return from its + /// thread start function or a JS event loop invocation and continue running in the JS event + /// loop. + /// + internal static KeepaliveToken KeepalivePush() => KeepaliveToken.Create(); + /// /// Start a thread that may be kept alive on its webworker after the start function returns, /// if the emscripten keepalive count is positive. Once the thread returns to the JS event @@ -30,4 +73,26 @@ internal static void StartExitable(Thread thread, bool captureContext) thread.HasExternalEventLoop = true; thread.UnsafeStart(); } + + /// returns true if the current thread has unsettled JS Interop promises + private static bool HasUnsettledInteropPromises => HasUnsettledInteropPromisesNative(); + + // FIXME: this could be a qcall with a SuppressGCTransitionAttribute + [MethodImpl(MethodImplOptions.InternalCall)] + private static extern bool HasUnsettledInteropPromisesNative(); + + /// returns true if the current WebWorker has JavaScript objects that depend on the + /// current managed thread. + /// + /// If this returns false, the runtime is allowed to allow the current managed thread + /// to exit and for the WebWorker to be recycled by Emscripten for another managed + /// thread. + internal static bool HasJavaScriptInteropDependents + { + // + // FIXME: + // https://github.com/dotnet/runtime/issues/85052 - unsettled promises are not the only relevant + // reasons for keeping a worker thread alive. We will need to add other conditions here. + get => HasUnsettledInteropPromises; + } } diff --git a/src/mono/browser/build/BrowserWasmApp.targets b/src/mono/browser/build/BrowserWasmApp.targets index aa231c45d8b7cc..6174769ef02ea4 100644 --- a/src/mono/browser/build/BrowserWasmApp.targets +++ b/src/mono/browser/build/BrowserWasmApp.targets @@ -64,6 +64,7 @@ <_BoolPropertiesThatTriggerRelinking Include="WasmEnableSIMD" DefaultValueInRuntimePack="true" /> <_BoolPropertiesThatTriggerRelinking Include="WasmEnableExceptionHandling" DefaultValueInRuntimePack="true" /> + <_BoolPropertiesThatTriggerRelinking Include="WasmNativeStrip" DefaultValueInRuntimePack="true" /> diff --git a/src/mono/browser/debugger/BrowserDebugProxy/MonoProxy.cs b/src/mono/browser/debugger/BrowserDebugProxy/MonoProxy.cs index 503f196c17f058..4ae7fdd5e15de9 100644 --- a/src/mono/browser/debugger/BrowserDebugProxy/MonoProxy.cs +++ b/src/mono/browser/debugger/BrowserDebugProxy/MonoProxy.cs @@ -911,18 +911,16 @@ protected async Task EvaluateCondition(SessionId sessionId, ExecutionConte return true; } } - catch (ReturnAsErrorException ree) + catch (ReturnAsErrorException raee) { - logger.LogDebug($"Unable to evaluate breakpoint condition '{condition}': {ree}"); - SendLog(sessionId, $"Unable to evaluate breakpoint condition '{condition}': {ree.Message}", token, type: "error"); + logger.LogDebug($"Unable to evaluate breakpoint condition '{condition}': {raee}"); + SendLog(sessionId, $"Unable to evaluate breakpoint condition '{condition}': {raee.Message}", token, type: "error"); bp.ConditionAlreadyEvaluatedWithError = true; - SendExceptionToTelemetry(ree, "EvaluateCondition", sessionId, token); } catch (Exception e) { Log("info", $"Unable to evaluate breakpoint condition '{condition}': {e}"); bp.ConditionAlreadyEvaluatedWithError = true; - SendExceptionToTelemetry(e, "EvaluateCondition", sessionId, token); } return false; } @@ -1521,29 +1519,17 @@ private async Task OnEvaluateOnCallFrame(MessageId msg_id, int scopeId, st catch (ReturnAsErrorException ree) { SendResponse(msg_id, AddCallStackInfoToException(ree.Error, context, scopeId), token); - SendExceptionToTelemetry(ree, "OnEvaluateOnCallFrame", msg_id, token); } catch (Exception e) { logger.LogDebug($"Error in EvaluateOnCallFrame for expression '{expression}' with '{e}."); - var ree = new ReturnAsErrorException(e.Message, e.GetType().Name); - SendResponse(msg_id, AddCallStackInfoToException(ree.Error, context, scopeId), token); - SendExceptionToTelemetry(e, "OnEvaluateOnCallFrame", msg_id, token); + var exc = new ReturnAsErrorException(e.Message, e.GetType().Name); + SendResponse(msg_id, AddCallStackInfoToException(exc.Error, context, scopeId), token); } return true; } - private void SendExceptionToTelemetry(Exception exc, string callingFunction, SessionId msg_id, CancellationToken token) - { - JObject reportBlazorDebugError = JObject.FromObject(new - { - exceptionType = "uncaughtException", - error = $"{exc.Message} at {callingFunction}", - }); - SendEvent(msg_id, "DotnetDebugger.reportBlazorDebugError", reportBlazorDebugError, token); - } - internal async Task GetScopeProperties(SessionId msg_id, int scopeId, CancellationToken token) { try diff --git a/src/mono/browser/debugger/BrowserDebugProxy/MonoSDBHelper.cs b/src/mono/browser/debugger/BrowserDebugProxy/MonoSDBHelper.cs index 81871b356e16d4..69e1cd780c0561 100644 --- a/src/mono/browser/debugger/BrowserDebugProxy/MonoSDBHelper.cs +++ b/src/mono/browser/debugger/BrowserDebugProxy/MonoSDBHelper.cs @@ -525,9 +525,7 @@ private unsafe T ReadBigEndian() where T : struct { data.Reverse(); } -#pragma warning disable CS8500 // takes address of managed type - data.CopyTo(new Span(&ret, data.Length)); -#pragma warning restore CS8500 + data.CopyTo(new Span(Unsafe.AsPointer(ref ret), data.Length)); return ret; } } @@ -548,9 +546,7 @@ public override void Write(string val) private unsafe void WriteBigEndian(T val) where T : struct { Span data = stackalloc byte[Unsafe.SizeOf()]; -#pragma warning disable CS8500 // takes address of managed type - new Span(&val, data.Length).CopyTo(data); -#pragma warning restore CS8500 + new Span(Unsafe.AsPointer(ref val), data.Length).CopyTo(data); if (BitConverter.IsLittleEndian) { data.Reverse(); diff --git a/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj b/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj index 7283603a955e1c..87cfb17bbc0477 100644 --- a/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj +++ b/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj @@ -81,7 +81,7 @@ BeforeTargets="CopyTestZipForHelix" DependsOnTargets="_GenerateRunSettingsFile"> - + @@ -89,15 +89,12 @@ <_Regex>^ *(DebuggerTests[^\($]+) - <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" /> - + <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', $(_Regex)))" /> + - - + Lines="@(TestClassName->Distinct())" /> diff --git a/src/mono/browser/runtime/cancelable-promise.ts b/src/mono/browser/runtime/cancelable-promise.ts index cf0b048a2c60cd..0d8d7be66842c0 100644 --- a/src/mono/browser/runtime/cancelable-promise.ts +++ b/src/mono/browser/runtime/cancelable-promise.ts @@ -1,16 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -import WasmEnableThreads from "consts:wasmEnableThreads"; - -import { _lookup_js_owned_object, teardown_managed_proxy, upgrade_managed_proxy_to_strong_ref } from "./gc-handles"; +import { _lookup_js_owned_object } from "./gc-handles"; import { createPromiseController, loaderHelpers, mono_assert } from "./globals"; -import { ControllablePromise, GCHandle, MarshalerToCs } from "./types/internal"; -import { ManagedObject } from "./marshal"; -import { compareExchangeI32, forceThreadMemoryViewRefresh } from "./memory"; -import { mono_log_debug } from "./logging"; -import { complete_task } from "./managed-exports"; -import { marshal_cs_object_to_cs } from "./marshal-to-cs"; +import { mono_log_warn } from "./logging"; +import { PromiseHolder } from "./marshal-to-cs"; +import { ControllablePromise, GCHandle } from "./types/internal"; export const _are_promises_supported = ((typeof Promise === "object") || (typeof Promise === "function")) && (typeof Promise.resolve === "function"); @@ -35,148 +30,19 @@ export function wrap_as_cancelable(inner: Promise): ControllablePromise } export function mono_wasm_cancel_promise(task_holder_gc_handle: GCHandle): void { - if (!loaderHelpers.is_runtime_running()) { - mono_log_debug("This promise can't be canceled, mono runtime already exited."); - return; - } const holder = _lookup_js_owned_object(task_holder_gc_handle) as PromiseHolder; mono_assert(!!holder, () => `Expected Promise for GCHandle ${task_holder_gc_handle}`); - holder.cancel(); -} - -// NOTE: layout has to match PromiseHolderState in JSHostImplementation.Types.cs -const enum PromiseHolderState { - IsResolving = 0, -} - -const promise_holder_symbol = Symbol.for("wasm promise_holder"); - -export class PromiseHolder extends ManagedObject { - public isResolved = false; - public isPosted = false; - public isPostponed = false; - public data: any = null; - public reason: any = null; - public constructor(public promise: Promise, - private gc_handle: GCHandle, - private promiseHolderPtr: number, // could be null for GCV_handle - private res_converter?: MarshalerToCs) { - super(); - } - - // returns false if the promise is being canceled by another thread in managed code - setIsResolving(): boolean { - if (!WasmEnableThreads || this.promiseHolderPtr === 0) { - return true; - } - forceThreadMemoryViewRefresh(); - if (compareExchangeI32(this.promiseHolderPtr + PromiseHolderState.IsResolving, 1, 0) === 0) { - return true; - } - return false; - } - - resolve(data: any) { - if (!loaderHelpers.is_runtime_running()) { - mono_log_debug("This promise resolution can't be propagated to managed code, mono runtime already exited."); - return; - } - mono_assert(!this.isResolved, "resolve could be called only once"); - mono_assert(!this.isDisposed, "resolve is already disposed."); - if (WasmEnableThreads && !this.setIsResolving()) { - // we know that cancelation is in flight - // because we need to keep the GCHandle alive until until the cancelation arrives - // we skip the this resolve and let the cancelation to reject the Task - // we store the original data and use it later - this.data = data; - this.isPostponed = true; - // but after the promise is resolved, nothing holds the weak reference to the PromiseHolder anymore - // we know that cancelation is in flight, so we upgrade the weak reference to strong for the meantime - upgrade_managed_proxy_to_strong_ref(this, this.gc_handle); - return; - } - this.isResolved = true; - this.complete_task_wrapper(data, null); - } - - reject(reason: any) { - if (!loaderHelpers.is_runtime_running()) { - mono_log_debug("This promise rejection can't be propagated to managed code, mono runtime already exited."); - return; - } - mono_assert(!this.isResolved, "reject could be called only once"); - mono_assert(!this.isDisposed, "resolve is already disposed."); - const isCancelation = reason && reason[promise_holder_symbol] === this; - if (WasmEnableThreads && !isCancelation && !this.setIsResolving()) { - // we know that cancelation is in flight - // because we need to keep the GCHandle alive until until the cancelation arrives - // we skip the this reject and let the cancelation to reject the Task - // we store the original reason and use it later - this.reason = reason; - this.isPostponed = true; - - // but after the promise is resolved, nothing holds the weak reference to the PromiseHolder anymore - // we know that cancelation is in flight, so we upgrade the weak reference to strong for the meantime - upgrade_managed_proxy_to_strong_ref(this, this.gc_handle); - return; - } - this.isResolved = true; - this.complete_task_wrapper(null, reason); - } - - cancel() { - if (!loaderHelpers.is_runtime_running()) { - mono_log_debug("This promise cancelation can't be propagated to managed code, mono runtime already exited."); - return; - } - mono_assert(!this.isResolved, "cancel could be called only once"); - mono_assert(!this.isDisposed, "resolve is already disposed."); - - if (this.isPostponed) { - // there was racing resolve/reject which was postponed, to retain valid GCHandle - // in this case we just finish the original resolve/reject - // and we need to use the postponed data/reason - this.isResolved = true; - if (this.reason !== undefined) { - this.complete_task_wrapper(null, this.reason); - } else { - this.complete_task_wrapper(this.data, null); - } - } else { - // there is no racing resolve/reject, we can reject/cancel the promise - const promise = this.promise; - loaderHelpers.assertIsControllablePromise(promise); - const promise_control = loaderHelpers.getPromiseController(promise); - - const reason = new Error("OperationCanceledException") as any; - reason[promise_holder_symbol] = this; - promise_control.reject(reason); - } + const promise = holder.promise; + loaderHelpers.assertIsControllablePromise(promise); + const promise_control = loaderHelpers.getPromiseController(promise); + if (holder.isResolved) { + // FIXME: something needs to free the GCHandle + mono_log_warn("Canceling a promise that has already resolved."); + return; } + mono_assert(!holder.isCanceled, "This promise already canceled."); + holder.isCanceled = true; + promise_control.reject(new Error("OperationCanceledException")); +} - // we can do this just once, because it will be dispose the GCHandle - complete_task_wrapper(data: any, reason: any) { - try { - mono_assert(!this.isPosted, "Promise is already posted to managed."); - this.isPosted = true; - if (WasmEnableThreads) { - forceThreadMemoryViewRefresh(); - } - - // we can unregister the GC handle just on JS side - teardown_managed_proxy(this, this.gc_handle, /*skipManaged: */ true); - // order of operations with teardown_managed_proxy matters - // so that managed user code running in the continuation could allocate the same GCHandle number and the local registry would be already ok with that - complete_task(this.gc_handle, reason, data, this.res_converter || marshal_cs_object_to_cs); - } - catch (ex) { - try { - loaderHelpers.mono_exit(1, ex); - } - catch (ex2) { - // there is no point to propagate the exception into the unhandled promise rejection - } - } - } -} \ No newline at end of file diff --git a/src/mono/browser/runtime/corebindings.c b/src/mono/browser/runtime/corebindings.c index 1b0c0a809f63ef..7329fb4bcd1924 100644 --- a/src/mono/browser/runtime/corebindings.c +++ b/src/mono/browser/runtime/corebindings.c @@ -42,7 +42,6 @@ void mono_wasm_resolve_or_reject_promise_post (pthread_t target_tid, void *args) void mono_wasm_cancel_promise_post (pthread_t target_tid, int task_holder_gc_handle); extern void mono_wasm_install_js_worker_interop (int context_gc_handle); -void mono_wasm_install_js_worker_interop_wrapper (int context_gc_handle, void* beforeSyncJSImport, void* afterSyncJSImport); extern void mono_wasm_uninstall_js_worker_interop (); extern void mono_wasm_invoke_jsimport (void* signature, void* args); void mono_wasm_invoke_jsimport_async_post (pthread_t target_tid, void* signature, void* args); @@ -78,7 +77,7 @@ void bindings_initialize_internals (void) #ifndef DISABLE_THREADS mono_add_internal_call ("Interop/Runtime::ReleaseCSOwnedObjectPost", mono_wasm_release_cs_owned_object_post); mono_add_internal_call ("Interop/Runtime::ResolveOrRejectPromisePost", mono_wasm_resolve_or_reject_promise_post); - mono_add_internal_call ("Interop/Runtime::InstallWebWorkerInterop", mono_wasm_install_js_worker_interop_wrapper); + mono_add_internal_call ("Interop/Runtime::InstallWebWorkerInterop", mono_wasm_install_js_worker_interop); mono_add_internal_call ("Interop/Runtime::UninstallWebWorkerInterop", mono_wasm_uninstall_js_worker_interop); mono_add_internal_call ("Interop/Runtime::InvokeJSImportSync", mono_wasm_invoke_jsimport); mono_add_internal_call ("Interop/Runtime::InvokeJSImportSyncSend", mono_wasm_invoke_jsimport_sync_send); @@ -254,47 +253,31 @@ void mono_wasm_get_assembly_export (char *assembly_name, char *namespace, char * #ifndef DISABLE_THREADS -void* before_sync_js_import; -void* after_sync_js_import; - -void mono_wasm_install_js_worker_interop_wrapper (int context_gc_handle, void* beforeSyncJSImport, void* afterSyncJSImport) -{ - before_sync_js_import = beforeSyncJSImport; - after_sync_js_import = afterSyncJSImport; - mono_wasm_install_js_worker_interop (context_gc_handle); -} - -// async void mono_wasm_release_cs_owned_object_post (pthread_t target_tid, int js_handle) { mono_threads_wasm_async_run_in_target_thread_vi (target_tid, (void (*) (gpointer))mono_wasm_release_cs_owned_object, (gpointer)js_handle); } -// async void mono_wasm_resolve_or_reject_promise_post (pthread_t target_tid, void* args) { mono_threads_wasm_async_run_in_target_thread_vi (target_tid, (void (*) (gpointer))mono_wasm_resolve_or_reject_promise, (gpointer)args); } -// async void mono_wasm_cancel_promise_post (pthread_t target_tid, int task_holder_gc_handle) { mono_threads_wasm_async_run_in_target_thread_vi (target_tid, (void (*) (gpointer))mono_wasm_cancel_promise, (gpointer)task_holder_gc_handle); } -// async void mono_wasm_invoke_jsimport_async_post (pthread_t target_tid, void* signature, void* args) { mono_threads_wasm_async_run_in_target_thread_vii (target_tid, (void (*) (gpointer, gpointer))mono_wasm_invoke_jsimport, (gpointer)signature, (gpointer)args); } -// sync void mono_wasm_invoke_jsimport_sync_send (pthread_t target_tid, void* signature, void* args) { mono_threads_wasm_sync_run_in_target_thread_vii (target_tid, (void (*) (gpointer, gpointer))mono_wasm_invoke_jsimport, (gpointer)signature, (gpointer)args); } -// sync void mono_wasm_invoke_js_function_send (pthread_t target_tid, int function_js_handle, void *args) { mono_threads_wasm_sync_run_in_target_thread_vii (target_tid, (void (*) (gpointer, gpointer))mono_wasm_invoke_js_function, (gpointer)function_js_handle, (gpointer)args); diff --git a/src/mono/browser/runtime/cwraps.ts b/src/mono/browser/runtime/cwraps.ts index 11168751ebb3bb..dbb0babce82319 100644 --- a/src/mono/browser/runtime/cwraps.ts +++ b/src/mono/browser/runtime/cwraps.ts @@ -27,12 +27,6 @@ const threading_cwraps: SigLine[] = WasmEnableThreads ? [ [false, "mono_wasm_init_finalizer_thread", null, []], [false, "mono_wasm_invoke_jsexport_async_post", "void", ["number", "number", "number"]], [false, "mono_wasm_invoke_jsexport_sync_send", "void", ["number", "number", "number"]], - [false, "mono_wasm_invoke_jsexport_sync", "void", ["number", "number"]], - [true, "mono_wasm_create_deputy_thread", "number", []], - [true, "mono_wasm_create_io_thread", "number", []], - [true, "mono_wasm_register_ui_thread", "void", []], - [true, "mono_wasm_register_io_thread", "void", []], - [true, "mono_wasm_print_thread_dump", "void", []], ] : []; // when the method is assigned/cached at usage, instead of being invoked directly from cwraps, it can't be marked lazy, because it would be re-bound on each call @@ -150,12 +144,6 @@ export interface t_ThreadingCwraps { mono_wasm_init_finalizer_thread(): void; mono_wasm_invoke_jsexport_async_post(targetTID: PThreadPtr, method: MonoMethod, args: VoidPtr): void; mono_wasm_invoke_jsexport_sync_send(targetTID: PThreadPtr, method: MonoMethod, args: VoidPtr): void; - mono_wasm_invoke_jsexport_sync(method: MonoMethod, args: VoidPtr): void; - mono_wasm_create_deputy_thread(): PThreadPtr; - mono_wasm_create_io_thread(): PThreadPtr; - mono_wasm_register_ui_thread(): void; - mono_wasm_register_io_thread(): void; - mono_wasm_print_thread_dump(): void; } export interface t_ProfilerCwraps { diff --git a/src/mono/browser/runtime/debug.ts b/src/mono/browser/runtime/debug.ts index 1cbe85aff718ac..74c0128f2e4e3b 100644 --- a/src/mono/browser/runtime/debug.ts +++ b/src/mono/browser/runtime/debug.ts @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +import BuildConfiguration from "consts:configuration"; + import { INTERNAL, Module, loaderHelpers, runtimeHelpers } from "./globals"; import { toBase64StringImpl } from "./base64"; import cwraps from "./cwraps"; @@ -362,6 +364,11 @@ export function mono_wasm_debugger_log(level: number, message_ptr: CharPtr): voi INTERNAL.logging.debugger(level, message); return; } + + if (BuildConfiguration === "Debug") { + // eslint-disable-next-line no-console + console.debug(`Debugger.Debug: ${message}`); + } } type CallDetails = { diff --git a/src/mono/browser/runtime/diagnostics/server_pthread/stream-queue.ts b/src/mono/browser/runtime/diagnostics/server_pthread/stream-queue.ts index 5ab42c2d8b4d96..c7b9f13db7e393 100644 --- a/src/mono/browser/runtime/diagnostics/server_pthread/stream-queue.ts +++ b/src/mono/browser/runtime/diagnostics/server_pthread/stream-queue.ts @@ -2,7 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. import { VoidPtr } from "../../types/emscripten"; -import { getI32, notifyI32, setI32, storeI32 } from "../../memory"; +import * as Memory from "../../memory"; /// One-reader, one-writer, size 1 queue for messages from an EventPipe streaming thread to @@ -68,22 +68,22 @@ export class StreamQueue { } private onWorkAvailable(this: StreamQueue /*,event: Event */): void { - const buf = getI32(this.buf_addr) as unknown as VoidPtr; + const buf = Memory.getI32(this.buf_addr) as unknown as VoidPtr; const intptr_buf = buf as unknown as number; if (intptr_buf === STREAM_CLOSE_SENTINEL) { // special value signaling that the streaming thread closed the queue. this.syncSendClose(); } else { - const count = getI32(this.count_addr); - setI32(this.buf_addr, 0); + const count = Memory.getI32(this.count_addr); + Memory.setI32(this.buf_addr, 0); if (count > 0) { this.syncSendBuffer(buf, count); } } /* buffer is now not full */ - storeI32(this.buf_full_addr, 0); + Memory.Atomics.storeI32(this.buf_full_addr, 0); /* wake up the writer thread */ - notifyI32(this.buf_full_addr, 1); + Memory.Atomics.notifyI32(this.buf_full_addr, 1); } } diff --git a/src/mono/browser/runtime/dotnet.d.ts b/src/mono/browser/runtime/dotnet.d.ts index 2fdf8e16d09f3e..7b901972e8b8f0 100644 --- a/src/mono/browser/runtime/dotnet.d.ts +++ b/src/mono/browser/runtime/dotnet.d.ts @@ -446,13 +446,6 @@ type APIType = { * @returns exit code of the Main() method. */ runMainAndExit: (mainAssemblyName?: string, args?: string[]) => Promise; - /** - * Exits the runtime. - * Note: after the runtime exits, it would reject all further calls to the API. - * @param code "process" exit code. - * @param reason could be a string or an Error object. - */ - exit: (code: number, reason?: any) => void; /** * Sets the environment variable for the "process" * @param name diff --git a/src/mono/browser/runtime/driver.c b/src/mono/browser/runtime/driver.c index 366fec9d60b70e..d76a8bacd92823 100644 --- a/src/mono/browser/runtime/driver.c +++ b/src/mono/browser/runtime/driver.c @@ -257,25 +257,14 @@ mono_wasm_invoke_jsexport (MonoMethod *method, void* args) extern void mono_threads_wasm_async_run_in_target_thread_vii (void* target_thread, void (*func) (gpointer, gpointer), gpointer user_data1, gpointer user_data2); extern void mono_threads_wasm_sync_run_in_target_thread_vii (void* target_thread, void (*func) (gpointer, gpointer), gpointer user_data1, gpointer user_data2); -extern void mono_print_thread_dump (void *sigctx); -EMSCRIPTEN_KEEPALIVE void -mono_wasm_print_thread_dump (void) -{ - mono_print_thread_dump (NULL); -} - -// this is running on the target thread static void mono_wasm_invoke_jsexport_async_post_cb (MonoMethod *method, void* args) { mono_wasm_invoke_jsexport (method, args); - if (args) { - MonoBoolean *is_receiver_should_free = (MonoBoolean *)(args + 20/*JSMarshalerArgumentOffsets.ReceiverShouldFree*/); - if(*is_receiver_should_free != 0){ - free (args); - } - } + // TODO assert receiver_should_free ? + if (args) + free (args); } // async @@ -285,25 +274,11 @@ mono_wasm_invoke_jsexport_async_post (void* target_thread, MonoMethod *method, v mono_threads_wasm_async_run_in_target_thread_vii(target_thread, (void (*)(gpointer, gpointer))mono_wasm_invoke_jsexport_async_post_cb, method, args); } - -typedef void (*js_interop_event)(void* args); -extern js_interop_event before_sync_js_import; -extern js_interop_event after_sync_js_import; - -// this is running on the target thread -EMSCRIPTEN_KEEPALIVE void -mono_wasm_invoke_jsexport_sync (MonoMethod *method, void* args) -{ - before_sync_js_import (args); - mono_wasm_invoke_jsexport (method, args); - after_sync_js_import (args); -} - // sync EMSCRIPTEN_KEEPALIVE void mono_wasm_invoke_jsexport_sync_send (void* target_thread, MonoMethod *method, void* args /*JSMarshalerArguments*/) { - mono_threads_wasm_sync_run_in_target_thread_vii(target_thread, (void (*)(gpointer, gpointer))mono_wasm_invoke_jsexport_sync, method, args); + mono_threads_wasm_sync_run_in_target_thread_vii(target_thread, (void (*)(gpointer, gpointer))mono_wasm_invoke_jsexport, method, args); } #endif /* DISABLE_THREADS */ @@ -434,9 +409,7 @@ mono_wasm_init_finalizer_thread (void) { // in the single threaded build, finalizers periodically run on the main thread instead. #ifndef DISABLE_THREADS - MONO_ENTER_GC_UNSAFE; mono_gc_init_finalizer_thread (); - MONO_EXIT_GC_UNSAFE; #endif } @@ -494,19 +467,11 @@ EMSCRIPTEN_KEEPALIVE int mono_wasm_f64_to_i52 (int64_t *destination, double valu // JS is responsible for freeing this EMSCRIPTEN_KEEPALIVE const char * mono_wasm_method_get_full_name (MonoMethod *method) { - const char *res; - MONO_ENTER_GC_UNSAFE; - res = mono_method_get_full_name (method); - MONO_EXIT_GC_UNSAFE; - return res; + return mono_method_get_full_name(method); } EMSCRIPTEN_KEEPALIVE const char * mono_wasm_method_get_name (MonoMethod *method) { - const char *res; - MONO_ENTER_GC_UNSAFE; - res = mono_method_get_name (method); - MONO_EXIT_GC_UNSAFE; - return res; + return mono_method_get_name(method); } EMSCRIPTEN_KEEPALIVE float mono_wasm_get_f32_unaligned (const float *src) { diff --git a/src/mono/browser/runtime/export-api.ts b/src/mono/browser/runtime/export-api.ts index 623c4222218f01..1b4b596bdb6e33 100644 --- a/src/mono/browser/runtime/export-api.ts +++ b/src/mono/browser/runtime/export-api.ts @@ -14,7 +14,6 @@ export function export_api(): any { const api: APIType = { runMain: mono_run_main, runMainAndExit: mono_run_main_and_exit, - exit: loaderHelpers.mono_exit, setEnvironmentVariable: mono_wasm_setenv, getAssemblyExports: mono_wasm_get_assembly_exports, setModuleImports: mono_wasm_set_module_imports, diff --git a/src/mono/browser/runtime/exports-binding.ts b/src/mono/browser/runtime/exports-binding.ts index d2ff55bc088a70..6ab2b430ab0481 100644 --- a/src/mono/browser/runtime/exports-binding.ts +++ b/src/mono/browser/runtime/exports-binding.ts @@ -10,6 +10,7 @@ import { mono_interp_tier_prepare_jiterpreter, mono_jiterp_free_method_data_js } import { mono_interp_jit_wasm_entry_trampoline, mono_interp_record_interp_entry } from "./jiterpreter-interp-entry"; import { mono_interp_jit_wasm_jit_call_trampoline, mono_interp_invoke_wasm_jit_call_trampoline, mono_interp_flush_jitcall_queue } from "./jiterpreter-jit-call"; import { mono_wasm_resolve_or_reject_promise } from "./marshal-to-js"; +import { mono_wasm_eventloop_has_unsettled_interop_promises } from "./pthreads"; import { mono_wasm_schedule_timer, schedule_background_exec } from "./scheduling"; import { mono_wasm_asm_loaded } from "./startup"; import { mono_wasm_diagnostic_server_on_server_thread_created } from "./diagnostics/server_pthread"; @@ -26,13 +27,8 @@ import { mono_wasm_get_first_day_of_week, mono_wasm_get_first_week_of_year } fro import { mono_wasm_browser_entropy } from "./crypto"; import { mono_wasm_cancel_promise } from "./cancelable-promise"; -import { - mono_wasm_start_deputy_thread_async, - mono_wasm_pthread_on_pthread_attached, mono_wasm_pthread_on_pthread_unregistered, - mono_wasm_pthread_on_pthread_registered, mono_wasm_pthread_set_name, mono_wasm_install_js_worker_interop, mono_wasm_uninstall_js_worker_interop, mono_wasm_start_io_thread_async -} from "./pthreads"; -import { mono_wasm_dump_threads } from "./pthreads/ui-thread"; - +import { mono_wasm_pthread_on_pthread_attached, mono_wasm_pthread_on_pthread_unregistered, mono_wasm_pthread_on_pthread_registered, mono_wasm_pthread_set_name } from "./pthreads"; +import { mono_wasm_install_js_worker_interop, mono_wasm_uninstall_js_worker_interop } from "./pthreads"; // the JS methods would be visible to EMCC linker and become imports of the WASM module @@ -42,11 +38,9 @@ export const mono_wasm_threads_imports = !WasmEnableThreads ? [] : [ mono_wasm_pthread_on_pthread_attached, mono_wasm_pthread_on_pthread_unregistered, mono_wasm_pthread_set_name, - mono_wasm_start_deputy_thread_async, - mono_wasm_start_io_thread_async, - // mono-threads.c - mono_wasm_dump_threads, + // threads.c + mono_wasm_eventloop_has_unsettled_interop_promises, // diagnostics_server.c mono_wasm_diagnostic_server_on_server_thread_created, mono_wasm_diagnostic_server_on_runtime_server_init, diff --git a/src/mono/browser/runtime/exports-internal.ts b/src/mono/browser/runtime/exports-internal.ts index 3158b553713e4f..80c52669fb3ad6 100644 --- a/src/mono/browser/runtime/exports-internal.ts +++ b/src/mono/browser/runtime/exports-internal.ts @@ -4,13 +4,13 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; import { MonoObjectNull, type MonoObject } from "./types/internal"; -import cwraps, { profiler_c_functions, threads_c_functions as twraps } from "./cwraps"; +import cwraps, { profiler_c_functions } from "./cwraps"; import { mono_wasm_send_dbg_command_with_parms, mono_wasm_send_dbg_command, mono_wasm_get_dbg_command_info, mono_wasm_get_details, mono_wasm_release_object, mono_wasm_call_function_on, mono_wasm_debugger_resume, mono_wasm_detach_debugger, mono_wasm_raise_debug_event, mono_wasm_change_debugger_log_level, mono_wasm_debugger_attached } from "./debug"; import { http_wasm_supports_streaming_request, http_wasm_supports_streaming_response, http_wasm_create_controller, http_wasm_abort_request, http_wasm_abort_response, http_wasm_transform_stream_write, http_wasm_transform_stream_close, http_wasm_fetch, http_wasm_fetch_stream, http_wasm_fetch_bytes, http_wasm_get_response_header_names, http_wasm_get_response_header_values, http_wasm_get_response_bytes, http_wasm_get_response_length, http_wasm_get_streamed_response_bytes, http_wasm_get_response_type, http_wasm_get_response_status } from "./http"; import { exportedRuntimeAPI, Module, runtimeHelpers } from "./globals"; import { get_property, set_property, has_property, get_typeof_property, get_global_this, dynamic_import } from "./invoke-js"; import { mono_wasm_stringify_as_error_with_stack } from "./logging"; -import { ws_wasm_create, ws_wasm_open, ws_wasm_send, ws_wasm_receive, ws_wasm_close, ws_wasm_abort, ws_get_state } from "./web-socket"; +import { ws_wasm_create, ws_wasm_open, ws_wasm_send, ws_wasm_receive, ws_wasm_close, ws_wasm_abort } from "./web-socket"; import { mono_wasm_get_loaded_files } from "./assets"; import { jiterpreter_dump_stats } from "./jiterpreter"; import { interp_pgo_load_data, interp_pgo_save_data } from "./interp-pgo"; @@ -23,14 +23,14 @@ import { mono_wasm_get_func_id_to_name_mappings } from "./logging"; import { monoStringToStringUnsafe } from "./strings"; import { mono_wasm_bind_cs_function } from "./invoke-cs"; -import { mono_wasm_dump_threads, thread_available } from "./pthreads"; +import { dumpThreads, thread_available } from "./pthreads"; export function export_internal(): any { return { // tests mono_wasm_exit: (exit_code: number) => { Module.err("early exit " + exit_code); }, forceDisposeProxies, - mono_wasm_dump_threads: WasmEnableThreads ? mono_wasm_dump_threads : undefined, + dumpThreads: WasmEnableThreads ? dumpThreads : undefined, // with mono_wasm_debugger_log and mono_wasm_trace_logger logging: undefined, @@ -71,7 +71,6 @@ export function export_internal(): any { ws_wasm_receive, ws_wasm_close, ws_wasm_abort, - ws_get_state, // BrowserHttpHandler http_wasm_supports_streaming_request, @@ -120,7 +119,6 @@ export function cwraps_internal(internal: any): void { mono_wasm_profiler_init_aot: profiler_c_functions.mono_wasm_profiler_init_aot, mono_wasm_profiler_init_browser: profiler_c_functions.mono_wasm_profiler_init_browser, mono_wasm_exec_regression: cwraps.mono_wasm_exec_regression, - mono_wasm_print_thread_dump: WasmEnableThreads ? twraps.mono_wasm_print_thread_dump : undefined, }); } diff --git a/src/mono/browser/runtime/exports.ts b/src/mono/browser/runtime/exports.ts index 2a4dc08320a0bc..8b2984f393b1f0 100644 --- a/src/mono/browser/runtime/exports.ts +++ b/src/mono/browser/runtime/exports.ts @@ -10,7 +10,7 @@ import WasmEnableExceptionHandling from "consts:wasmEnableExceptionHandling"; import type { RuntimeAPI } from "./types"; import { Module, exportedRuntimeAPI, loaderHelpers, passEmscriptenInternals, runtimeHelpers, setRuntimeGlobals, } from "./globals"; -import { GlobalObjects, RuntimeHelpers } from "./types/internal"; +import { GlobalObjects } from "./types/internal"; import { configureEmscriptenStartup, configureRuntimeStartup, configureWorkerStartup } from "./startup"; import { create_weak_ref } from "./weak-ref"; @@ -22,7 +22,7 @@ import { mono_wasm_stringify_as_error_with_stack } from "./logging"; import { instantiate_asset, instantiate_symbols_asset, instantiate_segmentation_rules_asset } from "./assets"; import { jiterpreter_dump_stats } from "./jiterpreter"; import { forceDisposeProxies } from "./gc-handles"; -import { mono_wasm_dump_threads } from "./pthreads"; +import { dumpThreads } from "./pthreads"; export let runtimeList: RuntimeList; @@ -32,19 +32,19 @@ function initializeExports(globalObjects: GlobalObjects): RuntimeAPI { const globalThisAny = globalThis as any; Object.assign(globals.internal, export_internal()); - const rh: Partial = { + Object.assign(runtimeHelpers, { stringify_as_error_with_stack: mono_wasm_stringify_as_error_with_stack, instantiate_symbols_asset, instantiate_asset, jiterpreter_dump_stats, forceDisposeProxies, instantiate_segmentation_rules_asset, - - }; + }); if (WasmEnableThreads) { - rh.dumpThreads = mono_wasm_dump_threads; + Object.assign(runtimeHelpers, { + dumpThreads, + }); } - Object.assign(runtimeHelpers, rh); const API = export_api(); Object.assign(exportedRuntimeAPI, { diff --git a/src/mono/browser/runtime/gc-handles.ts b/src/mono/browser/runtime/gc-handles.ts index cf39ddc822a917..5fe3a27a50fe1a 100644 --- a/src/mono/browser/runtime/gc-handles.ts +++ b/src/mono/browser/runtime/gc-handles.ts @@ -9,7 +9,7 @@ import { assert_js_interop, js_import_wrapper_by_fn_handle } from "./invoke-js"; import { mono_log_info, mono_log_warn } from "./logging"; import { bound_cs_function_symbol, imported_js_function_symbol, proxy_debug_symbol } from "./marshal"; import { GCHandle, GCHandleNull, JSHandle, WeakRefInternal } from "./types/internal"; -import { _use_weak_ref, create_strong_ref, create_weak_ref } from "./weak-ref"; +import { _use_weak_ref, create_weak_ref } from "./weak-ref"; import { exportsByAssembly } from "./invoke-cs"; import { release_js_owned_object_by_gc_handle } from "./managed-exports"; @@ -137,14 +137,6 @@ export function setup_managed_proxy(owner: any, gc_handle: GCHandle): void { _js_owned_object_table.set(gc_handle, wr); } -export function upgrade_managed_proxy_to_strong_ref(owner: any, gc_handle: GCHandle): void { - const sr = create_strong_ref(owner); - if (_use_finalization_registry) { - _js_owned_object_registry.unregister(owner); - } - _js_owned_object_table.set(gc_handle, sr); -} - export function teardown_managed_proxy(owner: any, gc_handle: GCHandle, skipManaged?: boolean): void { assert_js_interop(); // The JS object associated with this gc_handle has been collected by the JS GC. @@ -160,7 +152,7 @@ export function teardown_managed_proxy(owner: any, gc_handle: GCHandle, skipMana } } if (gc_handle !== GCHandleNull && _js_owned_object_table.delete(gc_handle) && !skipManaged) { - if (loaderHelpers.is_runtime_running() && !force_dispose_proxies_in_progress) { + if (loaderHelpers.is_runtime_running()) { release_js_owned_object_by_gc_handle(gc_handle); } } @@ -204,14 +196,11 @@ export function assertNoProxies(): void { mono_assert(js_import_wrapper_by_fn_handle.length === 1, "There should be no imports on this thread."); } -let force_dispose_proxies_in_progress = false; - // when we arrive here from UninstallWebWorkerInterop, the C# will unregister the handles too. // when called from elsewhere, C# side could be unbalanced!! export function forceDisposeProxies(disposeMethods: boolean, verbose: boolean): void { let keepSomeCsAlive = false; let keepSomeJsAlive = false; - force_dispose_proxies_in_progress = true; let doneImports = 0; let doneExports = 0; diff --git a/src/mono/browser/runtime/globals.ts b/src/mono/browser/runtime/globals.ts index 9513254cc55aa4..15ea06b82c6d07 100644 --- a/src/mono/browser/runtime/globals.ts +++ b/src/mono/browser/runtime/globals.ts @@ -65,7 +65,6 @@ export function setRuntimeGlobals(globalObjects: GlobalObjects) { afterPreRun: createPromiseController(), beforeOnRuntimeInitialized: createPromiseController(), afterMonoStarted: createPromiseController(), - afterIOStarted: createPromiseController(), afterOnRuntimeInitialized: createPromiseController(), afterPostRun: createPromiseController(), nativeAbort: (reason: any) => { throw reason || new Error("abort"); }, diff --git a/src/mono/browser/runtime/http.ts b/src/mono/browser/runtime/http.ts index 0e45e2ec819752..9f606f481d3956 100644 --- a/src/mono/browser/runtime/http.ts +++ b/src/mono/browser/runtime/http.ts @@ -25,11 +25,7 @@ function commonAsserts(controller: HttpController) { mono_assert(controller, "expected controller"); } -let http_wasm_supports_streaming_request_cached: boolean | undefined; export function http_wasm_supports_streaming_request(): boolean { - if (http_wasm_supports_streaming_request_cached !== undefined) { - return http_wasm_supports_streaming_request_cached; - } // Detecting streaming request support works like this: // If the browser doesn't support a particular body type, it calls toString() on the object and uses the result as the body. // So, if the browser doesn't support request streams, the request body becomes the string "[object ReadableStream]". @@ -47,20 +43,13 @@ export function http_wasm_supports_streaming_request(): boolean { return "half"; }, } as RequestInit /* https://github.com/microsoft/TypeScript-DOM-lib-generator/issues/1483 */).headers.has("Content-Type"); - http_wasm_supports_streaming_request_cached = duplexAccessed && !hasContentType; - } else { - http_wasm_supports_streaming_request_cached = false; + return duplexAccessed && !hasContentType; } - return http_wasm_supports_streaming_request_cached; + return false; } -let http_wasm_supports_streaming_response_cached: boolean | undefined; export function http_wasm_supports_streaming_response(): boolean { - if (http_wasm_supports_streaming_response_cached !== undefined) { - return http_wasm_supports_streaming_response_cached; - } - http_wasm_supports_streaming_response_cached = typeof Response !== "undefined" && "body" in Response.prototype && typeof ReadableStream === "function"; - return http_wasm_supports_streaming_response_cached; + return typeof Response !== "undefined" && "body" in Response.prototype && typeof ReadableStream === "function"; } export function http_wasm_create_controller(): HttpController { diff --git a/src/mono/browser/runtime/interp-pgo.ts b/src/mono/browser/runtime/interp-pgo.ts index c4214b4fe5d6a3..79ea1e29ab5df1 100644 --- a/src/mono/browser/runtime/interp-pgo.ts +++ b/src/mono/browser/runtime/interp-pgo.ts @@ -193,6 +193,7 @@ export async function getCacheKey(prefix: string): Promise { delete inputs.forwardConsoleLogsToWS; delete inputs.diagnosticTracing; delete inputs.appendElementOnExit; + delete inputs.assertAfterExit; delete inputs.interopCleanupOnExit; delete inputs.dumpThreadsOnNonZeroExit; delete inputs.logExitCode; @@ -206,9 +207,6 @@ export async function getCacheKey(prefix: string): Promise { delete inputs.enableDownloadRetry; delete inputs.extensions; delete inputs.runtimeId; - delete inputs.mainThreadingMode; - delete inputs.jsThreadBlockingMode; - delete inputs.jsThreadInteropMode; inputs.GitHash = loaderHelpers.gitHash; inputs.ProductVersion = ProductVersion; diff --git a/src/mono/browser/runtime/invoke-cs.ts b/src/mono/browser/runtime/invoke-cs.ts index 893245041d27ad..287ee5ca6f6812 100644 --- a/src/mono/browser/runtime/invoke-cs.ts +++ b/src/mono/browser/runtime/invoke-cs.ts @@ -100,9 +100,7 @@ export function mono_wasm_bind_cs_function(method: MonoMethod, assemblyName: str // in Release configuration, it would be a trimmed by rollup if (BuildConfiguration === "Debug" && !runtimeHelpers.cspPolicy) { try { - const url = `//# sourceURL=https://dotnet/JSExport/${methodName}`; - const body = `return (function JSExport_${methodName}(){ return fn.apply(this, arguments)});`; - bound_fn = new Function("fn", url + "\r\n" + body)(bound_fn); + bound_fn = new Function("fn", "return (function JSExport_" + methodName + "(){ return fn.apply(this, arguments)});")(bound_fn); } catch (ex) { runtimeHelpers.cspPolicy = true; @@ -125,8 +123,7 @@ function bind_fn_0V(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 2; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(2); // call C# side invoke_sync_jsexport(method, args); } finally { @@ -147,8 +144,7 @@ function bind_fn_1V(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); marshaler1(args, arg1); // call C# side @@ -172,8 +168,7 @@ function bind_fn_1R(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); marshaler1(args, arg1); // call C# side @@ -200,15 +195,14 @@ function bind_fn_1RA(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); marshaler1(args, arg1); // pre-allocate the promise let promise = res_converter(args); // call C# side - invoke_async_jsexport(runtimeHelpers.managedThreadTID, method, args, size); + invoke_async_jsexport(method, args, 3); // in case the C# side returned synchronously promise = end_marshal_task_to_js(args, undefined, promise); @@ -234,8 +228,7 @@ function bind_fn_2R(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 4; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(4); marshaler1(args, arg1); marshaler2(args, arg2); @@ -264,8 +257,7 @@ function bind_fn_2RA(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 4; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(4); marshaler1(args, arg1); marshaler2(args, arg2); @@ -273,7 +265,7 @@ function bind_fn_2RA(closure: BindingClosure) { let promise = res_converter(args); // call C# side - invoke_async_jsexport(runtimeHelpers.managedThreadTID, method, args, size); + invoke_async_jsexport(method, args, 4); // in case the C# side returned synchronously promise = end_marshal_task_to_js(args, undefined, promise); @@ -301,8 +293,7 @@ function bind_fn(closure: BindingClosure) { mono_assert(!WasmEnableThreads || !closure.isDisposed, "The function was already disposed"); const sp = Module.stackSave(); try { - const size = 2 + args_count; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(2 + args_count); for (let index = 0; index < args_count; index++) { const marshaler = arg_marshalers[index]; if (marshaler) { @@ -318,13 +309,13 @@ function bind_fn(closure: BindingClosure) { // call C# side if (is_async) { - invoke_async_jsexport(runtimeHelpers.managedThreadTID, method, args, size); + invoke_async_jsexport(method, args, 2 + args_count); // in case the C# side returned synchronously js_result = end_marshal_task_to_js(args, undefined, js_result); } else if (is_discard_no_wait) { // call C# side, fire and forget - invoke_async_jsexport(runtimeHelpers.managedThreadTID, method, args, size); + invoke_async_jsexport(method, args, 2 + args_count); } else { invoke_sync_jsexport(method, args); diff --git a/src/mono/browser/runtime/invoke-js.ts b/src/mono/browser/runtime/invoke-js.ts index cb26b89a73b79f..75d7c96b9c737e 100644 --- a/src/mono/browser/runtime/invoke-js.ts +++ b/src/mono/browser/runtime/invoke-js.ts @@ -5,7 +5,7 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; import BuildConfiguration from "consts:configuration"; import { marshal_exception_to_cs, bind_arg_marshal_to_cs } from "./marshal-to-cs"; -import { get_signature_argument_count, bound_js_function_symbol, get_sig, get_signature_version, get_signature_type, imported_js_function_symbol, get_signature_handle, get_signature_function_name, get_signature_module_name, is_receiver_should_free, get_caller_native_tid } from "./marshal"; +import { get_signature_argument_count, bound_js_function_symbol, get_sig, get_signature_version, get_signature_type, imported_js_function_symbol, get_signature_handle, get_signature_function_name, get_signature_module_name, is_receiver_should_free } from "./marshal"; import { setI32_unchecked, receiveWorkerHeapViews, forceThreadMemoryViewRefresh } from "./memory"; import { stringToMonoStringRoot } from "./strings"; import { MonoObject, MonoObjectRef, JSFunctionSignature, JSMarshalerArguments, WasmRoot, BoundMarshalerToJs, JSFnHandle, BoundMarshalerToCs, JSHandle, MarshalerType } from "./types/internal"; @@ -53,7 +53,6 @@ export function mono_wasm_invoke_jsimport(signature: JSFunctionSignature, args: export function mono_wasm_invoke_jsimport_ST(function_handle: JSFnHandle, args: JSMarshalerArguments): void { if (WasmEnableThreads) return; - loaderHelpers.assert_runtime_running(); const bound_fn = js_import_wrapper_by_fn_handle[function_handle]; mono_assert(bound_fn, () => `Imported function handle expected ${function_handle}`); bound_fn(args); @@ -133,17 +132,31 @@ function bind_js_import(signature: JSFunctionSignature): Function { } } + // this is just to make debugging easier by naming the function in the stack trace. + // It's not CSP compliant and possibly not performant, that's why it's only enabled in debug builds + // in Release configuration, it would be a trimmed by rollup + if (BuildConfiguration === "Debug" && !runtimeHelpers.cspPolicy) { + try { + bound_fn = new Function("fn", "return (function JSImport_" + js_function_name.replaceAll(".", "_") + "(){ return fn.apply(this, arguments)});")(bound_fn); + } + catch (ex) { + runtimeHelpers.cspPolicy = true; + } + } + function async_bound_fn(args: JSMarshalerArguments): void { - forceThreadMemoryViewRefresh(); + if (WasmEnableThreads) { + forceThreadMemoryViewRefresh(); + } bound_fn(args); } - function sync_bound_fn(args: JSMarshalerArguments): void { const previous = runtimeHelpers.isPendingSynchronousCall; try { - forceThreadMemoryViewRefresh(); - const caller_tid = get_caller_native_tid(args); - runtimeHelpers.isPendingSynchronousCall = runtimeHelpers.managedThreadTID === caller_tid; + runtimeHelpers.isPendingSynchronousCall = true; + if (WasmEnableThreads) { + forceThreadMemoryViewRefresh(); + } bound_fn(args); } finally { @@ -151,29 +164,12 @@ function bind_js_import(signature: JSFunctionSignature): Function { } } - let wrapped_fn: WrappedJSFunction = bound_fn; - if (WasmEnableThreads) { - if (is_async || is_discard_no_wait) { - wrapped_fn = async_bound_fn; - } - else { - wrapped_fn = sync_bound_fn; - } + let wrapped_fn: WrappedJSFunction; + if (is_async || is_discard_no_wait) { + wrapped_fn = async_bound_fn; } - - // this is just to make debugging easier by naming the function in the stack trace. - // It's not CSP compliant and possibly not performant, that's why it's only enabled in debug builds - // in Release configuration, it would be a trimmed by rollup - if (BuildConfiguration === "Debug" && !runtimeHelpers.cspPolicy) { - try { - const fname = js_function_name.replaceAll(".", "_"); - const url = `//# sourceURL=https://dotnet/JSImport/${fname}`; - const body = `return (function JSImport_${fname}(){ return fn.apply(this, arguments)});`; - wrapped_fn = new Function("fn", url + "\r\n" + body)(wrapped_fn); - } - catch (ex) { - runtimeHelpers.cspPolicy = true; - } + else { + wrapped_fn = sync_bound_fn; } (wrapped_fn)[imported_js_function_symbol] = closure; @@ -337,7 +333,6 @@ type BindingClosure = { } export function mono_wasm_invoke_js_function(bound_function_js_handle: JSHandle, args: JSMarshalerArguments): void { - loaderHelpers.assert_runtime_running(); const bound_fn = mono_wasm_get_jsobj_from_js_handle(bound_function_js_handle); mono_assert(bound_fn && typeof (bound_fn) === "function" && bound_fn[bound_js_function_symbol], () => `Bound function handle expected ${bound_function_js_handle}`); bound_fn(args); diff --git a/src/mono/browser/runtime/jiterpreter-enums.ts b/src/mono/browser/runtime/jiterpreter-enums.ts index 07a3f815b83127..1e65f7b3cec39e 100644 --- a/src/mono/browser/runtime/jiterpreter-enums.ts +++ b/src/mono/browser/runtime/jiterpreter-enums.ts @@ -44,8 +44,6 @@ export const enum JiterpMember { ClassRank, ClassElementClass, BoxedValueData, - BackwardBranchTaken, - BailoutOpcodeCount, } // keep in sync with jiterpreter.c, see mono_jiterp_write_number_unaligned diff --git a/src/mono/browser/runtime/jiterpreter-support.ts b/src/mono/browser/runtime/jiterpreter-support.ts index 1dbbac2eaaf649..998056d0aa145c 100644 --- a/src/mono/browser/runtime/jiterpreter-support.ts +++ b/src/mono/browser/runtime/jiterpreter-support.ts @@ -12,7 +12,7 @@ import { localHeapViewU8, localHeapViewU32 } from "./memory"; import { utf8ToString } from "./strings"; import { JiterpNumberMode, BailoutReason, JiterpreterTable, - JiterpCounter, JiterpMember, OpcodeInfoType + JiterpCounter, JiterpMember } from "./jiterpreter-enums"; export const maxFailures = 2, @@ -104,7 +104,6 @@ export class WasmBuilder { backBranchOffsets: Array = []; callHandlerReturnAddresses: Array = []; nextConstantSlot = 0; - backBranchTraceLevel = 0; compressImportNames = false; lockImports = false; @@ -1140,11 +1139,8 @@ class Cfg { backBranchTargets: Uint16Array | null = null; base!: MintOpcodePtr; ip!: MintOpcodePtr; - // The address of the prepare point entryIp!: MintOpcodePtr; exitIp!: MintOpcodePtr; - // The address of the first actual opcode in the trace - firstOpcodeIp!: MintOpcodePtr; lastSegmentStartIp!: MintOpcodePtr; lastSegmentEnd = 0; overheadBytes = 0; @@ -1152,7 +1148,7 @@ class Cfg { blockStack: Array = []; backDispatchOffsets: Array = []; dispatchTable = new Map(); - observedBackBranchTargets = new Set(); + observedBranchTargets = new Set(); trace = 0; constructor(builder: WasmBuilder) { @@ -1165,11 +1161,11 @@ class Cfg { this.startOfBody = startOfBody; this.backBranchTargets = backBranchTargets; this.base = this.builder.base; - this.ip = this.lastSegmentStartIp = this.firstOpcodeIp = this.builder.base; + this.ip = this.lastSegmentStartIp = this.builder.base; this.lastSegmentEnd = 0; this.overheadBytes = 10; // epilogue this.dispatchTable.clear(); - this.observedBackBranchTargets.clear(); + this.observedBranchTargets.clear(); this.trace = trace; this.backDispatchOffsets.length = 0; } @@ -1177,9 +1173,6 @@ class Cfg { // We have a header containing the table of locals and we need to preserve it entry(ip: MintOpcodePtr) { this.entryIp = ip; - // Skip over the enter opcode - const enterSizeU16 = cwraps.mono_jiterp_get_opcode_info(MintOpcode.MINT_TIER_ENTER_JITERPRETER, OpcodeInfoType.Length); - this.firstOpcodeIp = ip + (enterSizeU16 * 2); this.appendBlob(); mono_assert(this.segments.length === 1, "expected 1 segment"); mono_assert(this.segments[0].type === "blob", "expected blob"); @@ -1190,7 +1183,6 @@ class Cfg { this.overheadBytes += 20; // some extra padding for the dispatch br_table this.overheadBytes += this.backBranchTargets.length; // one byte for each target in the table } - return this.firstOpcodeIp; } appendBlob() { @@ -1220,9 +1212,7 @@ class Cfg { } branch(target: MintOpcodePtr, isBackward: boolean, branchType: CfgBranchType) { - if (isBackward) - this.observedBackBranchTargets.add(target); - + this.observedBranchTargets.add(target); this.appendBlob(); this.segments.push({ type: "branch", @@ -1234,21 +1224,21 @@ class Cfg { // some branches will generate bailouts instead so we allocate 4 bytes per branch // to try and balance this out and avoid underestimating too much this.overheadBytes += 4; // forward branches are a constant br + depth (optimally 2 bytes) - if (isBackward) { + // get_local + // i32_const 1 + // i32_store 0 0 // i32.const // set_local - this.overheadBytes += 4; + this.overheadBytes += 11; } - if (WasmEnableThreads) { - // Account for the size of the safepoint - if ( - (branchType === CfgBranchType.SafepointConditional) || - (branchType === CfgBranchType.SafepointUnconditional) - ) { - this.overheadBytes += 17; - } + // Account for the size of the safepoint + if ( + (branchType === CfgBranchType.SafepointConditional) || + (branchType === CfgBranchType.SafepointUnconditional) + ) { + this.overheadBytes += 17; } } @@ -1276,9 +1266,8 @@ class Cfg { // We wrap the entire trace in a loop that starts with a dispatch br_table in order to support // backwards branches. if (this.backBranchTargets) { - // unnecessary, the local is default initialized to zero - // this.builder.i32_const(0); - // this.builder.local("disp", WasmOpcode.set_local); + this.builder.i32_const(0); + this.builder.local("disp", WasmOpcode.set_local); this.builder.block(WasmValtype.void, WasmOpcode.loop); } @@ -1309,7 +1298,7 @@ class Cfg { const breakDepth = this.blockStack.indexOf(offset); if (breakDepth < 0) continue; - if (!this.observedBackBranchTargets.has(offset)) + if (!this.observedBranchTargets.has(offset)) continue; this.dispatchTable.set(offset, this.backDispatchOffsets.length + 1); @@ -1327,8 +1316,7 @@ class Cfg { mono_log_info(`Exactly one back dispatch offset and it was 0x${(this.backDispatchOffsets[0]).toString(16)}`); } - // if (disp) - // goto back_branch_target; + // if (disp) goto back_branch_target else fallthrough this.builder.local("disp"); this.builder.appendU8(WasmOpcode.br_if); this.builder.appendULeb(this.blockStack.indexOf(this.backDispatchOffsets[0])); @@ -1342,7 +1330,6 @@ class Cfg { // We wrap it in an additional block so we can have a trap for unexpected disp values this.builder.block(WasmValtype.void); this.builder.block(WasmValtype.void); - // switch (disp) { this.builder.local("disp"); this.builder.appendU8(WasmOpcode.br_table); @@ -1392,16 +1379,23 @@ class Cfg { case "branch": { const lookupTarget = segment.isBackward ? dispatchIp : segment.target; let indexInStack = this.blockStack.indexOf(lookupTarget), - successfulBackBranch = false, - disp : number | undefined = undefined; + successfulBackBranch = false; // Back branches will target the dispatcher loop so we need to update the dispatch index // which will be used by the loop dispatch br_table to jump to the correct location if (segment.isBackward) { if (this.dispatchTable.has(segment.target)) { - disp = this.dispatchTable.get(segment.target)!; + const disp = this.dispatchTable.get(segment.target)!; if (this.trace > 1) mono_log_info(`backward br from ${(segment.from).toString(16)} to ${(segment.target).toString(16)}: disp=${disp}`); + + // Set the back branch taken flag local so it will get flushed on monitoring exit + this.builder.i32_const(1); + this.builder.local("backbranched", WasmOpcode.set_local); + + // set the dispatch index for the br_table + this.builder.i32_const(disp); + this.builder.local("disp", WasmOpcode.set_local); successfulBackBranch = true; } else { if (this.trace > 0) @@ -1415,40 +1409,20 @@ class Cfg { switch (segment.branchType) { case CfgBranchType.SafepointUnconditional: append_safepoint(this.builder, segment.from); - if (disp !== undefined) { - this.builder.i32_const(disp); - this.builder.local("disp", WasmOpcode.set_local); - } this.builder.appendU8(WasmOpcode.br); break; case CfgBranchType.SafepointConditional: // Wrap the safepoint + branch in an if this.builder.block(WasmValtype.void, WasmOpcode.if_); append_safepoint(this.builder, segment.from); - if (disp !== undefined) { - this.builder.i32_const(disp); - this.builder.local("disp", WasmOpcode.set_local); - } this.builder.appendU8(WasmOpcode.br); offset = 1; break; case CfgBranchType.Unconditional: - if (disp !== undefined) { - this.builder.i32_const(disp); - this.builder.local("disp", WasmOpcode.set_local); - } this.builder.appendU8(WasmOpcode.br); break; case CfgBranchType.Conditional: - if (disp !== undefined) { - this.builder.block(WasmValtype.void, WasmOpcode.if_); - this.builder.i32_const(disp); - this.builder.local("disp", WasmOpcode.set_local); - offset = 1; - this.builder.appendU8(WasmOpcode.br); - } else { - this.builder.appendU8(WasmOpcode.br_if); - } + this.builder.appendU8(WasmOpcode.br_if); break; default: throw new Error("Unimplemented branch type"); @@ -1518,9 +1492,6 @@ export const _now = (globalThis.performance && globalThis.performance.now) let scratchBuffer: NativePointer = 0; export function append_safepoint(builder: WasmBuilder, ip: MintOpcodePtr) { - // safepoints are never triggered in a single-threaded build - if (!WasmEnableThreads) - return; // Check whether a safepoint is required builder.ptr_const(cwraps.mono_jiterp_get_polling_required_address()); builder.appendU8(WasmOpcode.i32_load); @@ -1546,32 +1517,20 @@ export function append_bailout(builder: WasmBuilder, ip: MintOpcodePtr, reason: // generate a bailout that is recorded for the monitoring phase as a possible early exit. export function append_exit(builder: WasmBuilder, ip: MintOpcodePtr, opcodeCounter: number, reason: BailoutReason) { - /* - * disp will always be nonzero once we've taken at least one backward branch. - * if (cinfo) { - * cinfo->backward_branch_taken = disp; - * if (opcodeCounter <= threshold) - * cinfo->opcode_count = opcodeCounter; - * } - */ - - builder.local("cinfo"); - builder.block(WasmValtype.void, WasmOpcode.if_); - - builder.local("cinfo"); - builder.local("disp"); - builder.appendU8(WasmOpcode.i32_store); - builder.appendMemarg(getMemberOffset(JiterpMember.BackwardBranchTaken), 0); - if (opcodeCounter <= (builder.options.monitoringLongDistance + 2)) { builder.local("cinfo"); builder.i32_const(opcodeCounter); builder.appendU8(WasmOpcode.i32_store); - builder.appendMemarg(getMemberOffset(JiterpMember.BailoutOpcodeCount), 0); + builder.appendMemarg(4, 0); // bailout_opcode_count + // flush the backward branch taken flag into the cinfo so that the monitoring phase + // knows we took a backward branch. this is unfortunate but unavoidable overhead + // we just make it a flag instead of an increment to reduce the cost + builder.local("cinfo"); + builder.local("backbranched"); + builder.appendU8(WasmOpcode.i32_store); + builder.appendMemarg(0, 0); // JiterpreterCallInfo.backward_branch_taken } - builder.endBlock(); - builder.ip_const(ip); if (builder.options.countBailouts) { builder.i32_const(builder.traceIndex); diff --git a/src/mono/browser/runtime/jiterpreter-tables.ts b/src/mono/browser/runtime/jiterpreter-tables.ts index a345844e9d9048..0ab6ce675267be 100644 --- a/src/mono/browser/runtime/jiterpreter-tables.ts +++ b/src/mono/browser/runtime/jiterpreter-tables.ts @@ -9,8 +9,16 @@ import { } from "./mintops"; export const ldcTable: { [opcode: number]: [WasmOpcode, number] } = { + [MintOpcode.MINT_LDC_I4_M1]: [WasmOpcode.i32_const, -1], [MintOpcode.MINT_LDC_I4_0]: [WasmOpcode.i32_const, 0], [MintOpcode.MINT_LDC_I4_1]: [WasmOpcode.i32_const, 1], + [MintOpcode.MINT_LDC_I4_2]: [WasmOpcode.i32_const, 2], + [MintOpcode.MINT_LDC_I4_3]: [WasmOpcode.i32_const, 3], + [MintOpcode.MINT_LDC_I4_4]: [WasmOpcode.i32_const, 4], + [MintOpcode.MINT_LDC_I4_5]: [WasmOpcode.i32_const, 5], + [MintOpcode.MINT_LDC_I4_6]: [WasmOpcode.i32_const, 6], + [MintOpcode.MINT_LDC_I4_7]: [WasmOpcode.i32_const, 7], + [MintOpcode.MINT_LDC_I4_8]: [WasmOpcode.i32_const, 8], }; // operator, loadOperator, storeOperator @@ -85,16 +93,6 @@ export const unopTable: { [opcode: number]: OpRec3 | undefined } = { [MintOpcode.MINT_CLZ_I8]: [WasmOpcode.i64_clz, WasmOpcode.i64_load, WasmOpcode.i64_store], [MintOpcode.MINT_CTZ_I8]: [WasmOpcode.i64_ctz, WasmOpcode.i64_load, WasmOpcode.i64_store], [MintOpcode.MINT_POPCNT_I8]: [WasmOpcode.i64_popcnt, WasmOpcode.i64_load, WasmOpcode.i64_store], - - [MintOpcode.MINT_ADD_I4_IMM2]: [WasmOpcode.i32_add, WasmOpcode.i32_load, WasmOpcode.i32_store], - [MintOpcode.MINT_MUL_I4_IMM2]: [WasmOpcode.i32_mul, WasmOpcode.i32_load, WasmOpcode.i32_store], - [MintOpcode.MINT_ADD_I8_IMM2]: [WasmOpcode.i64_add, WasmOpcode.i64_load, WasmOpcode.i64_store], - [MintOpcode.MINT_MUL_I8_IMM2]: [WasmOpcode.i64_mul, WasmOpcode.i64_load, WasmOpcode.i64_store], - - [MintOpcode.MINT_AND_I4_IMM]: [WasmOpcode.i32_and, WasmOpcode.i32_load, WasmOpcode.i32_store], - [MintOpcode.MINT_AND_I4_IMM2]: [WasmOpcode.i32_and, WasmOpcode.i32_load, WasmOpcode.i32_store], - [MintOpcode.MINT_OR_I4_IMM]: [WasmOpcode.i32_or, WasmOpcode.i32_load, WasmOpcode.i32_store], - [MintOpcode.MINT_OR_I4_IMM2]: [WasmOpcode.i32_or, WasmOpcode.i32_load, WasmOpcode.i32_store], }; // HACK: Generating correct wasm for these is non-trivial so we hand them off to C. diff --git a/src/mono/browser/runtime/jiterpreter-trace-generator.ts b/src/mono/browser/runtime/jiterpreter-trace-generator.ts index 1a629223b3a96b..a2edb883820281 100644 --- a/src/mono/browser/runtime/jiterpreter-trace-generator.ts +++ b/src/mono/browser/runtime/jiterpreter-trace-generator.ts @@ -37,7 +37,7 @@ import { emitPadding, traceBranchDisplacements, traceEip, nullCheckValidation, traceNullCheckOptimizations, - nullCheckCaching, defaultTraceBackBranches, + nullCheckCaching, traceBackBranches, maxCallHandlerReturnAddresses, mostRecentOptions, @@ -142,8 +142,6 @@ export function generateBackwardBranchTable( // IP of the start of the trace in U16s, relative to startOfBody. const rbase16 = (ip - startOfBody) / 2; - // FIXME: This will potentially scan the entire method and record branches that won't - // ever run since the trace compilation will end before we reach them. while (ip < endOfBody) { // IP of the current opcode in U16s, relative to startOfBody. This is what the back branch table uses const rip16 = (ip - startOfBody) / 2; @@ -168,24 +166,17 @@ export function generateBackwardBranchTable( break; } - // Only record *backward* branches - // We will filter this down further in the Cfg because it takes note of which branches it sees, - // but it is also beneficial to have a null table (further down) due to seeing no potential - // back branch targets at all, as it allows the Cfg to skip additional code generation entirely - // if it knows there will never be any backwards branches in a given trace - if (displacement < 0) { - const rtarget16 = rip16 + (displacement); - if (rtarget16 < 0) { - mono_log_info(`opcode @${ip}'s displacement of ${displacement} goes before body: ${rtarget16}. aborting backbranch table generation`); - break; - } - - // If the relative target is before the start of the trace, don't record it. - // The trace will be unable to successfully branch to it so it would just make the table bigger. - if (rtarget16 >= rbase16) - table.push(rtarget16); + const rtarget16 = rip16 + (displacement); + if (rtarget16 < 0) { + mono_log_info(`opcode @${ip}'s displacement of ${displacement} goes before body: ${rtarget16}. aborting backbranch table generation`); + break; } + // If the relative target is before the start of the trace, don't record it. + // The trace will be unable to successfully branch to it so it would just make the table bigger. + if (rtarget16 >= rbase16) + table.push(rtarget16); + switch (opcode) { case MintOpcode.MINT_CALL_HANDLER: case MintOpcode.MINT_CALL_HANDLER_S: @@ -219,17 +210,14 @@ export function generateWasmBody( let result = 0, prologueOpcodeCounter = 0, conditionalOpcodeCounter = 0; - eraseInferredState(); - // If a trace is instrumented, also activate back branch tracing - builder.backBranchTraceLevel = instrumentedTraceId - ? 2 - : defaultTraceBackBranches; + // Skip over the enter opcode + const enterSizeU16 = cwraps.mono_jiterp_get_opcode_info(MintOpcode.MINT_TIER_ENTER_JITERPRETER, OpcodeInfoType.Length); + ip += (enterSizeU16 * 2); + let rip = ip; - // Record the address of our prepare_jiterpreter opcode as the entry point, not the opcode after it. - // Some back-branches will target prepare_jiterpreter directly, and we need them to work. - let rip = builder.cfg.entry(ip); + builder.cfg.entry(ip); while (ip) { // This means some code went 'ip = abort; continue' @@ -304,7 +292,7 @@ export function generateWasmBody( // We record the offset of each backward branch we encounter, so that later branch // opcodes know that it's available by branching to the top of the dispatch loop if (isBackBranchTarget) { - if (builder.backBranchTraceLevel > 1) + if (traceBackBranches > 1) mono_log_info(`${traceName} recording back branch target 0x${(ip).toString(16)}`); builder.backBranchOffsets.push(ip); } @@ -381,20 +369,7 @@ export function generateWasmBody( builder.callImport("localloc"); break; } - case MintOpcode.MINT_ZEROBLK: { - // dest - append_ldloc(builder, getArgU16(ip, 1), WasmOpcode.i32_load); - // value - builder.i32_const(0); - // count - append_ldloc(builder, getArgU16(ip, 2), WasmOpcode.i32_load); - // memset - builder.appendU8(WasmOpcode.PREFIX_sat); - builder.appendU8(11); - builder.appendU8(0); - break; - } - case MintOpcode.MINT_ZEROBLK_IMM: { + case MintOpcode.MINT_INITOBJ: { append_ldloc(builder, getArgU16(ip, 1), WasmOpcode.i32_load); append_memset_dest(builder, 0, getArgU16(ip, 2)); break; @@ -1528,7 +1503,7 @@ export function generateWasmBody( } else ip = abort; } else if ( - (opcode >= MintOpcode.MINT_LDC_I4_0) && + (opcode >= MintOpcode.MINT_LDC_I4_M1) && (opcode <= MintOpcode.MINT_LDC_R8) ) { if (!emit_ldc(builder, ip, opcode)) @@ -2613,8 +2588,6 @@ function emit_unop(builder: WasmBuilder, ip: MintOpcodePtr, opcode: MintOpcode): case MintOpcode.MINT_ADD_I4_IMM: case MintOpcode.MINT_MUL_I4_IMM: - case MintOpcode.MINT_AND_I4_IMM: - case MintOpcode.MINT_OR_I4_IMM: case MintOpcode.MINT_SHL_I4_IMM: case MintOpcode.MINT_SHR_I4_IMM: case MintOpcode.MINT_SHR_UN_I4_IMM: @@ -2624,14 +2597,6 @@ function emit_unop(builder: WasmBuilder, ip: MintOpcodePtr, opcode: MintOpcode): builder.i32_const(getArgI16(ip, 3)); break; - case MintOpcode.MINT_ADD_I4_IMM2: - case MintOpcode.MINT_MUL_I4_IMM2: - case MintOpcode.MINT_AND_I4_IMM2: - case MintOpcode.MINT_OR_I4_IMM2: - append_ldloc(builder, getArgU16(ip, 2), loadOp); - builder.i32_const(getArgI32(ip, 3)); - break; - case MintOpcode.MINT_ADD_I8_IMM: case MintOpcode.MINT_MUL_I8_IMM: case MintOpcode.MINT_SHL_I8_IMM: @@ -2643,12 +2608,6 @@ function emit_unop(builder: WasmBuilder, ip: MintOpcodePtr, opcode: MintOpcode): builder.i52_const(getArgI16(ip, 3)); break; - case MintOpcode.MINT_ADD_I8_IMM2: - case MintOpcode.MINT_MUL_I8_IMM2: - append_ldloc(builder, getArgU16(ip, 2), loadOp); - builder.i52_const(getArgI32(ip, 3)); - break; - default: append_ldloc(builder, getArgU16(ip, 2), loadOp); break; @@ -2742,8 +2701,8 @@ function emit_branch( // We found a backward branch target we can branch to, so we branch out // to the top of the loop body // append_safepoint(builder, ip); - if (builder.backBranchTraceLevel > 1) - mono_log_info(`0x${(ip).toString(16)} performing backward branch to 0x${destination.toString(16)}`); + if (traceBackBranches > 1) + mono_log_info(`performing backward branch to 0x${destination.toString(16)}`); if (isCallHandler) append_call_handler_store_ret_ip(builder, ip, frame, opcode); builder.cfg.branch(destination, true, CfgBranchType.Unconditional); @@ -2751,9 +2710,9 @@ function emit_branch( return true; } else { if (destination < builder.cfg.entryIp) { - if ((builder.backBranchTraceLevel > 1) || (builder.cfg.trace > 1)) - mono_log_info(`0x${(ip).toString(16)} ${getOpcodeName(opcode)} target 0x${destination.toString(16)} before start of trace`); - } else if ((builder.backBranchTraceLevel > 0) || (builder.cfg.trace > 0)) + if ((traceBackBranches > 1) || (builder.cfg.trace > 1)) + mono_log_info(`${getOpcodeName(opcode)} target 0x${destination.toString(16)} before start of trace`); + } else if ((traceBackBranches > 0) || (builder.cfg.trace > 0)) mono_log_info(`0x${(ip).toString(16)} ${getOpcodeName(opcode)} target 0x${destination.toString(16)} not found in list ` + builder.backBranchOffsets.map(bbo => "0x" + (bbo).toString(16)).join(", ") ); @@ -2823,15 +2782,15 @@ function emit_branch( if (builder.backBranchOffsets.indexOf(destination) >= 0) { // We found a backwards branch target we can reach via our outer trace loop, so // we update eip and branch out to the top of the loop block - if (builder.backBranchTraceLevel > 1) - mono_log_info(`0x${(ip).toString(16)} performing conditional backward branch to 0x${destination.toString(16)}`); + if (traceBackBranches > 1) + mono_log_info(`performing conditional backward branch to 0x${destination.toString(16)}`); builder.cfg.branch(destination, true, isSafepoint ? CfgBranchType.SafepointConditional : CfgBranchType.Conditional); modifyCounter(JiterpCounter.BackBranchesEmitted, 1); } else { if (destination < builder.cfg.entryIp) { - if ((builder.backBranchTraceLevel > 1) || (builder.cfg.trace > 1)) - mono_log_info(`0x${(ip).toString(16)} ${getOpcodeName(opcode)} target 0x${destination.toString(16)} before start of trace`); - } else if ((builder.backBranchTraceLevel > 0) || (builder.cfg.trace > 0)) + if ((traceBackBranches > 1) || (builder.cfg.trace > 1)) + mono_log_info(`${getOpcodeName(opcode)} target 0x${destination.toString(16)} before start of trace`); + } else if ((traceBackBranches > 0) || (builder.cfg.trace > 0)) mono_log_info(`0x${(ip).toString(16)} ${getOpcodeName(opcode)} target 0x${destination.toString(16)} not found in list ` + builder.backBranchOffsets.map(bbo => "0x" + (bbo).toString(16)).join(", ") ); diff --git a/src/mono/browser/runtime/jiterpreter.ts b/src/mono/browser/runtime/jiterpreter.ts index 188ead0c51e99c..9d47c2e39b1eab 100644 --- a/src/mono/browser/runtime/jiterpreter.ts +++ b/src/mono/browser/runtime/jiterpreter.ts @@ -56,7 +56,7 @@ export const traceNullCheckOptimizations = false, // Print diagnostic information when generating backward branches // 1 = failures only, 2 = full detail - defaultTraceBackBranches = 0, + traceBackBranches = 0, // Enable generating conditional backward branches for ENDFINALLY opcodes if we saw some CALL_HANDLER // opcodes previously, up to this many potential return addresses. If a trace contains more potential // return addresses than this we will not emit code for the ENDFINALLY opcode @@ -779,6 +779,7 @@ function generate_wasm( "math_rhs64": WasmValtype.i64, "temp_f32": WasmValtype.f32, "temp_f64": WasmValtype.f64, + "backbranched": WasmValtype.i32, }; if (builder.options.enableSimd) { traceLocals["v128_zero"] = WasmValtype.v128; @@ -890,7 +891,7 @@ function generate_wasm( // suites or benchmarks if you've enabled stats const tracesCompiled = getCounter(JiterpCounter.TracesCompiled); if (builder.options.enableStats && tracesCompiled && (tracesCompiled % autoDumpInterval) === 0) - jiterpreter_dump_stats(true); + jiterpreter_dump_stats(false, true); return idx; } catch (exc: any) { @@ -1031,7 +1032,7 @@ export function mono_interp_tier_prepare_jiterpreter( const threshold = (ip - startOfBody) / 2; let foundReachableBranchTarget = false; for (let i = 0; i < backwardBranchTable.length; i++) { - if (backwardBranchTable[i] >= threshold) { + if (backwardBranchTable[i] > threshold) { foundReachableBranchTarget = true; break; } @@ -1073,14 +1074,14 @@ export function mono_jiterp_free_method_data_js( mono_jiterp_free_method_data_jit_call(method); } -export function jiterpreter_dump_stats(concise?: boolean): void { +export function jiterpreter_dump_stats(b?: boolean, concise?: boolean) { if (!runtimeHelpers.runtimeReady) { return; } - if (!mostRecentOptions) + if (!mostRecentOptions || (b !== undefined)) mostRecentOptions = getOptions(); - if (!mostRecentOptions.enableStats) + if (!mostRecentOptions.enableStats && (b !== undefined)) return; const backBranchesEmitted = getCounter(JiterpCounter.BackBranchesEmitted), @@ -1242,4 +1243,10 @@ export function jiterpreter_dump_stats(concise?: boolean): void { for (const k in simdFallbackCounters) mono_log_info(`// simd ${k}: ${simdFallbackCounters[k]} fallback insn(s)`); + + if ((typeof (globalThis.setTimeout) === "function") && (b !== undefined)) + setTimeout( + () => jiterpreter_dump_stats(b), + 15000 + ); } diff --git a/src/mono/browser/runtime/loader/assets.ts b/src/mono/browser/runtime/loader/assets.ts index 2416f931b52f1d..49d2f0f8ac6d8c 100644 --- a/src/mono/browser/runtime/loader/assets.ts +++ b/src/mono/browser/runtime/loader/assets.ts @@ -737,6 +737,7 @@ export async function streamingCompileWasm() { loaderHelpers.wasmCompilePromise.promise_control.reject(err); } } + export function preloadWorkers() { if (!WasmEnableThreads) return; const jsModuleWorker = resolve_single_asset_path("js-module-threads"); diff --git a/src/mono/browser/runtime/loader/config.ts b/src/mono/browser/runtime/loader/config.ts index 07b7750a2a2c65..8aec3557437212 100644 --- a/src/mono/browser/runtime/loader/config.ts +++ b/src/mono/browser/runtime/loader/config.ts @@ -4,15 +4,14 @@ import BuildConfiguration from "consts:configuration"; import WasmEnableThreads from "consts:wasmEnableThreads"; -import { MainThreadingMode, type DotnetModuleInternal, type MonoConfigInternal, JSThreadBlockingMode, JSThreadInteropMode } from "../types/internal"; +import type { DotnetModuleInternal, MonoConfigInternal } from "../types/internal"; import type { DotnetModuleConfig, MonoConfig, ResourceGroups, ResourceList } from "../types"; -import { exportedRuntimeAPI, loaderHelpers, runtimeHelpers } from "./globals"; +import { ENVIRONMENT_IS_WEB, exportedRuntimeAPI, loaderHelpers, runtimeHelpers } from "./globals"; import { mono_log_error, mono_log_debug } from "./logging"; import { importLibraryInitializers, invokeLibraryInitializers } from "./libraryInitializers"; import { mono_exit } from "./exit"; import { makeURLAbsoluteWithApplicationBase } from "./polyfills"; import { appendUniqueQuery } from "./assets"; -import { mono_log_warn } from "./logging"; export function deep_merge_config(target: MonoConfigInternal, source: MonoConfigInternal): MonoConfigInternal { // no need to merge the same object @@ -178,6 +177,8 @@ export function normalizeConfig() { } } + loaderHelpers.assertAfterExit = config.assertAfterExit = config.assertAfterExit || !ENVIRONMENT_IS_WEB; + if (config.debugLevel === undefined && BuildConfiguration === "Debug") { config.debugLevel = -1; } @@ -187,52 +188,14 @@ export function normalizeConfig() { } // ActiveIssue https://github.com/dotnet/runtime/issues/75602 - if (WasmEnableThreads) { - - if (!Number.isInteger(config.pthreadPoolInitialSize)) { - config.pthreadPoolInitialSize = 7; - } - if (!Number.isInteger(config.pthreadPoolUnusedSize)) { - config.pthreadPoolUnusedSize = 3; - } - if (!Number.isInteger(config.finalizerThreadStartDelayMs)) { - config.finalizerThreadStartDelayMs = 200; - } - if (config.mainThreadingMode == undefined) { - config.mainThreadingMode = MainThreadingMode.DeputyAndIOThreads; - } - if (config.jsThreadBlockingMode == undefined) { - config.jsThreadBlockingMode = JSThreadBlockingMode.AllowBlockingWaitInAsyncCode; - } - if (config.jsThreadInteropMode == undefined) { - config.jsThreadInteropMode = JSThreadInteropMode.SimpleSynchronousJSInterop; - } - let validModes = false; - if (config.mainThreadingMode == MainThreadingMode.DeputyThread - && config.jsThreadBlockingMode == JSThreadBlockingMode.NoBlockingWait - && config.jsThreadInteropMode == JSThreadInteropMode.SimpleSynchronousJSInterop - ) { - validModes = true; - } - else if (config.mainThreadingMode == MainThreadingMode.DeputyAndIOThreads - && config.jsThreadBlockingMode == JSThreadBlockingMode.AllowBlockingWaitInAsyncCode - && config.jsThreadInteropMode == JSThreadInteropMode.SimpleSynchronousJSInterop - ) { - validModes = true; - } - else if (config.mainThreadingMode == MainThreadingMode.DeputyThread - && config.jsThreadBlockingMode == JSThreadBlockingMode.AllowBlockingWait - && config.jsThreadInteropMode == JSThreadInteropMode.SimpleSynchronousJSInterop - ) { - validModes = true; - } - if (!validModes) { - mono_log_warn("Unsupported threading configuration", { - mainThreadingMode: config.mainThreadingMode, - jsThreadBlockingMode: config.jsThreadBlockingMode, - jsThreadInteropMode: config.jsThreadInteropMode - }); - } + if (WasmEnableThreads && !Number.isInteger(config.pthreadPoolInitialSize)) { + config.pthreadPoolInitialSize = 7; + } + if (WasmEnableThreads && !Number.isInteger(config.pthreadPoolUnusedSize)) { + config.pthreadPoolUnusedSize = 3; + } + if (WasmEnableThreads && !Number.isInteger(config.finalizerThreadStartDelayMs)) { + config.finalizerThreadStartDelayMs = 200; } // this is how long the Mono GC will try to wait for all threads to be suspended before it gives up and aborts the process diff --git a/src/mono/browser/runtime/loader/exit.ts b/src/mono/browser/runtime/loader/exit.ts index 1f4ad75f52faab..7f62ef7b560666 100644 --- a/src/mono/browser/runtime/loader/exit.ts +++ b/src/mono/browser/runtime/loader/exit.ts @@ -15,11 +15,14 @@ export function is_runtime_running() { } export function assert_runtime_running() { - mono_assert(!is_exited(), () => `.NET runtime already exited with ${loaderHelpers.exitCode} ${loaderHelpers.exitReason}. You can use runtime.runMain() which doesn't exit the runtime.`); - if (WasmEnableThreads && ENVIRONMENT_IS_WORKER) { - mono_assert(runtimeHelpers.runtimeReady, "The WebWorker is not attached to the runtime. See https://github.com/dotnet/runtime/blob/main/src/mono/wasm/threads.md#JS-interop-on-dedicated-threads"); + if (!is_exited()) { + if (WasmEnableThreads && ENVIRONMENT_IS_WORKER) { + mono_assert(runtimeHelpers.runtimeReady, "The WebWorker is not attached to the runtime. See https://github.com/dotnet/runtime/blob/main/src/mono/wasm/threads.md#JS-interop-on-dedicated-threads"); + } else { + mono_assert(runtimeHelpers.runtimeReady, ".NET runtime didn't start yet. Please call dotnet.create() first."); + } } else { - mono_assert(runtimeHelpers.runtimeReady, ".NET runtime didn't start yet. Please call dotnet.create() first."); + mono_assert(!loaderHelpers.assertAfterExit, () => `.NET runtime already exited with ${loaderHelpers.exitCode} ${loaderHelpers.exitReason}. You can use runtime.runMain() which doesn't exit the runtime.`); } } diff --git a/src/mono/browser/runtime/loader/globals.ts b/src/mono/browser/runtime/loader/globals.ts index 940051627f3046..7e9708845e65b0 100644 --- a/src/mono/browser/runtime/loader/globals.ts +++ b/src/mono/browser/runtime/loader/globals.ts @@ -86,6 +86,7 @@ export function setLoaderGlobals( maxParallelDownloads: 16, enableDownloadRetry: true, + assertAfterExit: !ENVIRONMENT_IS_WEB, _loaded_files: [], loadedFiles: [], diff --git a/src/mono/browser/runtime/loader/run.ts b/src/mono/browser/runtime/loader/run.ts index e0c9bd3cded56f..730a692b4ebf5b 100644 --- a/src/mono/browser/runtime/loader/run.ts +++ b/src/mono/browser/runtime/loader/run.ts @@ -138,6 +138,19 @@ export class HostBuilder implements DotnetHostBuilder { } } + // internal + withAssertAfterExit(): DotnetHostBuilder { + try { + deep_merge_config(monoConfig, { + assertAfterExit: true + }); + return this; + } catch (err) { + mono_exit(1, err); + throw err; + } + } + // internal // todo fallback later by debugLevel withWaitingForDebugger(level: number): DotnetHostBuilder { diff --git a/src/mono/browser/runtime/managed-exports.ts b/src/mono/browser/runtime/managed-exports.ts index df99745d6af376..c8aabae88e37b7 100644 --- a/src/mono/browser/runtime/managed-exports.ts +++ b/src/mono/browser/runtime/managed-exports.ts @@ -3,18 +3,17 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; -import { GCHandle, GCHandleNull, JSMarshalerArguments, JSThreadInteropMode, MarshalerToCs, MarshalerToJs, MarshalerType, MonoMethod, PThreadPtr } from "./types/internal"; -import cwraps, { threads_c_functions as twraps } from "./cwraps"; +import { GCHandle, GCHandleNull, JSMarshalerArguments, MarshalerToCs, MarshalerToJs, MarshalerType, MonoMethod } from "./types/internal"; +import cwraps from "./cwraps"; import { runtimeHelpers, Module, loaderHelpers, mono_assert } from "./globals"; -import { JavaScriptMarshalerArgSize, alloc_stack_frame, get_arg, get_arg_gc_handle, is_args_exception, set_arg_i32, set_arg_intptr, set_arg_type, set_gc_handle, set_receiver_should_free } from "./marshal"; +import { JavaScriptMarshalerArgSize, alloc_stack_frame, get_arg, get_arg_gc_handle, is_args_exception, set_arg_intptr, set_arg_type, set_gc_handle } from "./marshal"; import { marshal_array_to_cs, marshal_array_to_cs_impl, marshal_bool_to_cs, marshal_exception_to_cs, marshal_intptr_to_cs, marshal_string_to_cs } from "./marshal-to-cs"; import { marshal_int32_to_js, end_marshal_task_to_js, marshal_string_to_js, begin_marshal_task_to_js, marshal_exception_to_js } from "./marshal-to-js"; -import { do_not_force_dispose, is_gcv_handle } from "./gc-handles"; +import { do_not_force_dispose } from "./gc-handles"; import { assert_c_interop, assert_js_interop } from "./invoke-js"; -import { monoThreadInfo, mono_wasm_main_thread_ptr } from "./pthreads"; -import { _zero_region, copyBytes } from "./memory"; +import { mono_wasm_main_thread_ptr } from "./pthreads"; +import { _zero_region } from "./memory"; import { stringToUTF8Ptr } from "./strings"; -import { mono_log_debug } from "./logging"; const managedExports: ManagedExports = {} as any; @@ -48,8 +47,7 @@ export function call_entry_point(main_assembly_name: string, program_args: strin loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 5; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(5); const res = get_arg(args, 1); const arg1 = get_arg(args, 2); const arg2 = get_arg(args, 3); @@ -62,7 +60,7 @@ export function call_entry_point(main_assembly_name: string, program_args: strin // because this is async, we could pre-allocate the promise let promise = begin_marshal_task_to_js(res, MarshalerType.TaskPreCreated, marshal_int32_to_js); - invoke_async_jsexport(runtimeHelpers.managedThreadTID, managedExports.CallEntrypoint, args, size); + invoke_async_jsexport(managedExports.CallEntrypoint, args, 5); // in case the C# side returned synchronously promise = end_marshal_task_to_js(args, marshal_int32_to_js, promise); @@ -80,11 +78,9 @@ export function call_entry_point(main_assembly_name: string, program_args: strin // the marshaled signature is: void LoadSatelliteAssembly(byte[] dll) export function load_satellite_assembly(dll: Uint8Array): void { - loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); const arg1 = get_arg(args, 2); set_arg_type(arg1, MarshalerType.Array); marshal_array_to_cs(arg1, dll, MarshalerType.Byte); @@ -96,11 +92,9 @@ export function load_satellite_assembly(dll: Uint8Array): void { // the marshaled signature is: void LoadLazyAssembly(byte[] dll, byte[] pdb) export function load_lazy_assembly(dll: Uint8Array, pdb: Uint8Array | null): void { - loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 4; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(4); const arg1 = get_arg(args, 2); const arg2 = get_arg(args, 3); set_arg_type(arg1, MarshalerType.Array); @@ -119,43 +113,40 @@ export function release_js_owned_object_by_gc_handle(gc_handle: GCHandle) { loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); const arg1 = get_arg(args, 2); set_arg_type(arg1, MarshalerType.Object); set_gc_handle(arg1, gc_handle); - if (!WasmEnableThreads || is_gcv_handle(gc_handle) || !monoThreadInfo.isUI) { - // this must stay synchronous for free_gcv_handle sake, to not use-after-free - // also on JSWebWorker, because the message could arrive after the worker is terminated and the GCHandle of JSProxyContext is already freed - invoke_sync_jsexport(managedExports.ReleaseJSOwnedObjectByGCHandle, args); - } else { - invoke_async_jsexport(runtimeHelpers.ioThreadTID, managedExports.ReleaseJSOwnedObjectByGCHandle, args, size); - } + // this must stay synchronous for free_gcv_handle sake + invoke_sync_jsexport(managedExports.ReleaseJSOwnedObjectByGCHandle, args); } finally { Module.stackRestore(sp); } } // the marshaled signature is: void CompleteTask(GCHandle holder, Exception? exceptionResult, T? result) -export function complete_task(holder_gc_handle: GCHandle, error?: any, data?: any, res_converter?: MarshalerToCs) { +export function complete_task(holder_gc_handle: GCHandle, isCanceling: boolean, error?: any, data?: any, res_converter?: MarshalerToCs) { loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 5; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(5); + const res = get_arg(args, 1); const arg1 = get_arg(args, 2); set_arg_type(arg1, MarshalerType.Object); set_gc_handle(arg1, holder_gc_handle); const arg2 = get_arg(args, 3); if (error) { marshal_exception_to_cs(arg2, error); + if (isCanceling) { + set_arg_type(res, MarshalerType.Discard); + } } else { set_arg_type(arg2, MarshalerType.None); const arg3 = get_arg(args, 4); mono_assert(res_converter, "res_converter missing"); res_converter(arg3, data); } - invoke_async_jsexport(runtimeHelpers.ioThreadTID, managedExports.CompleteTask, args, size); + invoke_async_jsexport(managedExports.CompleteTask, args, 4); } finally { Module.stackRestore(sp); } @@ -164,18 +155,9 @@ export function complete_task(holder_gc_handle: GCHandle, error?: any, data?: an // the marshaled signature is: TRes? CallDelegate(GCHandle callback, T1? arg1, T2? arg2, T3? arg3) export function call_delegate(callback_gc_handle: GCHandle, arg1_js: any, arg2_js: any, arg3_js: any, res_converter?: MarshalerToJs, arg1_converter?: MarshalerToCs, arg2_converter?: MarshalerToCs, arg3_converter?: MarshalerToCs) { loaderHelpers.assert_runtime_running(); - if (WasmEnableThreads) { - if (runtimeHelpers.config.jsThreadInteropMode == JSThreadInteropMode.NoSyncJSInterop) { - throw new Error("Cannot call synchronous C# methods."); - } - else if (runtimeHelpers.isPendingSynchronousCall) { - throw new Error("Cannot call synchronous C# method from inside a synchronous call to a JS method."); - } - } const sp = Module.stackSave(); try { - const size = 6; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(6); const arg1 = get_arg(args, 2); set_arg_type(arg1, MarshalerType.Object); @@ -211,8 +193,7 @@ export function get_managed_stack_trace(exception_gc_handle: GCHandle) { loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); const arg1 = get_arg(args, 2); set_arg_type(arg1, MarshalerType.Exception); @@ -226,26 +207,21 @@ export function get_managed_stack_trace(exception_gc_handle: GCHandle) { } } -// GCHandle InstallMainSynchronizationContext(nint jsNativeTID, JSThreadBlockingMode jsThreadBlockingMode, JSThreadInteropMode jsThreadInteropMode, MainThreadingMode mainThreadingMode) -export function install_main_synchronization_context(jsThreadBlockingMode: number, jsThreadInteropMode: number, mainThreadingMode: number): GCHandle { +// GCHandle InstallMainSynchronizationContext(nint jsNativeTID) +export function install_main_synchronization_context(): GCHandle { if (!WasmEnableThreads) return GCHandleNull; assert_c_interop(); + const sp = Module.stackSave(); try { // this block is like alloc_stack_frame() but without set_args_context() - const bytes = JavaScriptMarshalerArgSize * 6; + const bytes = JavaScriptMarshalerArgSize * 3; const args = Module.stackAlloc(bytes) as any; _zero_region(args, bytes); const res = get_arg(args, 1); const arg1 = get_arg(args, 2); - const arg2 = get_arg(args, 3); - const arg3 = get_arg(args, 4); - const arg4 = get_arg(args, 5); set_arg_intptr(arg1, mono_wasm_main_thread_ptr() as any); - set_arg_i32(arg2, jsThreadBlockingMode); - set_arg_i32(arg3, jsThreadInteropMode); - set_arg_i32(arg4, mainThreadingMode); // this block is like invoke_sync_jsexport() but without assert_js_interop() cwraps.mono_wasm_invoke_jsexport(managedExports.InstallMainSynchronizationContext!, args); @@ -254,48 +230,45 @@ export function install_main_synchronization_context(jsThreadBlockingMode: numbe throw marshal_exception_to_js(exc); } return get_arg_gc_handle(res) as any; - } catch (e) { - mono_log_debug("install_main_synchronization_context failed", e); - throw e; + } finally { + Module.stackRestore(sp); } } -export function invoke_async_jsexport(managedTID: PThreadPtr, method: MonoMethod, args: JSMarshalerArguments, size: number): void { +export function invoke_async_jsexport(method: MonoMethod, args: JSMarshalerArguments, size: number): void { assert_js_interop(); - if (!WasmEnableThreads || runtimeHelpers.isManagedRunningOnCurrentThread) { + if (!WasmEnableThreads || runtimeHelpers.isCurrentThread) { cwraps.mono_wasm_invoke_jsexport(method, args as any); if (is_args_exception(args)) { const exc = get_arg(args, 0); throw marshal_exception_to_js(exc); } } else { + throw new Error("Should be unreachable until we implement deputy." + size); + /* set_receiver_should_free(args); const bytes = JavaScriptMarshalerArgSize * size; const cpy = Module._malloc(bytes) as any; copyBytes(args as any, cpy, bytes); - twraps.mono_wasm_invoke_jsexport_async_post(managedTID, method, cpy); + twraps.mono_wasm_invoke_jsexport_async_post(runtimeHelpers.managedThreadTID, method, cpy); + */ } } export function invoke_sync_jsexport(method: MonoMethod, args: JSMarshalerArguments): void { assert_js_interop(); - if (!WasmEnableThreads) { + if (!WasmEnableThreads || runtimeHelpers.isCurrentThread) { cwraps.mono_wasm_invoke_jsexport(method, args as any); } else { - if (runtimeHelpers.config.jsThreadInteropMode == JSThreadInteropMode.NoSyncJSInterop) { - throw new Error("Cannot call synchronous C# methods."); - } - else if (runtimeHelpers.isPendingSynchronousCall) { + throw new Error("Should be unreachable until we implement deputy."); + /* + if (!runtimeHelpers.isCurrentThread && runtimeHelpers.isPendingSynchronousCall) { throw new Error("Cannot call synchronous C# method from inside a synchronous call to a JS method."); } - if (runtimeHelpers.isManagedRunningOnCurrentThread) { - twraps.mono_wasm_invoke_jsexport_sync(method, args as any); - } else { - // this is blocking too - twraps.mono_wasm_invoke_jsexport_sync_send(runtimeHelpers.managedThreadTID, method, args as any); - } + // this is blocking too + twraps.mono_wasm_invoke_jsexport_sync_send(runtimeHelpers.managedThreadTID, method, args as any); + */ } - if (is_args_exception(args)) { const exc = get_arg(args, 0); throw marshal_exception_to_js(exc); @@ -307,8 +280,7 @@ export function bind_assembly_exports(assemblyName: string): Promise { loaderHelpers.assert_runtime_running(); const sp = Module.stackSave(); try { - const size = 3; - const args = alloc_stack_frame(size); + const args = alloc_stack_frame(3); const res = get_arg(args, 1); const arg1 = get_arg(args, 2); marshal_string_to_cs(arg1, assemblyName); @@ -316,7 +288,7 @@ export function bind_assembly_exports(assemblyName: string): Promise { // because this is async, we could pre-allocate the promise let promise = begin_marshal_task_to_js(res, MarshalerType.TaskPreCreated); - invoke_async_jsexport(runtimeHelpers.managedThreadTID, managedExports.BindAssemblyExports, args, size); + invoke_async_jsexport(managedExports.BindAssemblyExports, args, 3); // in case the C# side returned synchronously promise = end_marshal_task_to_js(args, marshal_int32_to_js, promise); diff --git a/src/mono/browser/runtime/marshal-to-cs.ts b/src/mono/browser/runtime/marshal-to-cs.ts index f408e5bebd2b1c..a42eb724a92151 100644 --- a/src/mono/browser/runtime/marshal-to-cs.ts +++ b/src/mono/browser/runtime/marshal-to-cs.ts @@ -5,23 +5,26 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; import BuildConfiguration from "consts:configuration"; import WasmEnableJsInteropByValue from "consts:wasmEnableJsInteropByValue"; -import { PromiseHolder, isThenable } from "./cancelable-promise"; +import { isThenable } from "./cancelable-promise"; import cwraps from "./cwraps"; -import { alloc_gcv_handle, assert_not_disposed, cs_owned_js_handle_symbol, js_owned_gc_handle_symbol, mono_wasm_get_js_handle, setup_managed_proxy } from "./gc-handles"; -import { Module, mono_assert, runtimeHelpers } from "./globals"; +import { alloc_gcv_handle, assert_not_disposed, cs_owned_js_handle_symbol, js_owned_gc_handle_symbol, mono_wasm_get_js_handle, setup_managed_proxy, teardown_managed_proxy } from "./gc-handles"; +import { Module, loaderHelpers, mono_assert, runtimeHelpers } from "./globals"; import { ManagedError, - set_gc_handle, set_js_handle, set_arg_type, set_arg_i32, set_arg_f64, set_arg_i52, set_arg_f32, set_arg_i16, set_arg_u8, set_arg_bool, set_arg_date, + set_gc_handle, set_js_handle, set_arg_type, set_arg_i32, set_arg_f64, set_arg_i52, set_arg_f32, set_arg_i16, set_arg_u8, set_arg_b8, set_arg_date, set_arg_length, get_arg, get_signature_arg1_type, get_signature_arg2_type, js_to_cs_marshalers, get_signature_res_type, bound_js_function_symbol, set_arg_u16, array_element_size, get_string_root, Span, ArraySegment, MemoryViewType, get_signature_arg3_type, set_arg_i64_big, set_arg_intptr, - set_arg_element_type, ManagedObject, JavaScriptMarshalerArgSize, proxy_debug_symbol, get_arg_gc_handle, get_arg_type, set_arg_proxy_context, get_arg_intptr + set_arg_element_type, ManagedObject, JavaScriptMarshalerArgSize, proxy_debug_symbol, get_arg_gc_handle, get_arg_type } from "./marshal"; import { get_marshaler_to_js_by_type } from "./marshal-to-js"; -import { _zero_region, localHeapViewF64, localHeapViewI32, localHeapViewU8 } from "./memory"; +import { _zero_region, forceThreadMemoryViewRefresh, localHeapViewF64, localHeapViewI32, localHeapViewU8 } from "./memory"; import { stringToMonoStringRoot, stringToUTF16 } from "./strings"; import { JSMarshalerArgument, JSMarshalerArguments, JSMarshalerType, MarshalerToCs, MarshalerToJs, BoundMarshalerToCs, MarshalerType } from "./types/internal"; import { TypedArray } from "./types/emscripten"; +import { addUnsettledPromise, settleUnsettledPromise } from "./pthreads"; +import { mono_log_debug } from "./logging"; +import { complete_task } from "./managed-exports"; import { gc_locked } from "./gc-lock"; export const jsinteropDoc = "For more information see https://aka.ms/dotnet-wasm-jsinterop"; @@ -47,7 +50,7 @@ export function initialize_marshalers_to_cs(): void { js_to_cs_marshalers.set(MarshalerType.Exception, marshal_exception_to_cs); js_to_cs_marshalers.set(MarshalerType.JSException, marshal_exception_to_cs); js_to_cs_marshalers.set(MarshalerType.JSObject, marshal_js_object_to_cs); - js_to_cs_marshalers.set(MarshalerType.Object, marshal_cs_object_to_cs); + js_to_cs_marshalers.set(MarshalerType.Object, _marshal_cs_object_to_cs); js_to_cs_marshalers.set(MarshalerType.Task, _marshal_task_to_cs); js_to_cs_marshalers.set(MarshalerType.TaskResolved, _marshal_task_to_cs); js_to_cs_marshalers.set(MarshalerType.TaskRejected, _marshal_task_to_cs); @@ -102,7 +105,7 @@ export function marshal_bool_to_cs(arg: JSMarshalerArgument, value: any): void { } else { set_arg_type(arg, MarshalerType.Boolean); - set_arg_bool(arg, value); + set_arg_b8(arg, value); } } @@ -282,7 +285,7 @@ function _marshal_function_to_cs(arg: JSMarshalerArgument, value: Function, _?: if (arg3_converter) { arg3_js = arg3_converter(arg3); } - runtimeHelpers.isPendingSynchronousCall = true; // this is always synchronous call for now + runtimeHelpers.isPendingSynchronousCall = true; // this is alway synchronous call for now const res_js = value(arg1_js, arg2_js, arg3_js); if (res_converter) { res_converter(res, res_js); @@ -306,6 +309,13 @@ function _marshal_function_to_cs(arg: JSMarshalerArgument, value: Function, _?: set_arg_type(arg, MarshalerType.Function);//TODO or action ? } +export class PromiseHolder extends ManagedObject { + public isResolved = false; + public isCanceled = false; + public constructor(public promise: Promise) { + super(); + } +} function _marshal_task_to_cs(arg: JSMarshalerArgument, value: Promise, _?: MarshalerType, res_converter?: MarshalerToCs) { const handleIsPreallocated = get_arg_type(arg) == MarshalerType.TaskPreCreated; @@ -325,20 +335,78 @@ function _marshal_task_to_cs(arg: JSMarshalerArgument, value: Promise, _?: mono_check(isThenable(value), "Value is not a Promise"); const gc_handle = handleIsPreallocated ? get_arg_gc_handle(arg) : alloc_gcv_handle(); - const promiseHolderPtr = WasmEnableThreads && handleIsPreallocated ? get_arg_intptr(arg) : 0; if (!handleIsPreallocated) { set_gc_handle(arg, gc_handle); set_arg_type(arg, MarshalerType.Task); } - - const holder = new PromiseHolder(value, gc_handle, promiseHolderPtr, res_converter); + const holder = new PromiseHolder(value); setup_managed_proxy(holder, gc_handle); - if (BuildConfiguration === "Debug") { (holder as any)[proxy_debug_symbol] = `PromiseHolder with GCHandle ${gc_handle}`; } - value.then(data => holder.resolve(data), reason => holder.reject(reason)); + if (WasmEnableThreads) + addUnsettledPromise(); + + function resolve(data: any) { + if (!loaderHelpers.is_runtime_running()) { + mono_log_debug("This promise can't be propagated to managed code, mono runtime already exited."); + return; + } + try { + mono_assert(!holder.isDisposed, "This promise can't be propagated to managed code, because the Task was already freed."); + mono_assert(!holder.isResolved, "This promise already resolved."); + mono_assert(!holder.isCanceled, "This promise already canceled."); + holder.isResolved = true; + if (WasmEnableThreads) { + forceThreadMemoryViewRefresh(); + settleUnsettledPromise(); + } + // we can unregister the GC handle just on JS side + teardown_managed_proxy(holder, gc_handle, /*skipManaged: */ true); + // order of operations with teardown_managed_proxy matters + // so that managed user code running in the continuation could allocate the same GCHandle number and the local registry would be already ok with that + complete_task(gc_handle, false, null, data, res_converter || _marshal_cs_object_to_cs); + } + catch (ex) { + try { + loaderHelpers.mono_exit(1, ex); + } + catch (ex2) { + // there is no point to propagate the exception into the unhandled promise rejection + } + } + } + + function reject(reason: any) { + if (!loaderHelpers.is_runtime_running()) { + mono_log_debug("This promise can't be propagated to managed code, mono runtime already exited.", reason); + return; + } + try { + mono_assert(!holder.isDisposed, "This promise can't be propagated to managed code, because the Task was already freed."); + mono_assert(!holder.isResolved, "This promise already resolved."); + holder.isResolved = true; + if (WasmEnableThreads) { + forceThreadMemoryViewRefresh(); + settleUnsettledPromise(); + } + // we can unregister the GC handle just on JS side + teardown_managed_proxy(holder, gc_handle, /*skipManaged: */ true); + // order of operations with teardown_managed_proxy matters + complete_task(gc_handle, holder.isCanceled, reason, null, undefined); + } + catch (ex) { + try { + loaderHelpers.mono_exit(1, ex); + } + catch (ex2) { + // there is no point to propagate the exception into the unhandled promise rejection + } + } + } + + value.then(resolve).catch(reject); } export function marshal_exception_to_cs(arg: JSMarshalerArgument, value: any): void { @@ -373,7 +441,6 @@ export function marshal_exception_to_cs(arg: JSMarshalerArgument, value: any): v export function marshal_js_object_to_cs(arg: JSMarshalerArgument, value: any): void { if (value === undefined || value === null) { set_arg_type(arg, MarshalerType.None); - set_arg_proxy_context(arg); } else { // if value was ManagedObject, it would be double proxied, but the C# signature requires that @@ -389,10 +456,9 @@ export function marshal_js_object_to_cs(arg: JSMarshalerArgument, value: any): v } } -export function marshal_cs_object_to_cs(arg: JSMarshalerArgument, value: any): void { +function _marshal_cs_object_to_cs(arg: JSMarshalerArgument, value: any): void { if (value === undefined || value === null) { set_arg_type(arg, MarshalerType.None); - set_arg_proxy_context(arg); } else { const gc_handle = value[js_owned_gc_handle_symbol]; @@ -412,7 +478,7 @@ export function marshal_cs_object_to_cs(arg: JSMarshalerArgument, value: any): v } else if (js_type === "boolean") { set_arg_type(arg, MarshalerType.Boolean); - set_arg_bool(arg, value); + set_arg_b8(arg, value); } else if (value instanceof Date) { set_arg_type(arg, MarshalerType.DateTime); @@ -515,7 +581,7 @@ export function marshal_array_to_cs_impl(arg: JSMarshalerArgument, value: Array< } for (let index = 0; index < length; index++) { const element_arg = get_arg(buffer_ptr, index); - marshal_cs_object_to_cs(element_arg, value[index]); + _marshal_cs_object_to_cs(element_arg, value[index]); } } else if (element_type == MarshalerType.JSObject) { diff --git a/src/mono/browser/runtime/marshal-to-js.ts b/src/mono/browser/runtime/marshal-to-js.ts index db6c36fe9927e0..f02c81765e6a47 100644 --- a/src/mono/browser/runtime/marshal-to-js.ts +++ b/src/mono/browser/runtime/marshal-to-js.ts @@ -11,7 +11,7 @@ import { Module, loaderHelpers, mono_assert } from "./globals"; import { ManagedObject, ManagedError, get_arg_gc_handle, get_arg_js_handle, get_arg_type, get_arg_i32, get_arg_f64, get_arg_i52, get_arg_i16, get_arg_u8, get_arg_f32, - get_arg_bool, get_arg_date, get_arg_length, get_arg, set_arg_type, + get_arg_b8, get_arg_date, get_arg_length, get_arg, set_arg_type, get_signature_arg2_type, get_signature_arg1_type, cs_to_js_marshalers, get_signature_res_type, get_arg_u16, array_element_size, get_string_root, ArraySegment, Span, MemoryViewType, get_signature_arg3_type, get_arg_i64_big, get_arg_intptr, get_arg_element_type, JavaScriptMarshalerArgSize, proxy_debug_symbol, set_js_handle, is_receiver_should_free @@ -23,7 +23,6 @@ import { get_marshaler_to_cs_by_type, jsinteropDoc, marshal_exception_to_cs } fr import { localHeapViewF64, localHeapViewI32, localHeapViewU8 } from "./memory"; import { call_delegate } from "./managed-exports"; import { gc_locked } from "./gc-lock"; -import { mono_log_debug } from "./logging"; export function initialize_marshalers_to_js(): void { if (cs_to_js_marshalers.size == 0) { @@ -102,7 +101,7 @@ function _marshal_bool_to_js(arg: JSMarshalerArgument): boolean | null { if (type == MarshalerType.None) { return null; } - return get_arg_bool(arg); + return get_arg_b8(arg); } function _marshal_byte_to_js(arg: JSMarshalerArgument): number | null { @@ -338,10 +337,6 @@ function create_task_holder(res_converter?: MarshalerToJs) { } export function mono_wasm_resolve_or_reject_promise(args: JSMarshalerArguments): void { - if (!loaderHelpers.is_runtime_running()) { - mono_log_debug("This promise resolution/rejection can't be propagated to managed code, mono runtime already exited."); - return; - } const exc = get_arg(args, 0); const receiver_should_free = WasmEnableThreads && is_receiver_should_free(args); try { diff --git a/src/mono/browser/runtime/marshal.ts b/src/mono/browser/runtime/marshal.ts index 4c531eec070097..c3c07d1c188955 100644 --- a/src/mono/browser/runtime/marshal.ts +++ b/src/mono/browser/runtime/marshal.ts @@ -7,7 +7,7 @@ import { js_owned_gc_handle_symbol, teardown_managed_proxy } from "./gc-handles" import { Module, loaderHelpers, mono_assert, runtimeHelpers } from "./globals"; import { getF32, getF64, getI16, getI32, getI64Big, getU16, getU32, getU8, setF32, setF64, setI16, setI32, setI64Big, setU16, setU32, setU8, localHeapViewF64, localHeapViewI32, localHeapViewU8, _zero_region, getB32, setB32, forceThreadMemoryViewRefresh } from "./memory"; import { mono_wasm_new_external_root } from "./roots"; -import { GCHandle, JSHandle, MonoObject, MonoString, GCHandleNull, JSMarshalerArguments, JSFunctionSignature, JSMarshalerType, JSMarshalerArgument, MarshalerToJs, MarshalerToCs, WasmRoot, MarshalerType, PThreadPtr, PThreadPtrNull } from "./types/internal"; +import { GCHandle, JSHandle, MonoObject, MonoString, GCHandleNull, JSMarshalerArguments, JSFunctionSignature, JSMarshalerType, JSMarshalerArgument, MarshalerToJs, MarshalerToCs, WasmRoot, MarshalerType } from "./types/internal"; import { TypedArray, VoidPtr } from "./types/emscripten"; import { utf16ToString } from "./strings"; import { get_managed_stack_trace } from "./managed-exports"; @@ -38,7 +38,6 @@ const enum JSMarshalerArgumentOffsets { ElementType = 13, ContextHandle = 16, ReceiverShouldFree = 20, - CallerNativeTID = 24, } export const JSMarshalerTypeSize = 32; // keep in sync with JSFunctionBinding.JSBindingType @@ -86,17 +85,11 @@ export function is_args_exception(args: JSMarshalerArguments): boolean { } export function is_receiver_should_free(args: JSMarshalerArguments): boolean { - if (!WasmEnableThreads) return false; + if (WasmEnableThreads) return false; mono_assert(args, "Null args"); return getB32(args + JSMarshalerArgumentOffsets.ReceiverShouldFree); } -export function get_caller_native_tid(args: JSMarshalerArguments): PThreadPtr { - if (!WasmEnableThreads) return PThreadPtrNull; - mono_assert(args, "Null args"); - return getI32(args + JSMarshalerArgumentOffsets.CallerNativeTID) as any; -} - export function set_receiver_should_free(args: JSMarshalerArguments): void { mono_assert(args, "Null args"); setB32(args + JSMarshalerArgumentOffsets.ReceiverShouldFree, true); @@ -200,7 +193,7 @@ export function set_arg_element_type(arg: JSMarshalerArgument, type: MarshalerTy setU8(arg + JSMarshalerArgumentOffsets.ElementType, type); } -export function get_arg_bool(arg: JSMarshalerArgument): boolean { +export function get_arg_b8(arg: JSMarshalerArgument): boolean { mono_assert(arg, "Null arg"); return !!getU8(arg); } @@ -258,10 +251,10 @@ export function get_arg_f64(arg: JSMarshalerArgument): number { return getF64(arg); } -export function set_arg_bool(arg: JSMarshalerArgument, value: boolean): void { +export function set_arg_b8(arg: JSMarshalerArgument, value: boolean): void { mono_assert(arg, "Null arg"); mono_check(typeof value === "boolean", () => `Value is not a Boolean: ${value} (${typeof (value)})`); - setB32(arg, value ? 1 : 0); + setU8(arg, value ? 1 : 0); } export function set_arg_u8(arg: JSMarshalerArgument, value: number): void { diff --git a/src/mono/browser/runtime/memory.ts b/src/mono/browser/runtime/memory.ts index 585c6e377a1c1e..ab4beebf1285aa 100644 --- a/src/mono/browser/runtime/memory.ts +++ b/src/mono/browser/runtime/memory.ts @@ -6,7 +6,7 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; import { MemOffset, NumberOrPointer } from "./types/internal"; import { VoidPtr, CharPtr } from "./types/emscripten"; import cwraps, { I52Error } from "./cwraps"; -import { Module, mono_assert, runtimeHelpers } from "./globals"; +import { Module, runtimeHelpers } from "./globals"; import { utf8ToString } from "./strings"; const alloca_stack: Array = []; @@ -322,29 +322,19 @@ export function getEnv(name: string): string | null { } } -export function compareExchangeI32(offset: MemOffset, value: number, expected: number): number { - mono_assert((offset & 3) === 0, () => `compareExchangeI32: offset must be 4-byte aligned, got ${offset}`); - if (!WasmEnableThreads) { - const actual = getI32(offset); - if (actual === expected) { - setI32(offset, value); - } - return actual; - } - return globalThis.Atomics.compareExchange(localHeapViewI32(), offset >>> 2, expected, value); -} - -export function storeI32(offset: MemOffset, value: number): void { - mono_assert((offset & 3) === 0, () => `storeI32: offset must be 4-byte aligned, got ${offset}`); - if (!WasmEnableThreads) return setI32(offset, value); - globalThis.Atomics.store(localHeapViewI32(), offset >>> 2, value); -} +const BuiltinAtomics = globalThis.Atomics; -export function notifyI32(offset: MemOffset, count: number): void { - mono_assert((offset & 3) === 0, () => `notifyI32: offset must be 4-byte aligned, got ${offset}`); - if (!WasmEnableThreads) return; - globalThis.Atomics.notify(localHeapViewI32(), offset >>> 2, count); -} +export const Atomics = WasmEnableThreads ? { + storeI32(offset: MemOffset, value: number): void { + BuiltinAtomics.store(localHeapViewI32(), offset >>> 2, value); + }, + notifyI32(offset: MemOffset, count: number): void { + BuiltinAtomics.notify(localHeapViewI32(), offset >>> 2, count); + } +} : { + storeI32: setI32, + notifyI32: () => { /*empty*/ } +}; // returns memory view which is valid within current synchronous call stack export function localHeapViewI8(): Int8Array { diff --git a/src/mono/browser/runtime/multi-threading.md b/src/mono/browser/runtime/multi-threading.md deleted file mode 100644 index e4b3985923d503..00000000000000 --- a/src/mono/browser/runtime/multi-threading.md +++ /dev/null @@ -1,52 +0,0 @@ -# Multi-threading with JavaScript interop - -## Meaningful configurations are: - - * Single-threaded mode as you know it since .Net 6 - - default, safe, tested, supported - - from .Net 8 it could be easily started also as a web worker, but you need your own messaging between main and worker - * `MainThreadingMode.DeputyThread` + `JSThreadBlockingMode.NoBlockingWait` + `JSThreadInteropMode.SimpleSynchronousJSInterop` - + **default threading**, safe, tested, supported - + blocking `.Wait` is allowed on thread pool and new threads - - blocking `.Wait` throws `PlatformNotSupportedException` on `JSWebWorker` and main thread - - DOM events like `onClick` need to be asynchronous, if the handler needs use synchronous `[JSImport]` - - synchronous calls to `[JSImport]`/`[JSExport]` can't synchronously call back - - * `MainThreadingMode.DeputyAndIOThreads` + `JSThreadBlockingMode.AllowBlockingWaitInAsyncCode` + `JSThreadInteropMode.SimpleSynchronousJSInterop` - + **default threading**, safe, tested, supported - + blocking `.Wait` is allowed on thread pool and new threads - - blocking `.Wait` throws `PlatformNotSupportedException` on `JSWebWorker` and main thread only when they are called from JS via synchronous `JSExport` - - DOM events like `onClick` need to be asynchronous, if the handler needs use synchronous `[JSImport]` - - synchronous calls to `[JSImport]`/`[JSExport]` can't synchronously call back - - * `MainThreadingMode.DeputyThread` + `JSThreadBlockingMode.AllowBlockingWait` + `JSThreadInteropMode.SimpleSynchronousJSInterop` - + pragmatic for legacy codebase, which contains blocking code and can't be fully executed on thread pool or new threads - - ** could cause deadlocks !!!** - - Use your own judgment before you opt in. - - blocking .Wait is allowed on all threads! - - blocking .Wait on pending JS `Task`/`Promise` (like HTTP/WS requests) could cause deadlocks! - - reason is that blocked thread can't process the browser event loop - - so it can't resolve the promises - - even when it's longer `Promise`/`Task` chain - - DOM events like `onClick` need to be asynchronous, if the handler needs use synchronous `[JSImport]` - - synchronous calls to `[JSImport]`/`[JSExport]` can't synchronously call back - -## Unsupported combinations are: - * `MainThreadingMode.DeputyThread` + `JSThreadBlockingMode.NoBlockingWait` + `JSThreadInteropMode.NoSyncJSInterop` - + very safe - - HTTP/WS requests are not possible because it currently uses synchronous JS interop - - Blazor doesn't work because it currently uses synchronous JS interop - * `MainThreadingMode.UIThread` - - not recommended, not tested, not supported! - - can deadlock on creating new threads - - can deadlock on blocking `.Wait` for a pending JS `Promise`/`Task`, including HTTP/WS requests - - .Wait is spin-waiting - it blocks debugger, network, UI rendering, ... - + JS interop to UI is faster, synchronous and re-entrant - -### There could be more JSThreadInteropModes: - - allow re-entrant synchronous JS interop on `JSWebWorker`. - - This is possible because managed code is running on same thread as JS. - - But it's nuanced to debug it, when things go wrong. - - allow re-entrant synchronous JS interop also on deputy thread. - - This is not possible for deputy, because it would deadlock on call back to different thread. - - The thread receiving the callback is still blocked waiting for the first synchronous call to finish. diff --git a/src/mono/browser/runtime/polyfills.ts b/src/mono/browser/runtime/polyfills.ts index 26150c797c81ea..0f8700f84754ec 100644 --- a/src/mono/browser/runtime/polyfills.ts +++ b/src/mono/browser/runtime/polyfills.ts @@ -5,7 +5,7 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; import type { EmscriptenReplacements } from "./types/internal"; import type { TypedArray } from "./types/emscripten"; import { ENVIRONMENT_IS_NODE, ENVIRONMENT_IS_WORKER, INTERNAL, Module, loaderHelpers, runtimeHelpers } from "./globals"; -import { replaceEmscriptenTLSInit } from "./pthreads"; +import { replaceEmscriptenPThreadWorker } from "./pthreads"; import { replaceEmscriptenPThreadUI } from "./pthreads"; const dummyPerformance = { @@ -36,7 +36,7 @@ export function initializeReplacements(replacements: EmscriptenReplacements): vo // threads if (WasmEnableThreads && replacements.modulePThread) { if (ENVIRONMENT_IS_WORKER) { - replaceEmscriptenTLSInit(replacements.modulePThread); + replaceEmscriptenPThreadWorker(replacements.modulePThread); } else { replaceEmscriptenPThreadUI(replacements.modulePThread); } diff --git a/src/mono/browser/runtime/pthreads/deputy-thread.ts b/src/mono/browser/runtime/pthreads/deputy-thread.ts deleted file mode 100644 index 1020aaf98620d1..00000000000000 --- a/src/mono/browser/runtime/pthreads/deputy-thread.ts +++ /dev/null @@ -1,60 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -import WasmEnableThreads from "consts:wasmEnableThreads"; -import BuildConfiguration from "consts:configuration"; - -import { mono_log_error, mono_log_info } from "../logging"; -import { monoThreadInfo, postMessageToMain, update_thread_info } from "./shared"; -import { Module, loaderHelpers, runtimeHelpers } from "../globals"; -import { start_runtime } from "../startup"; -import { WorkerToMainMessageType } from "../types/internal"; - -export function mono_wasm_start_deputy_thread_async() { - if (!WasmEnableThreads) return; - - if (BuildConfiguration === "Debug" && globalThis.setInterval) globalThis.setInterval(() => { - mono_log_info("Deputy thread is alive!"); - }, 3000); - - try { - monoThreadInfo.isDeputy = true; - monoThreadInfo.threadName = "Managed Main Deputy"; - update_thread_info(); - postMessageToMain({ - monoCmd: WorkerToMainMessageType.deputyCreated, - info: monoThreadInfo, - }); - Module.runtimeKeepalivePush(); - Module.safeSetTimeout(async () => { - try { - - await start_runtime(); - - postMessageToMain({ - monoCmd: WorkerToMainMessageType.deputyStarted, - info: monoThreadInfo, - deputyProxyGCHandle: runtimeHelpers.proxyGCHandle, - }); - } - catch (err) { - postMessageToMain({ - monoCmd: WorkerToMainMessageType.deputyFailed, - info: monoThreadInfo, - error: "mono_wasm_start_deputy_thread_async() failed" + err, - }); - mono_log_error("mono_wasm_start_deputy_thread_async() failed", err); - loaderHelpers.mono_exit(1, err); - throw err; - } - }, 0); - } - catch (err) { - mono_log_error("mono_wasm_start_deputy_thread_async() failed", err); - loaderHelpers.mono_exit(1, err); - throw err; - } - - // same as emscripten_exit_with_live_runtime() - throw "unwind"; -} \ No newline at end of file diff --git a/src/mono/browser/runtime/pthreads/index.ts b/src/mono/browser/runtime/pthreads/index.ts index 0a5911605282d0..a7b5da11e03fbe 100644 --- a/src/mono/browser/runtime/pthreads/index.ts +++ b/src/mono/browser/runtime/pthreads/index.ts @@ -6,15 +6,13 @@ export { mono_wasm_pthread_ptr, update_thread_info, isMonoThreadMessage, monoThreadInfo, } from "./shared"; export { - mono_wasm_dump_threads, thread_available, cancelThreads, is_thread_available, + dumpThreads, thread_available, cancelThreads, is_thread_available, populateEmscriptenPool, mono_wasm_init_threads, init_finalizer_thread, waitForThread, replaceEmscriptenPThreadUI } from "./ui-thread"; +export { addUnsettledPromise, settleUnsettledPromise, mono_wasm_eventloop_has_unsettled_interop_promises } from "./worker-eventloop"; export { mono_wasm_pthread_on_pthread_attached, mono_wasm_pthread_on_pthread_unregistered, mono_wasm_pthread_on_pthread_registered, mono_wasm_pthread_set_name, currentWorkerThreadEvents, - dotnetPthreadCreated, initWorkerThreadEvents, replaceEmscriptenTLSInit, pthread_self + dotnetPthreadCreated, initWorkerThreadEvents, replaceEmscriptenPThreadWorker, pthread_self } from "./worker-thread"; - -export { mono_wasm_start_deputy_thread_async } from "./deputy-thread"; -export { mono_wasm_start_io_thread_async } from "./io-thread"; diff --git a/src/mono/browser/runtime/pthreads/io-thread.ts b/src/mono/browser/runtime/pthreads/io-thread.ts deleted file mode 100644 index ebbd2b82a9b791..00000000000000 --- a/src/mono/browser/runtime/pthreads/io-thread.ts +++ /dev/null @@ -1,40 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -import WasmEnableThreads from "consts:wasmEnableThreads"; -import BuildConfiguration from "consts:configuration"; - -import { mono_log_error, mono_log_info } from "../logging"; -import { monoThreadInfo, postMessageToMain, update_thread_info } from "./shared"; -import { Module, loaderHelpers } from "../globals"; -import { WorkerToMainMessageType } from "../types/internal"; -import { threads_c_functions as tcwraps } from "../cwraps"; - -export function mono_wasm_start_io_thread_async() { - if (!WasmEnableThreads) return; - - - if (BuildConfiguration === "Debug" && globalThis.setInterval) globalThis.setInterval(() => { - mono_log_info("I/O thread is alive!"); - }, 3000); - - try { - monoThreadInfo.isIo = true; - monoThreadInfo.threadName = "JS I/O Thread"; - update_thread_info(); - tcwraps.mono_wasm_register_io_thread(); - postMessageToMain({ - monoCmd: WorkerToMainMessageType.ioStarted, - info: monoThreadInfo, - }); - Module.runtimeKeepalivePush(); - } - catch (err) { - mono_log_error("mono_wasm_start_io_thread_async() failed", err); - loaderHelpers.mono_exit(1, err); - throw err; - } - - // same as emscripten_exit_with_live_runtime() - throw "unwind"; -} \ No newline at end of file diff --git a/src/mono/browser/runtime/pthreads/shared.ts b/src/mono/browser/runtime/pthreads/shared.ts index c83eb3b967e038..97c41197d35e04 100644 --- a/src/mono/browser/runtime/pthreads/shared.ts +++ b/src/mono/browser/runtime/pthreads/shared.ts @@ -38,8 +38,8 @@ export function mono_wasm_install_js_worker_interop(context_gc_handle: GCHandle) mono_assert(!runtimeHelpers.proxyGCHandle, "JS interop should not be already installed on this worker."); runtimeHelpers.proxyGCHandle = context_gc_handle; if (ENVIRONMENT_IS_PTHREAD) { - runtimeHelpers.managedThreadTID = runtimeHelpers.currentThreadTID; - runtimeHelpers.isManagedRunningOnCurrentThread = true; + runtimeHelpers.managedThreadTID = mono_wasm_pthread_ptr(); + runtimeHelpers.isCurrentThread = true; } Module.runtimeKeepalivePush(); monoThreadInfo.isDirtyBecauseOfInterop = true; @@ -70,16 +70,14 @@ export function update_thread_info(): void { if (!WasmEnableThreads) return; const threadType = !monoThreadInfo.isRegistered ? "emsc" : monoThreadInfo.isUI ? "-UI-" - : monoThreadInfo.isDeputy ? "dpty" - : monoThreadInfo.isIo ? "-IO-" - : monoThreadInfo.isTimer ? "timr" - : monoThreadInfo.isLongRunning ? "long" - : monoThreadInfo.isThreadPoolGate ? "gate" - : monoThreadInfo.isDebugger ? "dbgr" - : monoThreadInfo.isThreadPoolWorker ? "pool" - : monoThreadInfo.isExternalEventLoop ? "jsww" - : monoThreadInfo.isBackground ? "back" - : "norm"; + : monoThreadInfo.isTimer ? "timr" + : monoThreadInfo.isLongRunning ? "long" + : monoThreadInfo.isThreadPoolGate ? "gate" + : monoThreadInfo.isDebugger ? "dbgr" + : monoThreadInfo.isThreadPoolWorker ? "pool" + : monoThreadInfo.isExternalEventLoop ? "jsww" + : monoThreadInfo.isBackground ? "back" + : "norm"; const hexPtr = (monoThreadInfo.pthreadId as any).toString(16).padStart(8, "0"); const hexPrefix = monoThreadInfo.isRegistered ? "0x" : "--"; monoThreadInfo.threadPrefix = `${hexPrefix}${hexPtr}-${threadType}`; @@ -89,16 +87,11 @@ export function update_thread_info(): void { set_thread_prefix(monoThreadInfo.threadPrefix!); } - // this is just to make debugging easier by naming the thread debugger window. - // It's not CSP compliant and possibly not performant, that's why it's only enabled in debug builds - // in Release configuration, it would be a trimmed by rollup + (globalThis as any).monoThreadInfo = monoThreadInfo; if (WasmEnableThreads && BuildConfiguration === "Debug" && !runtimeHelpers.cspPolicy) { monoThreadInfo.updateCount++; try { - const url = `//# sourceURL=https://dotnet/thread/${monoThreadInfo.updateCount}-${monoThreadInfo.threadPrefix}`; - const infoJson = JSON.stringify(monoThreadInfo, null, 2); - const body = `const monoThreadInfo=${infoJson};\r\nconsole.log(monoThreadInfo);`; - (globalThis as any).monoThreadInfoFn = new Function(body + "\r\n" + url); + (globalThis as any).monoThreadInfoFn = new Function(`//# sourceURL=https://${monoThreadInfo.updateCount}WorkerInfo${monoThreadInfo.isAttached ? monoThreadInfo.threadPrefix : ""}/\r\nconsole.log("${JSON.stringify(monoThreadInfo)}");`); } catch (ex) { runtimeHelpers.cspPolicy = true; @@ -126,8 +119,6 @@ export interface MonoWorkerToMainMessage { monoCmd: WorkerToMainMessageType; info: PThreadInfo; port?: MessagePort; - error?: string; - deputyProxyGCHandle?: GCHandle; } /// Identification of the current thread executing on a worker diff --git a/src/mono/browser/runtime/pthreads/ui-thread.ts b/src/mono/browser/runtime/pthreads/ui-thread.ts index 9838615cd21636..878ee38eadbb9a 100644 --- a/src/mono/browser/runtime/pthreads/ui-thread.ts +++ b/src/mono/browser/runtime/pthreads/ui-thread.ts @@ -5,7 +5,7 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; import BuildConfiguration from "consts:configuration"; import { } from "../globals"; -import { mono_log_debug, mono_log_warn } from "../logging"; +import { mono_log_warn } from "../logging"; import { MonoWorkerToMainMessage, monoThreadInfo, mono_wasm_pthread_ptr, update_thread_info, worker_empty_prefix } from "./shared"; import { Module, ENVIRONMENT_IS_WORKER, createPromiseController, loaderHelpers, mono_assert, runtimeHelpers } from "../globals"; import { PThreadLibrary, MainToWorkerMessageType, MonoThreadMessage, PThreadInfo, PThreadPtr, PThreadPtrNull, PThreadWorker, PromiseAndController, PromiseController, Thread, WorkerToMainMessageType, monoMessageSymbol } from "../types/internal"; @@ -95,23 +95,12 @@ function monoWorkerMessageHandler(worker: PThreadWorker, ev: MessageEvent): worker.thread = thread; worker.info.isRunning = true; resolveThreadPromises(pthreadId, thread); - worker.info = Object.assign(worker.info!, message.info, {}); - break; - case WorkerToMainMessageType.deputyStarted: - runtimeHelpers.afterMonoStarted.promise_control.resolve(message.deputyProxyGCHandle); - break; - case WorkerToMainMessageType.ioStarted: - runtimeHelpers.afterIOStarted.promise_control.resolve(); - break; - case WorkerToMainMessageType.deputyFailed: - runtimeHelpers.afterMonoStarted.promise_control.reject(new Error(message.error)); break; case WorkerToMainMessageType.monoRegistered: case WorkerToMainMessageType.monoAttached: case WorkerToMainMessageType.enabledInterop: case WorkerToMainMessageType.monoUnRegistered: case WorkerToMainMessageType.updateInfo: - case WorkerToMainMessageType.deputyCreated: // just worker.info updates above break; default: @@ -159,11 +148,10 @@ export async function mono_wasm_init_threads() { if (!WasmEnableThreads) return; // setup the UI thread - runtimeHelpers.currentThreadTID = monoThreadInfo.pthreadId = mono_wasm_pthread_ptr(); + monoThreadInfo.pthreadId = mono_wasm_pthread_ptr(); monoThreadInfo.threadName = "UI Thread"; monoThreadInfo.isUI = true; monoThreadInfo.isRunning = true; - monoThreadInfo.workerNumber = 0; update_thread_info(); // wait until all workers in the pool are loaded - ready to be used as pthread synchronously @@ -185,7 +173,7 @@ export function cancelThreads() { } } -export function mono_wasm_dump_threads(): void { +export function dumpThreads(): void { if (!WasmEnableThreads) return; mono_log_info("Dumping web worker info as seen by UI thread, it could be stale: "); const emptyInfo: PThreadInfo = { @@ -223,11 +211,7 @@ export function init_finalizer_thread() { // we don't need it immediately, so we can wait a bit, to keep CPU working on normal startup setTimeout(() => { try { - if (loaderHelpers.is_runtime_running()) { - cwraps.mono_wasm_init_finalizer_thread(); - } else { - mono_log_debug("init_finalizer_thread skipped"); - } + cwraps.mono_wasm_init_finalizer_thread(); } catch (err) { mono_log_error("init_finalizer_thread() failed", err); @@ -281,7 +265,7 @@ export function replaceEmscriptenPThreadUI(modulePThread: PThreadLibrary): void } }; if (BuildConfiguration === "Debug") { - (globalThis as any).dumpThreads = mono_wasm_dump_threads; + (globalThis as any).dumpThreads = dumpThreads; (globalThis as any).getModulePThread = getModulePThread; } } diff --git a/src/mono/browser/runtime/pthreads/worker-eventloop.ts b/src/mono/browser/runtime/pthreads/worker-eventloop.ts new file mode 100644 index 00000000000000..7b8a42b05ac660 --- /dev/null +++ b/src/mono/browser/runtime/pthreads/worker-eventloop.ts @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +let perThreadUnsettledPromiseCount = 0; + +export function addUnsettledPromise() { + perThreadUnsettledPromiseCount++; +} + +export function settleUnsettledPromise() { + perThreadUnsettledPromiseCount--; +} + +/// Called from the C# threadpool worker loop to find out if there are any +/// unsettled JS promises that need to keep the worker alive +export function mono_wasm_eventloop_has_unsettled_interop_promises(): boolean { + return perThreadUnsettledPromiseCount > 0; +} diff --git a/src/mono/browser/runtime/pthreads/worker-thread.ts b/src/mono/browser/runtime/pthreads/worker-thread.ts index 5631c2475e6528..083f87d7de025e 100644 --- a/src/mono/browser/runtime/pthreads/worker-thread.ts +++ b/src/mono/browser/runtime/pthreads/worker-thread.ts @@ -5,9 +5,11 @@ import WasmEnableThreads from "consts:wasmEnableThreads"; -import { ENVIRONMENT_IS_PTHREAD, Module, loaderHelpers, mono_assert, runtimeHelpers } from "../globals"; +import { Module } from "../globals"; + +import { ENVIRONMENT_IS_PTHREAD, loaderHelpers, mono_assert, runtimeHelpers } from "../globals"; import { PThreadSelf, monoThreadInfo, mono_wasm_pthread_ptr, postMessageToMain, update_thread_info } from "./shared"; -import { PThreadLibrary, MonoThreadMessage, PThreadInfo, PThreadPtr, WorkerToMainMessageType } from "../types/internal"; +import { PThreadLibrary, MonoThreadMessage, PThreadInfo, PThreadPtr, WorkerToMainMessageType, is_nullish } from "../types/internal"; import { makeWorkerThreadEvent, dotnetPthreadCreated, @@ -69,7 +71,7 @@ function monoDedicatedChannelMessageFromMainToWorker(event: MessageEvent } export function on_emscripten_thread_init(pthread_ptr: PThreadPtr) { - runtimeHelpers.currentThreadTID = monoThreadInfo.pthreadId = pthread_ptr; + monoThreadInfo.pthreadId = pthread_ptr; forceThreadMemoryViewRefresh(); } @@ -80,9 +82,10 @@ export function mono_wasm_pthread_on_pthread_created(): void { if (!WasmEnableThreads) return; try { forceThreadMemoryViewRefresh(); - const pthread_id = mono_wasm_pthread_ptr(); - mono_assert(pthread_id == monoThreadInfo.pthreadId, `needs to match (mono_wasm_pthread_ptr ${pthread_id}, threadId from thread info ${monoThreadInfo.pthreadId})`); + const pthread_id = mono_wasm_pthread_ptr(); + mono_assert(!is_nullish(pthread_id), "pthread_self() returned null"); + monoThreadInfo.pthreadId = pthread_id; monoThreadInfo.reuseCount++; monoThreadInfo.updateCount++; monoThreadInfo.threadName = "pthread-assigned"; @@ -203,24 +206,18 @@ export function mono_wasm_pthread_on_pthread_unregistered(pthread_id: PThreadPtr } } -export function replaceEmscriptenTLSInit(modulePThread: PThreadLibrary): void { +export function replaceEmscriptenPThreadWorker(modulePThread: PThreadLibrary): void { if (!WasmEnableThreads) return; const originalThreadInitTLS = modulePThread.threadInitTLS; + const original_emscripten_thread_init = (Module as any)["__emscripten_thread_init"]; + (Module as any)["__emscripten_thread_init"] = (pthread_ptr: PThreadPtr, isMainBrowserThread: number, isMainRuntimeThread: number, canBlock: number) => { + on_emscripten_thread_init(pthread_ptr); + original_emscripten_thread_init(pthread_ptr, isMainBrowserThread, isMainRuntimeThread, canBlock); + }; modulePThread.threadInitTLS = (): void => { originalThreadInitTLS(); mono_wasm_pthread_on_pthread_created(); }; -} - -export function replaceEmscriptenPThreadInit(): void { - const original_emscripten_thread_init = Module["__emscripten_thread_init"]; - function emscripten_thread_init_wrapper(pthread_ptr: PThreadPtr, isMainBrowserThread: number, isMainRuntimeThread: number, canBlock: number) { - on_emscripten_thread_init(pthread_ptr); - original_emscripten_thread_init(pthread_ptr, isMainBrowserThread, isMainRuntimeThread, canBlock); - // re-install self - Module["__emscripten_thread_init"] = emscripten_thread_init_wrapper; - } - Module["__emscripten_thread_init"] = emscripten_thread_init_wrapper; -} +} \ No newline at end of file diff --git a/src/mono/browser/runtime/runtime.c b/src/mono/browser/runtime/runtime.c index bbd645e21e821c..9030c4a52ee556 100644 --- a/src/mono/browser/runtime/runtime.c +++ b/src/mono/browser/runtime/runtime.c @@ -322,14 +322,13 @@ mono_wasm_load_runtime_common (int debug_level, MonoLogCallback log_callback, co EMSCRIPTEN_KEEPALIVE MonoAssembly* mono_wasm_assembly_load (const char *name) { - MonoAssembly *res; assert (name); MonoImageOpenStatus status; - MONO_ENTER_GC_UNSAFE; MonoAssemblyName* aname = mono_assembly_name_new (name); - res = mono_assembly_load (aname, NULL, &status); + + MonoAssembly *res = mono_assembly_load (aname, NULL, &status); mono_assembly_name_free (aname); - MONO_EXIT_GC_UNSAFE; + return res; } diff --git a/src/mono/browser/runtime/startup.ts b/src/mono/browser/runtime/startup.ts index 1ae17e43edf191..9759e879254b08 100644 --- a/src/mono/browser/runtime/startup.ts +++ b/src/mono/browser/runtime/startup.ts @@ -2,11 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. import WasmEnableThreads from "consts:wasmEnableThreads"; -import BuildConfiguration from "consts:configuration"; -import { DotnetModuleInternal, CharPtrNull, MainThreadingMode } from "./types/internal"; -import { exportedRuntimeAPI, INTERNAL, loaderHelpers, Module, runtimeHelpers, createPromiseController, mono_assert } from "./globals"; -import cwraps, { init_c_exports, threads_c_functions as tcwraps } from "./cwraps"; +import { DotnetModuleInternal, CharPtrNull } from "./types/internal"; +import { ENVIRONMENT_IS_NODE, exportedRuntimeAPI, INTERNAL, loaderHelpers, Module, runtimeHelpers, createPromiseController, mono_assert, ENVIRONMENT_IS_WORKER } from "./globals"; +import cwraps, { init_c_exports } from "./cwraps"; import { mono_wasm_raise_debug_event, mono_wasm_runtime_ready } from "./debug"; import { toBase64StringImpl } from "./base64"; import { mono_wasm_init_aot_profiler, mono_wasm_init_browser_profiler } from "./profiler"; @@ -21,7 +20,7 @@ import { wait_for_all_assets } from "./assets"; import { replace_linker_placeholders } from "./exports-binding"; import { endMeasure, MeasuredBlock, startMeasure } from "./profiler"; import { interp_pgo_load_data, interp_pgo_save_data } from "./interp-pgo"; -import { mono_log_debug, mono_log_error, mono_log_info, mono_log_warn } from "./logging"; +import { mono_log_debug, mono_log_error, mono_log_warn } from "./logging"; // threads import { populateEmscriptenPool, mono_wasm_init_threads, init_finalizer_thread } from "./pthreads"; @@ -33,7 +32,6 @@ import { assertNoProxies } from "./gc-handles"; import { runtimeList } from "./exports"; import { nativeAbort, nativeExit } from "./run"; import { mono_wasm_init_diagnostics } from "./diagnostics"; -import { replaceEmscriptenPThreadInit } from "./pthreads/worker-thread"; export async function configureRuntimeStartup(): Promise { await init_polyfills_async(); @@ -127,7 +125,6 @@ async function instantiateWasmWorker( await loaderHelpers.afterConfigLoaded.promise; replace_linker_placeholders(imports); - replaceEmscriptenPThreadInit(); // Instantiate from the module posted from the main thread. // We can just use sync instantiation in the worker. @@ -270,39 +267,20 @@ async function onRuntimeInitializedAsync(userOnRuntimeInitialized: () => void) { Module.runtimeKeepalivePush(); - if (WasmEnableThreads && BuildConfiguration === "Debug" && globalThis.setInterval) globalThis.setInterval(() => { - mono_log_info("UI thread is alive!"); - }, 3000); - - if (WasmEnableThreads && - (runtimeHelpers.config.mainThreadingMode == MainThreadingMode.DeputyThread - || runtimeHelpers.config.mainThreadingMode == MainThreadingMode.DeputyAndIOThreads)) { - // this will create thread and call start_runtime() on it - runtimeHelpers.monoThreadInfo = monoThreadInfo; - runtimeHelpers.isManagedRunningOnCurrentThread = false; - update_thread_info(); - runtimeHelpers.managedThreadTID = tcwraps.mono_wasm_create_deputy_thread(); - runtimeHelpers.proxyGCHandle = await runtimeHelpers.afterMonoStarted.promise; - if (WasmEnableThreads && runtimeHelpers.config.mainThreadingMode == MainThreadingMode.DeputyAndIOThreads) { - runtimeHelpers.ioThreadTID = tcwraps.mono_wasm_create_io_thread(); - } + // load runtime and apply environment settings (if necessary) + await start_runtime(); - // TODO make UI thread not managed - tcwraps.mono_wasm_register_ui_thread(); - monoThreadInfo.isAttached = true; - monoThreadInfo.isRegistered = true; + if (runtimeHelpers.config.interpreterPgo) { + await interp_pgo_load_data(); + } - runtimeHelpers.runtimeReady = true; - update_thread_info(); - bindings_init(); - } else { - // load mono runtime and apply environment settings (if necessary) - await start_runtime(); + if (!ENVIRONMENT_IS_WORKER) { + Module.runtimeKeepalivePush(); } - if (WasmEnableThreads && runtimeHelpers.config.mainThreadingMode == MainThreadingMode.DeputyAndIOThreads) { - await runtimeHelpers.afterIOStarted.promise; + if (ENVIRONMENT_IS_NODE && !ENVIRONMENT_IS_WORKER) { + Module.runtimeKeepalivePush(); } runtimeList.registerRuntime(exportedRuntimeAPI); @@ -537,15 +515,12 @@ export async function start_runtime() { if (WasmEnableThreads) { monoThreadInfo.isAttached = true; - monoThreadInfo.isRunning = true; monoThreadInfo.isRegistered = true; - runtimeHelpers.currentThreadTID = monoThreadInfo.pthreadId = runtimeHelpers.managedThreadTID = mono_wasm_pthread_ptr(); + monoThreadInfo.pthreadId = runtimeHelpers.managedThreadTID = mono_wasm_pthread_ptr(); + monoThreadInfo.workerNumber = 0; update_thread_info(); - runtimeHelpers.proxyGCHandle = install_main_synchronization_context( - runtimeHelpers.config.jsThreadBlockingMode!, - runtimeHelpers.config.jsThreadInteropMode!, - runtimeHelpers.config.mainThreadingMode!); - runtimeHelpers.isManagedRunningOnCurrentThread = true; + runtimeHelpers.proxyGCHandle = install_main_synchronization_context(); + runtimeHelpers.isCurrentThread = true; // start finalizer thread, lazy init_finalizer_thread(); @@ -554,10 +529,6 @@ export async function start_runtime() { // get GCHandle of the ctx runtimeHelpers.afterMonoStarted.promise_control.resolve(runtimeHelpers.proxyGCHandle); - if (runtimeHelpers.config.interpreterPgo) { - await interp_pgo_load_data(); - } - endMeasure(mark, MeasuredBlock.startRuntime); } catch (err) { mono_log_error("start_runtime() failed", err); diff --git a/src/mono/browser/runtime/types/index.ts b/src/mono/browser/runtime/types/index.ts index c5754f6c89e8fc..8d9c8a28ba1480 100644 --- a/src/mono/browser/runtime/types/index.ts +++ b/src/mono/browser/runtime/types/index.ts @@ -415,13 +415,6 @@ export type APIType = { * @returns exit code of the Main() method. */ runMainAndExit: (mainAssemblyName?: string, args?: string[]) => Promise; - /** - * Exits the runtime. - * Note: after the runtime exits, it would reject all further calls to the API. - * @param code "process" exit code. - * @param reason could be a string or an Error object. - */ - exit: (code: number, reason?: any) => void; /** * Sets the environment variable for the "process" * @param name diff --git a/src/mono/browser/runtime/types/internal.ts b/src/mono/browser/runtime/types/internal.ts index 16967d26b9d299..ae4ce1ff4813a9 100644 --- a/src/mono/browser/runtime/types/internal.ts +++ b/src/mono/browser/runtime/types/internal.ts @@ -80,6 +80,7 @@ export type MonoConfigInternal = MonoConfig & { browserProfilerOptions?: BrowserProfilerOptions, // dictionary-style Object. If omitted, browser profiler will not be initialized. waitForDebugger?: number, appendElementOnExit?: boolean + assertAfterExit?: boolean // default true for shell/nodeJS interopCleanupOnExit?: boolean dumpThreadsOnNonZeroExit?: boolean logExitCode?: boolean @@ -94,10 +95,6 @@ export type MonoConfigInternal = MonoConfig & { resourcesHash?: string, GitHash?: string, ProductVersion?: string, - - mainThreadingMode?: MainThreadingMode, - jsThreadBlockingMode?: JSThreadBlockingMode, - jsThreadInteropMode?: JSThreadInteropMode, }; export type RunArguments = { @@ -122,6 +119,7 @@ export type LoaderHelpers = { maxParallelDownloads: number; enableDownloadRetry: boolean; + assertAfterExit: boolean; exitCode: number | undefined; exitReason: any; @@ -209,10 +207,8 @@ export type RuntimeHelpers = { monoThreadInfo: PThreadInfo, proxyGCHandle: GCHandle | undefined, managedThreadTID: PThreadPtr, - ioThreadTID: PThreadPtr, - currentThreadTID: PThreadPtr, - isManagedRunningOnCurrentThread: boolean, - isPendingSynchronousCall: boolean, // true when we are in the middle of a synchronous call from managed code from same thread + isCurrentThread: boolean, + isPendingSynchronousCall: boolean, // true when we are in the middle of a synchronous call from managed code with the same JSProxyContext cspPolicy: boolean, allAssetsInMemory: PromiseAndController, @@ -223,7 +219,6 @@ export type RuntimeHelpers = { afterPreRun: PromiseAndController, beforeOnRuntimeInitialized: PromiseAndController, afterMonoStarted: PromiseAndController, - afterIOStarted: PromiseAndController, afterOnRuntimeInitialized: PromiseAndController, afterPostRun: PromiseAndController, @@ -235,7 +230,7 @@ export type RuntimeHelpers = { instantiate_asset: (asset: AssetEntry, url: string, bytes: Uint8Array) => void, instantiate_symbols_asset: (pendingAsset: AssetEntryInternal) => Promise, instantiate_segmentation_rules_asset: (pendingAsset: AssetEntryInternal) => Promise, - jiterpreter_dump_stats?: (concise?: boolean) => void, + jiterpreter_dump_stats?: (x: boolean) => string, forceDisposeProxies: (disposeMethods: boolean, verbose: boolean) => void, dumpThreads: () => void, } @@ -431,7 +426,6 @@ export declare interface EmscriptenModuleInternal { runtimeKeepalivePush(): void; runtimeKeepalivePop(): void; maybeExit(): void; - __emscripten_thread_init(pthread_ptr: PThreadPtr, isMainBrowserThread: number, isMainRuntimeThread: number, canBlock: number): void; } /// A PromiseController encapsulates a Promise together with easy access to its resolve and reject functions. @@ -459,7 +453,6 @@ export type passEmscriptenInternalsType = (internals: EmscriptenInternals, emscr export type setGlobalObjectsType = (globalObjects: GlobalObjects) => void; export type initializeExportsType = (globalObjects: GlobalObjects) => RuntimeAPI; export type initializeReplacementsType = (replacements: EmscriptenReplacements) => void; -export type afterInitializeType = (module: EmscriptenModuleInternal) => void; export type configureEmscriptenStartupType = (module: DotnetModuleInternal) => void; export type configureRuntimeStartupType = () => Promise; export type configureWorkerStartupType = (module: DotnetModuleInternal) => Promise @@ -494,10 +487,6 @@ export const enum WorkerToMainMessageType { enabledInterop = "notify_enabled_interop", monoUnRegistered = "monoUnRegistered", pthreadCreated = "pthreadCreated", - deputyCreated = "createdDeputy", - deputyFailed = "deputyFailed", - deputyStarted = "monoStarted", - ioStarted = "ioStarted", preload = "preload", } @@ -527,8 +516,6 @@ export interface PThreadInfo { isRegistered?: boolean, isRunning?: boolean, isAttached?: boolean, - isDeputy?: boolean, - isIo?: boolean, isExternalEventLoop?: boolean, isUI?: boolean; isBackground?: boolean, @@ -568,37 +555,3 @@ export interface MonoThreadMessage { // A particular kind of message. For example, "started", "stopped", "stopped_with_error", etc. cmd: string; } - -// keep in sync with JSHostImplementation.Types.cs -export const enum MainThreadingMode { - // Running the managed main thread on UI thread. - // Managed GC and similar scenarios could be blocking the UI. - // Easy to deadlock. Not recommended for production. - UIThread = 0, - // Running the managed main thread on dedicated WebWorker. Marshaling all JavaScript calls to and from the main thread. - DeputyThread = 1, - // TODO comment - DeputyAndIOThreads = 2, -} - -// keep in sync with JSHostImplementation.Types.cs -export const enum JSThreadBlockingMode { - // throw PlatformNotSupportedException if blocking .Wait is called on threads with JS interop, like JSWebWorker and Main thread. - // Avoids deadlocks (typically with pending JS promises on the same thread) by throwing exceptions. - NoBlockingWait = 0, - // TODO comment - AllowBlockingWaitInAsyncCode = 1, - // allow .Wait on all threads. - // Could cause deadlocks with blocking .Wait on a pending JS Task/Promise on the same thread or similar Task/Promise chain. - AllowBlockingWait = 100, -} - -// keep in sync with JSHostImplementation.Types.cs -export const enum JSThreadInteropMode { - // throw PlatformNotSupportedException if synchronous JSImport/JSExport is called on threads with JS interop, like JSWebWorker and Main thread. - // calling synchronous JSImport on thread pool or new threads is allowed. - NoSyncJSInterop = 0, - // allow non-re-entrant synchronous blocking calls to and from JS on JSWebWorker on threads with JS interop, like JSWebWorker and Main thread. - // calling synchronous JSImport on thread pool or new threads is allowed. - SimpleSynchronousJSInterop = 1, -} \ No newline at end of file diff --git a/src/mono/browser/runtime/weak-ref.ts b/src/mono/browser/runtime/weak-ref.ts index fc69eb8a2b6ed5..ebb4ab7f8e08b1 100644 --- a/src/mono/browser/runtime/weak-ref.ts +++ b/src/mono/browser/runtime/weak-ref.ts @@ -11,17 +11,13 @@ export function create_weak_ref(js_obj: T): WeakRefInternal } else { // this is trivial WeakRef replacement, which holds strong refrence, instead of weak one, when the browser doesn't support it - return create_strong_ref(js_obj); + return { + deref: () => { + return js_obj; + }, + dispose: () => { + js_obj = null!; + } + }; } } - -export function create_strong_ref(js_obj: T): WeakRefInternal { - return { - deref: () => { - return js_obj; - }, - dispose: () => { - js_obj = null!; - } - }; -} \ No newline at end of file diff --git a/src/mono/browser/runtime/web-socket.ts b/src/mono/browser/runtime/web-socket.ts index c7ebdac242da58..6c558938cface6 100644 --- a/src/mono/browser/runtime/web-socket.ts +++ b/src/mono/browser/runtime/web-socket.ts @@ -44,17 +44,6 @@ function verifyEnvironment() { } } -export function ws_get_state(ws: WebSocketExtension) : number -{ - if (ws.readyState != WebSocket.CLOSED) - return ws.readyState ?? -1; - const receive_event_queue = ws[wasm_ws_pending_receive_event_queue]; - const queued_events_count = receive_event_queue.getLength(); - if (queued_events_count == 0) - return ws.readyState ?? -1; - return WebSocket.OPEN; -} - export function ws_wasm_create(uri: string, sub_protocols: string[] | null, receive_status_ptr: VoidPtr): WebSocketExtension { verifyEnvironment(); assert_js_interop(); diff --git a/src/mono/browser/test-main.js b/src/mono/browser/test-main.js index 3aacd8e2c67d68..78c697de19962d 100644 --- a/src/mono/browser/test-main.js +++ b/src/mono/browser/test-main.js @@ -250,6 +250,7 @@ function configureRuntime(dotnet, runArgs) { .withExitCodeLogging() .withElementOnExit() .withInteropCleanupOnExit() + .withAssertAfterExit() .withDumpThreadsOnNonZeroExit() .withConfig({ loadAllSatelliteResources: true diff --git a/src/mono/cmake/config.h.in b/src/mono/cmake/config.h.in index 5efec6ebe932d0..e484a802aed9b5 100644 --- a/src/mono/cmake/config.h.in +++ b/src/mono/cmake/config.h.in @@ -5,9 +5,9 @@ #include -#if _WIN32_WINNT < 0x0602 -#error "Mono requires Windows 8 or later." -#endif /* _WIN32_WINNT < 0x0602 */ +#if _WIN32_WINNT < 0x0601 +#error "Mono requires Windows 7 or later." +#endif /* _WIN32_WINNT < 0x0601 */ #ifndef HAVE_WINAPI_FAMILY_SUPPORT diff --git a/src/mono/mono.proj b/src/mono/mono.proj index 07930dc1b55aa9..3b0ade6bd89ccb 100644 --- a/src/mono/mono.proj +++ b/src/mono/mono.proj @@ -97,7 +97,7 @@ - + <_MonoCMakeArgs Include="-DENABLE_WERROR=1"/> @@ -385,8 +385,9 @@ <_MonoCPPFLAGS Include="-D_CRT_NONSTDC_NO_DEPRECATE" /> <_MonoCPPFLAGS Include="-DWIN32_THREADS" /> - <_MonoCPPFLAGS Include="-DWINVER=0x0602" /> - <_MonoCPPFLAGS Include="-D_WIN32_WINNT=0x0602" /> + <_MonoCPPFLAGS Include="-DWINVER=0x0601" /> + <_MonoCPPFLAGS Include="-D_WIN32_WINNT=0x0601" /> + <_MonoCPPFLAGS Include="-D_WIN32_IE=0x0501" /> <_MonoCPPFLAGS Include="-D_UNICODE" /> <_MonoCPPFLAGS Include="-DUNICODE" /> <_MonoCPPFLAGS Include="-DFD_SETSIZE=1024" /> @@ -764,8 +765,9 @@ <_MonoAOTCPPFLAGS Include="-D_CRT_NONSTDC_NO_DEPRECATE" /> <_MonoAOTCPPFLAGS Include="-DWIN32_THREADS" /> - <_MonoAOTCPPFLAGS Include="-DWINVER=0x0602" /> - <_MonoAOTCPPFLAGS Include="-D_WIN32_WINNT=0x0602" /> + <_MonoAOTCPPFLAGS Include="-DWINVER=0x0601" /> + <_MonoAOTCPPFLAGS Include="-D_WIN32_WINNT=0x0601" /> + <_MonoAOTCPPFLAGS Include="-D_WIN32_IE=0x0501" /> <_MonoAOTCPPFLAGS Include="-D_UNICODE" /> <_MonoAOTCPPFLAGS Include="-DUNICODE" /> <_MonoAOTCPPFLAGS Include="-DFD_SETSIZE=1024" /> diff --git a/src/mono/mono/component/diagnostics_server.c b/src/mono/mono/component/diagnostics_server.c index 02179f785eb352..4bea3d722625c5 100644 --- a/src/mono/mono/component/diagnostics_server.c +++ b/src/mono/mono/component/diagnostics_server.c @@ -250,7 +250,6 @@ queue_push_sync (WasmIpcStreamQueue *q, const uint8_t *buf, uint32_t buf_size, u gboolean is_browser_thread = FALSE; while (mono_atomic_load_i32 (&q->buf_full) != 0) { if (G_UNLIKELY (!is_browser_thread_inited)) { - // FIXME for deputy is_browser_thread = mono_threads_wasm_is_ui_thread (); is_browser_thread_inited = TRUE; } diff --git a/src/mono/mono/eglib/gfile-posix.c b/src/mono/mono/eglib/gfile-posix.c index f38db70c6a7b27..53414c659d780f 100644 --- a/src/mono/mono/eglib/gfile-posix.c +++ b/src/mono/mono/eglib/gfile-posix.c @@ -142,7 +142,6 @@ g_file_open_tmp (const gchar *tmpl, gchar **name_used, GError **gerror) } t = g_build_filename (g_get_tmp_dir (), tmpl, (const char*)NULL); - g_assert (t); #ifdef HOST_WASI g_critical ("g_file_open_tmp is not implemented for WASI"); diff --git a/src/mono/mono/eglib/gfile.c b/src/mono/mono/eglib/gfile.c index 5720a54a712658..4d5390c1932944 100644 --- a/src/mono/mono/eglib/gfile.c +++ b/src/mono/mono/eglib/gfile.c @@ -112,7 +112,6 @@ is_ascii_string (const gchar *str) while (*str) { if (!g_isascii (*str)) return FALSE; - str ++; } return TRUE; } diff --git a/src/mono/mono/eglib/glib.h b/src/mono/mono/eglib/glib.h index d6c1e2e59a9a9c..7ec9dcde9c8e67 100644 --- a/src/mono/mono/eglib/glib.h +++ b/src/mono/mono/eglib/glib.h @@ -69,18 +69,6 @@ #error Mono requires _Noreturn (C11 or newer) #endif -G_ATTR_NORETURN -static inline void eg_unreachable (void) { -#if defined(_MSC_VER) - __assume(0); -#elif defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 5))) - __builtin_unreachable(); -#else - for (;;) - ; -#endif -} - #ifdef __cplusplus #define g_cast monoeg_g_cast // in case not inlined (see eglib-remap.h) @@ -747,13 +735,14 @@ G_ATTR_NORETURN void const char * g_get_assertion_message (void); #ifndef DISABLE_ASSERT_MESSAGES -#define g_error(...) do { g_log (G_LOG_DOMAIN, G_LOG_LEVEL_ERROR, __VA_ARGS__); eg_unreachable (); } while (0) +/* The for (;;) tells gc thats g_error () doesn't return, avoiding warnings */ +#define g_error(...) do { g_log (G_LOG_DOMAIN, G_LOG_LEVEL_ERROR, __VA_ARGS__); for (;;); } while (0) #define g_critical(...) g_log (G_LOG_DOMAIN, G_LOG_LEVEL_CRITICAL, __VA_ARGS__) #define g_warning(...) g_log (G_LOG_DOMAIN, G_LOG_LEVEL_WARNING, __VA_ARGS__) #define g_message(...) g_log (G_LOG_DOMAIN, G_LOG_LEVEL_MESSAGE, __VA_ARGS__) #define g_debug(...) g_log (G_LOG_DOMAIN, G_LOG_LEVEL_DEBUG, __VA_ARGS__) #else -#define g_error(...) do { g_log_disabled (G_LOG_DOMAIN, G_LOG_LEVEL_ERROR, __FILE__, __LINE__); eg_unreachable (); } while (0) +#define g_error(...) do { g_log_disabled (G_LOG_DOMAIN, G_LOG_LEVEL_ERROR, __FILE__, __LINE__); for (;;); } while (0) #define g_critical(...) g_log_disabled (G_LOG_DOMAIN, G_LOG_LEVEL_CRITICAL, __FILE__, __LINE__) #define g_warning(...) g_log_disabled (G_LOG_DOMAIN, G_LOG_LEVEL_WARNING, __FILE__, __LINE__) #define g_message(...) g_log_disabled (G_LOG_DOMAIN, G_LOG_LEVEL_MESSAGE, __FILE__, __LINE__) @@ -796,6 +785,14 @@ gpointer g_convert_error_quark(void); #define G_UNLIKELY(x) (x) #endif +#if defined(_MSC_VER) +#define eg_unreachable() __assume(0) +#elif defined(__GNUC__) && ((__GNUC__ > 4) || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 5))) +#define eg_unreachable() __builtin_unreachable() +#else +#define eg_unreachable() +#endif + /* g_assert is a boolean expression; the precise value is not preserved, just true or false. */ #ifdef DISABLE_ASSERT_MESSAGES // This is smaller than the equivalent mono_assertion_message (..."disabled"); diff --git a/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c b/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c index a62ddf3aec9bad..381164d8f94039 100644 --- a/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c +++ b/src/mono/mono/eventpipe/ep-rt-mono-runtime-provider.c @@ -779,8 +779,6 @@ include_method (MonoMethod *method) return false; } else if (!m_method_is_wrapper (method)) { return true; - } else if (method->wrapper_type == MONO_WRAPPER_DYNAMIC_METHOD){ - return true; } else { WrapperInfo *wrapper = mono_marshal_get_wrapper_info (method); return (wrapper && wrapper->subtype == WRAPPER_SUBTYPE_PINVOKE) ? true : false; diff --git a/src/mono/mono/eventpipe/ep-rt-mono.c b/src/mono/mono/eventpipe/ep-rt-mono.c index 0aa3518df7af77..9b9f9e34c4bce2 100644 --- a/src/mono/mono/eventpipe/ep-rt-mono.c +++ b/src/mono/mono/eventpipe/ep-rt-mono.c @@ -852,11 +852,6 @@ ep_rt_mono_init_finish (void) void ep_rt_mono_fini (void) { - // Avoid cleaning up resources to prevent cleaning up out from under running - // threads. - if (!mono_runtime_is_shutting_down ()) - return; - ep_rt_mono_runtime_provider_fini (); ep_rt_mono_profiler_provider_fini (); diff --git a/src/mono/mono/metadata/appdomain.c b/src/mono/mono/metadata/appdomain.c index e41b4fb1ca8b48..3ed266c09798d9 100644 --- a/src/mono/mono/metadata/appdomain.c +++ b/src/mono/mono/metadata/appdomain.c @@ -612,7 +612,6 @@ try_load_from (MonoAssembly **assembly, *assembly = NULL; fullpath = g_build_filename (path1, path2, path3, path4, (const char*)NULL); - g_assert (fullpath); found = g_file_test (fullpath, G_FILE_TEST_IS_REGULAR); diff --git a/src/mono/mono/metadata/assembly.c b/src/mono/mono/metadata/assembly.c index 49eca9772f481e..63677c87b6f34f 100644 --- a/src/mono/mono/metadata/assembly.c +++ b/src/mono/mono/metadata/assembly.c @@ -336,7 +336,6 @@ load_in_path (const char *basename, const char** search_path, const MonoAssembly for (i = 0; search_path [i]; ++i) { fullpath = g_build_filename (search_path [i], basename, (const char*)NULL); - g_assert (fullpath); result = mono_assembly_request_open (fullpath, req, status); g_free (fullpath); if (result) @@ -1408,7 +1407,6 @@ absolute_dir (const gchar *filename) cwd = g_get_current_dir (); mixed = g_build_filename (cwd, filename, (const char*)NULL); - g_assert (mixed); parts = g_strsplit (mixed, G_DIR_SEPARATOR_S, 0); g_free (mixed); g_free (cwd); diff --git a/src/mono/mono/metadata/class-init.c b/src/mono/mono/metadata/class-init.c index ffefe2faa77a79..484eedb9038f17 100644 --- a/src/mono/mono/metadata/class-init.c +++ b/src/mono/mono/metadata/class-init.c @@ -2272,7 +2272,6 @@ mono_class_layout_fields (MonoClass *klass, int base_instance_size, int packing_ } size = mono_type_size (field->type, &align); - // keep in sync with marshal.c mono_marshal_load_type_info if (m_class_is_inlinearray (klass)) { // Limit the max size of array instance to 1MiB const guint32 struct_max_size = 1024 * 1024; @@ -2953,7 +2952,7 @@ mono_class_init_internal (MonoClass *klass) if (klass->inited || mono_class_has_failure (klass)) return !mono_class_has_failure (klass); - // g_print ("Init class %s\n", mono_type_get_full_name (klass)); + /*g_print ("Init class %s\n", mono_type_get_full_name (klass));*/ /* * This function can recursively call itself. diff --git a/src/mono/mono/metadata/class.c b/src/mono/mono/metadata/class.c index 05da1332e3012f..c5fcd2a8d7a18d 100644 --- a/src/mono/mono/metadata/class.c +++ b/src/mono/mono/metadata/class.c @@ -640,7 +640,10 @@ mono_type_is_valid_generic_argument (MonoType *type) { switch (type->type) { case MONO_TYPE_VOID: + case MONO_TYPE_TYPEDBYREF: return FALSE; + case MONO_TYPE_VALUETYPE: + return !m_class_is_byreflike (type->data.klass); default: return TRUE; } @@ -6785,13 +6788,10 @@ mono_method_get_base_method (MonoMethod *method, gboolean definition, MonoError if (mono_class_is_open_constructed_type (m_class_get_byval_arg (parent))) { parent = mono_class_inflate_generic_class_checked (parent, generic_inst, error); return_val_if_nok (error, NULL); - g_assert (parent); } - if (mono_class_is_ginst (parent)) { parent_inst = mono_class_get_context (parent); parent = mono_class_get_generic_class (parent)->container_class; - g_assert (parent); } mono_class_setup_vtable (parent); @@ -6811,7 +6811,6 @@ mono_method_get_base_method (MonoMethod *method, gboolean definition, MonoError if (mono_class_is_open_constructed_type (m_class_get_byval_arg (klass))) { klass = mono_class_inflate_generic_class_checked (klass, generic_inst, error); return_val_if_nok (error, NULL); - g_assert (klass); generic_inst = NULL; } @@ -6825,7 +6824,6 @@ mono_method_get_base_method (MonoMethod *method, gboolean definition, MonoError if (generic_inst) { klass = mono_class_inflate_generic_class_checked (klass, generic_inst, error); return_val_if_nok (error, NULL); - g_assert (klass); generic_inst = NULL; } @@ -6914,7 +6912,7 @@ mono_class_has_default_constructor (MonoClass *klass, gboolean public_only) * \param klass class in which the failure was detected * \param fmt \c printf -style error message string. * - * Sets a deferred failure in the class and prints a warning message. + * Sets a deferred failure in the class and prints a warning message. * The deferred failure allows the runtime to attempt setting up the class layout at runtime. * * LOCKING: Acquires the loader lock. diff --git a/src/mono/mono/metadata/icall-decl.h b/src/mono/mono/metadata/icall-decl.h index 6fd12507ec4905..1f4ad944a32691 100644 --- a/src/mono/mono/metadata/icall-decl.h +++ b/src/mono/mono/metadata/icall-decl.h @@ -194,6 +194,9 @@ ICALL_EXPORT void ves_icall_System_Threading_LowLevelLifoSemaphore_ReleaseIn ICALL_EXPORT gpointer ves_icall_System_Threading_LowLevelLifoAsyncWaitSemaphore_InitInternal (void); ICALL_EXPORT void ves_icall_System_Threading_LowLevelLifoAsyncWaitSemaphore_PrepareAsyncWaitInternal (gpointer sem_ptr, gint32 timeout_ms, gpointer success_cb, gpointer timeout_cb, intptr_t user_data); +ICALL_EXPORT MonoBoolean ves_icall_System_Threading_WebWorkerEventLoop_HasUnsettledInteropPromisesNative(void); +ICALL_EXPORT void ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePushInternal (void); +ICALL_EXPORT void ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePopInternal (void); #endif #ifdef TARGET_AMD64 diff --git a/src/mono/mono/metadata/icall-def.h b/src/mono/mono/metadata/icall-def.h index c741ccaa447320..06f3ab888b729d 100644 --- a/src/mono/mono/metadata/icall-def.h +++ b/src/mono/mono/metadata/icall-def.h @@ -615,6 +615,14 @@ HANDLES(THREAD_10, "SetState", ves_icall_System_Threading_Thread_SetState, void, HANDLES(THREAD_13, "StartInternal", ves_icall_System_Threading_Thread_StartInternal, void, 2, (MonoThreadObject, gint32)) NOHANDLES(ICALL(THREAD_14, "YieldInternal", ves_icall_System_Threading_Thread_YieldInternal)) +/* include these icalls if we're in the threaded wasm runtime, or if we're building a wasm-targeting cross compiler and we need to support --print-icall-table */ +#if (defined(HOST_BROWSER) && !defined(DISABLE_THREADS)) || (defined(TARGET_WASM) && defined(ENABLE_ICALL_SYMBOL_MAP)) +ICALL_TYPE(WEBWORKERLOOP, "System.Threading.WebWorkerEventLoop", WEBWORKERLOOP_1) +NOHANDLES(ICALL(WEBWORKERLOOP_1, "HasUnsettledInteropPromisesNative", ves_icall_System_Threading_WebWorkerEventLoop_HasUnsettledInteropPromisesNative)) +NOHANDLES(ICALL(WEBWORKERLOOP_2, "KeepalivePopInternal", ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePopInternal)) +NOHANDLES(ICALL(WEBWORKERLOOP_3, "KeepalivePushInternal", ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePushInternal)) +#endif + ICALL_TYPE(TYPE, "System.Type", TYPE_1) HANDLES(TYPE_1, "internal_from_handle", ves_icall_System_Type_internal_from_handle, MonoReflectionType, 1, (MonoType_ref)) diff --git a/src/mono/mono/metadata/icall.c b/src/mono/mono/metadata/icall.c index 6a3e33bc341af6..9c3ea1171106e3 100644 --- a/src/mono/mono/metadata/icall.c +++ b/src/mono/mono/metadata/icall.c @@ -3028,7 +3028,7 @@ ves_icall_RuntimeType_GetNamespace (MonoQCallTypeHandle type_handle, MonoObjectH MonoClass *klass = mono_class_from_mono_type_internal (type); MonoClass *elem; - while (!m_class_is_enumtype (klass) && + while (!m_class_is_enumtype (klass) && !mono_class_is_nullable (klass) && (klass != (elem = m_class_get_element_class (klass)))) klass = elem; @@ -4564,7 +4564,6 @@ ves_icall_System_Reflection_RuntimeAssembly_GetInfo (MonoQCallAssemblyHandle ass else absolute = g_build_filename (assembly->basedir, filename, (const char*)NULL); - g_assert (absolute); mono_icall_make_platform_path (absolute); const gchar *prepend = mono_icall_get_file_path_prefix (absolute); diff --git a/src/mono/mono/metadata/image.c b/src/mono/mono/metadata/image.c index 23333b1b2a9764..dbca4e24f48e34 100644 --- a/src/mono/mono/metadata/image.c +++ b/src/mono/mono/metadata/image.c @@ -2511,7 +2511,6 @@ mono_image_load_file_for_image_checked (MonoImage *image, uint32_t fileidx, Mono fname = mono_metadata_string_heap (image, fname_id); base_dir = g_path_get_dirname (image->name); name = g_build_filename (base_dir, fname, (const char*)NULL); - g_assert (name); res = mono_image_open (name, NULL); if (!res) goto done; diff --git a/src/mono/mono/metadata/loader.c b/src/mono/mono/metadata/loader.c index 5f9e5c0e7d1f1e..67c613b9631ae0 100644 --- a/src/mono/mono/metadata/loader.c +++ b/src/mono/mono/metadata/loader.c @@ -823,6 +823,7 @@ mono_method_search_in_array_class (MonoClass *klass, const char *name, MonoMetho int i; mono_class_setup_methods (klass); + g_assert (!mono_class_has_failure (klass)); /*FIXME this should not fail, right?*/ int mcount = mono_class_get_method_count (klass); MonoMethod **klass_methods = m_class_get_methods (klass); for (i = 0; i < mcount; ++i) { diff --git a/src/mono/mono/metadata/marshal.c b/src/mono/mono/metadata/marshal.c index e9cebef7a81824..c57fa32127a9f7 100644 --- a/src/mono/mono/metadata/marshal.c +++ b/src/mono/mono/metadata/marshal.c @@ -3294,7 +3294,7 @@ mono_emit_marshal (EmitMarshalContext *m, int argnum, MonoType *t, return mono_emit_disabled_marshal (m, argnum, t, spec, conv_arg, conv_arg_type, action); return mono_component_marshal_ilgen()->emit_marshal_ilgen(m, argnum, t, spec, conv_arg, conv_arg_type, action, get_marshal_cb()); -} +} static void mono_marshal_set_callconv_for_type(MonoType *type, MonoMethodSignature *csig, gboolean *skip_gc_trans /*out*/) @@ -3577,7 +3577,7 @@ mono_marshal_get_native_wrapper (MonoMethod *method, gboolean check_exceptions, if (G_UNLIKELY (pinvoke && mono_method_has_unmanaged_callers_only_attribute (method))) { /* - * In AOT mode and embedding scenarios, it is possible that the icall is not registered in the runtime doing the AOT compilation. + * In AOT mode and embedding scenarios, it is possible that the icall is not registered in the runtime doing the AOT compilation. * Emit a wrapper that throws a NotSupportedException. */ get_marshal_cb ()->mb_emit_exception (mb, "System", "NotSupportedException", "Method canot be marked with both DllImportAttribute and UnmanagedCallersOnlyAttribute"); @@ -3757,7 +3757,7 @@ mono_marshal_get_native_wrapper (MonoMethod *method, gboolean check_exceptions, } goto leave; - + emit_exception_for_error: mono_error_cleanup (emitted_error); info = mono_wrapper_info_create (mb, WRAPPER_SUBTYPE_NONE); @@ -3922,14 +3922,14 @@ mono_marshal_get_native_func_wrapper_indirect (MonoClass *caller_class, MonoMeth caller_class = mono_class_get_generic_type_definition (caller_class); MonoImage *image = m_class_get_image (caller_class); g_assert (sig->pinvoke); - g_assert (!sig->hasthis && !sig->explicit_this); + g_assert (!sig->hasthis && ! sig->explicit_this); g_assert (!sig->has_type_parameters); #if 0 /* * Since calli sigs are already part of ECMA-335, they were already used by C++/CLI, which * allowed non-blittable types. So the C# function pointers spec doesn't restrict this to - * blittable types only. + * blittable tyhpes only. */ g_assertf (type_is_blittable (sig->ret), "sig return type %s is not blittable\n", mono_type_full_name (sig->ret)); @@ -5231,7 +5231,7 @@ mono_marshal_get_unsafe_accessor_wrapper (MonoMethod *accessor_method, MonoUnsaf if (member_name == NULL && kind != MONO_UNSAFE_ACCESSOR_CTOR) member_name = accessor_method->name; - + /* * Check cache */ @@ -5827,20 +5827,11 @@ mono_marshal_load_type_info (MonoClass* klass) continue; } - size = mono_marshal_type_size (field->type, info->fields [j].mspec, - &align, TRUE, m_class_is_unicode (klass)); - - // Keep in sync with class-init.c mono_class_layout_fields - if (m_class_is_inlinearray (klass)) { - // Limit the max size of array instance to 1MiB - const int struct_max_size = 1024 * 1024; - size *= m_class_inlinearray_value (klass); - g_assert ((size > 0) && (size <= struct_max_size)); - } - switch (layout) { case TYPE_ATTRIBUTE_AUTO_LAYOUT: case TYPE_ATTRIBUTE_SEQUENTIAL_LAYOUT: + size = mono_marshal_type_size (field->type, info->fields [j].mspec, + &align, TRUE, m_class_is_unicode (klass)); align = m_class_get_packing_size (klass) ? MIN (m_class_get_packing_size (klass), align): align; min_align = MAX (align, min_align); info->fields [j].offset = info->native_size; @@ -5849,6 +5840,8 @@ mono_marshal_load_type_info (MonoClass* klass) info->native_size = info->fields [j].offset + size; break; case TYPE_ATTRIBUTE_EXPLICIT_LAYOUT: + size = mono_marshal_type_size (field->type, info->fields [j].mspec, + &align, TRUE, m_class_is_unicode (klass)); min_align = MAX (align, min_align); info->fields [j].offset = m_field_get_offset (field) - MONO_ABI_SIZEOF (MonoObject); info->native_size = MAX (info->native_size, info->fields [j].offset + size); diff --git a/src/mono/mono/metadata/metadata.c b/src/mono/mono/metadata/metadata.c index b432e7f07dc54d..f7160292232986 100644 --- a/src/mono/mono/metadata/metadata.c +++ b/src/mono/mono/metadata/metadata.c @@ -3942,7 +3942,7 @@ compare_type_literals (MonoImage *image, int class_type, int type_type, MonoErro if (class_type == MONO_TYPE_STRING || class_type == MONO_TYPE_OBJECT) return TRUE; //XXX stringify this argument - mono_error_set_type_load_name (error, NULL, NULL, "Expected reference type but got type kind %d", class_type); + mono_error_set_bad_image (error, image, "Expected reference type but got type kind %d", class_type); return FALSE; } @@ -3966,7 +3966,7 @@ compare_type_literals (MonoImage *image, int class_type, int type_type, MonoErro return TRUE; default: //XXX stringify this argument - mono_error_set_type_load_name (error, NULL, NULL, "Expected value type but got type kind %d", class_type); + mono_error_set_bad_image (error, image, "Expected value type but got type kind %d", class_type); return FALSE; } } diff --git a/src/mono/mono/metadata/object-internals.h b/src/mono/mono/metadata/object-internals.h index daaba307f17dbc..257d06a915da99 100644 --- a/src/mono/mono/metadata/object-internals.h +++ b/src/mono/mono/metadata/object-internals.h @@ -1439,17 +1439,16 @@ typedef struct { /* Keep in sync with System.GenericParameterAttributes */ typedef enum { - GENERIC_PARAMETER_ATTRIBUTE_NON_VARIANT = 0x0000, - GENERIC_PARAMETER_ATTRIBUTE_COVARIANT = 0x0001, - GENERIC_PARAMETER_ATTRIBUTE_CONTRAVARIANT = 0x0002, - GENERIC_PARAMETER_ATTRIBUTE_VARIANCE_MASK = 0x0003, - - GENERIC_PARAMETER_ATTRIBUTE_NO_SPECIAL_CONSTRAINT = 0x0000, - GENERIC_PARAMETER_ATTRIBUTE_REFERENCE_TYPE_CONSTRAINT = 0x0004, - GENERIC_PARAMETER_ATTRIBUTE_VALUE_TYPE_CONSTRAINT = 0x0008, - GENERIC_PARAMETER_ATTRIBUTE_CONSTRUCTOR_CONSTRAINT = 0x0010, - GENERIC_PARAMETER_ATTRIBUTE_ACCEPT_BYREFLIKE_CONSTRAINTS = 0x0020, // type argument can be ByRefLike - GENERIC_PARAMETER_ATTRIBUTE_SPECIAL_CONSTRAINTS_MASK = 0x003c + GENERIC_PARAMETER_ATTRIBUTE_NON_VARIANT = 0, + GENERIC_PARAMETER_ATTRIBUTE_COVARIANT = 1, + GENERIC_PARAMETER_ATTRIBUTE_CONTRAVARIANT = 2, + GENERIC_PARAMETER_ATTRIBUTE_VARIANCE_MASK = 3, + + GENERIC_PARAMETER_ATTRIBUTE_NO_SPECIAL_CONSTRAINT = 0, + GENERIC_PARAMETER_ATTRIBUTE_REFERENCE_TYPE_CONSTRAINT = 4, + GENERIC_PARAMETER_ATTRIBUTE_VALUE_TYPE_CONSTRAINT = 8, + GENERIC_PARAMETER_ATTRIBUTE_CONSTRUCTOR_CONSTRAINT = 16, + GENERIC_PARAMETER_ATTRIBUTE_SPECIAL_CONSTRAINTS_MASK = 28 } GenericParameterAttributes; typedef struct { diff --git a/src/mono/mono/metadata/object.c b/src/mono/mono/metadata/object.c index 4b14d889b9df6b..d6ddb751fe3a74 100644 --- a/src/mono/mono/metadata/object.c +++ b/src/mono/mono/metadata/object.c @@ -4243,8 +4243,6 @@ prepare_run_main (MonoMethod *method, int argc, char *argv[]) basename, (const char*)NULL); - g_assert (fullpath); - utf8_fullpath = utf8_from_external (fullpath); if(utf8_fullpath == NULL) { /* Printing the arg text will cause glib to @@ -5357,7 +5355,7 @@ MonoObjectHandle mono_object_new_handle (MonoClass *klass, MonoError *error) { MONO_REQ_GC_UNSAFE_MODE; - + if (MONO_CLASS_IS_IMPORT(klass)) { mono_error_set_not_supported (error, "Built-in COM interop is not supported on Mono."); return MONO_HANDLE_NEW (MonoObject, NULL); diff --git a/src/mono/mono/metadata/sgen-tarjan-bridge.c b/src/mono/mono/metadata/sgen-tarjan-bridge.c index b0c9cf1f83baef..6f1fb1ec10bd94 100644 --- a/src/mono/mono/metadata/sgen-tarjan-bridge.c +++ b/src/mono/mono/metadata/sgen-tarjan-bridge.c @@ -819,10 +819,8 @@ create_scc (ScanData *data) g_error ("Invalid state when building SCC %d", other->state); } - if (other->is_bridge) { - g_assert (color_data); + if (other->is_bridge) dyn_array_ptr_add (&color_data->bridges, other->obj); - } // Maybe we should make sure we are not adding duplicates here. It is not really a problem // since we will get rid of duplicates before submitting the SCCs to the client in gather_xrefs diff --git a/src/mono/mono/metadata/threads.c b/src/mono/mono/metadata/threads.c index 74f1db84f0518c..0fe313db637cf9 100644 --- a/src/mono/mono/metadata/threads.c +++ b/src/mono/mono/metadata/threads.c @@ -4996,6 +4996,26 @@ ves_icall_System_Threading_LowLevelLifoAsyncWaitSemaphore_PrepareAsyncWaitIntern mono_lifo_semaphore_asyncwait_prepare_wait (sem, timeout_ms, (LifoSemaphoreAsyncWaitCallbackFn)success_cb, (LifoSemaphoreAsyncWaitCallbackFn)timedout_cb, user_data); } +void +ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePushInternal (void) +{ + emscripten_runtime_keepalive_push(); +} + +void +ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePopInternal (void) +{ + emscripten_runtime_keepalive_pop(); +} + +extern int mono_wasm_eventloop_has_unsettled_interop_promises(void); + +MonoBoolean +ves_icall_System_Threading_WebWorkerEventLoop_HasUnsettledInteropPromisesNative(void) +{ + return !!mono_wasm_eventloop_has_unsettled_interop_promises(); +} + #endif /* HOST_BROWSER && !DISABLE_THREADS */ /* for the AOT cross compiler with --print-icall-table these don't need to be callable, they just @@ -5013,5 +5033,22 @@ ves_icall_System_Threading_LowLevelLifoAsyncWaitSemaphore_PrepareAsyncWaitIntern g_assert_not_reached (); } +void +ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePushInternal (void) +{ + g_assert_not_reached(); +} + +void +ves_icall_System_Threading_WebWorkerEventLoop_KeepalivePopInternal (void) +{ + g_assert_not_reached(); +} + +MonoBoolean +ves_icall_System_Threading_WebWorkerEventLoop_HasUnsettledInteropPromisesNative(void) +{ + g_assert_not_reached(); +} #endif /* defined(TARGET_WASM) && defined(ENABLE_ICALL_SYMBOL_MAP) */ diff --git a/src/mono/mono/metadata/verify.c b/src/mono/mono/metadata/verify.c index 621599fa4eeeb5..660f6226eb009c 100644 --- a/src/mono/mono/metadata/verify.c +++ b/src/mono/mono/metadata/verify.c @@ -87,9 +87,6 @@ is_valid_generic_instantiation (MonoGenericContainer *gc, MonoGenericContext *co return FALSE; } - if (m_class_is_byreflike (paramClass) && (param_info->flags & GENERIC_PARAMETER_ATTRIBUTE_ACCEPT_BYREFLIKE_CONSTRAINTS) == 0) - return FALSE; - if (!param_info->constraints && !(param_info->flags & GENERIC_PARAMETER_ATTRIBUTE_SPECIAL_CONSTRAINTS_MASK)) continue; diff --git a/src/mono/mono/mini/aot-compiler.c b/src/mono/mono/mini/aot-compiler.c index c4e318b4f56882..d8f80b0bc6a932 100644 --- a/src/mono/mono/mini/aot-compiler.c +++ b/src/mono/mono/mini/aot-compiler.c @@ -238,8 +238,6 @@ typedef struct MonoAotOptions { gboolean verbose; gboolean deterministic; gboolean allow_errors; - gboolean compile_in_child; - gboolean child; char *tool_prefix; char *as_prefix; char *ld_flags; @@ -260,8 +258,6 @@ typedef struct MonoAotOptions { char *clangxx; char *depfile; char *runtime_init_callback; - GPtrArray *runtime_args; - const char *aot_options; } MonoAotOptions; typedef enum { @@ -370,12 +366,10 @@ typedef struct MonoAotCompile { MonoDwarfWriter *dwarf; FILE *fp; char *tmpbasename; - char *asm_fname; + char *tmpfname; char *temp_dir_to_delete; char *llvm_sfile; char *llvm_ofile; - char *bc_fname; - char *optbc_fname; GSList *cie_program; GHashTable *unwind_info_offsets; GPtrArray *unwind_ops; @@ -454,7 +448,6 @@ static MonoAotCompile *llvm_acfg; static MonoAotCompile *current_acfg; static MonoAssembly *dedup_assembly; static GHashTable *dedup_methods; -static GPtrArray *dedup_methods_list; /* Cache of decoded method external icall symbol names. */ /* Owned by acfg, but kept in this static as well since it is */ @@ -488,15 +481,9 @@ get_patch_name (int info) static int aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options); -static void -set_paths (MonoAotCompile *acfg); - static int emit_aot_image (MonoAotCompile *acfg); -static int -assemble_link (MonoAotCompile *acfg); - static guint32 get_unwind_info_offset (MonoAotCompile *acfg, guint8 *encoded, guint32 encoded_len); @@ -521,9 +508,6 @@ is_direct_pinvoke_specified_for_method (MonoAotCompile *acfg, MonoMethod *method static inline const char* lookup_direct_pinvoke_symbol_name_aot (MonoAotCompile *acfg, MonoMethod *method); -static int -compile_assemblies_in_child (MonoAotOptions *aot_opts, MonoAssembly **assemblies, int nassemblies, GPtrArray *runtime_args, const char *aot_options); - static gboolean mono_aot_mode_is_full (MonoAotOptions *opts) { @@ -4367,13 +4351,11 @@ collect_dedup_method (MonoAotCompile *acfg, MonoMethod *method) return TRUE; // Remember for later g_assert (acfg->dedup_phase == DEDUP_COLLECT); - if (!g_hash_table_lookup (dedup_methods, method)) { + if (!g_hash_table_lookup (dedup_methods, method)) g_hash_table_insert (dedup_methods, method, method); - g_ptr_array_add (dedup_methods_list, method); - } else { + else // Already processed when compiling another assembly return TRUE; - } } return FALSE; } @@ -5508,10 +5490,10 @@ MONO_RESTORE_WARNING if (decoded_args->named_args_info [j].field && !strcmp (decoded_args->named_args_info [j].field->name, "EntryPoint")) { named = (const char *)decoded_args->named_args[j]->value.primitive; slen = mono_metadata_decode_value (named, &named); - + int prefix_len = (int)strlen (acfg->user_symbol_prefix); g_assert (prefix_len < 2); - + export_name = (char *)g_malloc (prefix_len + slen + 1); if (prefix_len == 1) export_name[0] = *acfg->user_symbol_prefix; @@ -5851,14 +5833,12 @@ add_generic_class_with_depth (MonoAotCompile *acfg, MonoClass *klass, int depth, icomparable_inst = mono_class_inflate_generic_class_checked (icomparable, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (icomparable_inst); if (mono_class_is_assignable_from_internal (icomparable_inst, tclass)) { MonoClass *gcomparer_inst; gcomparer = mono_class_load_from_name (mono_defaults.corlib, "System.Collections.Generic", "GenericComparer`1"); gcomparer_inst = mono_class_inflate_generic_class_checked (gcomparer, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (gcomparer_inst); add_generic_class (acfg, gcomparer_inst, FALSE, "Comparer"); } @@ -5880,7 +5860,6 @@ add_generic_class_with_depth (MonoAotCompile *acfg, MonoClass *klass, int depth, iface_inst = mono_class_inflate_generic_class_checked (iface, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (iface_inst); if (mono_class_is_assignable_from_internal (iface_inst, tclass)) { MonoClass *gcomparer_inst; @@ -5888,7 +5867,6 @@ add_generic_class_with_depth (MonoAotCompile *acfg, MonoClass *klass, int depth, gcomparer = mono_class_load_from_name (mono_defaults.corlib, "System.Collections.Generic", "GenericEqualityComparer`1"); gcomparer_inst = mono_class_inflate_generic_class_checked (gcomparer, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (gcomparer_inst); add_generic_class (acfg, gcomparer_inst, FALSE, "EqualityComparer"); } } @@ -5910,7 +5888,6 @@ add_generic_class_with_depth (MonoAotCompile *acfg, MonoClass *klass, int depth, enum_comparer = mono_class_load_from_name (mono_defaults.corlib, "System.Collections.Generic", "EnumEqualityComparer`1"); enum_comparer_inst = mono_class_inflate_generic_class_checked (enum_comparer, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (enum_comparer_inst); add_generic_class (acfg, enum_comparer_inst, FALSE, "EqualityComparer"); } } @@ -5932,7 +5909,6 @@ add_generic_class_with_depth (MonoAotCompile *acfg, MonoClass *klass, int depth, comparer = mono_class_load_from_name (mono_defaults.corlib, "System.Collections.Generic", "ObjectComparer`1"); comparer_inst = mono_class_inflate_generic_class_checked (comparer, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (comparer_inst); add_generic_class (acfg, comparer_inst, FALSE, "Comparer"); } } @@ -5956,7 +5932,6 @@ add_instances_of (MonoAotCompile *acfg, MonoClass *klass, MonoType **insts, int ctx.class_inst = mono_metadata_get_generic_inst (1, args); generic_inst = mono_class_inflate_generic_class_checked (klass, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (generic_inst); add_generic_class (acfg, generic_inst, force, ""); } } @@ -9095,10 +9070,6 @@ mono_aot_parse_options (const char *aot_options, MonoAotOptions *opts) // direct pinvokes (managed-to-native wrappers) and fallbacks to JIT for majority of managed methods. } else if (str_begins_with (arg, "wrappers-only")) { opts->wrappers_only = TRUE; - } else if (!strcmp (arg, "compile-in-child")) { - opts->compile_in_child = TRUE; - } else if (!strcmp (arg, "_child")) { - opts->child = TRUE; } else if (str_begins_with (arg, "help") || str_begins_with (arg, "?")) { printf ("Supported options for --aot:\n"); printf (" asmonly - \n"); @@ -9149,7 +9120,6 @@ mono_aot_parse_options (const char *aot_options, MonoAotOptions *opts) printf (" verbose - \n"); printf (" allow-errors - \n"); printf (" no-opt - \n"); - printf (" compile-in-child - \n"); printf (" llvmopts= - \n"); printf (" llvmllc= - \n"); printf (" clangxx= - \n"); @@ -9548,23 +9518,6 @@ compile_method (MonoAotCompile *acfg, MonoMethod *method) cfg = mini_method_compile (method, acfg->jit_opts, flags, 0, index); mono_time_track_end (&mono_jit_stats.jit_time, jit_time_start); - if (cfg->prefer_instances) { - /* - * Compile the original specific instances in addition to the gshared method - * for performance reasons, since gshared methods cannot implement some - * features like static virtual methods efficiently. - */ - /* Instances encountered later will be handled in add_extra_method_full () */ - g_hash_table_insert (acfg->prefer_instances, method, method); - GPtrArray *instances = g_hash_table_lookup (acfg->gshared_instances, method); - if (instances) { - for (guint i = 0; i < instances->len; ++i) { - MonoMethod *instance = (MonoMethod*)g_ptr_array_index (instances, i); - add_extra_method_full (acfg, instance, FALSE, 0); - } - } - } - if (cfg->exception_type == MONO_EXCEPTION_GENERIC_SHARING_FAILED) { if (acfg->aot_opts.print_skipped_methods) printf ("Skip (gshared failure): %s (%s)\n", mono_method_get_full_name (method), cfg->exception_message); @@ -9650,6 +9603,23 @@ compile_method (MonoAotCompile *acfg, MonoMethod *method) printf ("%s ### %d\n", mono_method_get_full_name (method), cfg->code_size); } + if (cfg->prefer_instances) { + /* + * Compile the original specific instances in addition to the gshared method + * for performance reasons, since gshared methods cannot implement some + * features like static virtual methods efficiently. + */ + /* Instances encountered later will be handled in add_extra_method_full () */ + g_hash_table_insert (acfg->prefer_instances, method, method); + GPtrArray *instances = g_hash_table_lookup (acfg->gshared_instances, method); + if (instances) { + for (guint i = 0; i < instances->len; ++i) { + MonoMethod *instance = (MonoMethod*)g_ptr_array_index (instances, i); + add_extra_method_full (acfg, instance, FALSE, 0); + } + } + } + /* Adds generic instances referenced by this method */ /* * The depth is used to avoid infinite loops when generic virtual recursion is @@ -10679,21 +10649,32 @@ execute_system (const char * command) #endif /* - * compile_llvm_file: + * emit_llvm_file: * - * Compile the llvm bitcode file using the LLVM tools. + * Emit the LLVM code into an LLVM bytecode file, and compile it using the LLVM + * tools. */ static gboolean -compile_llvm_file (MonoAotCompile *acfg) +emit_llvm_file (MonoAotCompile *acfg) { char *command, *opts, *tempbc, *optbc, *output_fname; + if (acfg->aot_opts.llvm_only && acfg->aot_opts.asm_only) { + if (acfg->aot_opts.no_opt) + tempbc = g_strdup (acfg->aot_opts.llvm_outfile); + else + tempbc = g_strdup_printf ("%s.bc", acfg->tmpbasename); + optbc = g_strdup (acfg->aot_opts.llvm_outfile); + } else { + tempbc = g_strdup_printf ("%s.bc", acfg->tmpbasename); + optbc = g_strdup_printf ("%s.opt.bc", acfg->tmpbasename); + } + + mono_llvm_emit_aot_module (tempbc, g_path_get_basename (acfg->image->name)); + if (acfg->aot_opts.no_opt) return TRUE; - tempbc = acfg->bc_fname; - optbc = acfg->optbc_fname; - #if (defined(TARGET_X86) || defined(TARGET_AMD64)) if (acfg->aot_opts.llvm_cpu_attr && strstr (acfg->aot_opts.llvm_cpu_attr, "sse4.2")) /* @@ -10816,7 +10797,7 @@ compile_llvm_file (MonoAotCompile *acfg) #if ( defined(TARGET_MACH) && defined(TARGET_ARM) ) || defined(TARGET_ORBIS) || defined(TARGET_X86_64_WIN32_MSVC) || defined(TARGET_ANDROID) g_string_append_printf (acfg->llc_args, " -relocation-model=pic"); #else - if (acfg->aot_opts.static_link) + if (llvm_acfg->aot_opts.static_link) g_string_append_printf (acfg->llc_args, " -relocation-model=static"); else g_string_append_printf (acfg->llc_args, " -relocation-model=pic"); @@ -11573,9 +11554,6 @@ emit_exception_info (MonoAotCompile *acfg) char *aot_file = g_strdup_printf("%s%s", image_basename, SEQ_POINT_AOT_EXT); char *aot_file_path = g_build_filename (dir, aot_file, (const char*)NULL); - g_assert (dir); - g_assert (aot_file_path); - if (g_ensure_directory_exists (aot_file_path) == FALSE) { fprintf (stderr, "AOT : failed to create msym directory: %s\n", aot_file_path); exit (1); @@ -13202,7 +13180,7 @@ compile_asm (MonoAotCompile *acfg) #endif if (acfg->aot_opts.asm_only) { - aot_printf (acfg, "Output file: '%s'.\n", acfg->asm_fname); + aot_printf (acfg, "Output file: '%s'.\n", acfg->tmpfname); if (acfg->aot_opts.static_link) aot_printf (acfg, "Linking symbol: '%s'.\n", acfg->static_linking_symbol); if (acfg->llvm) @@ -13216,7 +13194,7 @@ compile_asm (MonoAotCompile *acfg) else objfile = g_strdup_printf ("%s." AS_OBJECT_FILE_SUFFIX, acfg->image->name); } else { - objfile = g_strdup_printf ("%s." AS_OBJECT_FILE_SUFFIX, acfg->asm_fname); + objfile = g_strdup_printf ("%s." AS_OBJECT_FILE_SUFFIX, acfg->tmpfname); } #ifdef TARGET_OSX @@ -13225,7 +13203,7 @@ compile_asm (MonoAotCompile *acfg) command = g_strdup_printf ("\"%s%s\" %s %s -o %s %s", as_prefix, AS_NAME, AS_OPTIONS, acfg->as_args ? acfg->as_args->str : "", - wrap_path (objfile), wrap_path (acfg->asm_fname)); + wrap_path (objfile), wrap_path (acfg->tmpfname)); aot_printf (acfg, "Executing the native assembler: %s\n", command); if (execute_system (command) != 0) { g_free (command); @@ -13310,7 +13288,7 @@ compile_asm (MonoAotCompile *acfg) #endif g_string_append_printf (str, " -o %s %s %s %s", wrap_path (tmp_outfile_name), wrap_path (llvm_ofile), - wrap_path (g_strdup_printf ("%s." AS_OBJECT_FILE_SUFFIX, acfg->asm_fname)), ld_flags); + wrap_path (g_strdup_printf ("%s." AS_OBJECT_FILE_SUFFIX, acfg->tmpfname)), ld_flags); #if defined(TARGET_MACH) g_string_append_printf (str, " \"-Wl,-install_name,%s%s\"", g_path_get_basename (acfg->image->name), MONO_SOLIB_EXT); @@ -13377,7 +13355,7 @@ compile_asm (MonoAotCompile *acfg) if (acfg->aot_opts.save_temps) aot_printf (acfg, "Retained input file.\n"); else - g_unlink (acfg->asm_fname); + g_unlink (acfg->tmpfname); return 0; } @@ -14910,14 +14888,14 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) if (acfg->jit_opts & MONO_OPT_GSHAREDVT) mono_set_generic_sharing_vt_supported (TRUE); - if (acfg->dedup_phase != DEDUP_COLLECT && !acfg->aot_opts.child) + if (acfg->dedup_phase != DEDUP_COLLECT) aot_printf (acfg, "Mono Ahead of Time compiler - compiling assembly %s\n", image->name); if (!acfg->aot_opts.deterministic) generate_aotid ((guint8*) &acfg->image->aotid); char *aotid = mono_guid_to_string (acfg->image->aotid); - if (acfg->dedup_phase != DEDUP_COLLECT && !acfg->aot_opts.deterministic && !acfg->aot_opts.child) + if (acfg->dedup_phase != DEDUP_COLLECT && !acfg->aot_opts.deterministic) aot_printf (acfg, "AOTID %s\n", aotid); g_free (aotid); @@ -15072,7 +15050,6 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) arch_init (acfg); if (mono_use_llvm || acfg->aot_opts.llvm) { - acfg->llvm = TRUE; /* * Emit all LLVM code into a separate assembly/object file and link with it * normally. @@ -15087,8 +15064,6 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) } } - set_paths (acfg); - if (acfg->llvm && acfg->thumb_mixed) acfg->flags = (MonoAotFileFlags)(acfg->flags | MONO_AOT_FILE_FLAG_LLVM_THUMB); if (acfg->aot_opts.llvm_only) @@ -15109,21 +15084,6 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) acfg->llvm_eh_frame_symbol = g_strdup_printf ("%s_eh_frame", acfg->global_prefix); } - if (acfg->aot_opts.compile_in_child) { - if (acfg->aot_opts.dedup_include) { - if (acfg->image->assembly == dedup_assembly) - return assemble_link (acfg); - else - return 0; - } else { - res = compile_assemblies_in_child (&acfg->aot_opts, &acfg->image->assembly, 1, acfg->aot_opts.runtime_args, acfg->aot_opts.aot_options); - if (res) - return res; - - return assemble_link (acfg); - } - } - acfg->method_index = 1; if (mono_aot_mode_is_full (&acfg->aot_opts) || mono_aot_mode_is_hybrid (&acfg->aot_opts)) @@ -15139,10 +15099,13 @@ aot_assembly (MonoAssembly *ass, guint32 jit_opts, MonoAotOptions *aot_options) /* Add collected dedup-able methods */ aot_printf (acfg, "Adding %d dedup-ed methods.\n", g_hash_table_size (dedup_methods)); - for (guint i = 0; i < dedup_methods_list->len; ++i) { - MonoMethod *method = (MonoMethod*)g_ptr_array_index (dedup_methods_list, i); + GHashTableIter iter; + MonoMethod *key; + MonoMethod *method; + + g_hash_table_iter_init (&iter, dedup_methods); + while (g_hash_table_iter_next (&iter, (gpointer *)&key, (gpointer *)&method)) add_method_full (acfg, method, TRUE, 0); - } } { @@ -15325,18 +15288,24 @@ create_depfile (MonoAotCompile *acfg) fclose (depfile); } -static void -set_paths (MonoAotCompile *acfg) +static int +emit_aot_image (MonoAotCompile *acfg) { + int res; + TV_DECLARE (atv); + TV_DECLARE (btv); + + TV_GETTIME (atv); + #ifdef ENABLE_LLVM if (acfg->llvm) { if (acfg->aot_opts.asm_only) { if (acfg->aot_opts.outfile) { - acfg->asm_fname = g_strdup_printf ("%s", acfg->aot_opts.outfile); - acfg->tmpbasename = g_strdup (acfg->asm_fname); + acfg->tmpfname = g_strdup_printf ("%s", acfg->aot_opts.outfile); + acfg->tmpbasename = g_strdup (acfg->tmpfname); } else { acfg->tmpbasename = g_strdup_printf ("%s", acfg->image->name); - acfg->asm_fname = g_strdup_printf ("%s.s", acfg->tmpbasename); + acfg->tmpfname = g_strdup_printf ("%s.s", acfg->tmpbasename); } g_assert (acfg->aot_opts.llvm_outfile); acfg->llvm_sfile = g_strdup (acfg->aot_opts.llvm_outfile); @@ -15355,9 +15324,7 @@ set_paths (MonoAotCompile *acfg) } acfg->tmpbasename = g_build_filename (temp_path, "temp", (const char*)NULL); - g_assert (acfg->tmpbasename); - - acfg->asm_fname = g_strdup_printf ("%s.s", acfg->tmpbasename); + acfg->tmpfname = g_strdup_printf ("%s.s", acfg->tmpbasename); acfg->llvm_sfile = g_strdup_printf ("%s-llvm.s", acfg->tmpbasename); if (acfg->aot_opts.static_link) @@ -15367,90 +15334,26 @@ set_paths (MonoAotCompile *acfg) g_free (temp_path); } - - if (acfg->aot_opts.llvm_only && acfg->aot_opts.asm_only) { - if (acfg->aot_opts.no_opt) - acfg->bc_fname = g_strdup (acfg->aot_opts.llvm_outfile); - else - acfg->bc_fname = g_strdup_printf ("%s.bc", acfg->tmpbasename); - acfg->optbc_fname = g_strdup (acfg->aot_opts.llvm_outfile); - } else { - acfg->bc_fname = g_strdup_printf ("%s.bc", acfg->tmpbasename); - acfg->optbc_fname = g_strdup_printf ("%s.opt.bc", acfg->tmpbasename); - } } #endif if (acfg->aot_opts.asm_only && !acfg->aot_opts.llvm_only) { if (acfg->aot_opts.outfile) - acfg->asm_fname = g_strdup_printf ("%s", acfg->aot_opts.outfile); + acfg->tmpfname = g_strdup_printf ("%s", acfg->aot_opts.outfile); else - acfg->asm_fname = g_strdup_printf ("%s.s", acfg->image->name); + acfg->tmpfname = g_strdup_printf ("%s.s", acfg->image->name); + acfg->fp = g_fopen (acfg->tmpfname, "w+"); } else { if (strcmp (acfg->aot_opts.temp_path, "") == 0) { - /* Done later */ + acfg->fp = fdopen (g_file_open_tmp ("mono_aot_XXXXXX", &acfg->tmpfname, NULL), "w+"); } else { acfg->tmpbasename = g_build_filename (acfg->aot_opts.temp_path, "temp", (const char*)NULL); - g_assert (acfg->tmpbasename); - - acfg->asm_fname = g_strdup_printf ("%s.s", acfg->tmpbasename); - } - } -} - -/* Run external tools to assemble/link the aot image */ -static int -assemble_link (MonoAotCompile *acfg) -{ - int res; - TV_DECLARE (atv); - TV_DECLARE (btv); - - TV_GETTIME (atv); - -#ifdef ENABLE_LLVM - if (acfg->llvm) { - gboolean emit_res; - - emit_res = compile_llvm_file (acfg); - if (!emit_res) - return 1; - } -#endif - - if (!acfg->aot_opts.llvm_only) { - res = compile_asm (acfg); - if (res != 0) { - acfg_free (acfg); - return res; - } - } - TV_GETTIME (btv); - acfg->stats.link_time = GINT64_TO_INT (TV_ELAPSED (atv, btv)); - - return 0; -} - -static int -emit_aot_image (MonoAotCompile *acfg) -{ - int res; - TV_DECLARE (atv); - TV_DECLARE (btv); - - TV_GETTIME (atv); - - if (acfg->aot_opts.asm_only && !acfg->aot_opts.llvm_only) { - acfg->fp = g_fopen (acfg->asm_fname, "w+"); - } else { - if (strcmp (acfg->aot_opts.temp_path, "") == 0) { - acfg->fp = fdopen (g_file_open_tmp ("mono_aot_XXXXXX", &acfg->asm_fname, NULL), "w+"); - } else { - acfg->fp = g_fopen (acfg->asm_fname, "w+"); + acfg->tmpfname = g_strdup_printf ("%s.s", acfg->tmpbasename); + acfg->fp = g_fopen (acfg->tmpfname, "w+"); } } if (acfg->fp == 0 && !acfg->aot_opts.llvm_only) { - aot_printerrf (acfg, "Unable to open file '%s': %s\n", acfg->asm_fname, strerror (errno)); + aot_printerrf (acfg, "Unable to open file '%s': %s\n", acfg->tmpfname, strerror (errno)); return 1; } if (acfg->fp) @@ -15568,8 +15471,13 @@ emit_aot_image (MonoAotCompile *acfg) fclose (acfg->data_outfile); #ifdef ENABLE_LLVM - if (acfg->llvm) - mono_llvm_emit_aot_module (acfg->bc_fname, g_path_get_basename (acfg->image->name)); + if (acfg->llvm) { + gboolean emit_res; + + emit_res = emit_llvm_file (acfg); + if (!emit_res) + return 1; + } #endif emit_library_info (acfg); @@ -15581,13 +15489,26 @@ emit_aot_image (MonoAotCompile *acfg) if (!acfg->aot_opts.stats) aot_printf (acfg, "Compiled: %d/%d\n", acfg->stats.ccount, acfg->stats.mcount); + TV_GETTIME (atv); if (acfg->w) { res = mono_img_writer_emit_writeout (acfg->w); if (res != 0) { acfg_free (acfg); return res; } + res = compile_asm (acfg); + if (res != 0) { + acfg_free (acfg); + return res; + } } + TV_GETTIME (btv); + acfg->stats.link_time = GINT64_TO_INT (TV_ELAPSED (atv, btv)); + + if (acfg->aot_opts.stats) + print_stats (acfg); + + aot_printf (acfg, "JIT time: %d ms, Generation time: %d ms, Assembly+Link time: %d ms.\n", acfg->stats.jit_time / 1000, acfg->stats.gen_time / 1000, acfg->stats.link_time / 1000); if (acfg->aot_opts.depfile) create_depfile (acfg); @@ -15595,19 +15516,6 @@ emit_aot_image (MonoAotCompile *acfg) if (acfg->aot_opts.dump_json) aot_dump (acfg); - if (acfg->aot_opts.child) - /* The rest is done in the parent */ - return 0; - - res = assemble_link (acfg); - if (res) - return res; - - if (acfg->aot_opts.stats) - print_stats (acfg); - - aot_printf (acfg, "JIT time: %d ms, Generation time: %d ms, Assembly+Link time: %d ms.\n", acfg->stats.jit_time / 1000, acfg->stats.gen_time / 1000, acfg->stats.link_time / 1000); - if (!acfg->aot_opts.save_temps && acfg->temp_dir_to_delete) { char *command = g_strdup_printf ("rm -r %s", acfg->temp_dir_to_delete); execute_system (command); @@ -15619,96 +15527,14 @@ emit_aot_image (MonoAotCompile *acfg) return 0; } -static int -compile_assemblies_in_child (MonoAotOptions *aot_opts, MonoAssembly **assemblies, int nassemblies, GPtrArray *runtime_args, const char *aot_options) -{ - FILE *response = NULL; - char *response_fname = NULL; - - /* Find --aot argument */ - int aot_index = -1; - for (guint32 i = 1; i < runtime_args->len; ++i) { - const char *arg = (const char*)g_ptr_array_index (runtime_args, i); - if (strncmp (arg, "--aot=", strlen ("--aot=")) == 0) { - aot_index = i; - break; - } - } - g_assert (aot_index != -1); - -#ifdef HOST_WIN32 - response_fname = g_build_filename (aot_opts->temp_path, "temp.rsp", (const char*)NULL); - g_assert (response_fname); - response = fopen (response_fname, "w"); - g_assert (response); -#endif - - GString *command; - - command = g_string_new (""); - - g_string_append_printf (command, "%s", (const char*)g_ptr_array_index (runtime_args, 0)); - - for (guint32 i = 1; i < runtime_args->len; ++i) { - const char *arg = (const char*)g_ptr_array_index (runtime_args, i); - if (strncmp (arg, "--response=", strlen ("--response=")) == 0) - /* Already expanded */ - continue; - if (i != aot_index) { - if (response) - fprintf (response, "%s\n", arg); - else - g_string_append_printf (command, " \"%s\"", arg); - } - } - - /* Pass '_child' instead of 'compile-in-child' */ - GPtrArray *aot_split_args = mono_aot_split_options (aot_options); - GString *new_aot_args = g_string_new (""); - for (guint32 i = 0; i < aot_split_args->len; ++i) { - const char *aot_arg = (const char*)g_ptr_array_index (aot_split_args, i); - if (!strcmp (aot_arg, "compile-in-child")) - aot_arg = "_child"; - if (i > 0) - g_string_append_printf (new_aot_args, ","); - g_string_append_printf (new_aot_args, "%s", aot_arg); - } - - if (response) - fprintf (response, "\"--aot=%s\"\n", g_string_free (new_aot_args, FALSE)); - else - g_string_append_printf (command, " \"--aot=%s\"", g_string_free (new_aot_args, FALSE)); - - for (int i = 0; i < nassemblies; ++i) { - if (response) - fprintf (response, "\"%s\"\n", assemblies [i]->image->name); - else - g_string_append_printf (command, " \"%s\"", assemblies [i]->image->name); - } - - if (response) - fclose (response); - - if (response) - g_string_append_printf (command, " \"--response=%s\"", response_fname); - char *cmd = g_string_free (command, FALSE); - printf ("Executing: %s\n", cmd); - int res = execute_system (cmd); - g_free (cmd); - - return res; -} - int -mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 jit_opts, GPtrArray *runtime_args, const char *aot_options) +mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 jit_opts, const char *aot_options) { int res = 0; MonoAotOptions aot_opts; init_options (&aot_opts); mono_aot_parse_options (aot_options, &aot_opts); - aot_opts.runtime_args = runtime_args; - aot_opts.aot_options = aot_options; if (aot_opts.direct_extern_calls && !(aot_opts.llvm && aot_opts.static_link)) { fprintf (stderr, "The 'direct-extern-calls' option requires the 'llvm' and 'static' options.\n"); res = 1; @@ -15721,22 +15547,6 @@ mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 jit_opt goto early_exit; } - if (aot_opts.compile_in_child) { - if (aot_opts.temp_path [0] == '\0') { - fprintf (stderr, "The 'compile-in-child' option requires the 'temp-path=' option.\n"); - res = 1; - goto early_exit; - } - if (nassemblies > 1 && !aot_opts.dedup_include) - aot_opts.compile_in_child = FALSE; - } - - if (aot_opts.dedup_include && aot_opts.compile_in_child) { - res = compile_assemblies_in_child (&aot_opts, assemblies, nassemblies, aot_opts.runtime_args, aot_opts.aot_options); - if (res) - return res; - } - if (aot_opts.dedup_include) { /* Find the assembly which will contain the dedup-ed code */ int dedup_aindex = -1; @@ -15760,7 +15570,6 @@ mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 jit_opt assemblies [dedup_aindex] = atmp; dedup_methods = g_hash_table_new (NULL, NULL); - dedup_methods_list = g_ptr_array_new (); } if (aot_opts.trimming_eligible_methods_outfile) { @@ -15827,7 +15636,7 @@ mono_aot_get_method_index (MonoMethod *method) } int -mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 jit_opts, GPtrArray *runtime_args, const char *aot_options) +mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 opts, const char *aot_options) { return 0; } diff --git a/src/mono/mono/mini/aot-compiler.h b/src/mono/mono/mini/aot-compiler.h index 5856134b8eff02..7aaf862f2e92da 100644 --- a/src/mono/mono/mini/aot-compiler.h +++ b/src/mono/mono/mini/aot-compiler.h @@ -7,7 +7,7 @@ #include "mini.h" -int mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 opts, GPtrArray *runtime_args, const char *aot_options); +int mono_aot_assemblies (MonoAssembly **assemblies, int nassemblies, guint32 opts, const char *aot_options); void* mono_aot_readonly_field_override (MonoClassField *field); gboolean mono_aot_direct_icalls_enabled_for_method (MonoCompile *cfg, MonoMethod *method); gboolean mono_aot_is_shared_got_offset (int offset); diff --git a/src/mono/mono/mini/aot-runtime-wasm.c b/src/mono/mono/mini/aot-runtime-wasm.c index 30fde73c155bd5..cf1ab02392934d 100644 --- a/src/mono/mono/mini/aot-runtime-wasm.c +++ b/src/mono/mono/mini/aot-runtime-wasm.c @@ -15,12 +15,8 @@ #ifdef HOST_WASM static char -type_to_c (MonoType *t, gboolean *is_byref_return) +type_to_c (MonoType *t) { - g_assert (t); - - if (is_byref_return) - *is_byref_return = 0; if (m_type_is_byref (t)) return 'I'; @@ -52,7 +48,7 @@ type_to_c (MonoType *t, gboolean *is_byref_return) return 'L'; case MONO_TYPE_VOID: return 'V'; - case MONO_TYPE_VALUETYPE: { + case MONO_TYPE_VALUETYPE: if (m_class_is_enumtype (t->data.klass)) { t = mono_class_enum_basetype_internal (t->data.klass); goto handle_enum; @@ -64,27 +60,13 @@ type_to_c (MonoType *t, gboolean *is_byref_return) // FIXME: Handle the scenario where there are fields of struct types that contain no members MonoType *scalar_vtype; if (mini_wasm_is_scalar_vtype (t, &scalar_vtype)) - return type_to_c (scalar_vtype, NULL); - - if (is_byref_return) - *is_byref_return = 1; + return type_to_c (scalar_vtype); return 'I'; - } - case MONO_TYPE_GENERICINST: { - if (m_class_is_valuetype (t->data.klass)) { - MonoType *scalar_vtype; - if (mini_wasm_is_scalar_vtype (t, &scalar_vtype)) - return type_to_c (scalar_vtype, NULL); - - if (is_byref_return) - *is_byref_return = 1; - + case MONO_TYPE_GENERICINST: + if (m_class_is_valuetype (t->data.klass)) return 'S'; - } - return 'I'; - } default: g_warning ("CANT TRANSLATE %s", mono_type_full_name (t)); return 'X'; @@ -158,29 +140,18 @@ gpointer mono_wasm_get_interp_to_native_trampoline (MonoMethodSignature *sig) { char cookie [32]; - int c_count, offset = 1; - gboolean is_byref_return = 0; - - memset (cookie, 0, 32); - cookie [0] = type_to_c (sig->ret, &is_byref_return); + int c_count; - c_count = sig->param_count + sig->hasthis + is_byref_return + 1; + c_count = sig->param_count + sig->hasthis + 1; g_assert (c_count < sizeof (cookie)); //ensure we don't overflow the local - if (is_byref_return) { - cookie[0] = 'V'; - // return value address goes in arg0 - cookie[1] = 'I'; - offset += 1; - } - if (sig->hasthis) { - // thisptr goes in arg0/arg1 depending on return type - cookie [offset] = 'I'; - offset += 1; - } + cookie [0] = type_to_c (sig->ret); + if (sig->hasthis) + cookie [1] = 'I'; for (int i = 0; i < sig->param_count; ++i) { - cookie [offset + i] = type_to_c (sig->params [i], NULL); + cookie [1 + sig->hasthis + i] = type_to_c (sig->params [i]); } + cookie [c_count] = 0; void *p = mono_wasm_interp_to_native_callback (cookie); if (!p) diff --git a/src/mono/mono/mini/driver.c b/src/mono/mono/mini/driver.c index 56e20eeb5eb017..2077f30d412d29 100644 --- a/src/mono/mono/mini/driver.c +++ b/src/mono/mono/mini/driver.c @@ -1386,7 +1386,6 @@ typedef struct char **argv; guint32 opts; char *aot_options; - GPtrArray *runtime_args; } MainThreadArgs; static void @@ -1422,7 +1421,7 @@ main_thread_handler (gpointer user_data) assemblies [i] = assembly; } - res = mono_aot_assemblies (assemblies, main_args->argc, main_args->opts, main_args->runtime_args, main_args->aot_options); + res = mono_aot_assemblies (assemblies, main_args->argc, main_args->opts, main_args->aot_options); if (res) exit (1); return; @@ -1768,7 +1767,7 @@ mono_jit_parse_options (int argc, char * argv[]) memcpy (copy_argv, argv, sizeof (char*) * argc); argv = copy_argv; - mono_options_parse_options ((const char**)argv, argc, &argc, NULL, error); + mono_options_parse_options ((const char**)argv, argc, &argc, error); if (!is_ok (error)) { g_printerr ("%s", mono_error_get_message (error)); mono_error_cleanup (error); @@ -2068,7 +2067,6 @@ mono_main (int argc, char* argv[]) GPtrArray *agents = NULL; char *extra_bindings_config_file = NULL; GList *paths = NULL; - GPtrArray *args; #ifdef MONO_JIT_INFO_TABLE_TEST int test_jit_info_table = FALSE; #endif @@ -2097,9 +2095,7 @@ mono_main (int argc, char* argv[]) enable_debugging = TRUE; - args = g_ptr_array_new (); - - mono_options_parse_options ((const char**)argv + 1, argc - 1, &argc, args, error); + mono_options_parse_options ((const char**)argv + 1, argc - 1, &argc, error); argc ++; if (!is_ok (error)) { g_printerr ("%s", mono_error_get_message (error)); @@ -2107,11 +2103,9 @@ mono_main (int argc, char* argv[]) return 1; } - g_ptr_array_add (args, argv [0]); for (i = 1; i < argc; ++i) { if (argv [i] [0] != '-') break; - g_ptr_array_add (args, argv [i]); if (strcmp (argv [i], "--regression") == 0) { action = DO_REGRESSION; } else if (strncmp (argv [i], "--single-method=", 16) == 0) { @@ -2470,7 +2464,7 @@ mono_main (int argc, char* argv[]) /* Parse newly added options */ int n = argc; - mono_options_parse_options ((const char**)(argv + orig_argc), argc - orig_argc, &n, args, error); + mono_options_parse_options ((const char**)(argv + orig_argc), argc - orig_argc, &n, error); if (!is_ok (error)) { g_printerr ("%s", mono_error_get_message (error)); mono_error_cleanup (error); @@ -2655,7 +2649,6 @@ mono_main (int argc, char* argv[]) main_args.argv = argv + i; main_args.opts = opt; main_args.aot_options = aot_options; - main_args.runtime_args = args; main_thread_handler (&main_args); mono_thread_manage_internal (); diff --git a/src/mono/mono/mini/exceptions-ppc.c b/src/mono/mono/mini/exceptions-ppc.c index c7537a258acfbf..146fece236928e 100644 --- a/src/mono/mono/mini/exceptions-ppc.c +++ b/src/mono/mono/mini/exceptions-ppc.c @@ -838,8 +838,5 @@ mono_arch_setup_resume_sighandler_ctx (MonoContext *ctx, gpointer func) ctx->regs[2] = (gulong)handler_ftnptr->toc; #else MONO_CONTEXT_SET_IP(ctx, (unsigned long) func); -#ifdef TARGET_POWERPC64 - ctx->regs[12] = (gulong)func; -#endif #endif } diff --git a/src/mono/mono/mini/exceptions-s390x.c b/src/mono/mono/mini/exceptions-s390x.c index a5c92c01ec4c69..6c40214a13f742 100644 --- a/src/mono/mono/mini/exceptions-s390x.c +++ b/src/mono/mono/mini/exceptions-s390x.c @@ -515,6 +515,7 @@ mono_arch_unwind_frame (MonoJitTlsData *jit_tls, *new_ctx = *ctx; if (ji != NULL) { + uintptr_t address; guint8 *cfa; guint32 unwind_info_len; guint8 *unwind_info; @@ -527,6 +528,8 @@ mono_arch_unwind_frame (MonoJitTlsData *jit_tls, unwind_info = mono_jinfo_get_unwind_info (ji, &unwind_info_len); + address = (char *)ip - (char *)ji->code_start; + if (ji->has_arch_eh_info) epilog = (guint8*)ji->code_start + ji->code_size - mono_jinfo_get_epilog_size (ji); diff --git a/src/mono/mono/mini/interp/interp.c b/src/mono/mono/mini/interp/interp.c index cdbcf418fed4fa..411d4f1e6b3db0 100644 --- a/src/mono/mono/mini/interp/interp.c +++ b/src/mono/mono/mini/interp/interp.c @@ -1340,10 +1340,7 @@ typedef enum { PINVOKE_ARG_R8 = 3, PINVOKE_ARG_R4 = 4, PINVOKE_ARG_VTYPE = 5, - PINVOKE_ARG_SCALAR_VTYPE = 6, - // This isn't ifdefed so it's easier to write code that handles it without sprinkling - // 800 ifdefs in this file - PINVOKE_ARG_WASM_VALUETYPE_RESULT = 7, + PINVOKE_ARG_SCALAR_VTYPE = 6 } PInvokeArgType; typedef struct { @@ -1439,7 +1436,6 @@ get_build_args_from_sig_info (MonoMemoryManager *mem_manager, MonoMethodSignatur ilen++; break; case MONO_TYPE_GENERICINST: { - // FIXME: Should mini_wasm_is_scalar_vtype stuff go in here? MonoClass *container_class = type->data.generic_class->container_class; type = m_class_get_byval_arg (container_class); goto retry; @@ -1477,32 +1473,11 @@ get_build_args_from_sig_info (MonoMemoryManager *mem_manager, MonoMethodSignatur case MONO_TYPE_CLASS: case MONO_TYPE_OBJECT: case MONO_TYPE_STRING: - info->ret_pinvoke_type = PINVOKE_ARG_INT; - break; -#if SIZEOF_VOID_P == 8 case MONO_TYPE_I8: case MONO_TYPE_U8: -#endif - info->ret_pinvoke_type = PINVOKE_ARG_INT; - break; -#if SIZEOF_VOID_P == 4 - case MONO_TYPE_I8: - case MONO_TYPE_U8: - info->ret_pinvoke_type = PINVOKE_ARG_INT; - break; -#endif case MONO_TYPE_VALUETYPE: case MONO_TYPE_GENERICINST: info->ret_pinvoke_type = PINVOKE_ARG_INT; -#ifdef HOST_WASM - // This ISSTRUCT check is important, because the type could be an enum - if (MONO_TYPE_ISSTRUCT (info->ret_mono_type)) { - // The return type was already filtered previously, so if we get here - // we're returning a struct byref instead of as a scalar - info->ret_pinvoke_type = PINVOKE_ARG_WASM_VALUETYPE_RESULT; - info->ilen++; - } -#endif break; case MONO_TYPE_R4: case MONO_TYPE_R8: @@ -1528,15 +1503,6 @@ build_args_from_sig (InterpMethodArguments *margs, MonoMethodSignature *sig, Bui margs->ilen = info->ilen; margs->flen = info->flen; - size_t int_i = 0; - size_t int_f = 0; - - if (info->ret_pinvoke_type == PINVOKE_ARG_WASM_VALUETYPE_RESULT) { - // Allocate an empty arg0 for the address of the return value - // info->ilen was already increased earlier - int_i++; - } - if (margs->ilen > 0) { if (margs->ilen <= 8) margs->iargs = margs->iargs_buf; @@ -1551,6 +1517,9 @@ build_args_from_sig (InterpMethodArguments *margs, MonoMethodSignature *sig, Bui margs->fargs = g_malloc0 (sizeof (double) * margs->flen); } + size_t int_i = 0; + size_t int_f = 0; + for (int i = 0; i < sig->param_count; i++) { guint32 offset = get_arg_offset (frame->imethod, sig, i); stackval *sp_arg = STACK_ADD_BYTES (frame->stack, offset); @@ -1609,15 +1578,6 @@ build_args_from_sig (InterpMethodArguments *margs, MonoMethodSignature *sig, Bui } switch (info->ret_pinvoke_type) { - case PINVOKE_ARG_WASM_VALUETYPE_RESULT: - // We pass the return value address in arg0 so fill it in, we already - // reserved space for it earlier. - g_assert (frame->retval); - margs->iargs[0] = (gpointer*)frame->retval; - // The return type is void so retval should be NULL - margs->retval = NULL; - margs->is_float_ret = 0; - break; case PINVOKE_ARG_INT: margs->retval = (gpointer*)frame->retval; margs->is_float_ret = 0; @@ -1835,10 +1795,8 @@ ves_pinvoke_method ( g_free (ccontext.stack); #else // Only the vt address has been returned, we need to copy the entire content on interp stack - if (!context->has_resume_state && MONO_TYPE_ISSTRUCT (call_info->ret_mono_type)) { - if (call_info->ret_pinvoke_type != PINVOKE_ARG_WASM_VALUETYPE_RESULT) - stackval_from_data (call_info->ret_mono_type, frame.retval, (char*)frame.retval->data.p, sig->pinvoke && !sig->marshalling_disabled); - } + if (!context->has_resume_state && MONO_TYPE_ISSTRUCT (call_info->ret_mono_type)) + stackval_from_data (call_info->ret_mono_type, frame.retval, (char*)frame.retval->data.p, sig->pinvoke && !sig->marshalling_disabled); if (margs.iargs != margs.iargs_buf) g_free (margs.iargs); @@ -3863,6 +3821,11 @@ max_d (double lhs, double rhs) return fmax (lhs, rhs); } +#if HOST_BROWSER +// Dummy call info used outside of monitoring phase. We don't care what's in it +static JiterpreterCallInfo jiterpreter_call_info = { 0 }; +#endif + /* * If CLAUSE_ARGS is non-null, start executing from it. * The ERROR argument is used to avoid declaring an error object for every interp frame, its not used @@ -3992,13 +3955,36 @@ mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClause MINT_IN_BREAK; } +#define LDC(n) do { LOCAL_VAR (ip [1], gint32) = (n); ip += 2; } while (0) + MINT_IN_CASE(MINT_LDC_I4_M1) + LDC(-1); + MINT_IN_BREAK; MINT_IN_CASE(MINT_LDC_I4_0) - LOCAL_VAR (ip [1], gint32) = 0; - ip += 2; + LDC(0); MINT_IN_BREAK; MINT_IN_CASE(MINT_LDC_I4_1) - LOCAL_VAR (ip [1], gint32) = 1; - ip += 2; + LDC(1); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_2) + LDC(2); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_3) + LDC(3); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_4) + LDC(4); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_5) + LDC(5); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_6) + LDC(6); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_7) + LDC(7); + MINT_IN_BREAK; + MINT_IN_CASE(MINT_LDC_I4_8) + LDC(8); MINT_IN_BREAK; MINT_IN_CASE(MINT_LDC_I4_S) LOCAL_VAR (ip [1], gint32) = (short)ip [2]; @@ -4269,7 +4255,6 @@ mono_interp_exec_method (InterpFrame *frame, ThreadContext *context, FrameClause call_args_offset = ip [2]; this_arg = LOCAL_VAR (call_args_offset, MonoObject*); - NULL_CHECK (this_arg); slot = (gint16)ip [4]; ip += 5; @@ -5281,10 +5266,6 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) + (gint16)ip [3]; ip += 4; MINT_IN_BREAK; - MINT_IN_CASE(MINT_ADD_I4_IMM2) - LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) + (gint32)READ32 (ip + 3); - ip += 5; - MINT_IN_BREAK; MINT_IN_CASE(MINT_ADD1_I8) LOCAL_VAR (ip [1], gint64) = LOCAL_VAR (ip [2], gint64) + 1; ip += 3; @@ -5293,10 +5274,6 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; LOCAL_VAR (ip [1], gint64) = LOCAL_VAR (ip [2], gint64) + (gint16)ip [3]; ip += 4; MINT_IN_BREAK; - MINT_IN_CASE(MINT_ADD_I8_IMM2) - LOCAL_VAR (ip [1], gint64) = LOCAL_VAR (ip [2], gint64) + (gint32)READ32 (ip + 3); - ip += 5; - MINT_IN_BREAK; MINT_IN_CASE(MINT_SUB_I4) BINOP(gint32, -); MINT_IN_BREAK; @@ -5327,18 +5304,10 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) * (gint16)ip [3]; ip += 4; MINT_IN_BREAK; - MINT_IN_CASE(MINT_MUL_I4_IMM2) - LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) * (gint32)READ32 (ip + 3); - ip += 5; - MINT_IN_BREAK; MINT_IN_CASE(MINT_MUL_I8_IMM) LOCAL_VAR (ip [1], gint64) = LOCAL_VAR (ip [2], gint64) * (gint16)ip [3]; ip += 4; MINT_IN_BREAK; - MINT_IN_CASE(MINT_MUL_I8_IMM2) - LOCAL_VAR (ip [1], gint64) = LOCAL_VAR (ip [2], gint64) * (gint32)READ32 (ip + 3); - ip += 5; - MINT_IN_BREAK; MINT_IN_CASE(MINT_ADD_MUL_I4_IMM) LOCAL_VAR (ip [1], gint32) = (LOCAL_VAR (ip [2], gint32) + (gint16)ip [3]) * (gint16)ip [4]; ip += 5; @@ -5446,28 +5415,12 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; MINT_IN_CASE(MINT_AND_I4) BINOP(gint32, &); MINT_IN_BREAK; - MINT_IN_CASE(MINT_AND_I4_IMM) - LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) & (gint16)ip [3]; - ip += 4; - MINT_IN_BREAK; - MINT_IN_CASE(MINT_AND_I4_IMM2) - LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) & READ32 (ip + 3); - ip += 5; - MINT_IN_BREAK; MINT_IN_CASE(MINT_AND_I8) BINOP(gint64, &); MINT_IN_BREAK; MINT_IN_CASE(MINT_OR_I4) BINOP(gint32, |); MINT_IN_BREAK; - MINT_IN_CASE(MINT_OR_I4_IMM) - LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) | (gint16)ip [3]; - ip += 4; - MINT_IN_BREAK; - MINT_IN_CASE(MINT_OR_I4_IMM2) - LOCAL_VAR (ip [1], gint32) = LOCAL_VAR (ip [2], gint32) | READ32 (ip + 3); - ip += 5; - MINT_IN_BREAK; MINT_IN_CASE(MINT_OR_I8) BINOP(gint64, |); MINT_IN_BREAK; @@ -7623,11 +7576,7 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; /* top of stack is result of filter */ frame->retval->data.i = LOCAL_VAR (ip [1], gint32); goto exit_clause; - MINT_IN_CASE(MINT_ZEROBLK) - memset (LOCAL_VAR (ip [1], gpointer), 0, LOCAL_VAR (ip [2], gsize)); - ip += 3; - MINT_IN_BREAK; - MINT_IN_CASE(MINT_ZEROBLK_IMM) + MINT_IN_CASE(MINT_INITOBJ) memset (LOCAL_VAR (ip [1], gpointer), 0, ip [2]); ip += 3; MINT_IN_BREAK; @@ -7860,7 +7809,7 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; // now execute the trace // this isn't important for performance, but it makes it easier to use the // jiterpreter early in automated tests where code only runs once - offset = prepare_result (frame, locals, NULL, ip); + offset = prepare_result (frame, locals, &jiterpreter_call_info, ip); ip = (guint16*) (((guint8*)ip) + offset); break; } @@ -7883,7 +7832,7 @@ MINT_IN_CASE(MINT_BRTRUE_I8_SP) ZEROP_SP(gint64, !=); MINT_IN_BREAK; MINT_IN_CASE(MINT_TIER_ENTER_JITERPRETER) { // The fn ptr is encoded in a guint16 relative to the index of the first trace fn ptr, so compute the actual ptr JiterpreterThunk thunk = (JiterpreterThunk)(void *)(((JiterpreterOpcode *)ip)->relative_fn_ptr + mono_jiterp_first_trace_fn_ptr); - ptrdiff_t offset = thunk (frame, locals, NULL, ip); + ptrdiff_t offset = thunk (frame, locals, &jiterpreter_call_info, ip); ip = (guint16*) (((guint8*)ip) + offset); MINT_IN_BREAK; } diff --git a/src/mono/mono/mini/interp/jiterpreter-opcode-values.h b/src/mono/mono/mini/interp/jiterpreter-opcode-values.h index c3926ccb4c4791..bf9965dc148a5f 100644 --- a/src/mono/mono/mini/interp/jiterpreter-opcode-values.h +++ b/src/mono/mono/mini/interp/jiterpreter-opcode-values.h @@ -35,7 +35,7 @@ OPRANGE(MINT_RET_I4_IMM, MINT_RET_I8_IMM, ABORT_OUTSIDE_BRANCH_BLOCK_NONE) // High value because interp has to do a memory load for the immediate // but we can inline it into the trace -OPRANGE(MINT_LDC_I4_0, MINT_LDC_R8, HIGH) +OPRANGE(MINT_LDC_I4_M1, MINT_LDC_R8, HIGH) OPRANGE(MINT_MOV_I4_I1, MINT_MOV_4, NORMAL) // High value for large/complex moves @@ -43,10 +43,8 @@ OPRANGE(MINT_MOV_8, MINT_MOV_8_4, HIGH) // Binops. Assume most of them are not any faster in jiterp OPRANGE(MINT_ADD_I4, MINT_CLT_UN_R8, NORMAL) -// Unops. Most will not be faster in jiterp. -OPRANGE(MINT_ADD1_I4, MINT_CEQ0_I4, NORMAL) -// Some superinsns that will be faster in jiterp due to inline constants -OPRANGE(MINT_ADD_I4_IMM, MINT_ADD_MUL_I8_IMM, HIGH) +// Unops and some superinsns. Most will not be faster in jiterp. +OPRANGE(MINT_ADD1_I4, MINT_SHR_I8_IMM, NORMAL) // Math intrinsics. We implement most of these by calling libc or using wasm opcodes OPRANGE(MINT_ASIN, MINT_MAXF, NORMAL) // Field operations. Null check optimization makes these more efficient than interp @@ -134,8 +132,7 @@ OP(MINT_INTRINS_MEMORYMARSHAL_GETARRAYDATAREF, HIGH) OP(MINT_INITLOCAL, MASSIVE) OP(MINT_INITLOCALS, MASSIVE) OP(MINT_LOCALLOC, NORMAL) -OP(MINT_ZEROBLK, MASSIVE) -OP(MINT_ZEROBLK_IMM, HIGH) +OP(MINT_INITOBJ, MASSIVE) OP(MINT_INTRINS_RUNTIMEHELPERS_OBJECT_HAS_COMPONENT_SIZE, HIGH) OP(MINT_INTRINS_ENUM_HASFLAG, HIGH) OP(MINT_INTRINS_ORDINAL_IGNORE_CASE_ASCII, HIGH) diff --git a/src/mono/mono/mini/interp/jiterpreter.c b/src/mono/mono/mini/interp/jiterpreter.c index 12678e10e828aa..52a6f74f47d631 100644 --- a/src/mono/mono/mini/interp/jiterpreter.c +++ b/src/mono/mono/mini/interp/jiterpreter.c @@ -617,17 +617,16 @@ jiterp_get_opcode_value (InterpInst *ins, gboolean *inside_branch_block) initialize_opcode_value_table (); guint16 opcode = ins->opcode; - g_assert (opcode < MINT_LASTOP); + g_assert(opcode < MINT_LASTOP); int table_value = opcode_value_table[opcode]; - switch (table_value) { - case VALUE_ABORT_OUTSIDE_BRANCH_BLOCK: - return *inside_branch_block ? VALUE_LOW : VALUE_ABORT; - case VALUE_ABORT_OUTSIDE_BRANCH_BLOCK_NONE: - return *inside_branch_block ? VALUE_NONE : VALUE_ABORT; - case VALUE_BEGIN_BRANCH_BLOCK: - *inside_branch_block = TRUE; - return VALUE_NORMAL; + if (table_value == VALUE_ABORT_OUTSIDE_BRANCH_BLOCK) { + return *inside_branch_block ? VALUE_LOW : VALUE_ABORT; + } else if (table_value == VALUE_ABORT_OUTSIDE_BRANCH_BLOCK) { + return *inside_branch_block ? VALUE_NONE : VALUE_ABORT; + } else if (table_value == VALUE_BEGIN_BRANCH_BLOCK) { + *inside_branch_block = TRUE; + return VALUE_NORMAL; } switch (opcode) { @@ -885,10 +884,7 @@ jiterp_insert_entry_points (void *_imethod, void *_td) // Increase the instruction counter. If we inserted an entry point at the top of this bb, // the new instruction counter will be the number of instructions in the block, so if // it's big enough we'll be able to insert another entry point right away. - for (InterpInst * ins = bb->first_ins; ins != NULL; ins = ins->next) { - if (!MINT_IS_EMIT_NOP (ins->opcode)) - instruction_count++; - } + instruction_count += bb->in_count; build_address_taken_bitset (td, bb, bitset_size); } @@ -1003,7 +999,7 @@ mono_jiterp_parse_option (const char *option) const char *arr[2] = { option, NULL }; int temp; - mono_options_parse_options (arr, 1, &temp, NULL, NULL); + mono_options_parse_options (arr, 1, &temp, NULL); return TRUE; } @@ -1179,9 +1175,7 @@ enum { JITERP_MEMBER_VTABLE_KLASS, JITERP_MEMBER_CLASS_RANK, JITERP_MEMBER_CLASS_ELEMENT_CLASS, - JITERP_MEMBER_BOXED_VALUE_DATA, - JITERP_MEMBER_BACKWARD_BRANCH_TAKEN, - JITERP_MEMBER_BAILOUT_OPCODE_COUNT, + JITERP_MEMBER_BOXED_VALUE_DATA }; @@ -1224,10 +1218,6 @@ mono_jiterp_get_member_offset (int member) { // see mono_object_get_data case JITERP_MEMBER_BOXED_VALUE_DATA: return MONO_ABI_SIZEOF (MonoObject); - case JITERP_MEMBER_BACKWARD_BRANCH_TAKEN: - return offsetof (JiterpreterCallInfo, backward_branch_taken); - case JITERP_MEMBER_BAILOUT_OPCODE_COUNT: - return offsetof (JiterpreterCallInfo, bailout_opcode_count); default: g_assert_not_reached(); } @@ -1683,20 +1673,7 @@ mono_jiterp_tlqueue_clear (int queue) { // HACK: fix C4206 EMSCRIPTEN_KEEPALIVE -#else -int -mono_jiterp_is_enabled (void); #endif // HOST_BROWSER -int -mono_jiterp_is_enabled (void) { -#if HOST_BROWSER - return mono_opt_jiterpreter_traces_enabled; -#else - return 0; -#endif -} - -void -jiterp_preserve_module (void) { +void jiterp_preserve_module (void) { } diff --git a/src/mono/mono/mini/interp/jiterpreter.h b/src/mono/mono/mini/interp/jiterpreter.h index ed57b0a0e17b3e..26b05f64a0c85f 100644 --- a/src/mono/mono/mini/interp/jiterpreter.h +++ b/src/mono/mono/mini/interp/jiterpreter.h @@ -239,7 +239,4 @@ mono_jiterp_tlqueue_purge_all (gpointer item); #endif // HOST_BROWSER -int -mono_jiterp_is_enabled (void); - #endif // __MONO_MINI_JITERPRETER_H__ diff --git a/src/mono/mono/mini/interp/mintops.def b/src/mono/mono/mini/interp/mintops.def index d79d5cd30acac2..d45579ce40fe39 100644 --- a/src/mono/mono/mini/interp/mintops.def +++ b/src/mono/mono/mini/interp/mintops.def @@ -42,8 +42,16 @@ OPDEF(MINT_RET_U1, "ret.u1", 2, 0, 1, MintOpNoArgs) OPDEF(MINT_RET_I2, "ret.i2", 2, 0, 1, MintOpNoArgs) OPDEF(MINT_RET_U2, "ret.u2", 2, 0, 1, MintOpNoArgs) +OPDEF(MINT_LDC_I4_M1, "ldc.i4.m1", 2, 1, 0, MintOpNoArgs) OPDEF(MINT_LDC_I4_0, "ldc.i4.0", 2, 1, 0, MintOpNoArgs) OPDEF(MINT_LDC_I4_1, "ldc.i4.1", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_2, "ldc.i4.2", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_3, "ldc.i4.3", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_4, "ldc.i4.4", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_5, "ldc.i4.5", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_6, "ldc.i4.6", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_7, "ldc.i4.7", 2, 1, 0, MintOpNoArgs) +OPDEF(MINT_LDC_I4_8, "ldc.i4.8", 2, 1, 0, MintOpNoArgs) OPDEF(MINT_LDC_I4_S, "ldc.i4.s", 3, 1, 0, MintOpShortInt) OPDEF(MINT_LDC_I4, "ldc.i4", 4, 1, 0, MintOpInt) @@ -355,8 +363,7 @@ OPDEF(MINT_NEWOBJ_STRING, "newobj_string", 4, 1, 1, MintOpMethodToken) OPDEF(MINT_NEWOBJ, "newobj", 5, 1, 1, MintOpMethodToken) OPDEF(MINT_NEWOBJ_INLINED, "newobj_inlined", 3, 1, 0, MintOpVTableToken) OPDEF(MINT_NEWOBJ_VT, "newobj_vt", 5, 1, 1, MintOpMethodToken) -OPDEF(MINT_ZEROBLK, "zeroblk", 3, 0, 2, MintOpNoArgs) -OPDEF(MINT_ZEROBLK_IMM, "zeroblk_imm", 3, 0, 1, MintOpShortInt) +OPDEF(MINT_INITOBJ, "initobj", 3, 0, 1, MintOpShortInt) OPDEF(MINT_CASTCLASS, "castclass", 4, 1, 1, MintOpClassToken) OPDEF(MINT_ISINST, "isinst", 4, 1, 1, MintOpClassToken) OPDEF(MINT_CASTCLASS_INTERFACE, "castclass.interface", 4, 1, 1, MintOpClassToken) @@ -646,20 +653,13 @@ OPDEF(MINT_RET_I4_IMM, "ret.i4.imm", 2, 0, 0, MintOpShortInt) OPDEF(MINT_RET_I8_IMM, "ret.i8.imm", 2, 0, 0, MintOpShortInt) OPDEF(MINT_ADD_I4_IMM, "add.i4.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_ADD_I4_IMM2, "add.i4.imm2", 5, 1, 1, MintOpInt) OPDEF(MINT_ADD_I8_IMM, "add.i8.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_ADD_I8_IMM2, "add.i8.imm2", 5, 1, 1, MintOpInt) + +OPDEF(MINT_ADD_MUL_I4_IMM, "add.mul.i4.imm", 5, 1, 1, MintOpTwoShorts) +OPDEF(MINT_ADD_MUL_I8_IMM, "add.mul.i8.imm", 5, 1, 1, MintOpTwoShorts) OPDEF(MINT_MUL_I4_IMM, "mul.i4.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_MUL_I4_IMM2, "mul.i4.imm2", 5, 1, 1, MintOpInt) OPDEF(MINT_MUL_I8_IMM, "mul.i8.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_MUL_I8_IMM2, "mul.i8.imm2", 5, 1, 1, MintOpInt) - -OPDEF(MINT_AND_I4_IMM, "and.i4.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_AND_I4_IMM2, "and.i4.imm2", 5, 1, 1, MintOpInt) - -OPDEF(MINT_OR_I4_IMM, "or.i4.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_OR_I4_IMM2, "or.i4.imm2", 5, 1, 1, MintOpInt) OPDEF(MINT_SHR_UN_I4_IMM, "shr.un.i4.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_SHR_UN_I8_IMM, "shr.un.i8.imm", 4, 1, 1, MintOpShortInt) @@ -668,9 +668,6 @@ OPDEF(MINT_SHL_I8_IMM, "shl.i8.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_SHR_I4_IMM, "shr.i4.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_SHR_I8_IMM, "shr.i8.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_ADD_MUL_I4_IMM, "add.mul.i4.imm", 5, 1, 1, MintOpTwoShorts) -OPDEF(MINT_ADD_MUL_I8_IMM, "add.mul.i8.imm", 5, 1, 1, MintOpTwoShorts) - OPDEF(MINT_SHL_AND_I4, "shl.i4.and", 4, 1, 2, MintOpNoArgs) OPDEF(MINT_SHL_AND_I8, "shl.i8.and", 4, 1, 2, MintOpNoArgs) diff --git a/src/mono/mono/mini/interp/mintops.h b/src/mono/mono/mini/interp/mintops.h index 0062e8d81438e8..27e3821dbccf8c 100644 --- a/src/mono/mono/mini/interp/mintops.h +++ b/src/mono/mono/mini/interp/mintops.h @@ -220,11 +220,10 @@ typedef enum { #define MINT_IS_SUPER_BRANCH(op) ((op) >= MINT_BRFALSE_I4_SP && (op) <= MINT_BLT_UN_I8_IMM_SP) #define MINT_IS_CALL(op) ((op) >= MINT_CALL && (op) <= MINT_JIT_CALL) #define MINT_IS_PATCHABLE_CALL(op) ((op) >= MINT_CALL && (op) <= MINT_VCALL) -#define MINT_IS_LDC_I4(op) ((op) >= MINT_LDC_I4_0 && (op) <= MINT_LDC_I4) +#define MINT_IS_LDC_I4(op) ((op) >= MINT_LDC_I4_M1 && (op) <= MINT_LDC_I4) #define MINT_IS_LDC_I8(op) ((op) >= MINT_LDC_I8_0 && (op) <= MINT_LDC_I8) #define MINT_IS_UNOP(op) ((op) >= MINT_ADD1_I4 && (op) <= MINT_CEQ0_I4) #define MINT_IS_BINOP(op) ((op) >= MINT_ADD_I4 && (op) <= MINT_CLT_UN_R8) -#define MINT_IS_BINOP_IMM(op) ((op) >= MINT_ADD_I4_IMM && (op) <= MINT_SHR_I8_IMM) #define MINT_IS_BINOP_SHIFT(op) ((op) >= MINT_SHR_UN_I4 && (op) <= MINT_SHR_I8) #define MINT_IS_LDFLD(op) ((op) >= MINT_LDFLD_I1 && (op) <= MINT_LDFLD_O) #define MINT_IS_STFLD(op) ((op) >= MINT_STFLD_I1 && (op) <= MINT_STFLD_O) diff --git a/src/mono/mono/mini/interp/transform-opt.c b/src/mono/mono/mini/interp/transform-opt.c index 6a0d45868dcffe..9933dbcc09a7ad 100644 --- a/src/mono/mono/mini/interp/transform-opt.c +++ b/src/mono/mono/mini/interp/transform-opt.c @@ -4,7 +4,6 @@ #include "mintops.h" #include "transform.h" -#include "interp-intrins.h" /* * VAR OFFSET ALLOCATOR @@ -1625,14 +1624,6 @@ interp_merge_bblocks (TransformData *td, InterpBasicBlock *bb, InterpBasicBlock } } -#if defined(TARGET_WASM) - // Copy jiterpreter data - if (bbadd->backwards_branch_target) - bb->backwards_branch_target = TRUE; - if (bbadd->contains_call_instruction) - bb->contains_call_instruction = TRUE; -#endif - mark_bb_as_dead (td, bbadd, bb); } @@ -1690,16 +1681,6 @@ interp_remove_bblock (TransformData *td, InterpBasicBlock *bb, InterpBasicBlock mark_bb_as_dead (td, bb, bb->next_bb); } -static int -get_bb_links_capacity (int links) -{ - if (links <= 2) - return links; - // Return the next power of 2 bigger or equal to links - int leading_zero = interp_intrins_clz_i4 (links - 1); - return 1 << (32 - leading_zero); -} - void interp_link_bblocks (TransformData *td, InterpBasicBlock *from, InterpBasicBlock *to) { @@ -1713,15 +1694,12 @@ interp_link_bblocks (TransformData *td, InterpBasicBlock *from, InterpBasicBlock } } if (!found) { - int prev_capacity = get_bb_links_capacity (from->out_count); - int new_capacity = get_bb_links_capacity (from->out_count + 1); - if (new_capacity > prev_capacity) { - InterpBasicBlock **newa = (InterpBasicBlock**)mono_mempool_alloc (td->mempool, new_capacity * sizeof (InterpBasicBlock*)); - memcpy (newa, from->out_bb, from->out_count * sizeof (InterpBasicBlock*)); - from->out_bb = newa; - } - from->out_bb [from->out_count] = to; + InterpBasicBlock **newa = (InterpBasicBlock**)mono_mempool_alloc (td->mempool, sizeof (InterpBasicBlock*) * (from->out_count + 1)); + for (i = 0; i < from->out_count; ++i) + newa [i] = from->out_bb [i]; + newa [i] = to; from->out_count++; + from->out_bb = newa; } found = FALSE; @@ -1732,15 +1710,12 @@ interp_link_bblocks (TransformData *td, InterpBasicBlock *from, InterpBasicBlock } } if (!found) { - int prev_capacity = get_bb_links_capacity (to->in_count); - int new_capacity = get_bb_links_capacity (to->in_count + 1); - if (new_capacity > prev_capacity) { - InterpBasicBlock **newa = (InterpBasicBlock**)mono_mempool_alloc (td->mempool, new_capacity * sizeof (InterpBasicBlock*)); - memcpy (newa, to->in_bb, to->in_count * sizeof (InterpBasicBlock*)); - to->in_bb = newa; - } - to->in_bb [to->in_count] = from; + InterpBasicBlock **newa = (InterpBasicBlock**)mono_mempool_alloc (td->mempool, sizeof (InterpBasicBlock*) * (to->in_count + 1)); + for (i = 0; i < to->in_count; ++i) + newa [i] = to->in_bb [i]; + newa [i] = from; to->in_count++; + to->in_bb = newa; } } @@ -1967,8 +1942,7 @@ interp_reorder_bblocks (TransformData *td) InterpInst *last_ins = interp_last_ins (in_bb); if (last_ins && (MINT_IS_CONDITIONAL_BRANCH (last_ins->opcode) || MINT_IS_UNCONDITIONAL_BRANCH (last_ins->opcode)) && - last_ins->info.target_bb == bb && - in_bb != bb) { + last_ins->info.target_bb == bb) { InterpBasicBlock *target_bb = first->info.target_bb; last_ins->info.target_bb = target_bb; interp_unlink_bblocks (in_bb, bb); @@ -3030,12 +3004,12 @@ interp_cprop (TransformData *td) interp_dump_ins (ins, td->data_items); } } - } else if (opcode == MINT_ZEROBLK_IMM) { + } else if (opcode == MINT_INITOBJ) { InterpInst *ldloca = get_var_value_def (td, sregs [0]); if (ldloca != NULL && ldloca->opcode == MINT_LDLOCA_S) { int size = ins->data [0]; int local = ldloca->sregs [0]; - // Replace LDLOCA + ZEROBLK_IMM with or LDC + // Replace LDLOCA + INITOBJ with or LDC if (size <= 4) ins->opcode = MINT_LDC_I4_0; else if (size <= 8) @@ -3046,7 +3020,7 @@ interp_cprop (TransformData *td) ins->dreg = local; if (td->verbose_level) { - g_print ("Replace ldloca/zeroblk pair :\n\t"); + g_print ("Replace ldloca/initobj pair :\n\t"); interp_dump_ins (ins, td->data_items); } } @@ -3196,10 +3170,8 @@ mono_test_interp_cprop (TransformData *td) interp_cprop (td); } -// If sreg is constant, it returns the value in `imm` and the smallest -// containing type for it in `imm_mt`. static gboolean -get_sreg_imm (TransformData *td, int sreg, gint32 *imm, int *imm_mt) +get_sreg_imm (TransformData *td, int sreg, gint16 *imm, int result_mt) { if (var_has_indirects (td, sreg)) return FALSE; @@ -3215,15 +3187,32 @@ get_sreg_imm (TransformData *td, int sreg, gint32 *imm, int *imm_mt) ct = interp_get_const_from_ldc_i8 (def); else return FALSE; - if (ct >= G_MININT16 && ct <= G_MAXINT16) { + gint64 min_val, max_val; + // We only propagate the immediate only if it fits into the desired type, + // so we don't accidentaly handle conversions wrong + switch (result_mt) { + case MINT_TYPE_I1: + min_val = G_MININT8; + max_val = G_MAXINT8; + break; + case MINT_TYPE_I2: + min_val = G_MININT16; + max_val = G_MAXINT16; + break; + case MINT_TYPE_U1: + min_val = 0; + max_val = G_MAXUINT8; + break; + case MINT_TYPE_U2: + min_val = 0; + max_val = G_MAXINT16; + break; + default: + g_assert_not_reached (); + + } + if (ct >= min_val && ct <= max_val) { *imm = (gint16)ct; - if (imm_mt) - *imm_mt = MINT_TYPE_I2; - return TRUE; - } else if (ct >= G_MININT32 && ct <= G_MAXINT32) { - *imm = (gint32)ct; - if (imm_mt) - *imm_mt = MINT_TYPE_I4; return TRUE; } } @@ -3362,67 +3351,45 @@ interp_super_instructions (TransformData *td) if (opcode == MINT_RET || (opcode >= MINT_RET_I1 && opcode <= MINT_RET_U2)) { // ldc + ret -> ret.imm int sreg = ins->sregs [0]; - gint32 imm; - if (get_sreg_imm (td, sreg, &imm, NULL)) { - // compute the casting as done by the ret opcode - int ret_mt = (opcode == MINT_RET) ? MINT_TYPE_I8 : opcode - MINT_RET_I1; - if (ret_mt == MINT_TYPE_I1) - imm = (gint8)imm; - else if (ret_mt == MINT_TYPE_U1) - imm = (guint8)imm; - else if (ret_mt == MINT_TYPE_I2) - imm = (gint16)imm; - else if (ret_mt == MINT_TYPE_U2) - imm = (guint16)imm; - - if (imm >= G_MININT16 && imm <= G_MAXINT16) { - InterpInst *def = td->var_values [sreg].def; - int ret_op = MINT_IS_LDC_I4 (def->opcode) ? MINT_RET_I4_IMM : MINT_RET_I8_IMM; - InterpInst *new_inst = interp_insert_ins (td, ins, ret_op); - new_inst->data [0] = (gint16)imm; - interp_clear_ins (def); - interp_clear_ins (ins); - td->var_values [sreg].ref_count--; // 0 - if (td->verbose_level) { - g_print ("superins: "); - interp_dump_ins (new_inst, td->data_items); - } + gint16 imm; + if (get_sreg_imm (td, sreg, &imm, (opcode == MINT_RET) ? MINT_TYPE_I2 : opcode - MINT_RET_I1)) { + InterpInst *def = td->var_values [sreg].def; + int ret_op = MINT_IS_LDC_I4 (def->opcode) ? MINT_RET_I4_IMM : MINT_RET_I8_IMM; + InterpInst *new_inst = interp_insert_ins (td, ins, ret_op); + new_inst->data [0] = imm; + interp_clear_ins (def); + interp_clear_ins (ins); + td->var_values [sreg].ref_count--; // 0 + if (td->verbose_level) { + g_print ("superins: "); + interp_dump_ins (new_inst, td->data_items); } } } else if (opcode == MINT_ADD_I4 || opcode == MINT_ADD_I8 || - opcode == MINT_MUL_I4 || opcode == MINT_MUL_I8 || - opcode == MINT_OR_I4 || opcode == MINT_AND_I4) { + opcode == MINT_MUL_I4 || opcode == MINT_MUL_I8) { int sreg = -1; int sreg_imm = -1; - int imm_mt; - gint32 imm; - if (get_sreg_imm (td, ins->sregs [0], &imm, &imm_mt)) { + gint16 imm; + if (get_sreg_imm (td, ins->sregs [0], &imm, MINT_TYPE_I2)) { sreg = ins->sregs [1]; sreg_imm = ins->sregs [0]; - } else if (get_sreg_imm (td, ins->sregs [1], &imm, &imm_mt)) { + } else if (get_sreg_imm (td, ins->sregs [1], &imm, MINT_TYPE_I2)) { sreg = ins->sregs [0]; sreg_imm = ins->sregs [1]; } if (sreg != -1) { int binop; switch (opcode) { - case MINT_ADD_I4: binop = (imm_mt == MINT_TYPE_I2) ? MINT_ADD_I4_IMM : MINT_ADD_I4_IMM2; break; - case MINT_ADD_I8: binop = (imm_mt == MINT_TYPE_I2) ? MINT_ADD_I8_IMM : MINT_ADD_I8_IMM2; break; - case MINT_MUL_I4: binop = (imm_mt == MINT_TYPE_I2) ? MINT_MUL_I4_IMM : MINT_MUL_I4_IMM2; break; - case MINT_MUL_I8: binop = (imm_mt == MINT_TYPE_I2) ? MINT_MUL_I8_IMM : MINT_MUL_I8_IMM2; break; - case MINT_OR_I4: binop = (imm_mt == MINT_TYPE_I2) ? MINT_OR_I4_IMM : MINT_OR_I4_IMM2; break; - case MINT_AND_I4: binop = (imm_mt == MINT_TYPE_I2) ? MINT_AND_I4_IMM : MINT_AND_I4_IMM2; break; + case MINT_ADD_I4: binop = MINT_ADD_I4_IMM; break; + case MINT_ADD_I8: binop = MINT_ADD_I8_IMM; break; + case MINT_MUL_I4: binop = MINT_MUL_I4_IMM; break; + case MINT_MUL_I8: binop = MINT_MUL_I8_IMM; break; default: g_assert_not_reached (); } InterpInst *new_inst = interp_insert_ins (td, ins, binop); new_inst->dreg = ins->dreg; new_inst->sregs [0] = sreg; - if (imm_mt == MINT_TYPE_I2) - new_inst->data [0] = (gint16)imm; - else if (imm_mt == MINT_TYPE_I4) - WRITE32_INS (new_inst, 0, &imm); - else - g_assert_not_reached (); + new_inst->data [0] = imm; interp_clear_ins (td->var_values [sreg_imm].def); interp_clear_ins (ins); td->var_values [sreg_imm].ref_count--; // 0 @@ -3434,15 +3401,14 @@ interp_super_instructions (TransformData *td) } } else if (opcode == MINT_SUB_I4 || opcode == MINT_SUB_I8) { // ldc + sub -> add.-imm - gint32 imm; - int imm_mt; + gint16 imm; int sreg_imm = ins->sregs [1]; - if (get_sreg_imm (td, sreg_imm, &imm, &imm_mt) && imm_mt == MINT_TYPE_I2 && imm != G_MININT16) { + if (get_sreg_imm (td, sreg_imm, &imm, MINT_TYPE_I2) && imm != G_MININT16) { int add_op = opcode == MINT_SUB_I4 ? MINT_ADD_I4_IMM : MINT_ADD_I8_IMM; InterpInst *new_inst = interp_insert_ins (td, ins, add_op); new_inst->dreg = ins->dreg; new_inst->sregs [0] = ins->sregs [0]; - new_inst->data [0] = (gint16)-imm; + new_inst->data [0] = -imm; interp_clear_ins (td->var_values [sreg_imm].def); interp_clear_ins (ins); td->var_values [sreg_imm].ref_count--; // 0 @@ -3475,16 +3441,15 @@ interp_super_instructions (TransformData *td) } } } else if (MINT_IS_BINOP_SHIFT (opcode)) { - gint32 imm; - int imm_mt; + gint16 imm; int sreg_imm = ins->sregs [1]; - if (get_sreg_imm (td, sreg_imm, &imm, &imm_mt) && imm_mt == MINT_TYPE_I2) { + if (get_sreg_imm (td, sreg_imm, &imm, MINT_TYPE_I2)) { // ldc + sh -> sh.imm int shift_op = MINT_SHR_UN_I4_IMM + (opcode - MINT_SHR_UN_I4); InterpInst *new_inst = interp_insert_ins (td, ins, shift_op); new_inst->dreg = ins->dreg; new_inst->sregs [0] = ins->sregs [0]; - new_inst->data [0] = (gint16)imm; + new_inst->data [0] = imm; interp_clear_ins (td->var_values [sreg_imm].def); interp_clear_ins (ins); td->var_values [sreg_imm].ref_count--; // 0 @@ -3496,28 +3461,33 @@ interp_super_instructions (TransformData *td) } else if (opcode == MINT_SHL_I4 || opcode == MINT_SHL_I8) { int amount_var = ins->sregs [1]; InterpInst *amount_def = get_var_value_def (td, amount_var); - if (amount_def != NULL && td->var_values [amount_var].ref_count == 1 && amount_def->opcode == MINT_AND_I4_IMM) { - // and_imm + shl -> shl_and_imm - int new_opcode = -1; - if (opcode == MINT_SHL_I4 && amount_def->data [0] == 31) - new_opcode = MINT_SHL_AND_I4; - else if (opcode == MINT_SHL_I8 && amount_def->data [0] == 63) - new_opcode = MINT_SHL_AND_I8; - - if (new_opcode != -1) { - InterpInst *new_inst = interp_insert_ins (td, ins, new_opcode); - new_inst->dreg = ins->dreg; - new_inst->sregs [0] = ins->sregs [0]; - new_inst->sregs [1] = amount_def->sregs [0]; - - td->var_values [amount_var].ref_count--; // 0 - td->var_values [new_inst->dreg].def = new_inst; - - interp_clear_ins (amount_def); - interp_clear_ins (ins); - if (td->verbose_level) { - g_print ("superins: "); - interp_dump_ins (new_inst, td->data_items); + if (amount_def != NULL && td->var_values [amount_var].ref_count == 1 && amount_def->opcode == MINT_AND_I4) { + int mask_var = amount_def->sregs [1]; + if (get_sreg_imm (td, mask_var, &imm, MINT_TYPE_I2)) { + // ldc + and + shl -> shl_and_imm + int new_opcode = -1; + if (opcode == MINT_SHL_I4 && imm == 31) + new_opcode = MINT_SHL_AND_I4; + else if (opcode == MINT_SHL_I8 && imm == 63) + new_opcode = MINT_SHL_AND_I8; + + if (new_opcode != -1) { + InterpInst *new_inst = interp_insert_ins (td, ins, new_opcode); + new_inst->dreg = ins->dreg; + new_inst->sregs [0] = ins->sregs [0]; + new_inst->sregs [1] = amount_def->sregs [0]; + + td->var_values [amount_var].ref_count--; // 0 + td->var_values [mask_var].ref_count--; // 0 + td->var_values [new_inst->dreg].def = new_inst; + + interp_clear_ins (td->var_values [mask_var].def); + interp_clear_ins (amount_def); + interp_clear_ins (ins); + if (td->verbose_level) { + g_print ("superins: "); + interp_dump_ins (new_inst, td->data_items); + } } } } @@ -3668,10 +3638,9 @@ interp_super_instructions (TransformData *td) td->var_values [obj_sreg].ref_count--; } } else if (MINT_IS_BINOP_CONDITIONAL_BRANCH (opcode) && interp_is_short_offset (noe, ins->info.target_bb->native_offset_estimate)) { - gint32 imm; - int imm_mt; + gint16 imm; int sreg_imm = ins->sregs [1]; - if (get_sreg_imm (td, sreg_imm, &imm, &imm_mt) && imm_mt == MINT_TYPE_I2) { + if (get_sreg_imm (td, sreg_imm, &imm, MINT_TYPE_I2)) { int condbr_op = get_binop_condbr_imm_sp (opcode); if (condbr_op != MINT_NOP) { InterpInst *prev_ins = interp_prev_ins (ins); @@ -3680,7 +3649,7 @@ interp_super_instructions (TransformData *td) interp_clear_ins (prev_ins); InterpInst *new_ins = interp_insert_ins (td, ins, condbr_op); new_ins->sregs [0] = ins->sregs [0]; - new_ins->data [0] = (gint16)imm; + new_ins->data [0] = imm; new_ins->info.target_bb = ins->info.target_bb; interp_clear_ins (td->var_values [sreg_imm].def); interp_clear_ins (ins); @@ -3918,15 +3887,8 @@ interp_optimize_code (TransformData *td) // We run this after var deadce to detect more single use vars. This pass will clear // unnecessary instruction on the fly so deadce is no longer needed to run. - if (mono_interp_opt & INTERP_OPT_SUPER_INSTRUCTIONS) { - // This pass is enough to be called only once, after all cprop and other optimizations - // are done. The problem is that currently it needs to run over code in SSA form, so we - // can't just run it at the very end of optimization cycles. Also bblock optimization can - // lead to another optimization iteration, so we can still end up running it multiple times. - // Basic block optimization currently needs to run after we exited SSA. - if (!ssa_enabled_retry && !td->need_optimization_retry) - MONO_TIME_TRACK (mono_interp_stats.super_instructions_time, interp_super_instructions (td)); - } + if (mono_interp_opt & INTERP_OPT_SUPER_INSTRUCTIONS) + MONO_TIME_TRACK (mono_interp_stats.super_instructions_time, interp_super_instructions (td)); if (!td->disable_ssa) interp_exit_ssa (td); diff --git a/src/mono/mono/mini/interp/transform.c b/src/mono/mono/mini/interp/transform.c index d91e6b413b3367..ac654631313fe7 100644 --- a/src/mono/mono/mini/interp/transform.c +++ b/src/mono/mono/mini/interp/transform.c @@ -1604,16 +1604,24 @@ interp_ip_in_cbb (TransformData *td, int il_offset) static gboolean interp_ins_is_ldc (InterpInst *ins) { - return ins->opcode >= MINT_LDC_I4_0 && ins->opcode <= MINT_LDC_I8; + return ins->opcode >= MINT_LDC_I4_M1 && ins->opcode <= MINT_LDC_I8; } gint32 interp_get_const_from_ldc_i4 (InterpInst *ins) { switch (ins->opcode) { + case MINT_LDC_I4_M1: return -1; case MINT_LDC_I4_0: return 0; case MINT_LDC_I4_1: return 1; - case MINT_LDC_I4_S: return (gint32)(gint16)ins->data [0]; + case MINT_LDC_I4_2: return 2; + case MINT_LDC_I4_3: return 3; + case MINT_LDC_I4_4: return 4; + case MINT_LDC_I4_5: return 5; + case MINT_LDC_I4_6: return 6; + case MINT_LDC_I4_7: return 7; + case MINT_LDC_I4_8: return 8; + case MINT_LDC_I4_S: return (gint32)(gint8)ins->data [0]; case MINT_LDC_I4: return READ32 (&ins->data [0]); default: g_assert_not_reached (); @@ -1625,14 +1633,24 @@ InterpInst* interp_get_ldc_i4_from_const (TransformData *td, InterpInst *ins, gint32 ct, int dreg) { guint16 opcode; - if (!ct) - opcode = MINT_LDC_I4_0; - else if (ct == 1) - opcode = MINT_LDC_I4_1; - else if (ct >= G_MININT16 && ct <= G_MAXINT16) - opcode = MINT_LDC_I4_S; - else - opcode = MINT_LDC_I4; + switch (ct) { + case -1: opcode = MINT_LDC_I4_M1; break; + case 0: opcode = MINT_LDC_I4_0; break; + case 1: opcode = MINT_LDC_I4_1; break; + case 2: opcode = MINT_LDC_I4_2; break; + case 3: opcode = MINT_LDC_I4_3; break; + case 4: opcode = MINT_LDC_I4_4; break; + case 5: opcode = MINT_LDC_I4_5; break; + case 6: opcode = MINT_LDC_I4_6; break; + case 7: opcode = MINT_LDC_I4_7; break; + case 8: opcode = MINT_LDC_I4_8; break; + default: + if (ct >= -128 && ct <= 127) + opcode = MINT_LDC_I4_S; + else + opcode = MINT_LDC_I4; + break; + } int new_size = mono_interp_oplen [opcode]; @@ -1650,7 +1668,7 @@ interp_get_ldc_i4_from_const (TransformData *td, InterpInst *ins, gint32 ct, int interp_ins_set_dreg (ins, dreg); if (new_size == 3) - ins->data [0] = (gint16)ct; + ins->data [0] = (gint8)ct; else if (new_size == 4) WRITE32_INS (ins, 0, &ct); @@ -1920,21 +1938,9 @@ interp_handle_intrinsics (TransformData *td, MonoMethod *target_method, MonoClas } } else if (in_corlib && !strcmp (klass_name_space, "System") && - !strcmp (klass_name, "SpanHelpers")) { - if (!strcmp (tm, "ClearWithReferences")) { - *op = MINT_INTRINS_CLEAR_WITH_REFERENCES; - } else if (!strcmp (tm, "ClearWithoutReferences")) { - *op = MINT_ZEROBLK; - } else if (!strcmp (tm, "Fill") && csignature->param_count == 3) { - int align; - if (mono_type_size (csignature->params [2], &align) == 1) { - interp_add_ins (td, MINT_INITBLK); - td->sp -= 3; - interp_ins_set_sregs3 (td->last_ins, td->sp [0].var, td->sp [2].var, td->sp [1].var); - td->ip += 5; - return TRUE; - } - } + !strcmp (klass_name, "SpanHelpers") && + !strcmp (tm, "ClearWithReferences")) { + *op = MINT_INTRINS_CLEAR_WITH_REFERENCES; } else if (in_corlib && !strcmp (klass_name_space, "System") && !strcmp (klass_name, "Marvin")) { if (!strcmp (tm, "Block")) { InterpInst *ldloca2 = td->last_ins; @@ -2867,14 +2873,6 @@ interp_method_check_inlining (TransformData *td, MonoMethod *method, MonoMethodS if (td->prof_coverage) return FALSE; - /* - * doesnotreturn methods are not profitable to inline, since they almost certainly will not - * actually run during normal execution, and if they do they will only run once, so the - * upside to inlining them is effectively zero, and we'd waste time doing the inline - */ - if (has_doesnotreturn_attribute (method)) - return FALSE; - if (!is_metadata_update_disabled () && mono_metadata_update_no_inline (td->method, method)) return FALSE; @@ -3304,7 +3302,6 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target int need_null_check = is_virtual; int fp_sreg = -1, first_sreg = -1, dreg = -1; gboolean is_delegate_invoke = FALSE; - InterpInst *null_check = NULL; guint32 token = read32 (td->ip + 1); @@ -3557,9 +3554,6 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target interp_ins_set_sreg (td->last_ins, sp->var); set_type_and_var (td, sp, sp->type, sp->klass); interp_ins_set_dreg (td->last_ins, sp->var); - // If the call instruction will do a null check, then this instruction - // will be transformed into a simple MOV, so it can be optimized out - null_check = td->last_ins; } /* Offset the function pointer when emitting convert instructions */ @@ -3825,7 +3819,6 @@ interp_transform_call (TransformData *td, MonoMethod *method, MonoMethod *target } else if (is_virtual) { interp_add_ins (td, MINT_CALLVIRT_FAST); td->last_ins->data [1] = get_virt_method_slot (target_method); - null_check->opcode = MINT_MOV_P; } else { interp_add_ins (td, MINT_CALL); } @@ -5227,8 +5220,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, ++td->ip; break; case CEE_LDC_I4_M1: - interp_add_ins (td, MINT_LDC_I4_S); - td->last_ins->data [0] = (guint16)-1; + interp_add_ins (td, MINT_LDC_I4_M1); push_simple_type (td, STACK_TYPE_I4); interp_ins_set_dreg (td->last_ins, td->sp [-1].var); ++td->ip; @@ -5272,8 +5264,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, case CEE_LDC_I4_6: case CEE_LDC_I4_7: case CEE_LDC_I4_8: - interp_add_ins (td, MINT_LDC_I4_S); - td->last_ins->data [0] = *td->ip - CEE_LDC_I4_0; + interp_add_ins (td, (*td->ip - CEE_LDC_I4_0) + MINT_LDC_I4_0); push_simple_type (td, STACK_TYPE_I4); interp_ins_set_dreg (td->last_ins, td->sp [-1].var); ++td->ip; @@ -6905,7 +6896,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, td->ip += 5; } else { if (G_UNLIKELY (m_class_is_byreflike (klass))) { - mono_error_set_invalid_program (error, "Cannot box IsByRefLike type '%s.%s'", m_class_get_name_space (klass), m_class_get_name (klass)); + mono_error_set_bad_image (error, image, "Cannot box IsByRefLike type '%s.%s'", m_class_get_name_space (klass), m_class_get_name (klass)); goto exit; } @@ -8134,7 +8125,7 @@ generate_code (TransformData *td, MonoMethod *method, MonoMethodHeader *header, CHECK_TYPELOAD (klass); if (m_class_is_valuetype (klass)) { --td->sp; - interp_add_ins (td, MINT_ZEROBLK_IMM); + interp_add_ins (td, MINT_INITOBJ); interp_ins_set_sreg (td->last_ins, td->sp [0].var); i32 = mono_class_value_size (klass, NULL); g_assert (i32 < G_MAXUINT16); @@ -8395,14 +8386,6 @@ interp_compute_native_offset_estimates (TransformData *td) if (!td->optimized && bb->patchpoint_bb) noe += 2; -#if HOST_BROWSER - // We don't know in advance whether a bb will have a trace entry point, - // but we know that it will only ever have one trace entry point, so - // reserve space for it so we can correctly insert one later - if (mono_jiterp_is_enabled ()) - noe += 4; -#endif - for (ins = bb->first_ins; ins != NULL; ins = ins->next) { int opcode = ins->opcode; // Skip dummy opcodes for more precise offset computation @@ -8553,15 +8536,8 @@ emit_compacted_instruction (TransformData *td, guint16* start_ip, InterpInst *in gboolean is_short = interp_is_short_offset (br_offset, target_bb_estimated_offset); if (is_short) *start_ip = GINT_TO_OPCODE (get_short_brop (opcode)); - else if (MINT_IS_SUPER_BRANCH (opcode)) { - g_printf ( - "long superbranch detected with opcode %d (%s) in method %s.%s\n", - opcode, mono_interp_opname (opcode), - m_class_get_name (td->method->klass), td->method->name - ); - // FIXME missing handling for long branch - g_assert (FALSE); - } + else + g_assert (!MINT_IS_SUPER_BRANCH (opcode)); // FIXME missing handling for long branch // We don't know the in_offset of the target, add a reloc Reloc *reloc = (Reloc*)mono_mempool_alloc0 (td->mempool, sizeof (Reloc)); diff --git a/src/mono/mono/mini/intrinsics.c b/src/mono/mono/mini/intrinsics.c index 4835a84e2c9115..178da508ce2e05 100644 --- a/src/mono/mono/mini/intrinsics.c +++ b/src/mono/mono/mini/intrinsics.c @@ -82,9 +82,11 @@ mini_emit_inst_for_ctor (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignat if (!(cfg->opt & MONO_OPT_INTRINS)) return ins; - ins = mono_emit_simd_intrinsics (cfg, cmethod, fsig, args); - if (ins) - return ins; + if (cfg->opt & MONO_OPT_SIMD) { + ins = mono_emit_simd_intrinsics (cfg, cmethod, fsig, args); + if (ins) + return ins; + } ins = mono_emit_common_intrinsics (cfg, cmethod, fsig, args); if (ins) @@ -2091,9 +2093,11 @@ mini_emit_inst_for_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSign } } - ins = mono_emit_simd_intrinsics (cfg, cmethod, fsig, args); - if (ins) - return ins; + if (cfg->opt & MONO_OPT_SIMD) { + ins = mono_emit_simd_intrinsics (cfg, cmethod, fsig, args); + if (ins) + return ins; + } ins = mono_emit_common_intrinsics (cfg, cmethod, fsig, args); if (ins) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index d4c7deea9ee8a3..ff8dfa2998a335 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -386,12 +386,6 @@ INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_LD4LANE_V64, aarch64_neon_ld4lane, Arm64, INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_LD2LANE_V128, aarch64_neon_ld2lane, Arm64, AddPointer, V128 | I1 | I2 | I4 | I8 | R4 | R8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_LD3LANE_V128, aarch64_neon_ld3lane, Arm64, AddPointer, V128 | I1 | I2 | I4 | I8 | R4 | R8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_LD4LANE_V128, aarch64_neon_ld4lane, Arm64, AddPointer, V128 | I1 | I2 | I4 | I8 | R4 | R8) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST2LANE_V64, aarch64_neon_st2lane, Arm64, AddPointer, V64 | I1 | I2 | I4 | R4) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST3LANE_V64, aarch64_neon_st3lane, Arm64, AddPointer, V64 | I1 | I2 | I4 | R4) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST4LANE_V64, aarch64_neon_st4lane, Arm64, AddPointer, V64 | I1 | I2 | I4 | R4) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST2LANE_V128, aarch64_neon_st2lane, Arm64, AddPointer, V128 | I1 | I2 | I4 | I8 | R4 | R8) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST3LANE_V128, aarch64_neon_st3lane, Arm64, AddPointer, V128 | I1 | I2 | I4 | I8 | R4 | R8) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST4LANE_V128, aarch64_neon_st4lane, Arm64, AddPointer, V128 | I1 | I2 | I4 | I8 | R4 | R8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST1X2_V64, aarch64_neon_st1x2, Arm64, AddPointer, V64 | I1 | I2 | I4 | R4) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST1X3_V64, aarch64_neon_st1x3, Arm64, AddPointer, V64 | I1 | I2 | I4 | R4) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_ST1X4_V64, aarch64_neon_st1x4, Arm64, AddPointer, V64 | I1 | I2 | I4 | R4) diff --git a/src/mono/mono/mini/method-to-ir.c b/src/mono/mono/mini/method-to-ir.c index 187e1a61d5c7cc..c696cb634566a8 100644 --- a/src/mono/mono/mini/method-to-ir.c +++ b/src/mono/mono/mini/method-to-ir.c @@ -2320,13 +2320,16 @@ emit_type_load_failure (MonoCompile* cfg, MonoClass* klass) } static void -emit_invalid_program_with_msg (MonoCompile *cfg, char *error_msg) +emit_invalid_program_with_msg (MonoCompile *cfg, MonoError *error_msg, MonoMethod *caller, MonoMethod *callee) { + g_assert (!is_ok (error_msg)); + + char *str = mono_mem_manager_strdup (cfg->mem_manager, mono_error_get_message (error_msg)); MonoInst *iargs[1]; if (cfg->compile_aot) - EMIT_NEW_LDSTRLITCONST (cfg, iargs [0], error_msg); + EMIT_NEW_LDSTRLITCONST (cfg, iargs [0], str); else - EMIT_NEW_PCONST (cfg, iargs [0], error_msg); + EMIT_NEW_PCONST (cfg, iargs [0], str); mono_emit_jit_icall (cfg, mono_throw_invalid_program, iargs); } @@ -3413,8 +3416,8 @@ mini_emit_box (MonoCompile *cfg, MonoInst *val, MonoClass *klass, int context_us MonoInst *alloc, *ins; if (G_UNLIKELY (m_class_is_byreflike (klass))) { - mono_error_set_invalid_program (cfg->error, "Cannot box IsByRefLike type '%s.%s'", m_class_get_name_space (klass), m_class_get_name (klass)); - mono_cfg_set_exception (cfg, MONO_EXCEPTION_INVALID_PROGRAM); + mono_error_set_bad_image (cfg->error, m_class_get_image (cfg->method->klass), "Cannot box IsByRefLike type '%s.%s'", m_class_get_name_space (klass), m_class_get_name (klass)); + mono_cfg_set_exception (cfg, MONO_EXCEPTION_MONO_ERROR); return NULL; } @@ -4732,11 +4735,11 @@ mini_inline_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature * } static gboolean -aggressive_inline_method (MonoCompile *cfg, MonoMethod *cmethod) +aggressive_inline_method (MonoMethod *cmethod) { gboolean aggressive_inline = m_method_is_aggressive_inlining (cmethod); if (aggressive_inline) - aggressive_inline = !mono_simd_unsupported_aggressive_inline_intrinsic_type (cfg, cmethod); + aggressive_inline = !mono_simd_unsupported_aggressive_inline_intrinsic_type (cmethod); return aggressive_inline; } @@ -4865,7 +4868,7 @@ inline_method (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, cfg->disable_inline = prev_disable_inline; cfg->inline_depth --; - if ((costs >= 0 && costs < 60) || inline_always || (costs >= 0 && aggressive_inline_method (cfg, cmethod))) { + if ((costs >= 0 && costs < 60) || inline_always || (costs >= 0 && aggressive_inline_method (cmethod))) { if (cfg->verbose_level > 2) printf ("INLINE END %s -> %s\n", mono_method_full_name (cfg->method, TRUE), mono_method_full_name (cmethod, TRUE)); @@ -7545,11 +7548,6 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b if (cfg->compile_aot) cfg->pinvoke_calli_signatures = g_slist_prepend_mempool (cfg->mempool, cfg->pinvoke_calli_signatures, fsig); - if (fsig->has_type_parameters) { - cfg->prefer_instances = TRUE; - GENERIC_SHARING_FAILURE (CEE_CALLI); - } - /* Call the wrapper that will do the GC transition instead */ MonoMethod *wrapper = mono_marshal_get_native_func_wrapper_indirect (method->klass, fsig, cfg->compile_aot); @@ -9279,7 +9277,7 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b mono_save_token_info (cfg, image, token, cmethod); - if (mono_class_has_failure (cmethod->klass) || !mono_class_init_internal (cmethod->klass)) + if (!mono_class_init_internal (cmethod->klass)) TYPE_LOAD_ERROR (cmethod->klass); context_used = mini_method_check_context_used (cfg, cmethod); @@ -9992,8 +9990,8 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b case MONO_CEE_STSFLD: { MonoClassField *field; guint foffset; - gpointer addr = NULL; gboolean is_instance; + gpointer addr = NULL; gboolean is_special_static; MonoType *ftype; MonoInst *store_val = NULL; @@ -10078,7 +10076,6 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b is_instance = FALSE; } - context_used = mini_class_check_context_used (cfg, klass); if (il_op == MONO_CEE_LDSFLD) { @@ -11843,8 +11840,7 @@ mono_method_to_ir (MonoCompile *cfg, MonoMethod *method, MonoBasicBlock *start_b /* if we couldn't create a wrapper because cmethod isn't supposed to have an UnmanagedCallersOnly attribute, follow CoreCLR behavior and throw when the method with the ldftn is executing, not when it is being compiled. */ - char *err_msg = mono_mem_manager_strdup (cfg->mem_manager, mono_error_get_message (wrapper_error)); - emit_invalid_program_with_msg (cfg, err_msg); + emit_invalid_program_with_msg (cfg, wrapper_error, method, cmethod); mono_error_cleanup (wrapper_error); EMIT_NEW_PCONST (cfg, ins, NULL); *sp++ = ins; diff --git a/src/mono/mono/mini/mini-generic-sharing.c b/src/mono/mono/mini/mini-generic-sharing.c index 90e724bf417bf4..58440dcdc3599c 100644 --- a/src/mono/mono/mini/mini-generic-sharing.c +++ b/src/mono/mono/mini/mini-generic-sharing.c @@ -584,14 +584,12 @@ inflate_info (MonoMemoryManager *mem_manager, MonoRuntimeGenericContextInfoTempl if (m_class_get_byval_arg (inflated_class)->type == MONO_TYPE_ARRAY || m_class_get_byval_arg (inflated_class)->type == MONO_TYPE_SZARRAY) { - g_assert (!mono_class_has_failure (inflated_class)); inflated_method = mono_method_search_in_array_class (inflated_class, method->name, method->signature); } else { inflated_method = mono_class_inflate_generic_method_checked (method, context, error); g_assert (is_ok (error)); /* FIXME don't swallow the error */ } - g_assert (inflated_method); mono_class_init_internal (inflated_method->klass); g_assert (inflated_method->klass == inflated_class); return inflated_method; @@ -650,14 +648,12 @@ inflate_info (MonoMemoryManager *mem_manager, MonoRuntimeGenericContextInfoTempl if (m_class_get_byval_arg (inflated_class)->type == MONO_TYPE_ARRAY || m_class_get_byval_arg (inflated_class)->type == MONO_TYPE_SZARRAY) { - g_assert (!mono_class_has_failure (inflated_class)); inflated_method = mono_method_search_in_array_class (inflated_class, method->name, method->signature); } else { inflated_method = mono_class_inflate_generic_method_checked (method, context, error); g_assert (is_ok (error)); /* FIXME don't swallow the error */ } - g_assert (inflated_method); mono_class_init_internal (inflated_method->klass); g_assert (inflated_method->klass == inflated_class); @@ -1312,7 +1308,6 @@ get_wrapper_shared_vtype (MonoType *t) MonoClass *tuple_inst = mono_class_inflate_generic_class_checked (tuple_class, &ctx, error); mono_error_assert_ok (error); - g_assert (tuple_inst); //printf ("T: %s\n", mono_class_full_name (tuple_inst)); @@ -1412,7 +1407,6 @@ get_wrapper_shared_type_full (MonoType *t, gboolean is_field) } klass = mono_class_inflate_generic_class_checked (mono_class_get_generic_class (klass)->container_class, &ctx, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (klass); t = m_class_get_byval_arg (klass); MonoType *shared_type = get_wrapper_shared_vtype (t); @@ -4351,7 +4345,6 @@ get_shared_type (MonoType *t, MonoType *type) k = mono_class_inflate_generic_class_checked (gclass->container_class, &context, error); mono_error_assert_ok (error); /* FIXME don't swallow the error */ - g_assert (k); return mini_get_shared_gparam (t, m_class_get_byval_arg (k)); } else if (MONO_TYPE_ISSTRUCT (type)) { diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index f7e59ef5acbbf6..5db5828eaaa91d 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -685,12 +685,12 @@ simd_valuetuple_to_llvm_type (EmitContext *ctx, MonoClass *klass) { const char *klass_name = m_class_get_name (klass); g_assert (strstr (klass_name, "ValueTuple") != NULL); - MonoGenericInst *class_inst = mono_class_get_generic_class (klass)->context.class_inst; - MonoType *etype = class_inst->type_argv [0]; + MonoGenericInst *classInst = mono_class_get_generic_class (klass)->context.class_inst; + MonoType *etype = classInst->type_argv [0]; g_assert (etype->type == MONO_TYPE_GENERICINST); MonoClass *eklass = etype->data.generic_class->cached_class; LLVMTypeRef ltype = simd_class_to_llvm_type (ctx, eklass); - return LLVMArrayType (ltype, class_inst->type_argc); + return LLVMArrayType (ltype, classInst->type_argc); } /* Return the 128 bit SIMD type corresponding to the mono type TYPE */ @@ -5473,9 +5473,7 @@ immediate_unroll_begin ( LLVMBasicBlockRef continuation = gen_bb (ctx, name); LLVMValueRef switch_ins = LLVMBuildSwitch (ctx->builder, switch_index, default_case, max_cases); LLVMPositionBuilderAtEnd (ctx->builder, continuation); - LLVMValueRef phi = NULL; - if (return_type != LLVMVoidType ()) - phi = LLVMBuildPhi (ctx->builder, return_type, name); + LLVMValueRef phi = LLVMBuildPhi (ctx->builder, return_type, name); ImmediateUnrollCtx ictx = { 0 }; ictx.ctx = ctx; ictx.bb = bb; @@ -5506,8 +5504,7 @@ immediate_unroll_commit (ImmediateUnrollCtx *ictx, int switch_const, LLVMValueRe { LLVMBuildBr (ictx->ctx->builder, ictx->continuation); LLVMAddCase (ictx->switch_ins, LLVMConstInt (ictx->switch_index_type, switch_const, FALSE), ictx->tmp_block); - if (ictx->phi) - LLVMAddIncoming (ictx->phi, &value, &ictx->tmp_block, 1); + LLVMAddIncoming (ictx->phi, &value, &ictx->tmp_block, 1); } static void @@ -5520,8 +5517,7 @@ static void immediate_unroll_commit_default (ImmediateUnrollCtx *ictx, LLVMValueRef value) { LLVMBuildBr (ictx->ctx->builder, ictx->continuation); - if (ictx->phi) - LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1); + LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1); } static void @@ -11637,7 +11633,7 @@ MONO_RESTORE_WARNING lhs = LLVMBuildLoad2 (builder, ret_t, addresses [ins->sreg1]->value, ""); - LLVMValueRef *args = g_newa0(LLVMValueRef, n_elem_tuple + 2); + LLVMValueRef *args = g_new0(LLVMValueRef, n_elem_tuple + 2); unsigned int idx = 0; for ( ; idx < n_elem_tuple; idx++) { args [idx] = LLVMBuildExtractValue (builder, lhs, idx, "extract_elem"); @@ -11754,7 +11750,7 @@ MONO_RESTORE_WARNING break; } case OP_ARM64_STM: { - LLVMTypeRef tuple_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef tuple_t = simd_valuetuple_to_llvm_type (ctx, ins->klass); LLVMTypeRef vec_t = LLVMGetElementType (tuple_t); IntrinsicId iid = (IntrinsicId) ins->inst_c0; @@ -11790,82 +11786,6 @@ MONO_RESTORE_WARNING mono_llvm_build_aligned_store (builder, val, address, FALSE, alignment); break; } - case OP_ARM64_STM_SCALAR: { - LLVMTypeRef tuple_t = simd_class_to_llvm_type (ctx, ins->klass); - LLVMTypeRef vec_t = LLVMGetElementType (tuple_t); - unsigned int n_elem_tuple = LLVMGetArrayLength (tuple_t); - unsigned int n_elem_vector = LLVMGetVectorSize (vec_t); - LLVMTypeRef elem_t = LLVMGetElementType (vec_t); - unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); - unsigned int vector_size = n_elem_vector * elem_bits; - IntrinsicId iid; - switch (vector_size) { - case 64: { - switch (n_elem_tuple) { - case 2: - iid = INTRINS_AARCH64_ADV_SIMD_ST2LANE_V64; - break; - case 3: - iid = INTRINS_AARCH64_ADV_SIMD_ST3LANE_V64; - break; - case 4: - iid = INTRINS_AARCH64_ADV_SIMD_ST4LANE_V64; - break; - default: - g_assert_not_reached (); - break; - } - break; - } - case 128: { - switch (n_elem_tuple) { - case 2: - iid = INTRINS_AARCH64_ADV_SIMD_ST2LANE_V128; - break; - case 3: - iid = INTRINS_AARCH64_ADV_SIMD_ST3LANE_V128; - break; - case 4: - iid = INTRINS_AARCH64_ADV_SIMD_ST4LANE_V128; - break; - default: - g_assert_not_reached (); - break; - } - break; - } - default: - g_assert_not_reached (); - break; - - } - - rhs = LLVMBuildLoad2 (builder, tuple_t, addresses [ins->sreg2]->value, ""); - - LLVMValueRef *args = g_newa0(LLVMValueRef, n_elem_tuple + 2); - unsigned int idx = 0; - for ( ; idx < n_elem_tuple; idx++) { - args [idx] = LLVMBuildExtractValue (builder, rhs, idx, "extract_elem"); - } - args [idx++] = arg3; - args [idx] = lhs; - - llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (vec_t); - - // convert arg3 to a constant - LLVMTypeRef ret_t = LLVMVoidType (); - ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, 16, arg3, ret_t, ""); - int i = 0; - while (immediate_unroll_next (&ictx, &i)) { - args [idx - 1] = const_int64 (i); - call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); - immediate_unroll_commit (&ictx, i, NULL); - } - immediate_unroll_default (&ictx); - immediate_unroll_commit_default (&ictx, NULL); - immediate_unroll_end (&ictx, &cbb); - break; - } case OP_ARM64_ADDHN: case OP_ARM64_ADDHN2: case OP_ARM64_SUBHN: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index ec82dd5de8a8c2..81d9dd29332adf 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -804,6 +804,8 @@ MINI_OP(OP_LDTOKEN_FIELD, "ldtoken_field", VREG, VREG, NONE) /* SIMD opcodes. */ +#if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) || defined(TARGET_ARM64) + MINI_OP(OP_ICONV_TO_R4_RAW, "iconv_to_r4_raw", FREG, IREG, NONE) /* Extract an element from a vector with a constant lane index. @@ -851,6 +853,8 @@ MINI_OP(OP_EXPAND_R4, "expand_r4", XREG, FREG, NONE) MINI_OP(OP_EXPAND_I8, "expand_i8", XREG, IREG, NONE) MINI_OP(OP_EXPAND_R8, "expand_r8", XREG, FREG, NONE) +#endif + // wasm specific SIMD v128 #if defined(TARGET_WASM) @@ -1660,7 +1664,6 @@ MINI_OP(OP_ARM64_UZP2, "arm64_uzp2", XREG, XREG, XREG) MINI_OP(OP_ARM64_ZIP1, "arm64_zip1", XREG, XREG, XREG) MINI_OP(OP_ARM64_ZIP2, "arm64_zip2", XREG, XREG, XREG) MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) -MINI_OP3(OP_ARM64_STM_SCALAR, "arm64_stm_scalar", NONE, IREG, VREG, IREG) MINI_OP3(OP_ARM64_STNP, "arm64_stnp", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) diff --git a/src/mono/mono/mini/mini-runtime.c b/src/mono/mono/mini/mini-runtime.c index 8274949414c275..82a9a5a0a9eff0 100644 --- a/src/mono/mono/mini/mini-runtime.c +++ b/src/mono/mono/mini/mini-runtime.c @@ -404,17 +404,14 @@ void *(mono_global_codeman_reserve) (int size) global_codeman = mono_code_manager_new (); else global_codeman = mono_code_manager_new_aot (); - ptr = mono_code_manager_reserve (global_codeman, size); + return mono_code_manager_reserve (global_codeman, size); } else { mono_jit_lock (); ptr = mono_code_manager_reserve (global_codeman, size); mono_jit_unlock (); + return ptr; } - - /* Virtually all call sites for this API assume it can't return NULL. */ - g_assert (ptr); - return ptr; } /* The callback shouldn't take any locks */ @@ -2170,7 +2167,7 @@ mono_emit_jit_dump (MonoJitInfo *jinfo, gpointer code) int i; memset (&rec, 0, sizeof (rec)); - + // populating info relating debug methods dmji = mono_debug_find_method (jinfo->d.method, NULL); @@ -4543,20 +4540,20 @@ mini_llvm_init (void) } #ifdef ENSURE_PRIMARY_STACK_SIZE -/*++ - Function: - EnsureStackSize - - Abstract: - This fixes a problem on MUSL where the initial stack size reported by the - pthread_attr_getstack is about 128kB, but this limit is not fixed and - the stack can grow dynamically. The problem is that it makes the - functions ReflectionInvocation::[Try]EnsureSufficientExecutionStack - to fail for real life scenarios like e.g. compilation of corefx. - Since there is no real fixed limit for the stack, the code below - ensures moving the stack limit to a value that makes reasonable - real life scenarios work. - +/*++ + Function: + EnsureStackSize + + Abstract: + This fixes a problem on MUSL where the initial stack size reported by the + pthread_attr_getstack is about 128kB, but this limit is not fixed and + the stack can grow dynamically. The problem is that it makes the + functions ReflectionInvocation::[Try]EnsureSufficientExecutionStack + to fail for real life scenarios like e.g. compilation of corefx. + Since there is no real fixed limit for the stack, the code below + ensures moving the stack limit to a value that makes reasonable + real life scenarios work. + --*/ static MONO_NO_OPTIMIZATION MONO_NEVER_INLINE void ensure_stack_size (size_t size) @@ -4740,7 +4737,7 @@ mini_init (const char *filename) mono_w32handle_init (); #endif -#ifdef ENSURE_PRIMARY_STACK_SIZE +#ifdef ENSURE_PRIMARY_STACK_SIZE ensure_stack_size (5 * 1024 * 1024); #endif // ENSURE_PRIMARY_STACK_SIZE diff --git a/src/mono/mono/mini/mini-s390x.c b/src/mono/mono/mini/mini-s390x.c index a5f228ea20f1fe..5c13a13420f7b2 100644 --- a/src/mono/mono/mini/mini-s390x.c +++ b/src/mono/mono/mini/mini-s390x.c @@ -3400,13 +3400,13 @@ mono_arch_output_basic_block (MonoCompile *cfg, MonoBasicBlock *bb) break; case OP_AOTCONST: { mono_add_patch_info (cfg, code - cfg->native_code, - (MonoJumpInfoType)(gsize)ins->inst_i1, ins->inst_p0); + (MonoJumpInfoType)ins->inst_i1, ins->inst_p0); S390_LOAD_TEMPLATE (code, ins->dreg); } break; case OP_JUMP_TABLE: { mono_add_patch_info (cfg, code - cfg->native_code, - (MonoJumpInfoType)(gsize)ins->inst_i1, ins->inst_p0); + (MonoJumpInfoType)ins->inst_i1, ins->inst_p0); S390_LOAD_TEMPLATE (code, ins->dreg); } break; diff --git a/src/mono/mono/mini/mini-wasm.c b/src/mono/mono/mini/mini-wasm.c index 991135288f63de..af943e24adece4 100644 --- a/src/mono/mono/mini/mini-wasm.c +++ b/src/mono/mono/mini/mini-wasm.c @@ -75,23 +75,17 @@ get_storage (MonoType *type, MonoType **etype, gboolean is_return) case MONO_TYPE_R8: return ArgOnStack; - case MONO_TYPE_GENERICINST: { + case MONO_TYPE_GENERICINST: if (!mono_type_generic_inst_is_valuetype (type)) return ArgOnStack; if (mini_is_gsharedvt_variable_type (type)) return ArgGsharedVTOnStack; - - if (mini_wasm_is_scalar_vtype (type, etype)) - return ArgVtypeAsScalar; - - return is_return ? ArgValuetypeAddrInIReg : ArgValuetypeAddrOnStack; - } + /* fall through */ case MONO_TYPE_VALUETYPE: case MONO_TYPE_TYPEDBYREF: { if (mini_wasm_is_scalar_vtype (type, etype)) return ArgVtypeAsScalar; - return is_return ? ArgValuetypeAddrInIReg : ArgValuetypeAddrOnStack; } case MONO_TYPE_VAR: @@ -600,9 +594,7 @@ mono_wasm_execute_timer (void) } background_job_cb cb = timer_handler; - MONO_ENTER_GC_UNSAFE; cb (); - MONO_EXIT_GC_UNSAFE; } #ifdef DISABLE_THREADS @@ -777,12 +769,7 @@ mini_wasm_is_scalar_vtype (MonoType *type, MonoType **etype) if (nfields > 1) return FALSE; MonoType *t = mini_get_underlying_type (field->type); - int align, field_size = mono_type_size (t, &align); - // inlinearray and fixed both work by having a single field that is bigger than its element type. - // we also don't want to scalarize a struct that has padding in its metadata, even if it would fit. - if (field_size != size) { - return FALSE; - } else if (MONO_TYPE_ISSTRUCT (t)) { + if (MONO_TYPE_ISSTRUCT (t)) { if (!mini_wasm_is_scalar_vtype (t, etype)) return FALSE; } else if (!((MONO_TYPE_IS_PRIMITIVE (t) || MONO_TYPE_IS_REFERENCE (t) || MONO_TYPE_IS_POINTER (t)))) { diff --git a/src/mono/mono/mini/mini.c b/src/mono/mono/mini/mini.c index 92bf21887c59f8..eb174000eabe7b 100644 --- a/src/mono/mono/mini/mini.c +++ b/src/mono/mono/mini/mini.c @@ -3037,9 +3037,6 @@ is_simd_supported (MonoCompile *cfg) { #ifdef DISABLE_SIMD return FALSE; -#endif -#ifndef MONO_ARCH_SIMD_INTRINSICS - return FALSE; #endif // FIXME: Clean this up #ifdef TARGET_WASM @@ -4322,7 +4319,6 @@ mini_handle_call_res_devirt (MonoMethod *cmethod) inst = mono_class_inflate_generic_class_checked (mono_class_get_iequatable_class (), &ctx, error); mono_error_assert_ok (error); - g_assert (inst); // EqualityComparer.Default returns specific types depending on T // FIXME: Special case more types: byte, string, nullable, enum ? diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index 70aab00b740f21..9982afc22e3f2f 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2962,7 +2962,7 @@ MonoInst* mono_emit_common_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoInst* mono_emit_simd_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args); MonoInst* mono_emit_simd_field_load (MonoCompile *cfg, MonoClassField *field, MonoInst *addr); void mono_simd_intrinsics_init (void); -gboolean mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoCompile *cfg, MonoMethod *cmethod); +gboolean mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoMethod *cmethod); MonoMethod* mini_method_to_shared (MonoMethod *method); // null if not shared diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index b0607db9b38b03..1d6a56676b0f02 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -19,6 +19,8 @@ #include #include +#if defined (MONO_ARCH_SIMD_INTRINSICS) + #if defined(DISABLE_JIT) void @@ -174,7 +176,7 @@ has_intrinsic_cattr (MonoMethod *method) } static gboolean -is_SIMD_feature_supported(MonoCompile *cfg, MonoCPUFeatures feature) +is_SIMD_feature_supported(MonoCompile *cfg, MonoCPUFeatures feature) { return mini_get_cpu_features (cfg) & feature; } @@ -317,7 +319,7 @@ emit_simd_ins_for_binary_op (MonoCompile *cfg, MonoClass *klass, MonoMethodSigna if (id == SN_BitwiseAnd || id == SN_BitwiseOr || id == SN_Xor || id == SN_op_BitwiseAnd || id == SN_op_BitwiseOr || id == SN_op_ExclusiveOr) { op = OP_XBINOP_FORCEINT; - + switch (id) { case SN_BitwiseAnd: case SN_op_BitwiseAnd: @@ -419,7 +421,7 @@ emit_simd_ins_for_binary_op (MonoCompile *cfg, MonoClass *klass, MonoMethodSigna if (!COMPILE_LLVM (cfg)) return NULL; #endif - if (fsig->params [1]->type != MONO_TYPE_GENERICINST) + if (fsig->params [1]->type != MONO_TYPE_GENERICINST) return handle_mul_div_by_scalar (cfg, klass, arg_type, args [1]->dreg, args [0]->dreg, OP_IMUL); else if (fsig->params [0]->type != MONO_TYPE_GENERICINST) return handle_mul_div_by_scalar (cfg, klass, arg_type, args [0]->dreg, args [1]->dreg, OP_IMUL); @@ -564,7 +566,7 @@ emit_xequal (MonoCompile *cfg, MonoClass *klass, MonoTypeEnum element_type, Mono } else { return emit_simd_ins (cfg, klass, OP_XEQUAL, arg1->dreg, arg2->dreg); } -#else +#else MonoInst *ins = emit_simd_ins (cfg, klass, OP_XEQUAL, arg1->dreg, arg2->dreg); if (!COMPILE_LLVM (cfg)) ins->inst_c1 = mono_class_get_context (klass)->class_inst->type_argv [0]->type; @@ -649,12 +651,12 @@ emit_sum_vector (MonoCompile *cfg, MonoType *vector_type, MonoTypeEnum element_t MonoClass *vector_class = mono_class_from_mono_type_internal (vector_type); int vector_size = mono_class_value_size (vector_class, NULL); int element_size; - + guint32 nelems; mini_get_simd_type_info (vector_class, &nelems); // Override nelems for Vector3, with actual number of elements, instead of treating it as a 4-element vector (three elements + zero). - const char *klass_name = m_class_get_name (vector_class); + const char *klass_name = m_class_get_name (vector_class); if (!strcmp (klass_name, "Vector3")) nelems = 3; @@ -726,7 +728,7 @@ emit_sum_vector (MonoCompile *cfg, MonoType *vector_type, MonoTypeEnum element_t case MONO_TYPE_U1: // byte, sbyte not supported yet return NULL; - case MONO_TYPE_I2: + case MONO_TYPE_I2: case MONO_TYPE_U2: instc0 = INTRINS_SSE_PHADDW; break; @@ -758,12 +760,12 @@ emit_sum_vector (MonoCompile *cfg, MonoType *vector_type, MonoTypeEnum element_t default: { return NULL; } - } - + } + // Check if necessary SIMD intrinsics are supported on the current machine MonoCPUFeatures feature = type_enum_is_float (element_type) ? MONO_CPU_X86_SSE3 : MONO_CPU_X86_SSSE3; if (!is_SIMD_feature_supported (cfg, feature)) - return NULL; + return NULL; int vector_size = mono_class_value_size (vector_class, NULL); MonoType *etype = mono_class_get_context (vector_class)->class_inst->type_argv [0]; @@ -1062,7 +1064,7 @@ emit_hardware_intrinsics ( static MonoInst* emit_vector_insert_element ( - MonoCompile* cfg, MonoClass* vklass, MonoInst* ins, MonoTypeEnum type, MonoInst* element, + MonoCompile* cfg, MonoClass* vklass, MonoInst* ins, MonoTypeEnum type, MonoInst* element, int index, gboolean is_zero_inited) { int op = type_to_insert_op (type); @@ -1070,7 +1072,7 @@ emit_vector_insert_element ( if (is_zero_inited && is_zero_const (element)) { // element already set to zero #ifdef TARGET_ARM64 - } else if (!COMPILE_LLVM (cfg) && element->opcode == type_to_extract_op (type) && + } else if (!COMPILE_LLVM (cfg) && element->opcode == type_to_extract_op (type) && (type == MONO_TYPE_R4 || type == MONO_TYPE_R8)) { // OP_INSERT_Ix inserts from GP reg, not SIMD. Cannot optimize for int types. ins = emit_simd_ins (cfg, vklass, op, ins->dreg, element->sreg1); @@ -1099,6 +1101,8 @@ emit_vector_create_elementwise ( return ins; } +#if defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) + static int type_to_xinsert_op (MonoTypeEnum type) { @@ -1175,6 +1179,20 @@ create_class_instance (const char* name_space, const char *name, MonoType *param return ivector_inst; } +static gboolean +is_supported_vector_primitive_type (MonoType *type) +{ + gboolean constrained_generic_param = (type->type == MONO_TYPE_VAR || type->type == MONO_TYPE_MVAR); + + if (constrained_generic_param && type->data.generic_param->gshared_constraint && MONO_TYPE_IS_VECTOR_PRIMITIVE (type->data.generic_param->gshared_constraint)) + return TRUE; + + if (MONO_TYPE_IS_VECTOR_PRIMITIVE (type)) + return TRUE; + + return FALSE; +} + static guint16 sri_vector_methods [] = { SN_Abs, SN_Add, @@ -1468,7 +1486,7 @@ emit_dot (MonoCompile *cfg, MonoClass *klass, MonoType *vector_type, MonoTypeEnu */ static MonoInst* emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) -{ +{ int id = lookup_intrins (sri_vector_methods, sizeof (sri_vector_methods), cmethod); if (id == -1) { //check_no_intrinsic_cattr (cmethod); @@ -1492,9 +1510,6 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi if (vector_size == 256 || vector_size == 512) return NULL; - if (!(cfg->opt & MONO_OPT_SIMD)) - return NULL; - // FIXME: This limitation could be removed once everything here are supported by mini JIT on arm64 #ifdef TARGET_ARM64 if (!COMPILE_LLVM (cfg)) { @@ -1557,9 +1572,9 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi } else { if (COMPILE_LLVM (cfg)) return emit_simd_ins_for_sig (cfg, klass, OP_VECTOR_IABS, -1, arg0_type, fsig, args); - + // SSSE3 does not support i64 - if (is_SIMD_feature_supported (cfg, MONO_CPU_X86_SSSE3) && + if (is_SIMD_feature_supported (cfg, MONO_CPU_X86_SSSE3) && !(arg0_type == MONO_TYPE_I8 || (TARGET_SIZEOF_VOID_P == 8 && arg0_type == MONO_TYPE_I))) return emit_simd_ins_for_sig (cfg, klass, OP_VECTOR_IABS, -1, arg0_type, fsig, args); @@ -1567,7 +1582,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi MonoInst *neg = emit_simd_ins (cfg, klass, OP_XBINOP, zero->dreg, args [0]->dreg); neg->inst_c0 = OP_ISUB; neg->inst_c1 = arg0_type; - + MonoInst *ins = emit_simd_ins (cfg, klass, OP_XBINOP, args [0]->dreg, neg->dreg); ins->inst_c0 = OP_IMAX; ins->inst_c1 = arg0_type; @@ -1596,7 +1611,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi return NULL; return emit_simd_ins_for_binary_op (cfg, klass, fsig, args, arg0_type, id); case SN_AndNot: { - if (!is_element_type_primitive (fsig->params [0])) + if (!is_element_type_primitive (fsig->params [0])) return NULL; #ifdef TARGET_ARM64 return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_BIC, -1, arg0_type, fsig, args); @@ -1671,8 +1686,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi return NULL; #if defined(TARGET_ARM64) if (!COMPILE_LLVM (cfg)) { - return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, - arg0_type == MONO_TYPE_I8 ? OP_CVT_SI_FP : OP_CVT_UI_FP, + return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, + arg0_type == MONO_TYPE_I8 ? OP_CVT_SI_FP : OP_CVT_UI_FP, MONO_TYPE_R8, fsig, args); } #endif @@ -1696,15 +1711,15 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi return NULL; #endif } - case SN_ConvertToInt32: + case SN_ConvertToInt32: case SN_ConvertToUInt32: { if (arg0_type != MONO_TYPE_R4) return NULL; #if defined(TARGET_ARM64) if (!COMPILE_LLVM (cfg)) { - return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, - id == SN_ConvertToInt32 ? OP_CVT_FP_SI : OP_CVT_FP_UI, - id == SN_ConvertToInt32 ? MONO_TYPE_I4 : MONO_TYPE_U4, + return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, + id == SN_ConvertToInt32 ? OP_CVT_FP_SI : OP_CVT_FP_UI, + id == SN_ConvertToInt32 ? MONO_TYPE_I4 : MONO_TYPE_U4, fsig, args); } #endif @@ -1728,9 +1743,9 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi return NULL; #if defined(TARGET_ARM64) if (!COMPILE_LLVM (cfg)) { - return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, - id == SN_ConvertToInt64 ? OP_CVT_FP_SI : OP_CVT_FP_UI, - id == SN_ConvertToInt64 ? MONO_TYPE_I8 : MONO_TYPE_U8, + return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, + id == SN_ConvertToInt64 ? OP_CVT_FP_SI : OP_CVT_FP_UI, + id == SN_ConvertToInt64 ? MONO_TYPE_I8 : MONO_TYPE_U8, fsig, args); } #endif @@ -1759,8 +1774,8 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi return NULL; #if defined(TARGET_ARM64) if (!COMPILE_LLVM (cfg)) { - return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, - arg0_type == MONO_TYPE_I4 ? OP_CVT_SI_FP : OP_CVT_UI_FP, + return emit_simd_ins_for_sig (cfg, klass, OP_XUNOP, + arg0_type == MONO_TYPE_I4 ? OP_CVT_SI_FP : OP_CVT_UI_FP, MONO_TYPE_R4, fsig, args); } #endif @@ -1892,7 +1907,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi } else { arg_class = mono_class_from_mono_type_internal (fsig->params [0]); } - + // FIXME: Add support for Vector64 on arm64 https://github.com/dotnet/runtime/issues/90402 int size = mono_class_value_size (arg_class, NULL); if (size != 16) @@ -1916,10 +1931,10 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi MonoInst* ext_low_vec = emit_simd_ins_for_sig (cfg, arg_class, OP_XLOWER, 8, arg0_type, fsig, &shift_res_vec); MonoInst* sum_low_vec = emit_sum_vector (cfg, fsig->params [0], arg0_type, ext_low_vec); - + MonoInst* ext_high_vec = emit_simd_ins_for_sig (cfg, arg_class, OP_XUPPER, 8, arg0_type, fsig, &shift_res_vec); - MonoInst* sum_high_vec = emit_sum_vector (cfg, fsig->params [0], arg0_type, ext_high_vec); - + MonoInst* sum_high_vec = emit_sum_vector (cfg, fsig->params [0], arg0_type, ext_high_vec); + MONO_EMIT_NEW_BIALU_IMM (cfg, OP_SHL_IMM, sum_high_vec->dreg, sum_high_vec->dreg, 8); EMIT_NEW_BIALU (cfg, result_ins, OP_IOR, sum_high_vec->dreg, sum_high_vec->dreg, sum_low_vec->dreg); } else { @@ -1932,9 +1947,9 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi switch (arg0_type) { case MONO_TYPE_U2: case MONO_TYPE_I2: { - if (!is_SIMD_feature_supported (cfg, MONO_CPU_X86_SSSE3)) + if (!is_SIMD_feature_supported (cfg, MONO_CPU_X86_SSSE3)) return NULL; - + type = type_enum_is_unsigned (arg0_type) ? MONO_TYPE_U1 : MONO_TYPE_I1; MonoClass* arg_class = mono_class_from_mono_type_internal (fsig->params [0]); @@ -2002,7 +2017,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi if (index < 0 || index >= elems) { MONO_EMIT_NEW_BIALU_IMM (cfg, OP_COMPARE_IMM, -1, args [1]->dreg, elems); MONO_EMIT_NEW_COND_EXC (cfg, GE_UN, "ArgumentOutOfRangeException"); - } + } // Bounds check is elided if we know the index is safe. int extract_op = type_to_extract_op (arg0_type); @@ -2088,7 +2103,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi case SN_GreaterThanAll: case SN_GreaterThanOrEqualAll: case SN_LessThanAll: - case SN_LessThanOrEqualAll: + case SN_LessThanOrEqualAll: is_all = TRUE; break; } @@ -2250,7 +2265,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi if (!is_element_type_primitive (fsig->params [0])) return NULL; return emit_simd_ins_for_unary_op (cfg, klass, fsig, args, arg0_type, id); - } + } case SN_Shuffle: { if (!is_element_type_primitive (fsig->params [0])) return NULL; @@ -2310,7 +2325,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi } case SN_WithElement: { int elems; - + if (!is_element_type_primitive (fsig->params [0])) return NULL; @@ -2338,7 +2353,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi } return emit_vector_insert_element (cfg, klass, args [0], arg0_type, args [2], index, FALSE); - } + } if (!COMPILE_LLVM (cfg) && fsig->params [0]->type != MONO_TYPE_GENERICINST) return NULL; @@ -2400,7 +2415,7 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi subop = is_upper ? OP_ARM64_UXTL2 : OP_ARM64_UXTL; else subop = is_upper ? OP_ARM64_SXTL2 : OP_ARM64_SXTL; - + MonoInst* ins = emit_simd_ins (cfg, klass, OP_XUNOP, args [0]->dreg, -1); ins->inst_c0 = subop; ins->inst_c1 = arg0_type; @@ -2484,6 +2499,12 @@ emit_sri_vector_t (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *f g_free (name); } + if (id == SN_get_IsSupported) { + MonoInst *ins; + EMIT_NEW_ICONST (cfg, ins, is_supported_vector_primitive_type (etype) ? 1 : 0); + return ins; + } + // Apart from filtering out non-primitive types this also filters out shared generic instance types like: T_BYTE which cannot be intrinsified if (!MONO_TYPE_IS_VECTOR_PRIMITIVE (etype)) { // Happens often in gshared code @@ -2527,9 +2548,6 @@ emit_sri_vector_t (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *f return ins; } - if (!(cfg->opt & MONO_OPT_SIMD)) - return NULL; - /* Vector256/Vector512 */ if (size == 32 || size == 64) return NULL; @@ -2650,7 +2668,7 @@ emit_sri_vector_t (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *f return NULL; arg0_type = fsig->param_count > 0 ? get_underlying_type (fsig->params [0]) : MONO_TYPE_VOID; return emit_simd_ins_for_binary_op (cfg, klass, fsig, args, arg0_type, id); - + } case SN_op_Equality: case SN_op_Inequality: { @@ -2754,15 +2772,13 @@ emit_vector_2_3_4 (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *f return NULL; #endif - if (!(cfg->opt & MONO_OPT_SIMD)) - return NULL; - if (cfg->verbose_level > 1) { char *name = mono_method_full_name (cmethod, TRUE); printf (" SIMD intrinsic %s\n", name); g_free (name); } + // Similar to the cases in emit_sys_numerics_vector_t () switch (id) { case SN_ctor: if (is_elementwise_ctor (fsig, etype)) { @@ -4148,7 +4164,7 @@ emit_arm64_intrinsics ( MONO_ADD_INS (cfg->cbb, ins); return ins; } - + default: g_assert_not_reached (); // if a new API is added we need to either implement it or change IsSupported to false } @@ -4299,13 +4315,9 @@ emit_arm64_intrinsics ( return ret; } case SN_StoreSelectedScalar: { - int store_op; - if (is_intrinsics_vector_type (fsig->params [1])) - store_op = OP_ARM64_ST1_SCALAR; - else - store_op = OP_ARM64_STM_SCALAR; - MonoClass* klass_tuple_var = mono_class_from_mono_type_internal (fsig->params [1]); - return emit_simd_ins_for_sig (cfg, klass_tuple_var, store_op, 0, arg0_type, fsig, args); + if (!is_intrinsics_vector_type (fsig->params [1])) + return NULL; + return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_ST1_SCALAR, 0, arg0_type, fsig, args); } case SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh: case SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh: @@ -5548,15 +5560,15 @@ emit_x86_intrinsics ( MONO_ADD_INS (cfg->cbb, ins); return ins; case SN_DivRem: { - g_assert (!(TARGET_SIZEOF_VOID_P == 4 && is_64bit)); // x86(no -64) cannot do divisions with 64-bit regs + g_assert (!(TARGET_SIZEOF_VOID_P == 4 && is_64bit)); // x86(no -64) cannot do divisions with 64-bit regs const MonoStackType divtype = is_64bit ? STACK_I8 : STACK_I4; const int storetype = is_64bit ? OP_STOREI8_MEMBASE_REG : OP_STOREI4_MEMBASE_REG; const int obj_size = MONO_ABI_SIZEOF (MonoObject); - // We must decide by the second argument, the first is always unsigned here + // We must decide by the second argument, the first is always unsigned here MonoTypeEnum arg1_type = fsig->param_count > 1 ? get_underlying_type (fsig->params [1]) : MONO_TYPE_VOID; MonoInst* div; - MonoInst* div2; + MonoInst* div2; if (type_enum_is_unsigned (arg1_type)) { MONO_INST_NEW (cfg, div, is_64bit ? OP_X86_LDIVREMU : OP_X86_IDIVREMU); @@ -5577,7 +5589,7 @@ emit_x86_intrinsics ( div2->dreg = is_64bit ? alloc_lreg (cfg) : alloc_ireg (cfg); div2->type = divtype; MONO_ADD_INS (cfg->cbb, div2); - + // TODO: Can the creation of tuple be elided? (e.g. if deconstruction is used) MonoInst* tuple = mono_compile_create_var (cfg, fsig->ret, OP_LOCAL); MonoInst* tuple_addr; @@ -6080,6 +6092,13 @@ arch_emit_simd_intrinsics (const char *class_ns, const char *class_name, MonoCom emit_arm64_intrinsics); } + if (!strcmp (class_ns, "System.Numerics")) { + if (!strcmp (class_name, "Vector")) + return emit_sri_vector (cfg, cmethod, fsig, args); + if (!strcmp (class_name, "Vector`1")) + return emit_sri_vector_t (cfg, cmethod, fsig, args); + } + return NULL; } #elif defined(TARGET_AMD64) @@ -6094,6 +6113,15 @@ arch_emit_simd_intrinsics (const char *class_ns, const char *class_name, MonoCom emit_x86_intrinsics); } + if (!strcmp (class_ns, "System.Numerics")) { + // FIXME: Shouldn't this call emit_sri_vector () ? + if (!strcmp (class_name, "Vector")) + return emit_sys_numerics_vector (cfg, cmethod, fsig, args); + // FIXME: Shouldn't this call emit_sri_vector_t () ? + if (!strcmp (class_name, "Vector`1")) + return emit_sys_numerics_vector_t (cfg, cmethod, fsig, args); + } + return NULL; } #elif defined(TARGET_WASM) @@ -6107,6 +6135,13 @@ arch_emit_simd_intrinsics (const char *class_ns, const char *class_name, MonoCom emit_wasm_supported_intrinsics); } + if (!strcmp (class_ns, "System.Numerics")) { + if (!strcmp (class_name, "Vector")) + return emit_sri_vector (cfg, cmethod, fsig, args); + if (!strcmp (class_name, "Vector`1")) + return emit_sri_vector_t (cfg, cmethod, fsig, args); + } + return NULL; } #else @@ -6149,11 +6184,9 @@ emit_simd_intrinsics (const char *class_ns, const char *class_name, MonoCompile { MonoInst *ins; - if (cfg->opt & MONO_OPT_SIMD) { - ins = arch_emit_simd_intrinsics (class_ns, class_name, cfg, cmethod, fsig, args); - if (ins) - return ins; - } + ins = arch_emit_simd_intrinsics (class_ns, class_name, cfg, cmethod, fsig, args); + if (ins) + return ins; if (!strcmp (class_ns, "System.Runtime.Intrinsics")) { if (!strcmp (class_name, "Vector64") || !strcmp (class_name, "Vector128") || !strcmp (class_name, "Vector256") || !strcmp (class_name, "Vector512")) @@ -6163,10 +6196,6 @@ emit_simd_intrinsics (const char *class_ns, const char *class_name, MonoCompile } if (!strcmp (class_ns, "System.Numerics")) { - if (!strcmp (class_name, "Vector")) - return emit_sri_vector (cfg, cmethod, fsig, args); - if (!strcmp (class_name, "Vector`1")) - return emit_sri_vector_t (cfg, cmethod, fsig, args); if (!strcmp (class_name, "Vector2") || !strcmp (class_name, "Vector3") || !strcmp (class_name, "Vector4") || !strcmp (class_name, "Quaternion") || !strcmp (class_name, "Plane")) return emit_vector_2_3_4 (cfg, cmethod, fsig, args); @@ -6292,17 +6321,8 @@ mono_simd_decompose_intrinsic (MonoCompile *cfg, MonoBasicBlock *bb, MonoInst *i } } -#else - -void -mono_simd_decompose_intrinsic (MonoCompile *cfg, MonoBasicBlock *bb, MonoInst *ins) -{ -} - -#endif /*defined(TARGET_WIN32) && defined(TARGET_AMD64)*/ - gboolean -mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoCompile *cfg, MonoMethod *cmethod) +mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoMethod *cmethod) { /* * If a method has been marked with aggressive inlining, check if we support @@ -6313,18 +6333,66 @@ mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoCompile *cfg, MonoMe if (!strcmp (m_class_get_name_space (cmethod->klass), "System.Runtime.Intrinsics")) { if (!strncmp(m_class_get_name (cmethod->klass), "Vector", 6)) { const char *vector_type = m_class_get_name (cmethod->klass) + 6; - if (!strcmp(vector_type, "256`1") || !strcmp(vector_type, "512`1") || !strcmp(vector_type, "256") || !strcmp(vector_type, "512")) - return TRUE; - if (!(cfg->opt & MONO_OPT_SIMD) && (!strcmp (vector_type, "128`1") || !strcmp (vector_type, "128") || !strcmp (vector_type, "64`1") || !strcmp (vector_type, "64"))) + if (!strcmp(vector_type, "256`1") || !strcmp(vector_type, "512`1")) return TRUE; } } + return FALSE; +} +#else +void +mono_simd_decompose_intrinsic (MonoCompile *cfg, MonoBasicBlock *bb, MonoInst *ins) +{ +} +gboolean +mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoMethod* cmethod) +{ return FALSE; } +#endif /*defined(TARGET_WIN32) && defined(TARGET_AMD64)*/ + #endif /* DISABLE_JIT */ +#else /* MONO_ARCH_SIMD_INTRINSICS */ + +void +mono_simd_intrinsics_init (void) +{ +} + +MonoInst* +mono_emit_simd_field_load (MonoCompile *cfg, MonoClassField *field, MonoInst *addr) +{ + return NULL; +} + +MonoInst* +mono_emit_simd_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) +{ + return NULL; +} + +MonoInst* +mono_emit_common_intrinsics (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) +{ + return NULL; +} + +void +mono_simd_decompose_intrinsic (MonoCompile *cfg, MonoBasicBlock *bb, MonoInst *ins) +{ +} + +gboolean +mono_simd_unsupported_aggressive_inline_intrinsic_type (MonoMethod* cmethod) +{ + return FALSE; +} + +#endif /* MONO_ARCH_SIMD_INTRINSICS */ + #if defined(TARGET_AMD64) void ves_icall_System_Runtime_Intrinsics_X86_X86Base___cpuidex (int abcd[4], int function_id, int subfunction_id) diff --git a/src/mono/mono/utils/mono-cgroup.c b/src/mono/mono/utils/mono-cgroup.c index 4c58f53257b97e..0a6c96678877d6 100644 --- a/src/mono/mono/utils/mono-cgroup.c +++ b/src/mono/mono/utils/mono-cgroup.c @@ -45,6 +45,7 @@ Module Name: #endif #define CGROUP2_SUPER_MAGIC 0x63677270 +#define TMPFS_MAGIC 0x01021994 #define PROC_MOUNTINFO_FILENAME "/proc/self/mountinfo" #define PROC_CGROUP_FILENAME "/proc/self/cgroup" @@ -218,13 +219,10 @@ findCGroupVersion(void) if (result != 0) return 0; - if (stats.f_type == CGROUP2_SUPER_MAGIC) { - return 2; - } else { - // Assume that if /sys/fs/cgroup exists and the file system type is not cgroup2fs, - // it is cgroup v1. Typically the file system type is tmpfs, but other values have - // been seen in the wild. - return 1; + switch (stats.f_type) { + case TMPFS_MAGIC: return 1; + case CGROUP2_SUPER_MAGIC: return 2; + default: return 0; } } diff --git a/src/mono/mono/utils/mono-path.c b/src/mono/mono/utils/mono-path.c index 616fa183d5a559..4632a74556c3b4 100644 --- a/src/mono/mono/utils/mono-path.c +++ b/src/mono/mono/utils/mono-path.c @@ -44,7 +44,6 @@ mono_path_canonicalize (const char *path) } else { gchar *tmpdir = g_get_current_dir (); abspath = g_build_filename (tmpdir, path, (const char*)NULL); - g_assert (abspath); g_free (tmpdir); } @@ -129,7 +128,6 @@ resolve_symlink (const char *path) if (!g_path_is_absolute (buffer)) { dir = g_path_get_dirname (p); concat = g_build_filename (dir, buffer, (const char*)NULL); - g_assert (concat); g_free (dir); } else { concat = g_strdup (buffer); diff --git a/src/mono/mono/utils/mono-threads-wasm.c b/src/mono/mono/utils/mono-threads-wasm.c index cfd411c712c385..67d3e7a1c6a790 100644 --- a/src/mono/mono/utils/mono-threads-wasm.c +++ b/src/mono/mono/utils/mono-threads-wasm.c @@ -458,7 +458,7 @@ mono_threads_platform_is_main_thread (void) #ifdef DISABLE_THREADS return TRUE; #else - return mono_threads_wasm_is_deputy_thread (); + return emscripten_is_main_runtime_thread (); #endif } @@ -496,7 +496,7 @@ mono_threads_wasm_on_thread_attached (pthread_t tid, const char* thread_name, gb #else if (mono_threads_wasm_is_ui_thread ()) { // FIXME: we should not be attaching UI thread with deputy design - // but right now we do + // but right now we do, because mono_wasm_load_runtime is running in UI thread // g_assert(!mono_threads_wasm_is_ui_thread ()); return; } @@ -541,107 +541,6 @@ mono_threads_wasm_on_thread_registered (void) } #ifndef DISABLE_THREADS -static pthread_t deputy_thread_tid; -static pthread_t io_thread_tid; -extern void mono_wasm_start_deputy_thread_async (void); -extern void mono_wasm_start_io_thread_async (void); -extern void mono_wasm_trace_logger (const char *log_domain, const char *log_level, const char *message, mono_bool fatal, void *user_data); -extern void mono_wasm_dump_threads (void); - -void mono_wasm_dump_threads_async (void) -{ - mono_threads_wasm_async_run_in_target_thread (mono_threads_wasm_ui_thread_tid (), mono_wasm_dump_threads); -} - -gboolean -mono_threads_wasm_is_deputy_thread (void) -{ - return pthread_self () == deputy_thread_tid; -} - -MonoNativeThreadId -mono_threads_wasm_deputy_thread_tid (void) -{ - return (MonoNativeThreadId) deputy_thread_tid; -} - -// this is running in deputy thread -static gsize -deputy_thread_fn (void* unused_arg G_GNUC_UNUSED) -{ - deputy_thread_tid = pthread_self (); - - // this will throw JS "unwind" - mono_wasm_start_deputy_thread_async(); - - return 0;// never reached -} - -EMSCRIPTEN_KEEPALIVE MonoNativeThreadId -mono_wasm_create_deputy_thread (void) -{ - pthread_create (&deputy_thread_tid, NULL, (void *(*)(void *)) deputy_thread_fn, NULL); - return deputy_thread_tid; -} - -gboolean -mono_threads_wasm_is_io_thread (void) -{ - return pthread_self () == io_thread_tid; -} - -MonoNativeThreadId -mono_threads_wasm_io_thread_tid (void) -{ - return (MonoNativeThreadId) io_thread_tid; -} - -// this is running in io thread -static gsize -io_thread_fn (void* unused_arg G_GNUC_UNUSED) -{ - io_thread_tid = pthread_self (); - - // this will throw JS "unwind" - mono_wasm_start_io_thread_async(); - - return 0;// never reached -} - -EMSCRIPTEN_KEEPALIVE MonoNativeThreadId -mono_wasm_create_io_thread (void) -{ - pthread_create (&io_thread_tid, NULL, (void *(*)(void *)) io_thread_fn, NULL); - return io_thread_tid; -} - -// TODO ideally we should not need to have UI thread registered as managed -EMSCRIPTEN_KEEPALIVE void -mono_wasm_register_ui_thread (void) -{ - MonoThread *thread = mono_thread_internal_attach (mono_get_root_domain ()); - mono_thread_set_state (thread, ThreadState_Background); - mono_thread_info_set_flags (MONO_THREAD_INFO_FLAGS_NONE); - - MonoThreadInfo *info = mono_thread_info_current_unchecked (); - g_assert (info); - info->runtime_thread = TRUE; - MONO_ENTER_GC_SAFE_UNBALANCED; -} - -// TODO ideally we should not need to have UI thread registered as managed -EMSCRIPTEN_KEEPALIVE void -mono_wasm_register_io_thread (void) -{ - MonoThread *thread = mono_thread_internal_attach (mono_get_root_domain ()); - mono_thread_set_state (thread, ThreadState_Background); - mono_thread_info_set_flags (MONO_THREAD_INFO_FLAGS_NONE); - - MonoThreadInfo *info = mono_thread_info_current_unchecked (); - g_assert (info); - info->runtime_thread = TRUE; - MONO_ENTER_GC_SAFE_UNBALANCED; -} void mono_threads_wasm_async_run_in_target_thread (pthread_t target_thread, void (*func) (void)) diff --git a/src/mono/mono/utils/mono-threads-wasm.h b/src/mono/mono/utils/mono-threads-wasm.h index ad9dc40b32db09..13d0b6cb76a6c8 100644 --- a/src/mono/mono/utils/mono-threads-wasm.h +++ b/src/mono/mono/utils/mono-threads-wasm.h @@ -28,33 +28,6 @@ mono_threads_wasm_ui_thread_tid (void); #ifndef DISABLE_THREADS -void -mono_wasm_dump_threads_async (void); - -gboolean -mono_threads_wasm_is_deputy_thread (void); - -gboolean -mono_threads_wasm_is_io_thread (void); - -MonoNativeThreadId -mono_threads_wasm_deputy_thread_tid (void); - -MonoNativeThreadId -mono_threads_wasm_io_thread_tid (void); - -MonoNativeThreadId -mono_wasm_create_deputy_thread (void); - -MonoNativeThreadId -mono_wasm_create_io_thread (void); - -void -mono_wasm_register_ui_thread (void); - -void -mono_wasm_register_io_thread (void); - void mono_threads_wasm_async_run_in_target_thread (pthread_t target_thread, void (*func) (void)); diff --git a/src/mono/mono/utils/mono-threads.c b/src/mono/mono/utils/mono-threads.c index 14a00bc9154291..515cde6eebad53 100644 --- a/src/mono/mono/utils/mono-threads.c +++ b/src/mono/mono/utils/mono-threads.c @@ -281,12 +281,6 @@ mono_threads_end_global_suspend (void) static void dump_threads (void) { -#ifdef HOST_BROWSER -#ifndef DISABLE_THREADS - mono_wasm_dump_threads_async (); -#endif -#endif - MonoThreadInfo *cur = mono_thread_info_current (); g_async_safe_printf ("STATE CUE CARD: (? means a positive number, usually 1 or 2, * means any number)\n"); diff --git a/src/mono/mono/utils/options.c b/src/mono/mono/utils/options.c index 2a8dc6f60480bd..bf092372828c35 100644 --- a/src/mono/mono/utils/options.c +++ b/src/mono/mono/utils/options.c @@ -121,13 +121,12 @@ get_option_hash (void) * Set options based on the command line arguments in ARGV/ARGC. * Remove processed arguments from ARGV and set *OUT_ARGC to the * number of remaining arguments. - * If PROCESSED is != NULL, add the processed arguments to it. * * NOTE: This only sets the variables, the caller might need to do * additional processing based on the new values of the variables. */ void -mono_options_parse_options (const char **argv, int argc, int *out_argc, GPtrArray *processed, MonoError *error) +mono_options_parse_options (const char **argv, int argc, int *out_argc, MonoError *error) { int aindex = 0; GHashTable *option_hash = NULL; @@ -188,8 +187,6 @@ mono_options_parse_options (const char **argv, int argc, int *out_argc, GPtrArra break; } *(gboolean*)option->addr = negate ? FALSE : TRUE; - if (processed) - g_ptr_array_add (processed, (gpointer)argv [aindex]); argv [aindex] = NULL; break; } @@ -205,18 +202,12 @@ mono_options_parse_options (const char **argv, int argc, int *out_argc, GPtrArra break; } value = argv [aindex + 1]; - if (processed) { - g_ptr_array_add (processed, (gpointer)argv [aindex]); - g_ptr_array_add (processed, (gpointer)argv [aindex + 1]); - } argv [aindex] = NULL; argv [aindex + 1] = NULL; aindex ++; } else if (equals_sign_index != -1) { // option=value value = arg + equals_sign_index + 1; - if (processed) - g_ptr_array_add (processed, (gpointer)argv [aindex]); argv [aindex] = NULL; } else { g_assert_not_reached (); diff --git a/src/mono/mono/utils/options.h b/src/mono/mono/utils/options.h index e7f2906eeb052c..41090e3897ca37 100644 --- a/src/mono/mono/utils/options.h +++ b/src/mono/mono/utils/options.h @@ -26,7 +26,7 @@ extern int mono_options_version; void mono_options_print_usage (void); -void mono_options_parse_options (const char **args, int argc, int *out_argc, GPtrArray *processed, MonoError *error); +void mono_options_parse_options (const char **args, int argc, int *out_argc, MonoError *error); /* returns a json blob representing the current values of all options */ char * mono_options_get_as_json (void); diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/README.md b/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/README.md index a611a20f7cd054..c8a46cb59461e0 100644 --- a/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/README.md +++ b/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/README.md @@ -5,34 +5,11 @@ Tasks, and targets to support workload testing in `dotnet` repositories. - `$(InstallWorkloadForTesting)` - required - `$(BuiltNuGetsDir)` - required - `$(DotNetInstallArgumentsForWorkloadsTesting)` - required -- `$(TemplateNuGetConfigPathForWorkloadTesting)` - required - - `$(TestUsingWorkloads)` - optional - `$(SkipTempDirectoryCleanup)` - optional - `$(VersionBandForManifestPackages)` - optional - `$(ExtraWorkloadInstallCommandArguments)` - optional -- `$(WorkloadInstallCommandOutputImportance)` - optional, defaults to `Normal` - -## `$(PackageSourceNameForBuiltPackages)` - optional - -`` - -Defaults to `nuget-local`. - -## `$(NuGetConfigPackageSourceMappingsForWorkloadTesting)` - optional - -For a value of `*Aspire*;Foo*`, a package source mapping will be added to the local nuget source -added for built nugets: - -```xml - - - - - - ... -``` -# items +## items - `@(DefaultPropertiesForNuGetBuild)` diff --git a/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/Sdk/WorkloadTesting.Core.targets b/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/Sdk/WorkloadTesting.Core.targets index d853cb881eb84d..dca02e0543769d 100755 --- a/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/Sdk/WorkloadTesting.Core.targets +++ b/src/mono/nuget/Microsoft.NET.Runtime.WorkloadTesting.Internal/Sdk/WorkloadTesting.Core.targets @@ -12,9 +12,6 @@ $(SdkWithWorkloadForTestingPath)version-$(SdkVersionForWorkloadTesting).stamp $(SdkWithWorkloadForTestingPath)workload.stamp - - $(RepoRoot)NuGet.config - ArchiveTests @@ -28,12 +25,12 @@ true $(InstallWorkloadUsingArtifactsDependsOn); - GetNuGetsToBuildForWorkloadTesting; - _PreparePackagesForWorkloadInstall; ProvisionDotNetForWorkloadTesting; _GetDotNetVersion; _FirstDotNetRun; _SetPackageVersionForWorkloadsTesting; + GetNuGetsToBuildForWorkloadTesting; + _PreparePackagesForWorkloadInstall; GetWorkloadInputs; _InstallWorkloads @@ -162,12 +159,8 @@ TaskName="Microsoft.Workload.Build.Tasks.InstallWorkloadFromArtifacts" AssemblyFile="$(WorkloadBuildTasksAssemblyPath)" /> - - @@ -205,11 +198,8 @@ VersionBandForManifestPackages="$(VersionBandForManifestPackages)" LocalNuGetsPath="$(BuiltNugetsDir)" ExtraWorkloadInstallCommandArguments="$(ExtraWorkloadInstallCommandArguments)" - PackageSourceNameForBuiltPackages="$(PackageSourceNameForBuiltPackages)" - TemplateNuGetConfigPath="$(TemplateNuGetConfigPathForWorkloadTesting)" - NuGetConfigPackageSourceMappings="$(NuGetConfigPackageSourceMappingsForWorkloadTesting)" + TemplateNuGetConfigPath="$(RepoRoot)NuGet.config" SdkWithNoWorkloadInstalledPath="$(_SdkWithNoWorkloadPath)" - WorkloadInstallCommandOutputImportance="$(WorkloadInstallCommandOutputImportance)" IntermediateOutputPath="$(ArtifactsObjDir)" SkipTempDirectoryCleanup="$(SkipTempDirectoryCleanup)" /> diff --git a/src/mono/nuget/Microsoft.NET.Sdk.WebAssembly.Pack/build/Microsoft.NET.Sdk.WebAssembly.Browser.targets b/src/mono/nuget/Microsoft.NET.Sdk.WebAssembly.Pack/build/Microsoft.NET.Sdk.WebAssembly.Browser.targets index 4f7513b8f26a22..f54c84ae9ad829 100644 --- a/src/mono/nuget/Microsoft.NET.Sdk.WebAssembly.Pack/build/Microsoft.NET.Sdk.WebAssembly.Browser.targets +++ b/src/mono/nuget/Microsoft.NET.Sdk.WebAssembly.Pack/build/Microsoft.NET.Sdk.WebAssembly.Browser.targets @@ -95,6 +95,10 @@ Copyright (c) .NET Foundation. All rights reserved. $(ResolveStaticWebAssetsInputsDependsOn); _AddWasmStaticWebAssets; + + _GenerateBuildWasmBootJson; + $(StaticWebAssetsPrepareForRunDependsOn) + $(ResolvePublishStaticWebAssetsDependsOn); ProcessPublishFilesForWasm; @@ -110,16 +114,18 @@ Copyright (c) .NET Foundation. All rights reserved. $(ResolveCompressedFilesDependsOn); ResolveWasmOutputs; - _GenerateBuildWasmBootJson; _AddWasmStaticWebAssets; $(ResolveCompressedFilesForPublishDependsOn); ProcessPublishFilesForWasm; ComputeWasmExtensions; - GeneratePublishWasmBootJson; _AddPublishWasmBootJsonToStaticWebAssets; + + $(CompressFilesDependsOn) + _GenerateBuildWasmBootJson; + $(CompressFilesForPublishDependsOn); GeneratePublishWasmBootJson; @@ -132,8 +138,11 @@ Copyright (c) .NET Foundation. All rights reserved. $(GenerateBuildWasmBootJsonDependsOn); - ResolveWasmOutputs; + ResolveStaticWebAssetsInputs; + + $(GeneratePublishWasmBootJsonDependsOn); + @@ -211,15 +220,12 @@ Copyright (c) .NET Foundation. All rights reserved. - - <_WasmNativeAssetFileNames>;@(WasmNativeAsset->'%(FileName)%(Extension)'); - - <_WasmConfigFileCandidates Include="@(StaticWebAsset)" Condition="'%(SourceType)' == 'Discovered'" /> - + @@ -290,6 +296,31 @@ Copyright (c) .NET Foundation. All rights reserved. + + <_WasmBuildBootJsonPath>$(IntermediateOutputPath)blazor.boot.json + + + + <_BuildWasmBootJson + Include="$(_WasmBuildBootJsonPath)" + RelativePath="_framework/blazor.boot.json" /> + + + + + @@ -352,33 +383,6 @@ Copyright (c) .NET Foundation. All rights reserved. - - - <_WasmBuildBootJsonPath>$(IntermediateOutputPath)blazor.boot.json - - - - <_BuildWasmBootJson - Include="$(_WasmBuildBootJsonPath)" - RelativePath="_framework/blazor.boot.json" /> - - - - - - @@ -446,23 +450,10 @@ Copyright (c) .NET Foundation. All rights reserved. - + - - - <_NewWebCilPublishStaticWebAssetsCandidatesNoMetadata - Include="@(_NewWebCilPublishStaticWebAssetsCandidates)" - RemoveMetadata="Integrity;Fingerprint" /> - - - - - - diff --git a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in index 86b8c527efa08c..8d15ecdc96dd0e 100644 --- a/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in +++ b/src/mono/nuget/Microsoft.NET.Workload.Mono.Toolchain.Current.Manifest/WorkloadManifest.targets.in @@ -54,6 +54,7 @@ <_WasmNativeWorkloadNeeded Condition=" + '$(WasmEnableThreads)' == 'true' or '$(_WasmPropertiesDifferFromRuntimePackThusNativeBuildNeeded)' == 'true' or '$(RunAOTCompilation)' == 'true' or '$(WasmBuildNative)' == 'true' or diff --git a/src/mono/sample/wasm/blazor-frame/blazor.csproj b/src/mono/sample/wasm/blazor-frame/blazor.csproj index 3061f77e83ac3a..5592718e07144e 100644 --- a/src/mono/sample/wasm/blazor-frame/blazor.csproj +++ b/src/mono/sample/wasm/blazor-frame/blazor.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj b/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj index 752927021914cb..bb523ade9a2a4c 100644 --- a/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj +++ b/src/mono/sample/wasm/browser-bench/Wasm.Browser.Bench.Sample.csproj @@ -84,36 +84,25 @@ DestinationFolder="$(MSBuildThisFileDirectory)/bin/$(Configuration)/AppBundle/blazor-template/%(RecursiveDir)" /> - + - - - - - - + + - - - - - - - - + - + - - -+ - - - diff --git a/src/mono/sample/wasm/browser-frame/runtimeconfig.template.json b/src/mono/sample/wasm/browser-frame/runtimeconfig.template.json new file mode 100644 index 00000000000000..b96a94320ba5ee --- /dev/null +++ b/src/mono/sample/wasm/browser-frame/runtimeconfig.template.json @@ -0,0 +1,10 @@ +{ + "wasmHostProperties": { + "perHostConfig": [ + { + "name": "browser", + "host": "browser" + } + ] + } +} \ No newline at end of file diff --git a/src/mono/sample/wasm/browser-frame/frame.js b/src/mono/sample/wasm/browser-frame/wwwroot/frame.js similarity index 100% rename from src/mono/sample/wasm/browser-frame/frame.js rename to src/mono/sample/wasm/browser-frame/wwwroot/frame.js diff --git a/src/mono/sample/wasm/browser-frame/wwwroot/index.html b/src/mono/sample/wasm/browser-frame/wwwroot/index.html new file mode 100644 index 00000000000000..8b8df7572c2b5e --- /dev/null +++ b/src/mono/sample/wasm/browser-frame/wwwroot/index.html @@ -0,0 +1,18 @@ + + + + + + + browser-frame + + + + + + + + + + + diff --git a/src/mono/sample/wasm/browser-frame/wwwroot/main.js b/src/mono/sample/wasm/browser-frame/wwwroot/main.js new file mode 100644 index 00000000000000..a073fc9cf70327 --- /dev/null +++ b/src/mono/sample/wasm/browser-frame/wwwroot/main.js @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { dotnet } from './_framework/dotnet.js' + +const { setModuleImports, getAssemblyExports, getConfig } = await dotnet + .withDiagnosticTracing(false) + .withApplicationArgumentsFromQuery() + .create(); + +setModuleImports('main.js', { + window: { + location: { + href: () => globalThis.window.location.href + } + } +}); + +const config = getConfig(); +const exports = await getAssemblyExports(config.mainAssemblyName); +const text = exports.MyClass.Greeting(); +console.log(text); + +document.getElementById('out').innerHTML = text; +await dotnet.run(); \ No newline at end of file diff --git a/src/mono/sample/wasm/browser-frame/wwwroot/start.html b/src/mono/sample/wasm/browser-frame/wwwroot/start.html new file mode 100644 index 00000000000000..9dad19ec0a1cd1 --- /dev/null +++ b/src/mono/sample/wasm/browser-frame/wwwroot/start.html @@ -0,0 +1,18 @@ + + + + + + + browser-frame + + + + + + + + + + + diff --git a/src/mono/sample/wasm/browser-frame/wwwroot/start.js b/src/mono/sample/wasm/browser-frame/wwwroot/start.js new file mode 100644 index 00000000000000..a073fc9cf70327 --- /dev/null +++ b/src/mono/sample/wasm/browser-frame/wwwroot/start.js @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +import { dotnet } from './_framework/dotnet.js' + +const { setModuleImports, getAssemblyExports, getConfig } = await dotnet + .withDiagnosticTracing(false) + .withApplicationArgumentsFromQuery() + .create(); + +setModuleImports('main.js', { + window: { + location: { + href: () => globalThis.window.location.href + } + } +}); + +const config = getConfig(); +const exports = await getAssemblyExports(config.mainAssemblyName); +const text = exports.MyClass.Greeting(); +console.log(text); + +document.getElementById('out').innerHTML = text; +await dotnet.run(); \ No newline at end of file diff --git a/src/mono/sample/wasm/browser-shutdown/main.js b/src/mono/sample/wasm/browser-shutdown/main.js index e3141c08976c1d..d727f7932a10d0 100644 --- a/src/mono/sample/wasm/browser-shutdown/main.js +++ b/src/mono/sample/wasm/browser-shutdown/main.js @@ -14,6 +14,7 @@ try { .withExitOnUnhandledError() .withExitCodeLogging() .withElementOnExit() + .withAssertAfterExit() .withOnConfigLoaded(() => { // you can test abort of the startup by opening http://localhost:8000/?throwError=true const params = new URLSearchParams(location.search); diff --git a/src/mono/sample/wasm/browser-threads/Program.cs b/src/mono/sample/wasm/browser-threads/Program.cs index 907b3d3356b3c4..148a002dfe1edd 100644 --- a/src/mono/sample/wasm/browser-threads/Program.cs +++ b/src/mono/sample/wasm/browser-threads/Program.cs @@ -8,72 +8,153 @@ using System.Threading.Tasks; using System.Collections.Generic; -namespace Sample; - -public partial class Test +namespace Sample { - private static int _animationCounter = 0; - private static int _callCounter = 0; - private static bool _isRunning = false; - private static readonly IReadOnlyList _animations = new string[] { "\u2680", "\u2681", "\u2682", "\u2683", "\u2684", "\u2685" }; - - public static int Main(string[] args) + public partial class Test { - Console.WriteLine("Hello, World!"); - return 0; - } + public static int Main(string[] args) + { + Console.WriteLine("Hello, World!"); + return 0; + } - [JSImport("globalThis.console.log")] - public static partial void ConsoleLog(string status); + [JSImport("globalThis.console.log")] + public static partial void ConsoleLog(string status); - [JSImport("Sample.Test.updateProgress", "main.js")] - private static partial Task updateProgress(string status); + [JSImport("Sample.Test.updateProgress", "main.js")] + static partial void updateProgress(string status); - [JSExport] - public static bool Progress() - { - updateProgress(""+_animations[_animationCounter++]); - if (_animationCounter >= _animations.Count) + internal static void UpdateProgress(string status) => updateProgress(status); + + static Demo _demo = null; + + [JSExport] + public static void Start(int n) { - _animationCounter = 0; + var comp = new ExpensiveComputation(n); + comp.Start(); + #pragma warning disable CS4014 + WaitForCompletion(comp); + _demo = new Demo(UpdateProgress, comp); + } + + public static async Task WaitForCompletion (ExpensiveComputation comp) { + Console.WriteLine($"WaitForCompletion started on thread {Thread.CurrentThread.ManagedThreadId}"); + await comp.Completion; + Console.WriteLine($"WaitForCompletion completed on thread {Thread.CurrentThread.ManagedThreadId}"); + UpdateProgress("\u270C\uFE0E"); } - return _isRunning; + + [JSExport] + public static int Progress() + { + if (_demo.Progress()) + return 0; /* done */ + else + return 1; /* continue */ + } + + [JSExport] + public static int GetAnswer() { return _demo.Result; } } - [JSExport] - [return: JSMarshalAs>] - public static Task Fib(int n) +} + +public class ExpensiveComputation +{ + private readonly TaskCompletionSource _tcs = new(); + private readonly int UpTo; + public ExpensiveComputation(int n) { UpTo = n; } + public long CallCounter { get; private set; } + public Task Completion => _tcs.Task; + + public void Start() { - return Task.Run(()=>{ - _isRunning = true; - var res = FibImpl(n); - _isRunning = false; - return Task.FromResult(res); - }); + new Thread((o) => ((ExpensiveComputation)o).Run()).Start(this); } - private static long FibImpl(int n) + public void Run() { - _callCounter++; + Console.WriteLine("Hello from ManagedThreadId " + Thread.CurrentThread.ManagedThreadId); + long result = Fib(UpTo); + if (result < (long)int.MaxValue) + _tcs.SetResult((int)result); + else + _tcs.SetException(new Exception("Fibonacci computation exceeded Int32.MaxValue")); + } + public long Fib(int n) + { + CallCounter++; // make some garbage every 1000 calls - if (_callCounter % 1000 == 0) + if (CallCounter % 1000 == 0) { AllocateGarbage(); } // and collect it every once in a while - if (_callCounter % 500000 == 0) + if (CallCounter % 500000 == 0) GC.Collect(); if (n < 2) return n; - return FibImpl(n - 1) + FibImpl(n - 2); + return Fib(n - 1) + Fib(n - 2); } [MethodImpl(MethodImplOptions.NoInlining)] - private static void AllocateGarbage() + private void AllocateGarbage() { object[] garbage = new object[200]; garbage[12] = new object(); garbage[197] = garbage; } + +} + +public class Demo +{ + public class Animation + { + private readonly Action _updateProgress; + private int _counter = 0; + + private readonly IReadOnlyList _animations = new string[] { "\u2680", "\u2681", "\u2682", "\u2683", "\u2684", "\u2685" }; + + public void Step(string suffix = "") + { + _updateProgress(_animations[_counter++] + suffix); + if (_counter >= _animations.Count) + { + _counter = 0; + } + } + + public Animation(Action updateProgress) + { + _updateProgress = updateProgress; + } + + + } + + private readonly Action _updateProgress; + private readonly Animation _animation; + private readonly ExpensiveComputation _expensiveComputation; + + public Demo(Action updateProgress, ExpensiveComputation comp) + { + _updateProgress = updateProgress; + _animation = new Animation(updateProgress); + _expensiveComputation = comp; + } + + public bool Progress() + { + if (!_expensiveComputation.Completion.IsCompleted) + { + _animation.Step($"{_expensiveComputation.CallCounter} calls"); + } + + return _expensiveComputation.Completion.IsCompleted; + } + + public int Result => _expensiveComputation.Completion.Result; } diff --git a/src/mono/sample/wasm/browser-threads/main.js b/src/mono/sample/wasm/browser-threads/main.js index 127e2402f78542..d53e59304e10d8 100644 --- a/src/mono/sample/wasm/browser-threads/main.js +++ b/src/mono/sample/wasm/browser-threads/main.js @@ -4,14 +4,54 @@ import { dotnet, exit } from './_framework/dotnet.js' let progressElement = null; -let inputElement = null; -let exports = null; + +function updateProgress(status) { + if (progressElement) { + progressElement.innerText = status; + } else { + console.log("Progress: " + status); + } +} + const assemblyName = "Wasm.Browser.Threads.Sample.dll"; -try { +function delay(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function Run(exports, N) { + while (true) { + await delay(50); + const p = exports.Sample.Test.Progress(); + if (p === 0) + break; + } + const answer = exports.Sample.Test.GetAnswer(); + document.getElementById("out").innerText = `Fib(${N}) = ${answer}`; +} + +async function doMathSlowly(exports) { progressElement = document.getElementById("progressElement"); - inputElement = document.getElementById("inputN"); + const N = parseInt(document.getElementById("inputN").value); + exports.Sample.Test.Start(N); + await Run(exports, N); +} + +function setEditable(inputElement, isEditable) { + inputElement.disabled = !isEditable; +} + +function onInputValueChanged(exports, inputElement) { + async function handler() { + setEditable(inputElement, false); + await doMathSlowly(exports); + setEditable(inputElement, true); + } + return handler; +} +try { + const inputElement = document.getElementById("inputN"); const { setModuleImports, getAssemblyExports, runMain } = await dotnet .withEnvironmentVariable("MONO_LOG_LEVEL", "debug") .withElementOnExit() @@ -27,52 +67,14 @@ try { } }); - exports = await getAssemblyExports(assemblyName); + const exports = await getAssemblyExports(assemblyName); - await doSlowMath(); - setEditable(true); - inputElement.addEventListener("change", onInputValueChanged); + await doMathSlowly(exports); + setEditable(inputElement, true); + inputElement.addEventListener("change", onInputValueChanged(exports, inputElement)); let exit_code = await runMain(assemblyName, []); - // comment out the following line for interactive testing, otherwise further call would be rejected by runtime exit(exit_code); } catch (err) { exit(2, err); } - -async function doSlowMath() { - const N = parseInt(document.getElementById("inputN").value); - const resultPromise = exports.Sample.Test.Fib(N); - - while (true) { - await delay(50); - const isRunning = exports.Sample.Test.Progress(); - if (!isRunning) - break; - } - const answer = await resultPromise; - document.getElementById("out").innerText = `Fib(${N}) = ${answer}`; - -} - -export async function updateProgress(status) { - if (progressElement) { - progressElement.innerText = status; - } else { - console.log("Progress: " + status); - } -} - -function delay(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -function setEditable(isEditable) { - inputElement.disabled = !isEditable; -} - -async function onInputValueChanged() { - setEditable(false); - await doSlowMath(exports); - setEditable(true); -} diff --git a/src/mono/sample/wasm/simple-raytracer/Program.cs b/src/mono/sample/wasm/simple-raytracer/Program.cs index 5ee605e8ef6835..f356394f0de1de 100644 --- a/src/mono/sample/wasm/simple-raytracer/Program.cs +++ b/src/mono/sample/wasm/simple-raytracer/Program.cs @@ -195,7 +195,7 @@ private static void renderPixel (int i, int j, ref Vec3f light, Intersector inte if (didHitZ && (hitZ > sphere.Center.z)) continue; - if (intersector.Intersect(ref pos, ref dir, &sphere, ref intersection_normal)) { + if (intersector.Intersect(ref pos, ref dir, Unsafe.AsPointer(ref sphere), ref intersection_normal)) { sampleEnv(ref intersection_normal, ref color); const float ambientScale = 0.2f; diff --git a/src/mono/wasi/build/WasiApp.targets b/src/mono/wasi/build/WasiApp.targets index b229d76dd71ff9..b31c573ea349b5 100644 --- a/src/mono/wasi/build/WasiApp.targets +++ b/src/mono/wasi/build/WasiApp.targets @@ -39,6 +39,7 @@ <_BoolPropertiesThatTriggerRelinking Include="WasmEnableSIMD" DefaultValueInRuntimePack="false" /> + <_BoolPropertiesThatTriggerRelinking Include="WasmNativeStrip" DefaultValueInRuntimePack="true" /> diff --git a/src/mono/wasm/Wasm.Build.Tests/Blazor/AppsettingsTests.cs b/src/mono/wasm/Wasm.Build.Tests/Blazor/AppsettingsTests.cs index db0607d226a8c3..8069ef424f1895 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Blazor/AppsettingsTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Blazor/AppsettingsTests.cs @@ -46,7 +46,7 @@ var builder await BlazorRunForBuildWithDotnetRun(new BlazorRunOptions() { Config = "debug", - OnConsoleMessage = (_, msg) => + OnConsoleMessage = msg => { if (msg.Text.Contains("appSettings Exists 'True'")) existsChecked = true; diff --git a/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs b/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs index 1b8145c69ae7d0..683524eac32235 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorRunOptions.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Playwright; @@ -14,15 +13,11 @@ public record BlazorRunOptions BlazorRunHost Host = BlazorRunHost.DotnetRun, bool DetectRuntimeFailures = true, bool CheckCounter = true, - Dictionary? ServerEnvironment = null, Func? Test = null, - Action? OnPageLoaded = null, - Action? OnConsoleMessage = null, - Action? OnServerMessage = null, + Action? OnConsoleMessage = null, Action? OnErrorMessage = null, string Config = "Debug", string? ExtraArgs = null, - string BrowserPath = "", string QueryString = "" ); diff --git a/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorWasmTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorWasmTestBase.cs index d66fa1d04a0fbd..2e3491374c7d91 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorWasmTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Blazor/BlazorWasmTestBase.cs @@ -191,25 +191,14 @@ public async Task BlazorRunTest(string runArgs, { if (!string.IsNullOrEmpty(runOptions.ExtraArgs)) runArgs += $" {runOptions.ExtraArgs}"; - - runOptions.ServerEnvironment?.ToList().ForEach( - kv => s_buildEnv.EnvVars[kv.Key] = kv.Value); - using var runCommand = new RunCommand(s_buildEnv, _testOutput) .WithWorkingDirectory(workingDirectory); await using var runner = new BrowserRunner(_testOutput); - var page = await runner.RunAsync( - runCommand, - runArgs, - onPageLoaded: runOptions.OnPageLoaded, - onConsoleMessage: OnConsoleMessage, - onServerMessage: runOptions.OnServerMessage, - onError: OnErrorMessage, - modifyBrowserUrl: browserUrl => browserUrl + runOptions.BrowserPath + runOptions.QueryString); + var page = await runner.RunAsync(runCommand, runArgs, onConsoleMessage: OnConsoleMessage, onError: OnErrorMessage, modifyBrowserUrl: browserUrl => browserUrl + runOptions.QueryString); _testOutput.WriteLine("Waiting for page to load"); - await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded, new () { Timeout = 1 * 60 * 1000 }); + await page.WaitForLoadStateAsync(LoadState.DOMContentLoaded); if (runOptions.CheckCounter) { @@ -218,7 +207,6 @@ public async Task BlazorRunTest(string runArgs, Assert.Equal("Current count: 0", txt); await page.Locator("text=\"Click me\"").ClickAsync(); - await Task.Delay(300); txt = await page.Locator("p[role='status']").InnerHTMLAsync(); Assert.Equal("Current count: 1", txt); } @@ -229,11 +217,11 @@ public async Task BlazorRunTest(string runArgs, _testOutput.WriteLine($"Waiting for additional 10secs to see if any errors are reported"); await Task.Delay(10_000); - void OnConsoleMessage(IPage page, IConsoleMessage msg) + void OnConsoleMessage(IConsoleMessage msg) { _testOutput.WriteLine($"[{msg.Type}] {msg.Text}"); - runOptions.OnConsoleMessage?.Invoke(page, msg); + runOptions.OnConsoleMessage?.Invoke(msg); if (runOptions.DetectRuntimeFailures) { @@ -249,12 +237,6 @@ void OnErrorMessage(string msg) } } - public string FindBlazorBinFrameworkDir(string config, bool forPublish, string framework = DefaultTargetFrameworkForBlazor, string? projectDir = null) - => _provider.FindBinFrameworkDir(config: config, forPublish: forPublish, framework: framework, projectDir: projectDir); - - public string FindBlazorHostedBinFrameworkDir(string config, bool forPublish, string clientDirRelativeToProjectDir, string framework = DefaultTargetFrameworkForBlazor) - { - string? clientProjectDir = _projectDir == null ? null : Path.Combine(_projectDir, clientDirRelativeToProjectDir); - return _provider.FindBinFrameworkDir(config: config, forPublish: forPublish, framework: framework, projectDir: clientProjectDir); - } + public string FindBlazorBinFrameworkDir(string config, bool forPublish, string framework = DefaultTargetFrameworkForBlazor) + => _provider.FindBinFrameworkDir(config: config, forPublish: forPublish, framework: framework); } diff --git a/src/mono/wasm/Wasm.Build.Tests/Blazor/SimpleMultiThreadedTests.cs b/src/mono/wasm/Wasm.Build.Tests/Blazor/SimpleMultiThreadedTests.cs index c92fc5f35bf980..5f1e1711151c09 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Blazor/SimpleMultiThreadedTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Blazor/SimpleMultiThreadedTests.cs @@ -38,7 +38,7 @@ public SimpleMultiThreadedTests(ITestOutputHelper output, SharedBuildPerTestClas // } [ConditionalTheory(typeof(BuildTestBase), nameof(IsWorkloadWithMultiThreadingForDefaultFramework))] - // [InlineData("Debug", false)] // ActiveIssue https://github.com/dotnet/runtime/issues/98758 + [InlineData("Debug", false)] // [InlineData("Debug", true)] [InlineData("Release", false)] // [InlineData("Release", true)] @@ -73,7 +73,7 @@ await BlazorRunForPublishWithWebServer( runOptions: new BlazorRunOptions( Config: config, ExtraArgs: "--web-server-use-cors --web-server-use-cop", - OnConsoleMessage: (_, message) => + OnConsoleMessage: (message) => { if (message.Text.Contains("WasmEnableThreads=true")) hasEmittedWasmEnableThreads = true; diff --git a/src/mono/wasm/Wasm.Build.Tests/Blazor/WorkloadRequiredTests.cs b/src/mono/wasm/Wasm.Build.Tests/Blazor/WorkloadRequiredTests.cs index 0fd604cfea6e24..deb8ad5def8e84 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Blazor/WorkloadRequiredTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Blazor/WorkloadRequiredTests.cs @@ -120,7 +120,7 @@ await BlazorRunTest(new BlazorRunOptions() { Config = config, Host = publish ? BlazorRunHost.WebServer : BlazorRunHost.DotnetRun, - OnConsoleMessage = (_, msg) => + OnConsoleMessage = msg => { sbOutput.AppendLine(msg.Text); } diff --git a/src/mono/wasm/Wasm.Build.Tests/BrowserRunner.cs b/src/mono/wasm/Wasm.Build.Tests/BrowserRunner.cs index 291c2d472ea068..4aad869d9c1760 100644 --- a/src/mono/wasm/Wasm.Build.Tests/BrowserRunner.cs +++ b/src/mono/wasm/Wasm.Build.Tests/BrowserRunner.cs @@ -36,8 +36,7 @@ internal class BrowserRunner : IAsyncDisposable public async Task StartServerAndGetUrlAsync( ToolCommand cmd, - string args, - Action? onServerMessage = null + string args ) { TaskCompletionSource urlAvailable = new(); Action outputHandler = msg => @@ -45,12 +44,8 @@ public async Task StartServerAndGetUrlAsync( if (string.IsNullOrEmpty(msg)) return; - onServerMessage?.Invoke(msg); - lock (OutputLines) - { OutputLines.Add(msg); - } Match m = s_appHostUrlRegex.Match(msg); if (!m.Success) @@ -75,17 +70,7 @@ public async Task StartServerAndGetUrlAsync( cmd.WithErrorDataReceived(outputHandler).WithOutputDataReceived(outputHandler); var runTask = cmd.ExecuteAsync(args); - var delayTask = Task.Delay(TimeSpan.FromSeconds(30)); - - await Task.WhenAny(runTask, urlAvailable.Task, delayTask); - if (delayTask.IsCompleted) - { - _testOutput.WriteLine("First 30s delay reached, scheduling next one"); - - delayTask = Task.Delay(TimeSpan.FromSeconds(30)); - await Task.WhenAny(runTask, urlAvailable.Task, delayTask); - } - + await Task.WhenAny(runTask, urlAvailable.Task, Task.Delay(TimeSpan.FromSeconds(30))); if (runTask.IsCompleted) { var res = await runTask; @@ -106,8 +91,7 @@ public async Task SpawnBrowserAsync( ) { var url = new Uri(browserUrl); Playwright = await Microsoft.Playwright.Playwright.CreateAsync(); - // codespaces: ignore certificate error -> Microsoft.Playwright.PlaywrightException : net::ERR_CERT_AUTHORITY_INVALID - string[] chromeArgs = new[] { $"--explicitly-allowed-ports={url.Port}", "--ignore-certificate-errors" }; + string[] chromeArgs = new[] { $"--explicitly-allowed-ports={url.Port}" }; _testOutput.WriteLine($"Launching chrome ('{s_chromePath.Value}') via playwright with args = {string.Join(',', chromeArgs)}"); return Browser = await Playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions{ ExecutablePath = s_chromePath.Value, @@ -121,24 +105,21 @@ public async Task RunAsync( ToolCommand cmd, string args, bool headless = true, - Action? onConsoleMessage = null, - Action? onPageLoaded = null, - Action? onServerMessage = null, + Action? onConsoleMessage = null, Action? onError = null, Func? modifyBrowserUrl = null) { - var urlString = await StartServerAndGetUrlAsync(cmd, args, onServerMessage); + var urlString = await StartServerAndGetUrlAsync(cmd, args); var browser = await SpawnBrowserAsync(urlString, headless); var context = await browser.NewContextAsync(); - return await RunAsync(context, urlString, headless, onPageLoaded, onConsoleMessage, onError, modifyBrowserUrl); + return await RunAsync(context, urlString, headless, onConsoleMessage, onError, modifyBrowserUrl); } public async Task RunAsync( IBrowserContext context, string browserUrl, bool headless = true, - Action? onPageLoaded = null, - Action? onConsoleMessage = null, + Action? onConsoleMessage = null, Action? onError = null, Func? modifyBrowserUrl = null, bool resetExitedState = false @@ -150,11 +131,8 @@ public async Task RunAsync( browserUrl = modifyBrowserUrl(browserUrl); IPage page = await context.NewPageAsync(); - if (onPageLoaded is not null) - page.Load += (_, _) => onPageLoaded(page); - if (onConsoleMessage is not null) - page.Console += (_, msg) => onConsoleMessage(page, msg); + page.Console += (_, msg) => onConsoleMessage(msg); onError ??= _testOutput.WriteLine; if (onError is not null) diff --git a/src/mono/wasm/Wasm.Build.Tests/Common/EnvironmentVariables.cs b/src/mono/wasm/Wasm.Build.Tests/Common/EnvironmentVariables.cs index 50bf882d072d31..841aa6a9b3456d 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Common/EnvironmentVariables.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Common/EnvironmentVariables.cs @@ -18,10 +18,10 @@ internal static class EnvironmentVariables internal static readonly string? BuiltNuGetsPath = Environment.GetEnvironmentVariable("BUILT_NUGETS_PATH"); internal static readonly string? BrowserPathForTests = Environment.GetEnvironmentVariable("BROWSER_PATH_FOR_TESTS"); internal static readonly string? V8PathForTests = Environment.GetEnvironmentVariable("V8_PATH_FOR_TESTS"); - internal static readonly bool IsRunningOnCI = Environment.GetEnvironmentVariable("IS_RUNNING_ON_CI") is "true"; - internal static readonly bool ShowBuildOutput = IsRunningOnCI || Environment.GetEnvironmentVariable("SHOW_BUILD_OUTPUT") is not null; + internal static readonly bool ShowBuildOutput = Environment.GetEnvironmentVariable("SHOW_BUILD_OUTPUT") is not null; internal static readonly bool UseWebcil = Environment.GetEnvironmentVariable("USE_WEBCIL_FOR_TESTS") is "true"; internal static readonly string? SdkDirName = Environment.GetEnvironmentVariable("SDK_DIR_NAME"); internal static readonly string? WasiSdkPath = Environment.GetEnvironmentVariable("WASI_SDK_PATH"); + internal static readonly bool IsRunningOnCI = Environment.GetEnvironmentVariable("IS_RUNNING_ON_CI") is "true"; } } diff --git a/src/mono/wasm/Wasm.Build.Tests/Common/TestOutputWrapper.cs b/src/mono/wasm/Wasm.Build.Tests/Common/TestOutputWrapper.cs index 03bef6c6ccb33c..a28657fa7bf0e5 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Common/TestOutputWrapper.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Common/TestOutputWrapper.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Text; using Xunit.Abstractions; #nullable enable @@ -11,12 +10,9 @@ namespace Wasm.Build.Tests; public class TestOutputWrapper(ITestOutputHelper baseOutput) : ITestOutputHelper { - private readonly StringBuilder _outputBuffer = new StringBuilder(); - public void WriteLine(string message) { baseOutput.WriteLine(message); - _outputBuffer.AppendLine(message); if (EnvironmentVariables.ShowBuildOutput) Console.WriteLine(message); } @@ -24,10 +20,7 @@ public void WriteLine(string message) public void WriteLine(string format, params object[] args) { baseOutput.WriteLine(format, args); - _outputBuffer.AppendFormat(format, args).AppendLine(); if (EnvironmentVariables.ShowBuildOutput) Console.WriteLine(format, args); } - - public override string ToString() => _outputBuffer.ToString(); } diff --git a/src/mono/wasm/Wasm.Build.Tests/Common/TestUtils.cs b/src/mono/wasm/Wasm.Build.Tests/Common/TestUtils.cs index 97aa019d454302..d9acfdaa37656b 100644 --- a/src/mono/wasm/Wasm.Build.Tests/Common/TestUtils.cs +++ b/src/mono/wasm/Wasm.Build.Tests/Common/TestUtils.cs @@ -52,6 +52,7 @@ public static void AssertFile(string file0, string file1, string? label = null, if (!same && finfo0.Length == finfo1.Length) throw new XunitException($"{label}:{Environment.NewLine} File sizes should not match for {file0} ({finfo0.Length}), and {file1} ({finfo1.Length})"); } + public static string FindSubDirIgnoringCase(string parentDir, string dirName) { IEnumerable matchingDirs = Directory.EnumerateDirectories(parentDir, diff --git a/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs b/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs index f7c7483a949d3f..ac8b567761f36a 100644 --- a/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/NativeBuildTests.cs @@ -93,24 +93,5 @@ public void IntermediateBitcodeToObjectFilesAreNotLLVMIR(BuildArgs buildArgs, st + " It might fail if it was incorrectly compiled to a bitcode file, instead of wasm."); } - [Theory] - [BuildAndRun(host: RunHost.None, aot: true)] - public void NativeBuildIsRequired(BuildArgs buildArgs, string id) - { - string projectName = $"native_build_{buildArgs.Config}_{buildArgs.AOT}"; - - buildArgs = buildArgs with { ProjectName = projectName, ExtraBuildArgs = "-p:WasmBuildNative=false -p:WasmSingleFileBundle=true" }; - buildArgs = ExpandBuildArgs(buildArgs); - - (_, string output) = BuildProject( - buildArgs, - id: id, - new BuildProjectOptions( - InitProject: () => File.WriteAllText(Path.Combine(_projectDir!, "Program.cs"), s_mainReturns42), - DotnetWasmFromRuntimePack: false, - ExpectSuccess: false)); - - Assert.Contains("WasmBuildNative is required", output); - } } } diff --git a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs index 8c0442a1b0d680..1d7a9740eba9e3 100644 --- a/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/PInvokeTableGeneratorTests.cs @@ -739,16 +739,6 @@ public struct Nested1 { public struct SingleI64Struct { public Int64 Value; } - public struct PairStruct { - public int A, B; - } - public unsafe struct MyFixedArray { - public fixed int elements[2]; - } - [System.Runtime.CompilerServices.InlineArray(2)] - public struct MyInlineArray { - public int element0; - } public class Test { @@ -775,35 +765,9 @@ public static unsafe int Main(string[] argv) var res = indirect(sds); Console.WriteLine(""s (s)="" + res.Value); - var pair = new PairStruct { A = 1, B = 2 }; - var paires = accept_and_return_pair(pair); - Console.WriteLine(""paires.B="" + paires.B); - - // This test is split into methods to simplify debugging issues with it - var ia = InlineArrayTest1(); - var iares = InlineArrayTest2(ia); - Console.WriteLine($""iares[0]={iares[0]} iares[1]={iares[1]}""); - - MyFixedArray fa = new (); - for (int i = 0; i < 2; i++) - fa.elements[i] = i; - var fares = accept_and_return_fixedarray(fa); - Console.WriteLine(""fares.elements[1]="" + fares.elements[1]); - return (int)res.Value; } - public static unsafe MyInlineArray InlineArrayTest1 () { - MyInlineArray ia = new (); - for (int i = 0; i < 2; i++) - ia[i] = i; - return ia; - } - - public static unsafe MyInlineArray InlineArrayTest2 (MyInlineArray ia) { - return accept_and_return_inlinearray(ia); - } - [DllImport(""wasm-abi"", EntryPoint=""accept_double_struct_and_return_float_struct"")] public static extern SingleFloatStruct indirect(SingleDoubleStruct arg); @@ -818,18 +782,9 @@ public static unsafe MyInlineArray InlineArrayTest2 (MyInlineArray ia) { [DllImport(""wasm-abi"", EntryPoint=""accept_and_return_i64_struct"")] public static extern Int64 direct64(Int64 arg); - - [DllImport(""wasm-abi"", EntryPoint=""accept_and_return_pair"")] - public static extern PairStruct accept_and_return_pair(PairStruct arg); - - [DllImport(""wasm-abi"", EntryPoint=""accept_and_return_fixedarray"")] - public static extern MyFixedArray accept_and_return_fixedarray(MyFixedArray arg); - - [DllImport(""wasm-abi"", EntryPoint=""accept_and_return_inlinearray"")] - public static extern MyInlineArray accept_and_return_inlinearray(MyInlineArray arg); }"; - var extraProperties = "true<_WasmDevel>falsefalse"; + var extraProperties = "true<_WasmDevel>true"; var extraItems = @""; buildArgs = ExpandBuildArgs(buildArgs, @@ -869,10 +824,6 @@ public static unsafe MyInlineArray InlineArrayTest2 (MyInlineArray ia) { Assert.Contains("f (d)=3.14", runOutput); Assert.Contains("f (s)=3.14", runOutput); Assert.Contains("s (s)=3.14", runOutput); - Assert.Contains("paires.B=4", runOutput); - Assert.Contains("iares[0]=32", runOutput); - Assert.Contains("iares[1]=2", runOutput); - Assert.Contains("fares.elements[1]=2", runOutput); } [Theory] diff --git a/src/mono/wasm/Wasm.Build.Tests/ProjectProviderBase.cs b/src/mono/wasm/Wasm.Build.Tests/ProjectProviderBase.cs index 581a187270ae4f..81c6a4894f1273 100644 --- a/src/mono/wasm/Wasm.Build.Tests/ProjectProviderBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/ProjectProviderBase.cs @@ -490,10 +490,10 @@ private void AssertFileNames(IEnumerable expected, IEnumerable a Assert.Equal(expected, actualFileNames); } - public virtual string FindBinFrameworkDir(string config, bool forPublish, string framework, string? bundleDirName = null, string? projectDir = null) + public virtual string FindBinFrameworkDir(string config, bool forPublish, string framework, string? bundleDirName = null) { EnsureProjectDirIsSet(); - string basePath = Path.Combine(projectDir ?? ProjectDir!, "bin", config, framework); + string basePath = Path.Combine(ProjectDir!, "bin", config, framework); if (forPublish) basePath = FindSubDirIgnoringCase(basePath, "publish"); diff --git a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/AppTestBase.cs b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/AppTestBase.cs index 7647b165862098..771daff1f5d469 100644 --- a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/AppTestBase.cs +++ b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/AppTestBase.cs @@ -31,48 +31,14 @@ protected void CopyTestAsset(string assetName, string generatedProjectNamePrefix LogPath = Path.Combine(s_buildEnv.LogRootPath, Id); Utils.DirectoryCopy(Path.Combine(BuildEnvironment.TestAssetsPath, assetName), Path.Combine(_projectDir!)); - switch(assetName) - { - case "WasmBasicTestApp": - // WasmBasicTestApp consists of App + Library projects - _projectDir = Path.Combine(_projectDir!, "App"); - break; - case "BlazorHostedApp": - // BlazorHostedApp consists of BlazorHosted.Client and BlazorHosted.Server projects - _projectDir = Path.Combine(_projectDir!, "BlazorHosted.Server"); - break; - } - } - - protected void BlazorHostedBuild( - string config, - string assetName, - string clientDirRelativeToProjectDir = "", - string? generatedProjectNamePrefix = null, - RuntimeVariant runtimeType = RuntimeVariant.SingleThreaded) - { - CopyTestAsset(assetName, generatedProjectNamePrefix); - string frameworkDir = FindBlazorHostedBinFrameworkDir(config, - forPublish: false, - clientDirRelativeToProjectDir: clientDirRelativeToProjectDir); - BuildProject(configuration: config, - binFrameworkDir: frameworkDir, - runtimeType: runtimeType); + // WasmBasicTestApp consists of App + Library projects + if (assetName == "WasmBasicTestApp") + _projectDir = Path.Combine(_projectDir!, "App"); } - protected void BuildProject( - string configuration, - string? binFrameworkDir = null, - RuntimeVariant runtimeType = RuntimeVariant.SingleThreaded, - bool assertAppBundle = true, - params string[] extraArgs) + protected void BuildProject(string configuration, params string[] extraArgs) { - (CommandResult result, _) = BlazorBuild(new BlazorBuildOptions( - Id: Id, - Config: configuration, - BinFrameworkDir: binFrameworkDir, - RuntimeType: runtimeType, - AssertAppBundle: assertAppBundle), extraArgs); + (CommandResult result, _) = BlazorBuild(new BlazorBuildOptions(Id, configuration), extraArgs); result.EnsureSuccessful(); } @@ -88,42 +54,37 @@ protected void PublishProject(string configuration, params string[] extraArgs) protected Task RunSdkStyleAppForBuild(RunOptions options) => RunSdkStyleApp(options, BlazorRunHost.DotnetRun); - + protected Task RunSdkStyleAppForPublish(RunOptions options) => RunSdkStyleApp(options, BlazorRunHost.WebServer); private async Task RunSdkStyleApp(RunOptions options, BlazorRunHost host = BlazorRunHost.DotnetRun) { - var query = options.BrowserQueryString ?? new Dictionary(); - if (!string.IsNullOrEmpty(options.TestScenario)) - query.Add("test", options.TestScenario); - - var queryString = query.Any() ? "?" + string.Join("&", query.Select(kvp => $"{kvp.Key}={kvp.Value}")) : ""; + string queryString = "?test=" + options.TestScenario; + if (options.BrowserQueryString != null) + queryString += "&" + string.Join("&", options.BrowserQueryString.Select(kvp => $"{kvp.Key}={kvp.Value}")); var tcs = new TaskCompletionSource(); List testOutput = new(); List consoleOutput = new(); - List serverOutput = new(); - Regex exitRegex = new Regex("(WASM EXIT (?[0-9]+)$)|(Program terminated with exit\\((?[0-9]+)\\))"); + Regex exitRegex = new Regex("WASM EXIT (?[0-9]+)$"); BlazorRunOptions blazorRunOptions = new( CheckCounter: false, Config: options.Configuration, - ServerEnvironment: options.ServerEnvironment, - OnPageLoaded: options.OnPageLoaded, OnConsoleMessage: OnConsoleMessage, - OnServerMessage: OnServerMessage, - BrowserPath: options.BrowserPath, QueryString: queryString, Host: host); await BlazorRunTest(blazorRunOptions); - void OnConsoleMessage(IPage page, IConsoleMessage msg) + void OnConsoleMessage(IConsoleMessage msg) { consoleOutput.Add(msg.Text); - OnTestOutput(msg.Text); + const string testOutputPrefix = "TestOutput -> "; + if (msg.Text.StartsWith(testOutputPrefix)) + testOutput.Add(msg.Text.Substring(testOutputPrefix.Length)); var exitMatch = exitRegex.Match(msg.Text); if (exitMatch.Success) @@ -133,23 +94,7 @@ void OnConsoleMessage(IPage page, IConsoleMessage msg) throw new Exception(msg.Text); if (options.OnConsoleMessage != null) - options.OnConsoleMessage(page, msg); - } - - void OnServerMessage(string msg) - { - serverOutput.Add(msg); - OnTestOutput(msg); - - if (options.OnServerMessage != null) - options.OnServerMessage(msg); - } - - void OnTestOutput(string msg) - { - const string testOutputPrefix = "TestOutput -> "; - if (msg.StartsWith(testOutputPrefix)) - testOutput.Add(msg.Substring(testOutputPrefix.Length)); + options.OnConsoleMessage(msg); } //TimeSpan timeout = TimeSpan.FromMinutes(2); @@ -161,25 +106,20 @@ void OnTestOutput(string msg) if (options.ExpectedExitCode != null && wasmExitCode != options.ExpectedExitCode) throw new Exception($"Expected exit code {options.ExpectedExitCode} but got {wasmExitCode}"); - return new(wasmExitCode, testOutput, consoleOutput, serverOutput); + return new(wasmExitCode, testOutput, consoleOutput); } protected record RunOptions( string Configuration, - string BrowserPath = "", - string? TestScenario = null, + string TestScenario, Dictionary BrowserQueryString = null, - Dictionary ServerEnvironment = null, - Action OnPageLoaded = null, - Action OnConsoleMessage = null, - Action OnServerMessage = null, + Action OnConsoleMessage = null, int? ExpectedExitCode = 0 ); protected record RunResult( int ExitCode, IReadOnlyCollection TestOutput, - IReadOnlyCollection ConsoleOutput, - IReadOnlyCollection ServerOutput + IReadOnlyCollection ConsoleOutput ); } diff --git a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DebugLevelTests.cs b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DebugLevelTests.cs index 1bbe8691d80db3..3dfe4c467f79f7 100644 --- a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DebugLevelTests.cs +++ b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/DebugLevelTests.cs @@ -20,7 +20,7 @@ public DebugLevelTests(ITestOutputHelper output, SharedBuildPerTestClassFixture { } - private void AssertDebugLevel(RunResult result, int value) + private void AssertDebugLevel(RunResult result, int value) { Assert.Collection( result.TestOutput, @@ -51,7 +51,7 @@ public async Task BuildWithDefaultLevel(string configuration) public async Task BuildWithExplicitValue(string configuration, int debugLevel) { CopyTestAsset("WasmBasicTestApp", $"DebugLevelTests_BuildWithExplicitValue_{configuration}"); - BuildProject(configuration: configuration, extraArgs: $"-p:WasmDebugLevel={debugLevel}"); + BuildProject(configuration, $"-p:WasmDebugLevel={debugLevel}"); var result = await RunSdkStyleAppForBuild(new( Configuration: configuration, diff --git a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/SignalRClientTests.cs b/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/SignalRClientTests.cs deleted file mode 100644 index b1e46d5110067d..00000000000000 --- a/src/mono/wasm/Wasm.Build.Tests/TestAppScenarios/SignalRClientTests.cs +++ /dev/null @@ -1,84 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Playwright; -using Xunit.Abstractions; -using Xunit; - -#nullable enable - -namespace Wasm.Build.Tests.TestAppScenarios; - -public class SignalRClientTests : AppTestBase -{ - public SignalRClientTests(ITestOutputHelper output, SharedBuildPerTestClassFixture buildContext) - : base(output, buildContext) - { - } - - [ConditionalTheory(typeof(BuildTestBase), nameof(IsWorkloadWithMultiThreadingForDefaultFramework))] - [InlineData("Debug", "LongPolling")] - [InlineData("Release", "LongPolling")] - [InlineData("Debug", "WebSockets")] - [InlineData("Release", "WebSockets")] - public async Task SignalRPassMessages(string config, string transport) - { - BlazorHostedBuild(config, - assetName: "BlazorHostedApp", - clientDirRelativeToProjectDir: "../BlazorHosted.Client", - generatedProjectNamePrefix: "SignalRClientTests", - runtimeType: RuntimeVariant.MultiThreaded); - - List consoleOutput = new(); - List serverOutput = new(); - - var result = await RunSdkStyleAppForBuild(new( - Configuration: config, - // We are using build (not publish), - // we need to instruct static web assets to use manifest file, - // because wwwroot in bin doesn't contain all files (for build) - ServerEnvironment: new Dictionary { ["ASPNETCORE_ENVIRONMENT"] = "Development" }, - BrowserPath: "/chat", - BrowserQueryString: new Dictionary { ["transport"] = transport, ["message"] = "ping" }, - OnPageLoaded: async page => await page.ClickAsync("button#connectButton"), - OnServerMessage: (msg) => { serverOutput.Add(msg); }, - OnConsoleMessage: async (page, msg) => - { - consoleOutput.Add(msg.Text); - if (msg.Text.Contains("TestOutput ->")) - _testOutput.WriteLine(msg.Text); - - if (msg.Text.Contains("SignalR connected")) - await page.ClickAsync("button#subscribeButton"); - - if (msg.Text.Contains("Subscribed to ReceiveMessage")) - await page.ClickAsync("button#sendMessageButton"); - - if (msg.Text.Contains("ReceiveMessage from server")) - await page.ClickAsync("button#exitProgramButton"); - } - )); - - string output = _testOutput.ToString() ?? ""; - Assert.NotEmpty(output); - // check sending and receiving threadId - string threadIdUsedForSending = GetThreadOfAction(output, @"SignalRPassMessages was sent by CurrentManagedThreadId=(\d+)", "signalR message was sent"); - string threadIdUsedForReceiving = GetThreadOfAction(output, @"ReceiveMessage from server on CurrentManagedThreadId=(\d+)", "signalR message was received"); - Assert.True("1" != threadIdUsedForSending || "1" != threadIdUsedForReceiving, - $"Expected to send/receive with signalR in non-UI threads, instead only CurrentManagedThreadId=1 was used. TestOutput: {output}."); - } - - private string GetThreadOfAction(string testOutput, string pattern, string actionDescription) - { - Match match = Regex.Match(testOutput, pattern); - Assert.True(match.Success, $"Expected to find a log that {actionDescription}. TestOutput: {testOutput}."); - return match.Groups[1].Value ?? ""; - } -} diff --git a/src/mono/wasm/Wasm.Build.Tests/TestMainJsProjectProvider.cs b/src/mono/wasm/Wasm.Build.Tests/TestMainJsProjectProvider.cs index 89a1ddc8c132b7..7042c4855549d2 100644 --- a/src/mono/wasm/Wasm.Build.Tests/TestMainJsProjectProvider.cs +++ b/src/mono/wasm/Wasm.Build.Tests/TestMainJsProjectProvider.cs @@ -108,9 +108,9 @@ public void AssertBundle(BuildArgs buildArgs, BuildProjectOptions buildProjectOp AssertBundle(assertOptions); } - public override string FindBinFrameworkDir(string config, bool forPublish, string framework, string? bundleDirName = null, string? projectDir = null) + public override string FindBinFrameworkDir(string config, bool forPublish, string framework, string? bundleDirName = null) { EnsureProjectDirIsSet(); - return Path.Combine(projectDir ?? ProjectDir!, "bin", config, framework, "browser-wasm", bundleDirName ?? this.BundleDirName, "_framework"); + return Path.Combine(ProjectDir!, "bin", config, framework, "browser-wasm", bundleDirName ?? this.BundleDirName, "_framework"); } } diff --git a/src/mono/wasm/Wasm.Build.Tests/WasmSdkBasedProjectProvider.cs b/src/mono/wasm/Wasm.Build.Tests/WasmSdkBasedProjectProvider.cs index 9cfbcca73843c9..0480473d249a68 100644 --- a/src/mono/wasm/Wasm.Build.Tests/WasmSdkBasedProjectProvider.cs +++ b/src/mono/wasm/Wasm.Build.Tests/WasmSdkBasedProjectProvider.cs @@ -105,14 +105,12 @@ public void AssertBundle(AssertWasmSdkBundleOptions assertOptions) _ => throw new ArgumentOutOfRangeException(nameof(assertOptions.ExpectedFileType)) }; string buildType = assertOptions.IsPublish ? "publish" : "build"; - var nativeFilesToCheck = new List() { "dotnet.native.wasm", "dotnet.native.js" }; - if (assertOptions.RuntimeType == RuntimeVariant.MultiThreaded) - nativeFilesToCheck.Add("dotnet.native.worker.js"); - foreach (string nativeFilename in nativeFilesToCheck) + foreach (string nativeFilename in new[] { "dotnet.native.wasm", "dotnet.native.js" }) { if (!actualDotnetFiles.TryGetValue(nativeFilename, out DotNetFileName? dotnetFile)) { throw new XunitException($"Could not find {nativeFilename}. Actual files on disk: {string.Join($"{Environment.NewLine} ", actualDotnetFiles.Values.Select(a => a.ActualPath).Order())}"); + } // For any *type*, check against the expected path TestUtils.AssertSameFile(Path.Combine(srcDirForNativeFileToCompareAgainst, nativeFilename), @@ -121,11 +119,6 @@ public void AssertBundle(AssertWasmSdkBundleOptions assertOptions) if (assertOptions.ExpectedFileType != NativeFilesType.FromRuntimePack) { - if (nativeFilename == "dotnet.native.worker.js") - { - Console.WriteLine($"Skipping the verification whether {nativeFilename} is from the runtime pack. The check wouldn't be meaningful as the runtime pack file has the same size as the relinked file"); - continue; - } // Confirm that it doesn't match the file from the runtime pack TestUtils.AssertNotSameFile(Path.Combine(runtimeNativeDir, nativeFilename), actualDotnetFiles[nativeFilename].ActualPath, diff --git a/src/mono/wasm/build/WasmApp.Common.targets b/src/mono/wasm/build/WasmApp.Common.targets index 2d2a6c03e43714..f0c0d4be6946ab 100644 --- a/src/mono/wasm/build/WasmApp.Common.targets +++ b/src/mono/wasm/build/WasmApp.Common.targets @@ -111,13 +111,6 @@ - - <_BoolPropertiesThatTriggerRelinking Include="InvariantTimezone" DefaultValueInRuntimePack="false" /> - <_BoolPropertiesThatTriggerRelinking Include="InvariantGlobalization" DefaultValueInRuntimePack="false" /> - <_BoolPropertiesThatTriggerRelinking Include="WasmNativeStrip" DefaultValueInRuntimePack="true" /> - <_BoolPropertiesThatTriggerRelinking Include="WasmSingleFileBundle" DefaultValueInRuntimePack="false" /> - - $(PrepareInputsForWasmBuildDependsOn); @@ -501,18 +494,14 @@ Text="$(_ToolchainMissingErrorMessage) SDK is required for AOT'ing assemblies." /> - <_ChangedBoolPropertiesThatTriggerRelinking Include="%(_BoolPropertiesThatTriggerRelinking.Identity)" Condition="'$(%(_BoolPropertiesThatTriggerRelinking.Identity))' != '' and - '$(%(_BoolPropertiesThatTriggerRelinking.Identity))' != '%(_BoolPropertiesThatTriggerRelinking.DefaultValueInRuntimePack)'" /> + <_BoolPropertiesThatTriggerRelinking Include="InvariantTimezone" DefaultValueInRuntimePack="false" /> + <_BoolPropertiesThatTriggerRelinking Include="InvariantGlobalization" DefaultValueInRuntimePack="false" /> - - <_WasmBuildNativeRequired Condition="@(_ChangedBoolPropertiesThatTriggerRelinking->Count()) > 0">true - - - - true + true @@ -521,6 +510,7 @@ true true + true @@ -528,6 +518,7 @@ true true + true false @@ -624,7 +615,6 @@ - diff --git a/src/mono/wasm/features.md b/src/mono/wasm/features.md index 53995e656a1a61..90c0744a8586c1 100644 --- a/src/mono/wasm/features.md +++ b/src/mono/wasm/features.md @@ -402,8 +402,8 @@ In Blazor, you can customize the startup in your index.html - - - - diff --git a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/BlazorHosted.Server.csproj b/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/BlazorHosted.Server.csproj deleted file mode 100644 index cc3ac1aae891e3..00000000000000 --- a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/BlazorHosted.Server.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - net9.0 - enable - enable - - CA2007 - - - - - - - - - - - diff --git a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/ChatHub.cs b/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/ChatHub.cs deleted file mode 100644 index 8b2e77807c6fbc..00000000000000 --- a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/ChatHub.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.AspNetCore.SignalR; - -namespace BlazorHosted.Server.Hubs; -public class ChatHub : Hub -{ - public async Task SendMessage(string message, int sendingThreadId) - { - Console.WriteLine($"Server: receives Message=[{message}] sent by treadID = {sendingThreadId} and sends it back."); - string changedMessage = $"{message}-pong"; - await Clients.All.SendAsync("ReceiveMessage", changedMessage).ConfigureAwait(false); - } -} diff --git a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/Program.cs b/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/Program.cs deleted file mode 100644 index fea18a9250cc15..00000000000000 --- a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/Program.cs +++ /dev/null @@ -1,52 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.Extensions.Configuration; -using System; -using Microsoft.Extensions.Logging; -using BlazorHosted.Server.Hubs; - -var builder = WebApplication.CreateBuilder(args); - -builder.Services.AddControllersWithViews(); -builder.Services.AddRazorPages(); -builder.Services.AddSignalR(options => -{ - options.KeepAliveInterval = TimeSpan.Zero; // minimize keep-alive messages -}); - -var app = builder.Build(); - -// Configure the HTTP request pipeline. -if (app.Environment.IsDevelopment()) -{ - app.UseWebAssemblyDebugging(); -} -else -{ - app.UseExceptionHandler("/Error"); - // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. - app.UseHsts(); -} - -// Add headers to enable SharedArrayBuffer -app.Use(async (context, next) => -{ - var response = context.Response; - response.Headers.Append("Cross-Origin-Opener-Policy", "same-origin"); - response.Headers.Append("Cross-Origin-Embedder-Policy", "require-corp"); - - await next(); -}); -app.UseBlazorFrameworkFiles(); -app.UseStaticFiles(); - -app.UseRouting(); - -app.MapRazorPages(); -app.MapControllers(); -app.MapFallbackToFile("index.html"); - -app.MapHub("/chathub"); - -app.Run(); diff --git a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/appsettings.json b/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/appsettings.json deleted file mode 100644 index 75b7c2aa1ecedb..00000000000000 --- a/src/mono/wasm/testassets/BlazorHostedApp/BlazorHosted.Server/appsettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - } -} \ No newline at end of file diff --git a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/README.md b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/README.md deleted file mode 100644 index 996992663189fb..00000000000000 --- a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/README.md +++ /dev/null @@ -1,15 +0,0 @@ -## WasmBasicTestApp - -This is a test application used by various Wasm.Build.Tests. The idea is to share a common behavior (so that we don't have to maintain many test apps) and tweak it for the test case. -It typically suits scenario where you need more than a plain template app. If the test case is too different, feel free to create another app. - -### Usage - -The app reads `test` query parameter and uses it to switch between test cases. Entrypoint is `main.js`. -There is common unit, then switch based on test case for modifying app startup, then app starts and executes next switch based on test case for actually running code. - -Some test cases passes additional parameters to differentiate behavior, see `src/mono/wasm/Wasm.Build.Tests/TestAppScenarios`. - -### Running out side of WBT - -One of the benefits is that you can copy the app out of intree and run the app without running Wasm.Build.Tests with just `dotnet run`. \ No newline at end of file diff --git a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js index 3a01053875c0a6..2e1b7ff8c84fe3 100644 --- a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js +++ b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js @@ -33,7 +33,6 @@ switch (testCase) { Math.floor(Math.random() * 5) + 5, Math.floor(Math.random() * 5) + 10 ]; - console.log(`Failing test at assembly indexes [${failAtAssemblyNumbers.join(", ")}]`); let alreadyFailed = []; dotnet.withDiagnosticTracing(true).withResourceLoader((type, name, defaultUri, integrity, behavior) => { if (type === "dotnetjs") { @@ -46,8 +45,8 @@ switch (testCase) { return defaultUri; } - const currentCounter = assemblyCounter++; - if (!failAtAssemblyNumbers.includes(currentCounter) || alreadyFailed.includes(defaultUri)) + assemblyCounter++; + if (!failAtAssemblyNumbers.includes(assemblyCounter) || alreadyFailed.includes(defaultUri)) return defaultUri; alreadyFailed.push(defaultUri); diff --git a/src/mono/wasm/testassets/native-libs/wasm-abi.c b/src/mono/wasm/testassets/native-libs/wasm-abi.c index 083bce6abe0c59..0ace2037daf2f9 100644 --- a/src/mono/wasm/testassets/native-libs/wasm-abi.c +++ b/src/mono/wasm/testassets/native-libs/wasm-abi.c @@ -1,7 +1,5 @@ #include -#define TRACING 0 - typedef struct { float value; } TRes; @@ -9,12 +7,10 @@ typedef struct { TRes accept_double_struct_and_return_float_struct ( struct { struct { double value; } value; } arg ) { -#if TRACING printf ( "&arg=%x (ulonglong)arg=%llx arg.value.value=%lf\n", (unsigned int)&arg, *(unsigned long long*)&arg, (double)arg.value.value ); -#endif TRes result = { arg.value.value }; return result; } @@ -24,48 +20,10 @@ typedef struct { } TResI64; TResI64 accept_and_return_i64_struct (TResI64 arg) { -#if TRACING printf ( "&arg=%x (ulonglong)arg=%llx\n", (unsigned int)&arg, *(unsigned long long*)&arg ); -#endif TResI64 result = { ~arg.value }; return result; } - -typedef struct { - int A, B; -} PairStruct; - -PairStruct accept_and_return_pair (PairStruct arg) { -#if TRACING - printf ( - "&arg=%d arg.A=%d arg.B=%d\n", - (unsigned int)&arg, arg.A, arg.B - ); -#endif - arg.A = 32; - arg.B *= 2; - return arg; -} - -typedef struct { - int elements[2]; -} MyInlineArray; - -MyInlineArray accept_and_return_inlinearray (MyInlineArray arg) { -#if TRACING - printf ( - "&arg=%d arg.elements[0]=%d arg.elements[1]=%d\n", - (unsigned int)&arg, arg.elements[0], arg.elements[1] - ); -#endif - arg.elements[0] = 32; - arg.elements[1] *= 2; - return arg; -} - -MyInlineArray accept_and_return_fixedarray (MyInlineArray arg) { - return accept_and_return_inlinearray (arg); -} diff --git a/src/native/corehost/apphost/static/singlefilehost.def b/src/native/corehost/apphost/static/singlefilehost.def index e1208056b83201..6052b832b0b054 100644 --- a/src/native/corehost/apphost/static/singlefilehost.def +++ b/src/native/corehost/apphost/static/singlefilehost.def @@ -13,8 +13,5 @@ CLRJitAttachState @3 data ; needed by SOS, WinDBG, and Watson. This must remain ordinal 4. DotNetRuntimeInfo @4 data -; DAC table export -g_dacTable = s_dacGlobals - ; Used by profilers MetaDataGetDispenser diff --git a/src/native/corehost/json_parser.h b/src/native/corehost/json_parser.h index d7393b0ae678d3..2c2845aac46bfe 100644 --- a/src/native/corehost/json_parser.h +++ b/src/native/corehost/json_parser.h @@ -8,22 +8,12 @@ // https://github.com/Tencent/rapidjson/issues/1596#issuecomment-548774663 #define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 -// see https://github.com/Tencent/rapidjson/issues/1448 -// including windows.h on purpose to provoke a compile time problem as GetObject is a -// macro that gets defined when windows.h is included -#ifdef _WIN32 -#define NOMINMAX -#include -#endif - #include "pal.h" #include #include #include #include "bundle/info.h" -#undef GetObject - class json_parser_t { public: #ifdef _WIN32 diff --git a/src/native/external/cgmanifest.json b/src/native/external/cgmanifest.json index e45d9d90029623..4f6264b3c7ab29 100644 --- a/src/native/external/cgmanifest.json +++ b/src/native/external/cgmanifest.json @@ -46,7 +46,7 @@ "Type": "git", "Git": { "RepositoryUrl": "https://github.com/madler/zlib", - "CommitHash": "51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf" + "CommitHash": "04f42ceca40f73e2978b50e93806c2a18c1281fc" } }, "DevelopmentDependency": false diff --git a/src/native/external/libunwind_extras/CMakeLists.txt b/src/native/external/libunwind_extras/CMakeLists.txt index 0b911094c767bb..07a4bb64db940d 100644 --- a/src/native/external/libunwind_extras/CMakeLists.txt +++ b/src/native/external/libunwind_extras/CMakeLists.txt @@ -140,7 +140,6 @@ if(CLR_CMAKE_HOST_WIN32) # Warnings in release builds add_compile_options(-wd4068) # ignore unknown pragma warnings (gcc pragmas) - add_compile_options(-wd4242) # possible loss of data add_compile_options(-wd4244) # possible loss of data add_compile_options(-wd4334) # 32-bit shift implicitly converted to 64 bits diff --git a/src/native/external/rapidjson-version.txt b/src/native/external/rapidjson-version.txt index b6f5f9532a7dba..0ccc08a3c22387 100644 --- a/src/native/external/rapidjson-version.txt +++ b/src/native/external/rapidjson-version.txt @@ -1,6 +1,6 @@ -3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d +d87b698d0fcc10a5f632ecbc80a9cb2a8fa094a5 -https://github.com/Tencent/rapidjson/commit/3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d +https://github.com/Tencent/rapidjson/commit/d87b698d0fcc10a5f632ecbc80a9cb2a8fa094a5 Note: This library is not using a proper release lifecycle. v1.1.0 was the last version released in 2016. - Therefore, we are pointing to a random commit from 2024 rather than a version tag. + Therefore, we are pointing to a random commit from 2019 rather than a version tag. diff --git a/src/native/external/rapidjson/README.TXT b/src/native/external/rapidjson/README.TXT index 9eff509a934d30..bc0a70382f4a43 100644 --- a/src/native/external/rapidjson/README.TXT +++ b/src/native/external/rapidjson/README.TXT @@ -1,2 +1,2 @@ -This directory contains selective files from -https://github.com/Tencent/rapidjson/tree/3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d/include/rapidjson +This directory contains the contents of `include/rapidjson` from +, commit hash d87b698d0fcc10. diff --git a/src/native/external/rapidjson/allocators.h b/src/native/external/rapidjson/allocators.h index 275417bd8b3770..cc67c8971323c0 100644 --- a/src/native/external/rapidjson/allocators.h +++ b/src/native/external/rapidjson/allocators.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -16,14 +16,6 @@ #define RAPIDJSON_ALLOCATORS_H_ #include "rapidjson.h" -#include "internal/meta.h" - -#include -#include - -#if RAPIDJSON_HAS_CXX11 -#include -#endif RAPIDJSON_NAMESPACE_BEGIN @@ -85,26 +77,19 @@ class CrtAllocator { static const bool kNeedFree = true; void* Malloc(size_t size) { if (size) // behavior of malloc(0) is implementation defined. - return RAPIDJSON_MALLOC(size); + return std::malloc(size); else return NULL; // standardize to returning NULL. } void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; if (newSize == 0) { - RAPIDJSON_FREE(originalPtr); + std::free(originalPtr); return NULL; } - return RAPIDJSON_REALLOC(originalPtr, newSize); - } - static void Free(void *ptr) RAPIDJSON_NOEXCEPT { RAPIDJSON_FREE(ptr); } - - bool operator==(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { - return true; - } - bool operator!=(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { - return false; + return std::realloc(originalPtr, newSize); } + static void Free(void *ptr) { std::free(ptr); } }; /////////////////////////////////////////////////////////////////////////////// @@ -128,64 +113,16 @@ class CrtAllocator { */ template class MemoryPoolAllocator { - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. - }; - - struct SharedData { - ChunkHeader *chunkHead; //!< Head of the chunk linked-list. Only the head chunk serves allocation. - BaseAllocator* ownBaseAllocator; //!< base allocator created by this object. - size_t refcount; - bool ownBuffer; - }; - - static const size_t SIZEOF_SHARED_DATA = RAPIDJSON_ALIGN(sizeof(SharedData)); - static const size_t SIZEOF_CHUNK_HEADER = RAPIDJSON_ALIGN(sizeof(ChunkHeader)); - - static inline ChunkHeader *GetChunkHead(SharedData *shared) - { - return reinterpret_cast(reinterpret_cast(shared) + SIZEOF_SHARED_DATA); - } - static inline uint8_t *GetChunkBuffer(SharedData *shared) - { - return reinterpret_cast(shared->chunkHead) + SIZEOF_CHUNK_HEADER; - } - - static const size_t kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. - public: static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) - static const bool kRefCounted = true; //!< Tell users that this allocator is reference counted on copy //! Constructor with chunkSize. /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. \param baseAllocator The allocator for allocating memory chunks. */ - explicit MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunk_capacity_(chunkSize), - baseAllocator_(baseAllocator ? baseAllocator : RAPIDJSON_NEW(BaseAllocator)()), - shared_(static_cast(baseAllocator_ ? baseAllocator_->Malloc(SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER) : 0)) + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) { - RAPIDJSON_ASSERT(baseAllocator_ != 0); - RAPIDJSON_ASSERT(shared_ != 0); - if (baseAllocator) { - shared_->ownBaseAllocator = 0; - } - else { - shared_->ownBaseAllocator = baseAllocator_; - } - shared_->chunkHead = GetChunkHead(shared_); - shared_->chunkHead->capacity = 0; - shared_->chunkHead->size = 0; - shared_->chunkHead->next = 0; - shared_->ownBuffer = true; - shared_->refcount = 1; } //! Constructor with user-supplied buffer. @@ -199,101 +136,41 @@ class MemoryPoolAllocator { \param baseAllocator The allocator for allocating memory chunks. */ MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunk_capacity_(chunkSize), - baseAllocator_(baseAllocator), - shared_(static_cast(AlignBuffer(buffer, size))) - { - RAPIDJSON_ASSERT(size >= SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER); - shared_->chunkHead = GetChunkHead(shared_); - shared_->chunkHead->capacity = size - SIZEOF_SHARED_DATA - SIZEOF_CHUNK_HEADER; - shared_->chunkHead->size = 0; - shared_->chunkHead->next = 0; - shared_->ownBaseAllocator = 0; - shared_->ownBuffer = false; - shared_->refcount = 1; - } - - MemoryPoolAllocator(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT : - chunk_capacity_(rhs.chunk_capacity_), - baseAllocator_(rhs.baseAllocator_), - shared_(rhs.shared_) - { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - ++shared_->refcount; - } - MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT + chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) { - RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); - ++rhs.shared_->refcount; - this->~MemoryPoolAllocator(); - baseAllocator_ = rhs.baseAllocator_; - chunk_capacity_ = rhs.chunk_capacity_; - shared_ = rhs.shared_; - return *this; + RAPIDJSON_ASSERT(buffer != 0); + RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); + chunkHead_ = reinterpret_cast(buffer); + chunkHead_->capacity = size - sizeof(ChunkHeader); + chunkHead_->size = 0; + chunkHead_->next = 0; } -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - MemoryPoolAllocator(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT : - chunk_capacity_(rhs.chunk_capacity_), - baseAllocator_(rhs.baseAllocator_), - shared_(rhs.shared_) - { - RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); - rhs.shared_ = 0; - } - MemoryPoolAllocator& operator=(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT - { - RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); - this->~MemoryPoolAllocator(); - baseAllocator_ = rhs.baseAllocator_; - chunk_capacity_ = rhs.chunk_capacity_; - shared_ = rhs.shared_; - rhs.shared_ = 0; - return *this; - } -#endif - //! Destructor. /*! This deallocates all memory chunks, excluding the user-supplied buffer. */ - ~MemoryPoolAllocator() RAPIDJSON_NOEXCEPT { - if (!shared_) { - // do nothing if moved - return; - } - if (shared_->refcount > 1) { - --shared_->refcount; - return; - } + ~MemoryPoolAllocator() { Clear(); - BaseAllocator *a = shared_->ownBaseAllocator; - if (shared_->ownBuffer) { - baseAllocator_->Free(shared_); - } - RAPIDJSON_DELETE(a); + RAPIDJSON_DELETE(ownBaseAllocator_); } - //! Deallocates all memory chunks, excluding the first/user one. - void Clear() RAPIDJSON_NOEXCEPT { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - for (;;) { - ChunkHeader* c = shared_->chunkHead; - if (!c->next) { - break; - } - shared_->chunkHead = c->next; - baseAllocator_->Free(c); + //! Deallocates all memory chunks, excluding the user-supplied buffer. + void Clear() { + while (chunkHead_ && chunkHead_ != userBuffer_) { + ChunkHeader* next = chunkHead_->next; + baseAllocator_->Free(chunkHead_); + chunkHead_ = next; } - shared_->chunkHead->size = 0; + if (chunkHead_ && chunkHead_ == userBuffer_) + chunkHead_->size = 0; // Clear user buffer } //! Computes the total capacity of allocated memory chunks. /*! \return total capacity in bytes. */ - size_t Capacity() const RAPIDJSON_NOEXCEPT { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + size_t Capacity() const { size_t capacity = 0; - for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) capacity += c->capacity; return capacity; } @@ -301,35 +178,25 @@ class MemoryPoolAllocator { //! Computes the memory blocks allocated. /*! \return total used bytes. */ - size_t Size() const RAPIDJSON_NOEXCEPT { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + size_t Size() const { size_t size = 0; - for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) + for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) size += c->size; return size; } - //! Whether the allocator is shared. - /*! \return true or false. - */ - bool Shared() const RAPIDJSON_NOEXCEPT { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - return shared_->refcount > 1; - } - //! Allocates a memory block. (concept Allocator) void* Malloc(size_t size) { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); if (!size) return NULL; size = RAPIDJSON_ALIGN(size); - if (RAPIDJSON_UNLIKELY(shared_->chunkHead->size + size > shared_->chunkHead->capacity)) + if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) return NULL; - void *buffer = GetChunkBuffer(shared_) + shared_->chunkHead->size; - shared_->chunkHead->size += size; + void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; + chunkHead_->size += size; return buffer; } @@ -338,7 +205,6 @@ class MemoryPoolAllocator { if (originalPtr == 0) return Malloc(newSize); - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); if (newSize == 0) return NULL; @@ -350,10 +216,10 @@ class MemoryPoolAllocator { return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == GetChunkBuffer(shared_) + shared_->chunkHead->size - originalSize) { + if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { size_t increment = static_cast(newSize - originalSize); - if (shared_->chunkHead->size + increment <= shared_->chunkHead->capacity) { - shared_->chunkHead->size += increment; + if (chunkHead_->size + increment <= chunkHead_->capacity) { + chunkHead_->size += increment; return originalPtr; } } @@ -369,324 +235,49 @@ class MemoryPoolAllocator { } //! Frees a memory block (concept Allocator) - static void Free(void *ptr) RAPIDJSON_NOEXCEPT { (void)ptr; } // Do nothing - - //! Compare (equality) with another MemoryPoolAllocator - bool operator==(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT { - RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); - RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); - return shared_ == rhs.shared_; - } - //! Compare (inequality) with another MemoryPoolAllocator - bool operator!=(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT { - return !operator==(rhs); - } + static void Free(void *ptr) { (void)ptr; } // Do nothing private: + //! Copy constructor is not permitted. + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; + //! Copy assignment operator is not permitted. + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. \return true if success. */ bool AddChunk(size_t capacity) { if (!baseAllocator_) - shared_->ownBaseAllocator = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); - if (ChunkHeader* chunk = static_cast(baseAllocator_->Malloc(SIZEOF_CHUNK_HEADER + capacity))) { + ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); + if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { chunk->capacity = capacity; chunk->size = 0; - chunk->next = shared_->chunkHead; - shared_->chunkHead = chunk; + chunk->next = chunkHead_; + chunkHead_ = chunk; return true; } else return false; } - static inline void* AlignBuffer(void* buf, size_t &size) - { - RAPIDJSON_NOEXCEPT_ASSERT(buf != 0); - const uintptr_t mask = sizeof(void*) - 1; - const uintptr_t ubuf = reinterpret_cast(buf); - if (RAPIDJSON_UNLIKELY(ubuf & mask)) { - const uintptr_t abuf = (ubuf + mask) & ~mask; - RAPIDJSON_ASSERT(size >= abuf - ubuf); - buf = reinterpret_cast(abuf); - size -= abuf - ubuf; - } - return buf; - } - - size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - SharedData *shared_; //!< The shared data of the allocator -}; - -namespace internal { - template - struct IsRefCounted : - public FalseType - { }; - template - struct IsRefCounted::Type> : - public TrueType - { }; -} - -template -inline T* Realloc(A& a, T* old_p, size_t old_n, size_t new_n) -{ - RAPIDJSON_NOEXCEPT_ASSERT(old_n <= (std::numeric_limits::max)() / sizeof(T) && new_n <= (std::numeric_limits::max)() / sizeof(T)); - return static_cast(a.Realloc(old_p, old_n * sizeof(T), new_n * sizeof(T))); -} - -template -inline T *Malloc(A& a, size_t n = 1) -{ - return Realloc(a, NULL, 0, n); -} - -template -inline void Free(A& a, T *p, size_t n = 1) -{ - static_cast(Realloc(a, p, n, 0)); -} - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) // std::allocator can safely be inherited -#endif - -template -class StdAllocator : - public std::allocator -{ - typedef std::allocator allocator_type; -#if RAPIDJSON_HAS_CXX11 - typedef std::allocator_traits traits_type; -#else - typedef allocator_type traits_type; -#endif - -public: - typedef BaseAllocator BaseAllocatorType; - - StdAllocator() RAPIDJSON_NOEXCEPT : - allocator_type(), - baseAllocator_() - { } - - StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : - allocator_type(rhs), - baseAllocator_(rhs.baseAllocator_) - { } - - template - StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : - allocator_type(rhs), - baseAllocator_(rhs.baseAllocator_) - { } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - StdAllocator(StdAllocator&& rhs) RAPIDJSON_NOEXCEPT : - allocator_type(std::move(rhs)), - baseAllocator_(std::move(rhs.baseAllocator_)) - { } -#endif -#if RAPIDJSON_HAS_CXX11 - using propagate_on_container_move_assignment = std::true_type; - using propagate_on_container_swap = std::true_type; -#endif - - /* implicit */ - StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT : - allocator_type(), - baseAllocator_(baseAllocator) - { } + static const int kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. - ~StdAllocator() RAPIDJSON_NOEXCEPT - { } - - template - struct rebind { - typedef StdAllocator other; - }; - - typedef typename traits_type::size_type size_type; - typedef typename traits_type::difference_type difference_type; - - typedef typename traits_type::value_type value_type; - typedef typename traits_type::pointer pointer; - typedef typename traits_type::const_pointer const_pointer; - -#if RAPIDJSON_HAS_CXX11 - - typedef typename std::add_lvalue_reference::type &reference; - typedef typename std::add_lvalue_reference::type>::type &const_reference; - - pointer address(reference r) const RAPIDJSON_NOEXCEPT - { - return std::addressof(r); - } - const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT - { - return std::addressof(r); - } - - size_type max_size() const RAPIDJSON_NOEXCEPT - { - return traits_type::max_size(*this); - } - - template - void construct(pointer p, Args&&... args) - { - traits_type::construct(*this, p, std::forward(args)...); - } - void destroy(pointer p) - { - traits_type::destroy(*this, p); - } - -#else // !RAPIDJSON_HAS_CXX11 - - typedef typename allocator_type::reference reference; - typedef typename allocator_type::const_reference const_reference; - - pointer address(reference r) const RAPIDJSON_NOEXCEPT - { - return allocator_type::address(r); - } - const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT - { - return allocator_type::address(r); - } - - size_type max_size() const RAPIDJSON_NOEXCEPT - { - return allocator_type::max_size(); - } - - void construct(pointer p, const_reference r) - { - allocator_type::construct(p, r); - } - void destroy(pointer p) - { - allocator_type::destroy(p); - } - -#endif // !RAPIDJSON_HAS_CXX11 - - template - U* allocate(size_type n = 1, const void* = 0) - { - return RAPIDJSON_NAMESPACE::Malloc(baseAllocator_, n); - } - template - void deallocate(U* p, size_type n = 1) - { - RAPIDJSON_NAMESPACE::Free(baseAllocator_, p, n); - } - - pointer allocate(size_type n = 1, const void* = 0) - { - return allocate(n); - } - void deallocate(pointer p, size_type n = 1) - { - deallocate(p, n); - } - -#if RAPIDJSON_HAS_CXX11 - using is_always_equal = std::is_empty; -#endif - - template - bool operator==(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT - { - return baseAllocator_ == rhs.baseAllocator_; - } - template - bool operator!=(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT - { - return !operator==(rhs); - } - - //! rapidjson Allocator concept - static const bool kNeedFree = BaseAllocator::kNeedFree; - static const bool kRefCounted = internal::IsRefCounted::Value; - void* Malloc(size_t size) - { - return baseAllocator_.Malloc(size); - } - void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) - { - return baseAllocator_.Realloc(originalPtr, originalSize, newSize); - } - static void Free(void *ptr) RAPIDJSON_NOEXCEPT - { - BaseAllocator::Free(ptr); - } - -private: - template - friend class StdAllocator; // access to StdAllocator.* - - BaseAllocator baseAllocator_; -}; - -#if !RAPIDJSON_HAS_CXX17 // std::allocator deprecated in C++17 -template -class StdAllocator : - public std::allocator -{ - typedef std::allocator allocator_type; - -public: - typedef BaseAllocator BaseAllocatorType; - - StdAllocator() RAPIDJSON_NOEXCEPT : - allocator_type(), - baseAllocator_() - { } - - StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : - allocator_type(rhs), - baseAllocator_(rhs.baseAllocator_) - { } - - template - StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : - allocator_type(rhs), - baseAllocator_(rhs.baseAllocator_) - { } - - /* implicit */ - StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT : - allocator_type(), - baseAllocator_(baseAllocator) - { } - - ~StdAllocator() RAPIDJSON_NOEXCEPT - { } - - template - struct rebind { - typedef StdAllocator other; + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. }; - typedef typename allocator_type::value_type value_type; - -private: - template - friend class StdAllocator; // access to StdAllocator.* - - BaseAllocator baseAllocator_; + ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. + void *userBuffer_; //!< User supplied buffer. + BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. + BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. }; -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif RAPIDJSON_NAMESPACE_END diff --git a/src/native/external/rapidjson/cursorstreamwrapper.h b/src/native/external/rapidjson/cursorstreamwrapper.h new file mode 100644 index 00000000000000..52c11a7c01d7d9 --- /dev/null +++ b/src/native/external/rapidjson/cursorstreamwrapper.h @@ -0,0 +1,78 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_CURSORSTREAMWRAPPER_H_ +#define RAPIDJSON_CURSORSTREAMWRAPPER_H_ + +#include "stream.h" + +#if defined(__GNUC__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4702) // unreachable code +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + + +//! Cursor stream wrapper for counting line and column number if error exists. +/*! + \tparam InputStream Any stream that implements Stream Concept +*/ +template > +class CursorStreamWrapper : public GenericStreamWrapper { +public: + typedef typename Encoding::Ch Ch; + + CursorStreamWrapper(InputStream& is): + GenericStreamWrapper(is), line_(1), col_(0) {} + + // counting line and column number + Ch Take() { + Ch ch = this->is_.Take(); + if(ch == '\n') { + line_ ++; + col_ = 0; + } else { + col_ ++; + } + return ch; + } + + //! Get the error line number, if error exists. + size_t GetLine() const { return line_; } + //! Get the error column number, if error exists. + size_t GetColumn() const { return col_; } + +private: + size_t line_; //!< Current Line + size_t col_; //!< Current Column +}; + +#if defined(_MSC_VER) && _MSC_VER <= 1800 +RAPIDJSON_DIAG_POP +#endif + +#if defined(__GNUC__) +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_CURSORSTREAMWRAPPER_H_ diff --git a/src/native/external/rapidjson/document.h b/src/native/external/rapidjson/document.h index 2cd9a70a60037f..74666e3423ee7d 100644 --- a/src/native/external/rapidjson/document.h +++ b/src/native/external/rapidjson/document.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -24,9 +24,6 @@ #include "encodedstream.h" #include // placement new #include -#ifdef __cpp_lib_three_way_comparison -#include -#endif RAPIDJSON_DIAG_PUSH #ifdef __clang__ @@ -42,21 +39,12 @@ RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible lo RAPIDJSON_DIAG_OFF(effc++) #endif // __GNUC__ -#ifdef GetObject -// see https://github.com/Tencent/rapidjson/issues/1448 -// a former included windows.h might have defined a macro called GetObject, which affects -// GetObject defined here. This ensures the macro does not get applied -#pragma push_macro("GetObject") -#define RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED -#undef GetObject -#endif - #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::random_access_iterator_tag #endif -#if RAPIDJSON_USE_MEMBERSMAP -#include // std::multimap +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move #endif RAPIDJSON_NAMESPACE_BEGIN @@ -68,48 +56,6 @@ class GenericValue; template class GenericDocument; -/*! \def RAPIDJSON_DEFAULT_ALLOCATOR - \ingroup RAPIDJSON_CONFIG - \brief Allows to choose default allocator. - - User can define this to use CrtAllocator or MemoryPoolAllocator. -*/ -#ifndef RAPIDJSON_DEFAULT_ALLOCATOR -#define RAPIDJSON_DEFAULT_ALLOCATOR ::RAPIDJSON_NAMESPACE::MemoryPoolAllocator<::RAPIDJSON_NAMESPACE::CrtAllocator> -#endif - -/*! \def RAPIDJSON_DEFAULT_STACK_ALLOCATOR - \ingroup RAPIDJSON_CONFIG - \brief Allows to choose default stack allocator for Document. - - User can define this to use CrtAllocator or MemoryPoolAllocator. -*/ -#ifndef RAPIDJSON_DEFAULT_STACK_ALLOCATOR -#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR ::RAPIDJSON_NAMESPACE::CrtAllocator -#endif - -/*! \def RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY - \ingroup RAPIDJSON_CONFIG - \brief User defined kDefaultObjectCapacity value. - - User can define this as any natural number. -*/ -#ifndef RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY -// number of objects that rapidjson::Value allocates memory for by default -#define RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY 16 -#endif - -/*! \def RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY - \ingroup RAPIDJSON_CONFIG - \brief User defined kDefaultArrayCapacity value. - - User can define this as any natural number. -*/ -#ifndef RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY -// number of array elements that rapidjson::Value allocates memory for by default -#define RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY 16 -#endif - //! Name-value pair in a JSON object value. /*! This class was internal to GenericValue. It used to be a inner struct. @@ -117,45 +63,15 @@ class GenericDocument; https://code.google.com/p/rapidjson/issues/detail?id=64 */ template -class GenericMember { -public: +struct GenericMember { GenericValue name; //!< name of member (must be a string) GenericValue value; //!< value of member. -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericMember(GenericMember&& rhs) RAPIDJSON_NOEXCEPT - : name(std::move(rhs.name)), - value(std::move(rhs.value)) - { - } - - //! Move assignment in C++11 - GenericMember& operator=(GenericMember&& rhs) RAPIDJSON_NOEXCEPT { - return *this = static_cast(rhs); - } -#endif - - //! Assignment with move semantics. - /*! \param rhs Source of the assignment. Its name and value will become a null value after assignment. - */ - GenericMember& operator=(GenericMember& rhs) RAPIDJSON_NOEXCEPT { - if (RAPIDJSON_LIKELY(this != &rhs)) { - name = rhs.name; - value = rhs.value; - } - return *this; - } - // swap() for std::sort() and other potential use in STL. friend inline void swap(GenericMember& a, GenericMember& b) RAPIDJSON_NOEXCEPT { a.name.Swap(b.name); a.value.Swap(b.value); } - -private: - //! Copy constructor is not permitted. - GenericMember(const GenericMember& rhs); }; /////////////////////////////////////////////////////////////////////////////// @@ -259,16 +175,12 @@ class GenericMemberIterator { //! @name relations //@{ - template bool operator==(const GenericMemberIterator& that) const { return ptr_ == that.ptr_; } - template bool operator!=(const GenericMemberIterator& that) const { return ptr_ != that.ptr_; } - template bool operator<=(const GenericMemberIterator& that) const { return ptr_ <= that.ptr_; } - template bool operator>=(const GenericMemberIterator& that) const { return ptr_ >= that.ptr_; } - template bool operator< (const GenericMemberIterator& that) const { return ptr_ < that.ptr_; } - template bool operator> (const GenericMemberIterator& that) const { return ptr_ > that.ptr_; } - -#ifdef __cpp_lib_three_way_comparison - template std::strong_ordering operator<=>(const GenericMemberIterator& that) const { return ptr_ <=> that.ptr_; } -#endif + bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } + bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } + bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } + bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } + bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } + bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } //@} //! @name dereference @@ -298,14 +210,12 @@ class GenericMemberIterator; //! non-const GenericMemberIterator template class GenericMemberIterator { -public: //! use plain pointer as iterator type typedef GenericMember* Iterator; }; //! const GenericMemberIterator template class GenericMemberIterator { -public: //! use plain const pointer as iterator type typedef const GenericMember* Iterator; }; @@ -664,7 +574,7 @@ template class GenericObject; \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) \tparam Allocator Allocator type for allocating memory of object, array and string. */ -template +template > class GenericValue { public: //! Name-value pair in an object. @@ -741,8 +651,18 @@ class GenericValue { template GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { switch (rhs.GetType()) { - case kObjectType: - DoCopyMembers(rhs, allocator, copyConstStrings); + case kObjectType: { + SizeType count = rhs.data_.o.size; + Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); + } + data_.f.flags = kObjectFlag; + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } break; case kArrayType: { SizeType count = rhs.data_.a.size; @@ -878,30 +798,25 @@ class GenericValue { /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { - // With RAPIDJSON_USE_MEMBERSMAP, the maps need to be destroyed to release - // their Allocator if it's refcounted (e.g. MemoryPoolAllocator). - if (Allocator::kNeedFree || (RAPIDJSON_USE_MEMBERSMAP+0 && - internal::IsRefCounted::Value)) { + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait switch(data_.f.flags) { case kArrayFlag: { GenericValue* e = GetElementsPointer(); for (GenericValue* v = e; v != e + data_.a.size; ++v) v->~GenericValue(); - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - Allocator::Free(e); - } + Allocator::Free(e); } break; case kObjectFlag: - DoFreeMembers(); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); break; case kCopyStringFlag: - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - Allocator::Free(const_cast(GetStringPointer())); - } + Allocator::Free(const_cast(GetStringPointer())); break; default: @@ -920,13 +835,8 @@ class GenericValue { */ GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { if (RAPIDJSON_LIKELY(this != &rhs)) { - // Can't destroy "this" before assigning "rhs", otherwise "rhs" - // could be used after free if it's an sub-Value of "this", - // hence the temporary danse. - GenericValue temp; - temp.RawAssign(rhs); this->~GenericValue(); - RawAssign(temp); + RawAssign(rhs); } return *this; } @@ -1078,7 +988,6 @@ class GenericValue { */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } -#ifndef __cpp_impl_three_way_comparison //! Not-equal-to operator /*! \return !(*this == rhs) */ @@ -1103,7 +1012,6 @@ class GenericValue { */ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } //@} -#endif //!@name Type //@{ @@ -1230,28 +1138,13 @@ class GenericValue { else { RAPIDJSON_ASSERT(false); // see above note -#if RAPIDJSON_HAS_CXX11 - // Use thread-local storage to prevent races between threads. - // Use static buffer and placement-new to prevent destruction, with - // alignas() to ensure proper alignment. - alignas(GenericValue) thread_local static char buffer[sizeof(GenericValue)]; - return *new (buffer) GenericValue(); -#elif defined(_MSC_VER) && _MSC_VER < 1900 - // There's no way to solve both thread locality and proper alignment - // simultaneously. - __declspec(thread) static char buffer[sizeof(GenericValue)]; + // This will generate -Wexit-time-destructors in clang + // static GenericValue NullValue; + // return NullValue; + + // Use static buffer and placement-new to prevent destruction + static char buffer[sizeof(GenericValue)]; return *new (buffer) GenericValue(); -#elif defined(__GNUC__) || defined(__clang__) - // This will generate -Wexit-time-destructors in clang, but that's - // better than having under-alignment. - __thread static GenericValue buffer; - return buffer; -#else - // Don't know what compiler this is, so don't know how to ensure - // thread-locality. - static GenericValue buffer; - return buffer; -#endif } } template @@ -1284,7 +1177,10 @@ class GenericValue { */ GenericValue& MemberReserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsObject()); - DoReserveMembers(newCapacity, allocator); + if (newCapacity > data_.o.capacity) { + SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), data_.o.capacity * sizeof(Member), newCapacity * sizeof(Member)))); + data_.o.capacity = newCapacity; + } return *this; } @@ -1358,7 +1254,11 @@ class GenericValue { MemberIterator FindMember(const GenericValue& name) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - return DoFindMember(name); + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; } template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } @@ -1387,7 +1287,14 @@ class GenericValue { GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - DoAddMember(name, value, allocator); + + ObjectData& o = data_.o; + if (o.size >= o.capacity) + MemberReserve(o.capacity == 0 ? kDefaultObjectCapacity : (o.capacity + (o.capacity + 1) / 2), allocator); + Member* members = GetMembersPointer(); + members[o.size].name.RawAssign(name); + members[o.size].value.RawAssign(value); + o.size++; return *this; } @@ -1521,7 +1428,9 @@ class GenericValue { */ void RemoveAllMembers() { RAPIDJSON_ASSERT(IsObject()); - DoClearMembers(); + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; } //! Remove a member in object by its name. @@ -1565,7 +1474,14 @@ class GenericValue { RAPIDJSON_ASSERT(data_.o.size > 0); RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - return DoRemoveMember(m); + + MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); + if (data_.o.size > 1 && m != last) + *m = *last; // Move the last one to this place + else + m->~Member(); // Only one left, just destroy + --data_.o.size; + return m; } //! Remove a member from an object by iterator. @@ -1597,7 +1513,13 @@ class GenericValue { RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); - return DoEraseMembers(first, last); + + MemberIterator pos = MemberBegin() + (first - MemberBegin()); + for (MemberIterator itr = pos; itr != last; ++itr) + itr->~Member(); + std::memmove(static_cast(&*pos), &*last, static_cast(MemberEnd() - last) * sizeof(Member)); + data_.o.size -= static_cast(last - first); + return pos; } //! Erase a member in object by its name. @@ -1626,9 +1548,7 @@ class GenericValue { } Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } - Object GetObj() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } - ConstObject GetObj() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } //@} @@ -1850,12 +1770,12 @@ class GenericValue { //!@name String //@{ - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return DataString(data_); } + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return DataStringLength(data_); } + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. @@ -1966,7 +1886,7 @@ class GenericValue { case kArrayType: if (RAPIDJSON_UNLIKELY(!handler.StartArray())) return false; - for (ConstValueIterator v = Begin(); v != End(); ++v) + for (const GenericValue* v = Begin(); v != End(); ++v) if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) return false; return handler.EndArray(data_.a.size); @@ -2002,26 +1922,25 @@ class GenericValue { // Initial flags of different types. kNullFlag = kNullType, - // These casts are added to suppress the warning on MSVC about bitwise operations between enums of different types. - kTrueFlag = static_cast(kTrueType) | static_cast(kBoolFlag), - kFalseFlag = static_cast(kFalseType) | static_cast(kBoolFlag), - kNumberIntFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kIntFlag | kInt64Flag), - kNumberUintFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag), - kNumberInt64Flag = static_cast(kNumberType) | static_cast(kNumberFlag | kInt64Flag), - kNumberUint64Flag = static_cast(kNumberType) | static_cast(kNumberFlag | kUint64Flag), - kNumberDoubleFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kDoubleFlag), - kNumberAnyFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag), - kConstStringFlag = static_cast(kStringType) | static_cast(kStringFlag), - kCopyStringFlag = static_cast(kStringType) | static_cast(kStringFlag | kCopyFlag), - kShortStringFlag = static_cast(kStringType) | static_cast(kStringFlag | kCopyFlag | kInlineStrFlag), + kTrueFlag = kTrueType | kBoolFlag, + kFalseFlag = kFalseType | kBoolFlag, + kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, + kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, + kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, + kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, + kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, + kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, + kConstStringFlag = kStringType | kStringFlag, + kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, + kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, kObjectFlag = kObjectType, kArrayFlag = kArrayType, kTypeMask = 0x07 }; - static const SizeType kDefaultArrayCapacity = RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY; - static const SizeType kDefaultObjectCapacity = RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY; + static const SizeType kDefaultArrayCapacity = 16; + static const SizeType kDefaultObjectCapacity = 16; struct Flag { #if RAPIDJSON_48BITPOINTER_OPTIMIZATION @@ -2104,13 +2023,6 @@ class GenericValue { Flag f; }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION - static RAPIDJSON_FORCEINLINE const Ch* DataString(const Data& data) { - return (data.f.flags & kInlineStrFlag) ? data.ss.str : RAPIDJSON_GETPOINTER(Ch, data.s.str); - } - static RAPIDJSON_FORCEINLINE SizeType DataStringLength(const Data& data) { - return (data.f.flags & kInlineStrFlag) ? data.ss.GetLength() : data.s.length; - } - RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } @@ -2118,286 +2030,6 @@ class GenericValue { RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } -#if RAPIDJSON_USE_MEMBERSMAP - - struct MapTraits { - struct Less { - bool operator()(const Data& s1, const Data& s2) const { - SizeType n1 = DataStringLength(s1), n2 = DataStringLength(s2); - int cmp = std::memcmp(DataString(s1), DataString(s2), sizeof(Ch) * (n1 < n2 ? n1 : n2)); - return cmp < 0 || (cmp == 0 && n1 < n2); - } - }; - typedef std::pair Pair; - typedef std::multimap > Map; - typedef typename Map::iterator Iterator; - }; - typedef typename MapTraits::Map Map; - typedef typename MapTraits::Less MapLess; - typedef typename MapTraits::Pair MapPair; - typedef typename MapTraits::Iterator MapIterator; - - // - // Layout of the members' map/array, re(al)located according to the needed capacity: - // - // {Map*}<>{capacity}<>{Member[capacity]}<>{MapIterator[capacity]} - // - // (where <> stands for the RAPIDJSON_ALIGN-ment, if needed) - // - - static RAPIDJSON_FORCEINLINE size_t GetMapLayoutSize(SizeType capacity) { - return RAPIDJSON_ALIGN(sizeof(Map*)) + - RAPIDJSON_ALIGN(sizeof(SizeType)) + - RAPIDJSON_ALIGN(capacity * sizeof(Member)) + - capacity * sizeof(MapIterator); - } - - static RAPIDJSON_FORCEINLINE SizeType &GetMapCapacity(Map* &map) { - return *reinterpret_cast(reinterpret_cast(&map) + - RAPIDJSON_ALIGN(sizeof(Map*))); - } - - static RAPIDJSON_FORCEINLINE Member* GetMapMembers(Map* &map) { - return reinterpret_cast(reinterpret_cast(&map) + - RAPIDJSON_ALIGN(sizeof(Map*)) + - RAPIDJSON_ALIGN(sizeof(SizeType))); - } - - static RAPIDJSON_FORCEINLINE MapIterator* GetMapIterators(Map* &map) { - return reinterpret_cast(reinterpret_cast(&map) + - RAPIDJSON_ALIGN(sizeof(Map*)) + - RAPIDJSON_ALIGN(sizeof(SizeType)) + - RAPIDJSON_ALIGN(GetMapCapacity(map) * sizeof(Member))); - } - - static RAPIDJSON_FORCEINLINE Map* &GetMap(Member* members) { - RAPIDJSON_ASSERT(members != 0); - return *reinterpret_cast(reinterpret_cast(members) - - RAPIDJSON_ALIGN(sizeof(SizeType)) - - RAPIDJSON_ALIGN(sizeof(Map*))); - } - - // Some compilers' debug mechanisms want all iterators to be destroyed, for their accounting.. - RAPIDJSON_FORCEINLINE MapIterator DropMapIterator(MapIterator& rhs) { -#if RAPIDJSON_HAS_CXX11 - MapIterator ret = std::move(rhs); -#else - MapIterator ret = rhs; -#endif - rhs.~MapIterator(); - return ret; - } - - Map* &DoReallocMap(Map** oldMap, SizeType newCapacity, Allocator& allocator) { - Map **newMap = static_cast(allocator.Malloc(GetMapLayoutSize(newCapacity))); - GetMapCapacity(*newMap) = newCapacity; - if (!oldMap) { - *newMap = new (allocator.Malloc(sizeof(Map))) Map(MapLess(), allocator); - } - else { - *newMap = *oldMap; - size_t count = (*oldMap)->size(); - std::memcpy(static_cast(GetMapMembers(*newMap)), - static_cast(GetMapMembers(*oldMap)), - count * sizeof(Member)); - MapIterator *oldIt = GetMapIterators(*oldMap), - *newIt = GetMapIterators(*newMap); - while (count--) { - new (&newIt[count]) MapIterator(DropMapIterator(oldIt[count])); - } - Allocator::Free(oldMap); - } - return *newMap; - } - - RAPIDJSON_FORCEINLINE Member* DoAllocMembers(SizeType capacity, Allocator& allocator) { - return GetMapMembers(DoReallocMap(0, capacity, allocator)); - } - - void DoReserveMembers(SizeType newCapacity, Allocator& allocator) { - ObjectData& o = data_.o; - if (newCapacity > o.capacity) { - Member* oldMembers = GetMembersPointer(); - Map **oldMap = oldMembers ? &GetMap(oldMembers) : 0, - *&newMap = DoReallocMap(oldMap, newCapacity, allocator); - RAPIDJSON_SETPOINTER(Member, o.members, GetMapMembers(newMap)); - o.capacity = newCapacity; - } - } - - template - MemberIterator DoFindMember(const GenericValue& name) { - if (Member* members = GetMembersPointer()) { - Map* &map = GetMap(members); - MapIterator mit = map->find(reinterpret_cast(name.data_)); - if (mit != map->end()) { - return MemberIterator(&members[mit->second]); - } - } - return MemberEnd(); - } - - void DoClearMembers() { - if (Member* members = GetMembersPointer()) { - Map* &map = GetMap(members); - MapIterator* mit = GetMapIterators(map); - for (SizeType i = 0; i < data_.o.size; i++) { - map->erase(DropMapIterator(mit[i])); - members[i].~Member(); - } - data_.o.size = 0; - } - } - - void DoFreeMembers() { - if (Member* members = GetMembersPointer()) { - GetMap(members)->~Map(); - for (SizeType i = 0; i < data_.o.size; i++) { - members[i].~Member(); - } - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait - Map** map = &GetMap(members); - Allocator::Free(*map); - Allocator::Free(map); - } - } - } - -#else // !RAPIDJSON_USE_MEMBERSMAP - - RAPIDJSON_FORCEINLINE Member* DoAllocMembers(SizeType capacity, Allocator& allocator) { - return Malloc(allocator, capacity); - } - - void DoReserveMembers(SizeType newCapacity, Allocator& allocator) { - ObjectData& o = data_.o; - if (newCapacity > o.capacity) { - Member* newMembers = Realloc(allocator, GetMembersPointer(), o.capacity, newCapacity); - RAPIDJSON_SETPOINTER(Member, o.members, newMembers); - o.capacity = newCapacity; - } - } - - template - MemberIterator DoFindMember(const GenericValue& name) { - MemberIterator member = MemberBegin(); - for ( ; member != MemberEnd(); ++member) - if (name.StringEqual(member->name)) - break; - return member; - } - - void DoClearMembers() { - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - data_.o.size = 0; - } - - void DoFreeMembers() { - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - Allocator::Free(GetMembersPointer()); - } - -#endif // !RAPIDJSON_USE_MEMBERSMAP - - void DoAddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { - ObjectData& o = data_.o; - if (o.size >= o.capacity) - DoReserveMembers(o.capacity ? (o.capacity + (o.capacity + 1) / 2) : kDefaultObjectCapacity, allocator); - Member* members = GetMembersPointer(); - Member* m = members + o.size; - m->name.RawAssign(name); - m->value.RawAssign(value); -#if RAPIDJSON_USE_MEMBERSMAP - Map* &map = GetMap(members); - MapIterator* mit = GetMapIterators(map); - new (&mit[o.size]) MapIterator(map->insert(MapPair(m->name.data_, o.size))); -#endif - ++o.size; - } - - MemberIterator DoRemoveMember(MemberIterator m) { - ObjectData& o = data_.o; - Member* members = GetMembersPointer(); -#if RAPIDJSON_USE_MEMBERSMAP - Map* &map = GetMap(members); - MapIterator* mit = GetMapIterators(map); - SizeType mpos = static_cast(&*m - members); - map->erase(DropMapIterator(mit[mpos])); -#endif - MemberIterator last(members + (o.size - 1)); - if (o.size > 1 && m != last) { -#if RAPIDJSON_USE_MEMBERSMAP - new (&mit[mpos]) MapIterator(DropMapIterator(mit[&*last - members])); - mit[mpos]->second = mpos; -#endif - *m = *last; // Move the last one to this place - } - else { - m->~Member(); // Only one left, just destroy - } - --o.size; - return m; - } - - MemberIterator DoEraseMembers(ConstMemberIterator first, ConstMemberIterator last) { - ObjectData& o = data_.o; - MemberIterator beg = MemberBegin(), - pos = beg + (first - beg), - end = MemberEnd(); -#if RAPIDJSON_USE_MEMBERSMAP - Map* &map = GetMap(GetMembersPointer()); - MapIterator* mit = GetMapIterators(map); -#endif - for (MemberIterator itr = pos; itr != last; ++itr) { -#if RAPIDJSON_USE_MEMBERSMAP - map->erase(DropMapIterator(mit[itr - beg])); -#endif - itr->~Member(); - } -#if RAPIDJSON_USE_MEMBERSMAP - if (first != last) { - // Move remaining members/iterators - MemberIterator next = pos + (last - first); - for (MemberIterator itr = pos; next != end; ++itr, ++next) { - std::memcpy(static_cast(&*itr), &*next, sizeof(Member)); - SizeType mpos = static_cast(itr - beg); - new (&mit[mpos]) MapIterator(DropMapIterator(mit[next - beg])); - mit[mpos]->second = mpos; - } - } -#else - std::memmove(static_cast(&*pos), &*last, - static_cast(end - last) * sizeof(Member)); -#endif - o.size -= static_cast(last - first); - return pos; - } - - template - void DoCopyMembers(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings) { - RAPIDJSON_ASSERT(rhs.GetType() == kObjectType); - - data_.f.flags = kObjectFlag; - SizeType count = rhs.data_.o.size; - Member* lm = DoAllocMembers(count, allocator); - const typename GenericValue::Member* rm = rhs.GetMembersPointer(); -#if RAPIDJSON_USE_MEMBERSMAP - Map* &map = GetMap(lm); - MapIterator* mit = GetMapIterators(map); -#endif - for (SizeType i = 0; i < count; i++) { - new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); - new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); -#if RAPIDJSON_USE_MEMBERSMAP - new (&mit[i]) MapIterator(map->insert(MapPair(lm[i].name.data_, i))); -#endif - } - data_.o.size = data_.o.capacity = count; - SetMembersPointer(lm); - } - // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { data_.f.flags = kArrayFlag; @@ -2415,16 +2047,9 @@ class GenericValue { void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { data_.f.flags = kObjectFlag; if (count) { - Member* m = DoAllocMembers(count, allocator); + Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); SetMembersPointer(m); std::memcpy(static_cast(m), members, count * sizeof(Member)); -#if RAPIDJSON_USE_MEMBERSMAP - Map* &map = GetMap(m); - MapIterator* mit = GetMapIterators(map); - for (SizeType i = 0; i < count; i++) { - new (&mit[i]) MapIterator(map->insert(MapPair(m[i].name.data_, i))); - } -#endif } else SetMembersPointer(0); @@ -2469,11 +2094,11 @@ class GenericValue { const SizeType len1 = GetStringLength(); const SizeType len2 = rhs.GetStringLength(); - if(len1 != len2) { return false; } + if (len1 != len2) { return false; } const Ch* const str1 = GetString(); const Ch* const str2 = rhs.GetString(); - if(str1 == str2) { return true; } // fast path for constant string + if (str1 == str2) { return true; } // fast path for constant string return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); } @@ -2495,13 +2120,12 @@ typedef GenericValue > Value; \tparam StackAllocator Allocator for allocating memory for stack during parsing. \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. */ -template +template , typename StackAllocator = CrtAllocator> class GenericDocument : public GenericValue { public: typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. typedef GenericValue ValueType; //!< Value type of the document. typedef Allocator AllocatorType; //!< Allocator type from template parameter. - typedef StackAllocator StackAllocatorType; //!< StackAllocator type from template parameter. //! Constructor /*! Creates an empty document of specified type. @@ -2546,13 +2170,6 @@ class GenericDocument : public GenericValue { #endif ~GenericDocument() { - // Clear the ::ValueType before ownAllocator is destroyed, ~ValueType() - // runs last and may access its elements or members which would be freed - // with an allocator like MemoryPoolAllocator (CrtAllocator does not - // free its data when destroyed, but MemoryPoolAllocator does). - if (ownAllocator_) { - ValueType::SetNull(); - } Destroy(); } @@ -2888,7 +2505,6 @@ class GenericDocument : public GenericValue { //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; - //! Helper class for accessing Value of array type. /*! Instance of this helper class is obtained by \c GenericValue::GetArray(). @@ -2913,7 +2529,6 @@ class GenericArray { GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } ~GenericArray() {} - operator ValueType&() const { return value_; } SizeType Size() const { return value_.Size(); } SizeType Capacity() const { return value_.Capacity(); } bool Empty() const { return value_.Empty(); } @@ -2969,7 +2584,6 @@ class GenericObject { GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } ~GenericObject() {} - operator ValueType&() const { return value_; } SizeType MemberCount() const { return value_.MemberCount(); } SizeType MemberCapacity() const { return value_.MemberCapacity(); } bool ObjectEmpty() const { return value_.ObjectEmpty(); } @@ -3035,9 +2649,4 @@ class GenericObject { RAPIDJSON_NAMESPACE_END RAPIDJSON_DIAG_POP -#ifdef RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED -#pragma pop_macro("GetObject") -#undef RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED -#endif - #endif // RAPIDJSON_DOCUMENT_H_ diff --git a/src/native/external/rapidjson/encodedstream.h b/src/native/external/rapidjson/encodedstream.h index cf046b89235f57..223601c0599b4b 100644 --- a/src/native/external/rapidjson/encodedstream.h +++ b/src/native/external/rapidjson/encodedstream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/encodings.h b/src/native/external/rapidjson/encodings.h index 50ad18bdc08cd0..0b244679501588 100644 --- a/src/native/external/rapidjson/encodings.h +++ b/src/native/external/rapidjson/encodings.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/error/en.h b/src/native/external/rapidjson/error/en.h index c87b04eb133ed5..2db838bff2399d 100644 --- a/src/native/external/rapidjson/error/en.h +++ b/src/native/external/rapidjson/error/en.h @@ -1,15 +1,15 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_EN_H_ @@ -39,13 +39,13 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); - + case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); - + case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); - + case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); @@ -65,108 +65,6 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro } } -//! Maps error code of validation into error message. -/*! - \ingroup RAPIDJSON_ERRORS - \param validateErrorCode Error code obtained from validator. - \return the error message. - \note User can make a copy of this function for localization. - Using switch-case is safer for future modification of error codes. -*/ -inline const RAPIDJSON_ERROR_CHARTYPE* GetValidateError_En(ValidateErrorCode validateErrorCode) { - switch (validateErrorCode) { - case kValidateErrors: return RAPIDJSON_ERROR_STRING("One or more validation errors have occurred"); - case kValidateErrorNone: return RAPIDJSON_ERROR_STRING("No error."); - - case kValidateErrorMultipleOf: return RAPIDJSON_ERROR_STRING("Number '%actual' is not a multiple of the 'multipleOf' value '%expected'."); - case kValidateErrorMaximum: return RAPIDJSON_ERROR_STRING("Number '%actual' is greater than the 'maximum' value '%expected'."); - case kValidateErrorExclusiveMaximum: return RAPIDJSON_ERROR_STRING("Number '%actual' is greater than or equal to the 'exclusiveMaximum' value '%expected'."); - case kValidateErrorMinimum: return RAPIDJSON_ERROR_STRING("Number '%actual' is less than the 'minimum' value '%expected'."); - case kValidateErrorExclusiveMinimum: return RAPIDJSON_ERROR_STRING("Number '%actual' is less than or equal to the 'exclusiveMinimum' value '%expected'."); - - case kValidateErrorMaxLength: return RAPIDJSON_ERROR_STRING("String '%actual' is longer than the 'maxLength' value '%expected'."); - case kValidateErrorMinLength: return RAPIDJSON_ERROR_STRING("String '%actual' is shorter than the 'minLength' value '%expected'."); - case kValidateErrorPattern: return RAPIDJSON_ERROR_STRING("String '%actual' does not match the 'pattern' regular expression."); - - case kValidateErrorMaxItems: return RAPIDJSON_ERROR_STRING("Array of length '%actual' is longer than the 'maxItems' value '%expected'."); - case kValidateErrorMinItems: return RAPIDJSON_ERROR_STRING("Array of length '%actual' is shorter than the 'minItems' value '%expected'."); - case kValidateErrorUniqueItems: return RAPIDJSON_ERROR_STRING("Array has duplicate items at indices '%duplicates' but 'uniqueItems' is true."); - case kValidateErrorAdditionalItems: return RAPIDJSON_ERROR_STRING("Array has an additional item at index '%disallowed' that is not allowed by the schema."); - - case kValidateErrorMaxProperties: return RAPIDJSON_ERROR_STRING("Object has '%actual' members which is more than 'maxProperties' value '%expected'."); - case kValidateErrorMinProperties: return RAPIDJSON_ERROR_STRING("Object has '%actual' members which is less than 'minProperties' value '%expected'."); - case kValidateErrorRequired: return RAPIDJSON_ERROR_STRING("Object is missing the following members required by the schema: '%missing'."); - case kValidateErrorAdditionalProperties: return RAPIDJSON_ERROR_STRING("Object has an additional member '%disallowed' that is not allowed by the schema."); - case kValidateErrorPatternProperties: return RAPIDJSON_ERROR_STRING("Object has 'patternProperties' that are not allowed by the schema."); - case kValidateErrorDependencies: return RAPIDJSON_ERROR_STRING("Object has missing property or schema dependencies, refer to following errors."); - - case kValidateErrorEnum: return RAPIDJSON_ERROR_STRING("Property has a value that is not one of its allowed enumerated values."); - case kValidateErrorType: return RAPIDJSON_ERROR_STRING("Property has a type '%actual' that is not in the following list: '%expected'."); - - case kValidateErrorOneOf: return RAPIDJSON_ERROR_STRING("Property did not match any of the sub-schemas specified by 'oneOf', refer to following errors."); - case kValidateErrorOneOfMatch: return RAPIDJSON_ERROR_STRING("Property matched more than one of the sub-schemas specified by 'oneOf', indices '%matches'."); - case kValidateErrorAllOf: return RAPIDJSON_ERROR_STRING("Property did not match all of the sub-schemas specified by 'allOf', refer to following errors."); - case kValidateErrorAnyOf: return RAPIDJSON_ERROR_STRING("Property did not match any of the sub-schemas specified by 'anyOf', refer to following errors."); - case kValidateErrorNot: return RAPIDJSON_ERROR_STRING("Property matched the sub-schema specified by 'not'."); - - case kValidateErrorReadOnly: return RAPIDJSON_ERROR_STRING("Property is read-only but has been provided when validation is for writing."); - case kValidateErrorWriteOnly: return RAPIDJSON_ERROR_STRING("Property is write-only but has been provided when validation is for reading."); - - default: return RAPIDJSON_ERROR_STRING("Unknown error."); - } -} - -//! Maps error code of schema document compilation into error message. -/*! - \ingroup RAPIDJSON_ERRORS - \param schemaErrorCode Error code obtained from compiling the schema document. - \return the error message. - \note User can make a copy of this function for localization. - Using switch-case is safer for future modification of error codes. -*/ - inline const RAPIDJSON_ERROR_CHARTYPE* GetSchemaError_En(SchemaErrorCode schemaErrorCode) { - switch (schemaErrorCode) { - case kSchemaErrorNone: return RAPIDJSON_ERROR_STRING("No error."); - - case kSchemaErrorStartUnknown: return RAPIDJSON_ERROR_STRING("Pointer '%value' to start of schema does not resolve to a location in the document."); - case kSchemaErrorRefPlainName: return RAPIDJSON_ERROR_STRING("$ref fragment '%value' must be a JSON pointer."); - case kSchemaErrorRefInvalid: return RAPIDJSON_ERROR_STRING("$ref must not be an empty string."); - case kSchemaErrorRefPointerInvalid: return RAPIDJSON_ERROR_STRING("$ref fragment '%value' is not a valid JSON pointer at offset '%offset'."); - case kSchemaErrorRefUnknown: return RAPIDJSON_ERROR_STRING("$ref '%value' does not resolve to a location in the target document."); - case kSchemaErrorRefCyclical: return RAPIDJSON_ERROR_STRING("$ref '%value' is cyclical."); - case kSchemaErrorRefNoRemoteProvider: return RAPIDJSON_ERROR_STRING("$ref is remote but there is no remote provider."); - case kSchemaErrorRefNoRemoteSchema: return RAPIDJSON_ERROR_STRING("$ref '%value' is remote but the remote provider did not return a schema."); - case kSchemaErrorRegexInvalid: return RAPIDJSON_ERROR_STRING("Invalid regular expression '%value' in 'pattern' or 'patternProperties'."); - case kSchemaErrorSpecUnknown: return RAPIDJSON_ERROR_STRING("JSON schema draft or OpenAPI version is not recognized."); - case kSchemaErrorSpecUnsupported: return RAPIDJSON_ERROR_STRING("JSON schema draft or OpenAPI version is not supported."); - case kSchemaErrorSpecIllegal: return RAPIDJSON_ERROR_STRING("Both JSON schema draft and OpenAPI version found in document."); - case kSchemaErrorReadOnlyAndWriteOnly: return RAPIDJSON_ERROR_STRING("Property must not be both 'readOnly' and 'writeOnly'."); - - default: return RAPIDJSON_ERROR_STRING("Unknown error."); - } - } - -//! Maps error code of pointer parse into error message. -/*! - \ingroup RAPIDJSON_ERRORS - \param pointerParseErrorCode Error code obtained from pointer parse. - \return the error message. - \note User can make a copy of this function for localization. - Using switch-case is safer for future modification of error codes. -*/ -inline const RAPIDJSON_ERROR_CHARTYPE* GetPointerParseError_En(PointerParseErrorCode pointerParseErrorCode) { - switch (pointerParseErrorCode) { - case kPointerParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); - - case kPointerParseErrorTokenMustBeginWithSolidus: return RAPIDJSON_ERROR_STRING("A token must begin with a '/'."); - case kPointerParseErrorInvalidEscape: return RAPIDJSON_ERROR_STRING("Invalid escape."); - case kPointerParseErrorInvalidPercentEncoding: return RAPIDJSON_ERROR_STRING("Invalid percent encoding in URI fragment."); - case kPointerParseErrorCharacterMustPercentEncode: return RAPIDJSON_ERROR_STRING("A character must be percent encoded in a URI fragment."); - - default: return RAPIDJSON_ERROR_STRING("Unknown error."); - } -} - RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/src/native/external/rapidjson/error/error.h b/src/native/external/rapidjson/error/error.h index cae345db36d2c1..9311d2f03bffeb 100644 --- a/src/native/external/rapidjson/error/error.h +++ b/src/native/external/rapidjson/error/error.h @@ -1,15 +1,15 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_ERROR_H_ @@ -42,7 +42,7 @@ RAPIDJSON_DIAG_OFF(padded) /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ERROR_STRING -//! Macro for converting string literal to \ref RAPIDJSON_ERROR_CHARTYPE[]. +//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. /*! \ingroup RAPIDJSON_ERRORS By default this conversion macro does nothing. On Windows, user can define this macro as \c _T(x) for supporting both @@ -152,130 +152,6 @@ struct ParseResult { */ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); -/////////////////////////////////////////////////////////////////////////////// -// ValidateErrorCode - -//! Error codes when validating. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericSchemaValidator -*/ -enum ValidateErrorCode { - kValidateErrors = -1, //!< Top level error code when kValidateContinueOnErrorsFlag set. - kValidateErrorNone = 0, //!< No error. - - kValidateErrorMultipleOf, //!< Number is not a multiple of the 'multipleOf' value. - kValidateErrorMaximum, //!< Number is greater than the 'maximum' value. - kValidateErrorExclusiveMaximum, //!< Number is greater than or equal to the 'maximum' value. - kValidateErrorMinimum, //!< Number is less than the 'minimum' value. - kValidateErrorExclusiveMinimum, //!< Number is less than or equal to the 'minimum' value. - - kValidateErrorMaxLength, //!< String is longer than the 'maxLength' value. - kValidateErrorMinLength, //!< String is longer than the 'maxLength' value. - kValidateErrorPattern, //!< String does not match the 'pattern' regular expression. - - kValidateErrorMaxItems, //!< Array is longer than the 'maxItems' value. - kValidateErrorMinItems, //!< Array is shorter than the 'minItems' value. - kValidateErrorUniqueItems, //!< Array has duplicate items but 'uniqueItems' is true. - kValidateErrorAdditionalItems, //!< Array has additional items that are not allowed by the schema. - - kValidateErrorMaxProperties, //!< Object has more members than 'maxProperties' value. - kValidateErrorMinProperties, //!< Object has less members than 'minProperties' value. - kValidateErrorRequired, //!< Object is missing one or more members required by the schema. - kValidateErrorAdditionalProperties, //!< Object has additional members that are not allowed by the schema. - kValidateErrorPatternProperties, //!< See other errors. - kValidateErrorDependencies, //!< Object has missing property or schema dependencies. - - kValidateErrorEnum, //!< Property has a value that is not one of its allowed enumerated values. - kValidateErrorType, //!< Property has a type that is not allowed by the schema. - - kValidateErrorOneOf, //!< Property did not match any of the sub-schemas specified by 'oneOf'. - kValidateErrorOneOfMatch, //!< Property matched more than one of the sub-schemas specified by 'oneOf'. - kValidateErrorAllOf, //!< Property did not match all of the sub-schemas specified by 'allOf'. - kValidateErrorAnyOf, //!< Property did not match any of the sub-schemas specified by 'anyOf'. - kValidateErrorNot, //!< Property matched the sub-schema specified by 'not'. - - kValidateErrorReadOnly, //!< Property is read-only but has been provided when validation is for writing - kValidateErrorWriteOnly //!< Property is write-only but has been provided when validation is for reading -}; - -//! Function pointer type of GetValidateError(). -/*! \ingroup RAPIDJSON_ERRORS - - This is the prototype for \c GetValidateError_X(), where \c X is a locale. - User can dynamically change locale in runtime, e.g.: -\code - GetValidateErrorFunc GetValidateError = GetValidateError_En; // or whatever - const RAPIDJSON_ERROR_CHARTYPE* s = GetValidateError(validator.GetInvalidSchemaCode()); -\endcode -*/ -typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetValidateErrorFunc)(ValidateErrorCode); - -/////////////////////////////////////////////////////////////////////////////// -// SchemaErrorCode - -//! Error codes when validating. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericSchemaValidator -*/ -enum SchemaErrorCode { - kSchemaErrorNone = 0, //!< No error. - - kSchemaErrorStartUnknown, //!< Pointer to start of schema does not resolve to a location in the document - kSchemaErrorRefPlainName, //!< $ref fragment must be a JSON pointer - kSchemaErrorRefInvalid, //!< $ref must not be an empty string - kSchemaErrorRefPointerInvalid, //!< $ref fragment is not a valid JSON pointer at offset - kSchemaErrorRefUnknown, //!< $ref does not resolve to a location in the target document - kSchemaErrorRefCyclical, //!< $ref is cyclical - kSchemaErrorRefNoRemoteProvider, //!< $ref is remote but there is no remote provider - kSchemaErrorRefNoRemoteSchema, //!< $ref is remote but the remote provider did not return a schema - kSchemaErrorRegexInvalid, //!< Invalid regular expression in 'pattern' or 'patternProperties' - kSchemaErrorSpecUnknown, //!< JSON schema draft or OpenAPI version is not recognized - kSchemaErrorSpecUnsupported, //!< JSON schema draft or OpenAPI version is not supported - kSchemaErrorSpecIllegal, //!< Both JSON schema draft and OpenAPI version found in document - kSchemaErrorReadOnlyAndWriteOnly //!< Property must not be both 'readOnly' and 'writeOnly' -}; - -//! Function pointer type of GetSchemaError(). -/*! \ingroup RAPIDJSON_ERRORS - - This is the prototype for \c GetSchemaError_X(), where \c X is a locale. - User can dynamically change locale in runtime, e.g.: -\code - GetSchemaErrorFunc GetSchemaError = GetSchemaError_En; // or whatever - const RAPIDJSON_ERROR_CHARTYPE* s = GetSchemaError(validator.GetInvalidSchemaCode()); -\endcode -*/ -typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetSchemaErrorFunc)(SchemaErrorCode); - -/////////////////////////////////////////////////////////////////////////////// -// PointerParseErrorCode - -//! Error code of JSON pointer parsing. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode -*/ -enum PointerParseErrorCode { - kPointerParseErrorNone = 0, //!< The parse is successful - - kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' - kPointerParseErrorInvalidEscape, //!< Invalid escape - kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment - kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment -}; - -//! Function pointer type of GetPointerParseError(). -/*! \ingroup RAPIDJSON_ERRORS - - This is the prototype for \c GetPointerParseError_X(), where \c X is a locale. - User can dynamically change locale in runtime, e.g.: -\code - GetPointerParseErrorFunc GetPointerParseError = GetPointerParseError_En; // or whatever - const RAPIDJSON_ERROR_CHARTYPE* s = GetPointerParseError(pointer.GetParseErrorCode()); -\endcode -*/ -typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetPointerParseErrorFunc)(PointerParseErrorCode); - - RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/src/native/external/rapidjson/filereadstream.h b/src/native/external/rapidjson/filereadstream.h new file mode 100644 index 00000000000000..6b343707ade08d --- /dev/null +++ b/src/native/external/rapidjson/filereadstream.h @@ -0,0 +1,99 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FILEREADSTREAM_H_ +#define RAPIDJSON_FILEREADSTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(unreachable-code) +RAPIDJSON_DIAG_OFF(missing-noreturn) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! File byte stream for input using fread(). +/*! + \note implements Stream concept +*/ +class FileReadStream { +public: + typedef char Ch; //!< Character type (byte). + + //! Constructor. + /*! + \param fp File pointer opened for read. + \param buffer user-supplied buffer. + \param bufferSize size of buffer in bytes. Must >=4 bytes. + */ + FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { + RAPIDJSON_ASSERT(fp_ != 0); + RAPIDJSON_ASSERT(bufferSize >= 4); + Read(); + } + + Ch Peek() const { return *current_; } + Ch Take() { Ch c = *current_; Read(); return c; } + size_t Tell() const { return count_ + static_cast(current_ - buffer_); } + + // Not implemented + void Put(Ch) { RAPIDJSON_ASSERT(false); } + void Flush() { RAPIDJSON_ASSERT(false); } + Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } + + // For encoding detection only. + const Ch* Peek4() const { + return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; + } + +private: + void Read() { + if (current_ < bufferLast_) + ++current_; + else if (!eof_) { + count_ += readCount_; + readCount_ = std::fread(buffer_, 1, bufferSize_, fp_); + bufferLast_ = buffer_ + readCount_ - 1; + current_ = buffer_; + + if (readCount_ < bufferSize_) { + buffer_[readCount_] = '\0'; + ++bufferLast_; + eof_ = true; + } + } + } + + std::FILE* fp_; + Ch *buffer_; + size_t bufferSize_; + Ch *bufferLast_; + Ch *current_; + size_t readCount_; + size_t count_; //!< Number of characters read + bool eof_; +}; + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/src/native/external/rapidjson/filewritestream.h b/src/native/external/rapidjson/filewritestream.h new file mode 100644 index 00000000000000..8b48fee197c43a --- /dev/null +++ b/src/native/external/rapidjson/filewritestream.h @@ -0,0 +1,104 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_FILEWRITESTREAM_H_ +#define RAPIDJSON_FILEWRITESTREAM_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(unreachable-code) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of C file stream for output using fwrite(). +/*! + \note implements Stream concept +*/ +class FileWriteStream { +public: + typedef char Ch; //!< Character type. Only support char. + + FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { + RAPIDJSON_ASSERT(fp_ != 0); + } + + void Put(char c) { + if (current_ >= bufferEnd_) + Flush(); + + *current_++ = c; + } + + void PutN(char c, size_t n) { + size_t avail = static_cast(bufferEnd_ - current_); + while (n > avail) { + std::memset(current_, c, avail); + current_ += avail; + Flush(); + n -= avail; + avail = static_cast(bufferEnd_ - current_); + } + + if (n > 0) { + std::memset(current_, c, n); + current_ += n; + } + } + + void Flush() { + if (current_ != buffer_) { + size_t result = std::fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); + if (result < static_cast(current_ - buffer_)) { + // failure deliberately ignored at this time + // added to avoid warn_unused_result build errors + } + current_ = buffer_; + } + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + // Prohibit copy constructor & assignment operator. + FileWriteStream(const FileWriteStream&); + FileWriteStream& operator=(const FileWriteStream&); + + std::FILE* fp_; + char *buffer_; + char *bufferEnd_; + char *current_; +}; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(FileWriteStream& stream, char c, size_t n) { + stream.PutN(c, n); +} + +RAPIDJSON_NAMESPACE_END + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/src/native/external/rapidjson/fwd.h b/src/native/external/rapidjson/fwd.h index d62f77f0ecfafd..e8104e841bcdca 100644 --- a/src/native/external/rapidjson/fwd.h +++ b/src/native/external/rapidjson/fwd.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -102,7 +102,7 @@ class PrettyWriter; // document.h template -class GenericMember; +struct GenericMember; template class GenericMemberIterator; diff --git a/src/native/external/rapidjson/internal/biginteger.h b/src/native/external/rapidjson/internal/biginteger.h index 4930043dc7c5f9..a31c8a88d6eb42 100644 --- a/src/native/external/rapidjson/internal/biginteger.h +++ b/src/native/external/rapidjson/internal/biginteger.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -17,13 +17,9 @@ #include "../rapidjson.h" -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && defined(_M_AMD64) +#if defined(_MSC_VER) && !__INTEL_COMPILER && defined(_M_AMD64) #include // for _umul128 -#if !defined(_ARM64EC_) #pragma intrinsic(_umul128) -#else -#pragma comment(lib,"softintrin") -#endif #endif RAPIDJSON_NAMESPACE_BEGIN @@ -41,8 +37,7 @@ class BigInteger { digits_[0] = u; } - template - BigInteger(const Ch* decimals, size_t length) : count_(1) { + BigInteger(const char* decimals, size_t length) : count_(1) { RAPIDJSON_ASSERT(length > 0); digits_[0] = 0; size_t i = 0; @@ -226,8 +221,7 @@ class BigInteger { bool IsZero() const { return count_ == 1 && digits_[0] == 0; } private: - template - void AppendDecimal64(const Ch* begin, const Ch* end) { + void AppendDecimal64(const char* begin, const char* end) { uint64_t u = ParseUint64(begin, end); if (IsZero()) *this = u; @@ -242,12 +236,11 @@ class BigInteger { digits_[count_++] = digit; } - template - static uint64_t ParseUint64(const Ch* begin, const Ch* end) { + static uint64_t ParseUint64(const char* begin, const char* end) { uint64_t r = 0; - for (const Ch* p = begin; p != end; ++p) { - RAPIDJSON_ASSERT(*p >= Ch('0') && *p <= Ch('9')); - r = r * 10u + static_cast(*p - Ch('0')); + for (const char* p = begin; p != end; ++p) { + RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); + r = r * 10u + static_cast(*p - '0'); } return r; } @@ -259,7 +252,7 @@ class BigInteger { if (low < k) (*outHigh)++; return low; -#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(a) * static_cast(b); p += k; diff --git a/src/native/external/rapidjson/internal/clzll.h b/src/native/external/rapidjson/internal/clzll.h deleted file mode 100644 index 8fc5118aa47b8e..00000000000000 --- a/src/native/external/rapidjson/internal/clzll.h +++ /dev/null @@ -1,71 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_CLZLL_H_ -#define RAPIDJSON_CLZLL_H_ - -#include "../rapidjson.h" - -#if defined(_MSC_VER) && !defined(UNDER_CE) -#include -#if defined(_WIN64) -#pragma intrinsic(_BitScanReverse64) -#else -#pragma intrinsic(_BitScanReverse) -#endif -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -inline uint32_t clzll(uint64_t x) { - // Passing 0 to __builtin_clzll is UB in GCC and results in an - // infinite loop in the software implementation. - RAPIDJSON_ASSERT(x != 0); - -#if defined(_MSC_VER) && !defined(UNDER_CE) - unsigned long r = 0; -#if defined(_WIN64) - _BitScanReverse64(&r, x); -#else - // Scan the high 32 bits. - if (_BitScanReverse(&r, static_cast(x >> 32))) - return 63 - (r + 32); - - // Scan the low 32 bits. - _BitScanReverse(&r, static_cast(x & 0xFFFFFFFF)); -#endif // _WIN64 - - return 63 - r; -#elif (defined(__GNUC__) && __GNUC__ >= 4) || RAPIDJSON_HAS_BUILTIN(__builtin_clzll) - // __builtin_clzll wrapper - return static_cast(__builtin_clzll(x)); -#else - // naive version - uint32_t r = 0; - while (!(x & (static_cast(1) << 63))) { - x <<= 1; - ++r; - } - - return r; -#endif // _MSC_VER -} - -#define RAPIDJSON_CLZLL RAPIDJSON_NAMESPACE::internal::clzll - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_CLZLL_H_ diff --git a/src/native/external/rapidjson/internal/diyfp.h b/src/native/external/rapidjson/internal/diyfp.h index 1f60fb60ca0437..b6c2cf5618d452 100644 --- a/src/native/external/rapidjson/internal/diyfp.h +++ b/src/native/external/rapidjson/internal/diyfp.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -20,16 +20,12 @@ #define RAPIDJSON_DIYFP_H_ #include "../rapidjson.h" -#include "clzll.h" #include #if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) #include -#if !defined(_ARM64EC_) +#pragma intrinsic(_BitScanReverse64) #pragma intrinsic(_umul128) -#else -#pragma comment(lib,"softintrin") -#endif #endif RAPIDJSON_NAMESPACE_BEGIN @@ -79,7 +75,7 @@ struct DiyFp { if (l & (uint64_t(1) << 63)) // rounding h++; return DiyFp(h, e + rhs.e + 64); -#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) +#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(f) * static_cast(rhs.f); uint64_t h = static_cast(p >> 64); @@ -104,8 +100,22 @@ struct DiyFp { } DiyFp Normalize() const { - int s = static_cast(clzll(f)); + RAPIDJSON_ASSERT(f != 0); // https://stackoverflow.com/a/26809183/291737 +#if defined(_MSC_VER) && defined(_M_AMD64) + unsigned long index; + _BitScanReverse64(&index, f); + return DiyFp(f << (63 - index), e - (63 - index)); +#elif defined(__GNUC__) && __GNUC__ >= 4 + int s = __builtin_clzll(f); return DiyFp(f << s, e - s); +#else + DiyFp res = *this; + while (!(res.f & (static_cast(1) << 63))) { + res.f <<= 1; + res.e--; + } + return res; +#endif } DiyFp NormalizeBoundary() const { diff --git a/src/native/external/rapidjson/internal/dtoa.h b/src/native/external/rapidjson/internal/dtoa.h index cd456721a71c03..bf2e9b2e59a495 100644 --- a/src/native/external/rapidjson/internal/dtoa.h +++ b/src/native/external/rapidjson/internal/dtoa.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -58,11 +58,7 @@ inline int CountDecimalDigit32(uint32_t n) { } inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint64_t kPow10[] = { 1ULL, 10ULL, 100ULL, 1000ULL, 10000ULL, 100000ULL, 1000000ULL, 10000000ULL, 100000000ULL, - 1000000000ULL, 10000000000ULL, 100000000000ULL, 1000000000000ULL, - 10000000000000ULL, 100000000000000ULL, 1000000000000000ULL, - 10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL, - 10000000000000000000ULL }; + static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); const DiyFp wp_w = Mp - W; uint32_t p1 = static_cast(Mp.f >> -one.e); @@ -90,7 +86,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff uint64_t tmp = (static_cast(p1) << -one.e) + p2; if (tmp <= delta) { *K += kappa; - GrisuRound(buffer, *len, delta, tmp, kPow10[kappa] << -one.e, wp_w.f); + GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); return; } } @@ -107,7 +103,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff if (p2 < delta) { *K += kappa; int index = -kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 20 ? kPow10[index] : 0)); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); return; } } diff --git a/src/native/external/rapidjson/internal/ieee754.h b/src/native/external/rapidjson/internal/ieee754.h index 68c9e96649b8aa..c2684ba2a35ffa 100644 --- a/src/native/external/rapidjson/internal/ieee754.h +++ b/src/native/external/rapidjson/internal/ieee754.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/itoa.h b/src/native/external/rapidjson/internal/itoa.h index 9fe8c932ffa6d0..9b1c45cc1b4a88 100644 --- a/src/native/external/rapidjson/internal/itoa.h +++ b/src/native/external/rapidjson/internal/itoa.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/meta.h b/src/native/external/rapidjson/internal/meta.h index 27092dc0d69c43..d401edf85150d8 100644 --- a/src/native/external/rapidjson/internal/meta.h +++ b/src/native/external/rapidjson/internal/meta.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/pow10.h b/src/native/external/rapidjson/internal/pow10.h index eae1a43ed1a06d..02f475d705fcbc 100644 --- a/src/native/external/rapidjson/internal/pow10.h +++ b/src/native/external/rapidjson/internal/pow10.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/regex.h b/src/native/external/rapidjson/internal/regex.h new file mode 100644 index 00000000000000..16e355921f884c --- /dev/null +++ b/src/native/external/rapidjson/internal/regex.h @@ -0,0 +1,740 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_INTERNAL_REGEX_H_ +#define RAPIDJSON_INTERNAL_REGEX_H_ + +#include "../allocators.h" +#include "../stream.h" +#include "stack.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +RAPIDJSON_DIAG_OFF(switch-enum) +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#if __GNUC__ >= 7 +RAPIDJSON_DIAG_OFF(implicit-fallthrough) +#endif +#endif + +#ifndef RAPIDJSON_REGEX_VERBOSE +#define RAPIDJSON_REGEX_VERBOSE 0 +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +/////////////////////////////////////////////////////////////////////////////// +// DecodedStream + +template +class DecodedStream { +public: + DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } + unsigned Peek() { return codepoint_; } + unsigned Take() { + unsigned c = codepoint_; + if (c) // No further decoding when '\0' + Decode(); + return c; + } + +private: + void Decode() { + if (!Encoding::Decode(ss_, &codepoint_)) + codepoint_ = 0; + } + + SourceStream& ss_; + unsigned codepoint_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericRegex + +static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 +static const SizeType kRegexInvalidRange = ~SizeType(0); + +template +class GenericRegexSearch; + +//! Regular expression engine with subset of ECMAscript grammar. +/*! + Supported regular expression syntax: + - \c ab Concatenation + - \c a|b Alternation + - \c a? Zero or one + - \c a* Zero or more + - \c a+ One or more + - \c a{3} Exactly 3 times + - \c a{3,} At least 3 times + - \c a{3,5} 3 to 5 times + - \c (ab) Grouping + - \c ^a At the beginning + - \c a$ At the end + - \c . Any character + - \c [abc] Character classes + - \c [a-c] Character class range + - \c [a-z0-9_] Character class combination + - \c [^abc] Negated character classes + - \c [^a-c] Negated character class range + - \c [\b] Backspace (U+0008) + - \c \\| \\\\ ... Escape characters + - \c \\f Form feed (U+000C) + - \c \\n Line feed (U+000A) + - \c \\r Carriage return (U+000D) + - \c \\t Tab (U+0009) + - \c \\v Vertical tab (U+000B) + + \note This is a Thompson NFA engine, implemented with reference to + Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", + https://swtch.com/~rsc/regexp/regexp1.html +*/ +template +class GenericRegex { +public: + typedef Encoding EncodingType; + typedef typename Encoding::Ch Ch; + template friend class GenericRegexSearch; + + GenericRegex(const Ch* source, Allocator* allocator = 0) : + ownAllocator_(allocator ? 0 : RAPIDJSON_NEW(Allocator)()), allocator_(allocator ? allocator : ownAllocator_), + states_(allocator_, 256), ranges_(allocator_, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), + anchorBegin_(), anchorEnd_() + { + GenericStringStream ss(source); + DecodedStream, Encoding> ds(ss); + Parse(ds); + } + + ~GenericRegex() + { + RAPIDJSON_DELETE(ownAllocator_); + } + + bool IsValid() const { + return root_ != kRegexInvalidState; + } + +private: + enum Operator { + kZeroOrOne, + kZeroOrMore, + kOneOrMore, + kConcatenation, + kAlternation, + kLeftParenthesis + }; + + static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' + static const unsigned kRangeCharacterClass = 0xFFFFFFFE; + static const unsigned kRangeNegationFlag = 0x80000000; + + struct Range { + unsigned start; // + unsigned end; + SizeType next; + }; + + struct State { + SizeType out; //!< Equals to kInvalid for matching state + SizeType out1; //!< Equals to non-kInvalid for split + SizeType rangeStart; + unsigned codepoint; + }; + + struct Frag { + Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} + SizeType start; + SizeType out; //!< link-list of all output states + SizeType minIndex; + }; + + State& GetState(SizeType index) { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + const State& GetState(SizeType index) const { + RAPIDJSON_ASSERT(index < stateCount_); + return states_.template Bottom()[index]; + } + + Range& GetRange(SizeType index) { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + const Range& GetRange(SizeType index) const { + RAPIDJSON_ASSERT(index < rangeCount_); + return ranges_.template Bottom()[index]; + } + + template + void Parse(DecodedStream& ds) { + Stack operandStack(allocator_, 256); // Frag + Stack operatorStack(allocator_, 256); // Operator + Stack atomCountStack(allocator_, 256); // unsigned (Atom per parenthesis) + + *atomCountStack.template Push() = 0; + + unsigned codepoint; + while (ds.Peek() != 0) { + switch (codepoint = ds.Take()) { + case '^': + anchorBegin_ = true; + break; + + case '$': + anchorEnd_ = true; + break; + + case '|': + while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + *operatorStack.template Push() = kAlternation; + *atomCountStack.template Top() = 0; + break; + + case '(': + *operatorStack.template Push() = kLeftParenthesis; + *atomCountStack.template Push() = 0; + break; + + case ')': + while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + if (operatorStack.Empty()) + return; + operatorStack.template Pop(1); + atomCountStack.template Pop(1); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '?': + if (!Eval(operandStack, kZeroOrOne)) + return; + break; + + case '*': + if (!Eval(operandStack, kZeroOrMore)) + return; + break; + + case '+': + if (!Eval(operandStack, kOneOrMore)) + return; + break; + + case '{': + { + unsigned n, m; + if (!ParseUnsigned(ds, &n)) + return; + + if (ds.Peek() == ',') { + ds.Take(); + if (ds.Peek() == '}') + m = kInfinityQuantifier; + else if (!ParseUnsigned(ds, &m) || m < n) + return; + } + else + m = n; + + if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') + return; + ds.Take(); + } + break; + + case '.': + PushOperand(operandStack, kAnyCharacterClass); + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '[': + { + SizeType range; + if (!ParseRange(ds, &range)) + return; + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); + GetState(s).rangeStart = range; + *operandStack.template Push() = Frag(s, s, s); + } + ImplicitConcatenation(atomCountStack, operatorStack); + break; + + case '\\': // Escape character + if (!CharacterEscape(ds, &codepoint)) + return; // Unsupported escape character + // fall through to default + + default: // Pattern character + PushOperand(operandStack, codepoint); + ImplicitConcatenation(atomCountStack, operatorStack); + } + } + + while (!operatorStack.Empty()) + if (!Eval(operandStack, *operatorStack.template Pop(1))) + return; + + // Link the operand to matching state. + if (operandStack.GetSize() == sizeof(Frag)) { + Frag* e = operandStack.template Pop(1); + Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); + root_ = e->start; + +#if RAPIDJSON_REGEX_VERBOSE + printf("root: %d\n", root_); + for (SizeType i = 0; i < stateCount_ ; i++) { + State& s = GetState(i); + printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); + } + printf("\n"); +#endif + } + } + + SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { + State* s = states_.template Push(); + s->out = out; + s->out1 = out1; + s->codepoint = codepoint; + s->rangeStart = kRegexInvalidRange; + return stateCount_++; + } + + void PushOperand(Stack& operandStack, unsigned codepoint) { + SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); + *operandStack.template Push() = Frag(s, s, s); + } + + void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { + if (*atomCountStack.template Top()) + *operatorStack.template Push() = kConcatenation; + (*atomCountStack.template Top())++; + } + + SizeType Append(SizeType l1, SizeType l2) { + SizeType old = l1; + while (GetState(l1).out != kRegexInvalidState) + l1 = GetState(l1).out; + GetState(l1).out = l2; + return old; + } + + void Patch(SizeType l, SizeType s) { + for (SizeType next; l != kRegexInvalidState; l = next) { + next = GetState(l).out; + GetState(l).out = s; + } + } + + bool Eval(Stack& operandStack, Operator op) { + switch (op) { + case kConcatenation: + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); + { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + Patch(e1.out, e2.start); + *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); + } + return true; + + case kAlternation: + if (operandStack.GetSize() >= sizeof(Frag) * 2) { + Frag e2 = *operandStack.template Pop(1); + Frag e1 = *operandStack.template Pop(1); + SizeType s = NewState(e1.start, e2.start, 0); + *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); + return true; + } + return false; + + case kZeroOrOne: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); + return true; + } + return false; + + case kZeroOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(s, s, e.minIndex); + return true; + } + return false; + + case kOneOrMore: + if (operandStack.GetSize() >= sizeof(Frag)) { + Frag e = *operandStack.template Pop(1); + SizeType s = NewState(kRegexInvalidState, e.start, 0); + Patch(e.out, s); + *operandStack.template Push() = Frag(e.start, s, e.minIndex); + return true; + } + return false; + + default: + // syntax error (e.g. unclosed kLeftParenthesis) + return false; + } + } + + bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { + RAPIDJSON_ASSERT(n <= m); + RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); + + if (n == 0) { + if (m == 0) // a{0} not support + return false; + else if (m == kInfinityQuantifier) + Eval(operandStack, kZeroOrMore); // a{0,} -> a* + else { + Eval(operandStack, kZeroOrOne); // a{0,5} -> a? + for (unsigned i = 0; i < m - 1; i++) + CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? + for (unsigned i = 0; i < m - 1; i++) + Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? + } + return true; + } + + for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a + CloneTopOperand(operandStack); + + if (m == kInfinityQuantifier) + Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ + else if (m > n) { + CloneTopOperand(operandStack); // a{3,5} -> a a a a + Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? + for (unsigned i = n; i < m - 1; i++) + CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? + for (unsigned i = n; i < m; i++) + Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? + } + + for (unsigned i = 0; i < n - 1; i++) + Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? + + return true; + } + + static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } + + void CloneTopOperand(Stack& operandStack) { + const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation + SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) + State* s = states_.template Push(count); + memcpy(s, &GetState(src.minIndex), count * sizeof(State)); + for (SizeType j = 0; j < count; j++) { + if (s[j].out != kRegexInvalidState) + s[j].out += count; + if (s[j].out1 != kRegexInvalidState) + s[j].out1 += count; + } + *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); + stateCount_ += count; + } + + template + bool ParseUnsigned(DecodedStream& ds, unsigned* u) { + unsigned r = 0; + if (ds.Peek() < '0' || ds.Peek() > '9') + return false; + while (ds.Peek() >= '0' && ds.Peek() <= '9') { + if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 + return false; // overflow + r = r * 10 + (ds.Take() - '0'); + } + *u = r; + return true; + } + + template + bool ParseRange(DecodedStream& ds, SizeType* range) { + bool isBegin = true; + bool negate = false; + int step = 0; + SizeType start = kRegexInvalidRange; + SizeType current = kRegexInvalidRange; + unsigned codepoint; + while ((codepoint = ds.Take()) != 0) { + if (isBegin) { + isBegin = false; + if (codepoint == '^') { + negate = true; + continue; + } + } + + switch (codepoint) { + case ']': + if (start == kRegexInvalidRange) + return false; // Error: nothing inside [] + if (step == 2) { // Add trailing '-' + SizeType r = NewRange('-'); + RAPIDJSON_ASSERT(current != kRegexInvalidRange); + GetRange(current).next = r; + } + if (negate) + GetRange(start).start |= kRangeNegationFlag; + *range = start; + return true; + + case '\\': + if (ds.Peek() == 'b') { + ds.Take(); + codepoint = 0x0008; // Escape backspace character + } + else if (!CharacterEscape(ds, &codepoint)) + return false; + // fall through to default + + default: + switch (step) { + case 1: + if (codepoint == '-') { + step++; + break; + } + // fall through to step 0 for other characters + + case 0: + { + SizeType r = NewRange(codepoint); + if (current != kRegexInvalidRange) + GetRange(current).next = r; + if (start == kRegexInvalidRange) + start = r; + current = r; + } + step = 1; + break; + + default: + RAPIDJSON_ASSERT(step == 2); + GetRange(current).end = codepoint; + step = 0; + } + } + } + return false; + } + + SizeType NewRange(unsigned codepoint) { + Range* r = ranges_.template Push(); + r->start = r->end = codepoint; + r->next = kRegexInvalidRange; + return rangeCount_++; + } + + template + bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { + unsigned codepoint; + switch (codepoint = ds.Take()) { + case '^': + case '$': + case '|': + case '(': + case ')': + case '?': + case '*': + case '+': + case '.': + case '[': + case ']': + case '{': + case '}': + case '\\': + *escapedCodepoint = codepoint; return true; + case 'f': *escapedCodepoint = 0x000C; return true; + case 'n': *escapedCodepoint = 0x000A; return true; + case 'r': *escapedCodepoint = 0x000D; return true; + case 't': *escapedCodepoint = 0x0009; return true; + case 'v': *escapedCodepoint = 0x000B; return true; + default: + return false; // Unsupported escape character + } + } + + Allocator* ownAllocator_; + Allocator* allocator_; + Stack states_; + Stack ranges_; + SizeType root_; + SizeType stateCount_; + SizeType rangeCount_; + + static const unsigned kInfinityQuantifier = ~0u; + + // For SearchWithAnchoring() + bool anchorBegin_; + bool anchorEnd_; +}; + +template +class GenericRegexSearch { +public: + typedef typename RegexType::EncodingType Encoding; + typedef typename Encoding::Ch Ch; + + GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : + regex_(regex), allocator_(allocator), ownAllocator_(0), + state0_(allocator, 0), state1_(allocator, 0), stateSet_() + { + RAPIDJSON_ASSERT(regex_.IsValid()); + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); + state0_.template Reserve(regex_.stateCount_); + state1_.template Reserve(regex_.stateCount_); + } + + ~GenericRegexSearch() { + Allocator::Free(stateSet_); + RAPIDJSON_DELETE(ownAllocator_); + } + + template + bool Match(InputStream& is) { + return SearchWithAnchoring(is, true, true); + } + + bool Match(const Ch* s) { + GenericStringStream is(s); + return Match(is); + } + + template + bool Search(InputStream& is) { + return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); + } + + bool Search(const Ch* s) { + GenericStringStream is(s); + return Search(is); + } + +private: + typedef typename RegexType::State State; + typedef typename RegexType::Range Range; + + template + bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { + DecodedStream ds(is); + + state0_.Clear(); + Stack *current = &state0_, *next = &state1_; + const size_t stateSetSize = GetStateSetSize(); + std::memset(stateSet_, 0, stateSetSize); + + bool matched = AddState(*current, regex_.root_); + unsigned codepoint; + while (!current->Empty() && (codepoint = ds.Take()) != 0) { + std::memset(stateSet_, 0, stateSetSize); + next->Clear(); + matched = false; + for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { + const State& sr = regex_.GetState(*s); + if (sr.codepoint == codepoint || + sr.codepoint == RegexType::kAnyCharacterClass || + (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) + { + matched = AddState(*next, sr.out) || matched; + if (!anchorEnd && matched) + return true; + } + if (!anchorBegin) + AddState(*next, regex_.root_); + } + internal::Swap(current, next); + } + + return matched; + } + + size_t GetStateSetSize() const { + return (regex_.stateCount_ + 31) / 32 * 4; + } + + // Return whether the added states is a match state + bool AddState(Stack& l, SizeType index) { + RAPIDJSON_ASSERT(index != kRegexInvalidState); + + const State& s = regex_.GetState(index); + if (s.out1 != kRegexInvalidState) { // Split + bool matched = AddState(l, s.out); + return AddState(l, s.out1) || matched; + } + else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { + stateSet_[index >> 5] |= (1u << (index & 31)); + *l.template PushUnsafe() = index; + } + return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. + } + + bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { + bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; + while (rangeIndex != kRegexInvalidRange) { + const Range& r = regex_.GetRange(rangeIndex); + if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) + return yes; + rangeIndex = r.next; + } + return !yes; + } + + const RegexType& regex_; + Allocator* allocator_; + Allocator* ownAllocator_; + Stack state0_; + Stack state1_; + uint32_t* stateSet_; +}; + +typedef GenericRegex > Regex; +typedef GenericRegexSearch RegexSearch; + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#if defined(__clang__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/src/native/external/rapidjson/internal/stack.h b/src/native/external/rapidjson/internal/stack.h index 73abd706e97692..45dca6a8b09eb4 100644 --- a/src/native/external/rapidjson/internal/stack.h +++ b/src/native/external/rapidjson/internal/stack.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/strfunc.h b/src/native/external/rapidjson/internal/strfunc.h index b698a8f43fa63b..226439a7673645 100644 --- a/src/native/external/rapidjson/internal/strfunc.h +++ b/src/native/external/rapidjson/internal/strfunc.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -45,20 +45,6 @@ inline SizeType StrLen(const wchar_t* s) { return SizeType(std::wcslen(s)); } -//! Custom strcmpn() which works on different character types. -/*! \tparam Ch Character type (e.g. char, wchar_t, short) - \param s1 Null-terminated input string. - \param s2 Null-terminated input string. - \return 0 if equal -*/ -template -inline int StrCmp(const Ch* s1, const Ch* s2) { - RAPIDJSON_ASSERT(s1 != 0); - RAPIDJSON_ASSERT(s2 != 0); - while(*s1 && (*s1 == *s2)) { s1++; s2++; } - return static_cast(*s1) < static_cast(*s2) ? -1 : static_cast(*s1) > static_cast(*s2); -} - //! Returns number of code points in a encoded string. template bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { diff --git a/src/native/external/rapidjson/internal/strtod.h b/src/native/external/rapidjson/internal/strtod.h index 55f0e380bfaa35..dfca22b65ac03b 100644 --- a/src/native/external/rapidjson/internal/strtod.h +++ b/src/native/external/rapidjson/internal/strtod.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -128,18 +128,17 @@ inline bool StrtodFast(double d, int p, double* result) { } // Compute an approximation and see if it is within 1/2 ULP -template -inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) { +inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) { uint64_t significand = 0; int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 for (; i < dLen; i++) { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || - (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5'))) + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) break; - significand = significand * 10u + static_cast(decimals[i] - Ch('0')); + significand = significand * 10u + static_cast(decimals[i] - '0'); } - if (i < dLen && decimals[i] >= Ch('5')) // Rounding + if (i < dLen && decimals[i] >= '5') // Rounding significand++; int remaining = dLen - i; @@ -206,8 +205,7 @@ inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } -template -inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int dExp) { +inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) { RAPIDJSON_ASSERT(dLen >= 0); const BigInteger dInt(decimals, static_cast(dLen)); Double a(approx); @@ -225,8 +223,7 @@ inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int return a.NextPositiveDouble(); } -template -inline double StrtodFullPrecision(double d, int p, const Ch* decimals, size_t length, size_t decimalPosition, int exp) { +inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { RAPIDJSON_ASSERT(d >= 0.0); RAPIDJSON_ASSERT(length >= 1); diff --git a/src/native/external/rapidjson/internal/swap.h b/src/native/external/rapidjson/internal/swap.h index 2cf92f93a1d371..666e49f97b68d9 100644 --- a/src/native/external/rapidjson/internal/swap.h +++ b/src/native/external/rapidjson/internal/swap.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/istreamwrapper.h b/src/native/external/rapidjson/istreamwrapper.h index 01437ec0127a01..c4950b9dcf8284 100644 --- a/src/native/external/rapidjson/istreamwrapper.h +++ b/src/native/external/rapidjson/istreamwrapper.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/memorybuffer.h b/src/native/external/rapidjson/memorybuffer.h new file mode 100644 index 00000000000000..39bee1dec1c036 --- /dev/null +++ b/src/native/external/rapidjson/memorybuffer.h @@ -0,0 +1,70 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_MEMORYBUFFER_H_ +#define RAPIDJSON_MEMORYBUFFER_H_ + +#include "stream.h" +#include "internal/stack.h" + +RAPIDJSON_NAMESPACE_BEGIN + +//! Represents an in-memory output byte stream. +/*! + This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. + + It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. + + Differences between MemoryBuffer and StringBuffer: + 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. + 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. + + \tparam Allocator type for allocating memory buffer. + \note implements Stream concept +*/ +template +struct GenericMemoryBuffer { + typedef char Ch; // byte + + GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} + + void Put(Ch c) { *stack_.template Push() = c; } + void Flush() {} + + void Clear() { stack_.Clear(); } + void ShrinkToFit() { stack_.ShrinkToFit(); } + Ch* Push(size_t count) { return stack_.template Push(count); } + void Pop(size_t count) { stack_.template Pop(count); } + + const Ch* GetBuffer() const { + return stack_.template Bottom(); + } + + size_t GetSize() const { return stack_.GetSize(); } + + static const size_t kDefaultCapacity = 256; + mutable internal::Stack stack_; +}; + +typedef GenericMemoryBuffer<> MemoryBuffer; + +//! Implement specialized version of PutN() with memset() for better performance. +template<> +inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { + std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); +} + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/src/native/external/rapidjson/memorystream.h b/src/native/external/rapidjson/memorystream.h index 77af6c999e977a..1d71d8a4f0e0ad 100644 --- a/src/native/external/rapidjson/memorystream.h +++ b/src/native/external/rapidjson/memorystream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/msinttypes/inttypes.h b/src/native/external/rapidjson/msinttypes/inttypes.h new file mode 100644 index 00000000000000..18111286bf55bc --- /dev/null +++ b/src/native/external/rapidjson/msinttypes/inttypes.h @@ -0,0 +1,316 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include "stdint.h" + +// miloyip: VC supports inttypes.h since VC2013 +#if _MSC_VER >= 1800 +#include +#else + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "I32d" +#define PRIi32 "I32i" +#define PRIdLEAST32 "I32d" +#define PRIiLEAST32 "I32i" +#define PRIdFAST32 "I32d" +#define PRIiFAST32 "I32i" + +#define PRId64 "I64d" +#define PRIi64 "I64i" +#define PRIdLEAST64 "I64d" +#define PRIiLEAST64 "I64i" +#define PRIdFAST64 "I64d" +#define PRIiFAST64 "I64i" + +#define PRIdMAX "I64d" +#define PRIiMAX "I64i" + +#define PRIdPTR "Id" +#define PRIiPTR "Ii" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "I32o" +#define PRIu32 "I32u" +#define PRIx32 "I32x" +#define PRIX32 "I32X" +#define PRIoLEAST32 "I32o" +#define PRIuLEAST32 "I32u" +#define PRIxLEAST32 "I32x" +#define PRIXLEAST32 "I32X" +#define PRIoFAST32 "I32o" +#define PRIuFAST32 "I32u" +#define PRIxFAST32 "I32x" +#define PRIXFAST32 "I32X" + +#define PRIo64 "I64o" +#define PRIu64 "I64u" +#define PRIx64 "I64x" +#define PRIX64 "I64X" +#define PRIoLEAST64 "I64o" +#define PRIuLEAST64 "I64u" +#define PRIxLEAST64 "I64x" +#define PRIXLEAST64 "I64X" +#define PRIoFAST64 "I64o" +#define PRIuFAST64 "I64u" +#define PRIxFAST64 "I64x" +#define PRIXFAST64 "I64X" + +#define PRIoMAX "I64o" +#define PRIuMAX "I64u" +#define PRIxMAX "I64x" +#define PRIXMAX "I64X" + +#define PRIoPTR "Io" +#define PRIuPTR "Iu" +#define PRIxPTR "Ix" +#define PRIXPTR "IX" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + +#endif // _MSC_VER >= 1800 + +#endif // _MSC_INTTYPES_H_ ] diff --git a/src/native/external/rapidjson/msinttypes/stdint.h b/src/native/external/rapidjson/msinttypes/stdint.h new file mode 100644 index 00000000000000..3d4477b9a024a8 --- /dev/null +++ b/src/native/external/rapidjson/msinttypes/stdint.h @@ -0,0 +1,300 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +// The above software in this distribution may have been modified by +// THL A29 Limited ("Tencent Modifications"). +// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. +#if _MSC_VER >= 1600 // [ +#include + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +#undef INT8_C +#undef INT16_C +#undef INT32_C +#undef INT64_C +#undef UINT8_C +#undef UINT16_C +#undef UINT32_C +#undef UINT64_C + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#else // ] _MSC_VER >= 1700 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we have to wrap include with 'extern "C++" {}' +// or compiler would give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#if defined(__cplusplus) && !defined(_M_ARM) +extern "C" { +#endif +# include +#if defined(__cplusplus) && !defined(_M_ARM) +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff --git a/src/native/external/rapidjson/ostreamwrapper.h b/src/native/external/rapidjson/ostreamwrapper.h new file mode 100644 index 00000000000000..6f4667c08ad7ba --- /dev/null +++ b/src/native/external/rapidjson/ostreamwrapper.h @@ -0,0 +1,81 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ +#define RAPIDJSON_OSTREAMWRAPPER_H_ + +#include "stream.h" +#include + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(padded) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. +/*! + The classes can be wrapped including but not limited to: + + - \c std::ostringstream + - \c std::stringstream + - \c std::wpstringstream + - \c std::wstringstream + - \c std::ifstream + - \c std::fstream + - \c std::wofstream + - \c std::wfstream + + \tparam StreamType Class derived from \c std::basic_ostream. +*/ + +template +class BasicOStreamWrapper { +public: + typedef typename StreamType::char_type Ch; + BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} + + void Put(Ch c) { + stream_.put(c); + } + + void Flush() { + stream_.flush(); + } + + // Not implemented + char Peek() const { RAPIDJSON_ASSERT(false); return 0; } + char Take() { RAPIDJSON_ASSERT(false); return 0; } + size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } + char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } + size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } + +private: + BasicOStreamWrapper(const BasicOStreamWrapper&); + BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); + + StreamType& stream_; +}; + +typedef BasicOStreamWrapper OStreamWrapper; +typedef BasicOStreamWrapper WOStreamWrapper; + +#ifdef __clang__ +RAPIDJSON_DIAG_POP +#endif + +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/src/native/external/rapidjson/pointer.h b/src/native/external/rapidjson/pointer.h new file mode 100644 index 00000000000000..063abab9a1703c --- /dev/null +++ b/src/native/external/rapidjson/pointer.h @@ -0,0 +1,1414 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_POINTER_H_ +#define RAPIDJSON_POINTER_H_ + +#include "document.h" +#include "internal/itoa.h" + +#ifdef __clang__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(switch-enum) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token + +//! Error code of parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode +*/ +enum PointerParseErrorCode { + kPointerParseErrorNone = 0, //!< The parse is successful + + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericPointer + +//! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. +/*! + This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" + (https://tools.ietf.org/html/rfc6901). + + A JSON pointer is for identifying a specific value in a JSON document + (GenericDocument). It can simplify coding of DOM tree manipulation, because it + can access multiple-level depth of DOM tree with single API call. + + After it parses a string representation (e.g. "/foo/0" or URI fragment + representation (e.g. "#/foo/0") into its internal representation (tokens), + it can be used to resolve a specific value in multiple documents, or sub-tree + of documents. + + Contrary to GenericValue, Pointer can be copy constructed and copy assigned. + Apart from assignment, a Pointer cannot be modified after construction. + + Although Pointer is very convenient, please aware that constructing Pointer + involves parsing and dynamic memory allocation. A special constructor with user- + supplied tokens eliminates these. + + GenericPointer depends on GenericDocument and GenericValue. + + \tparam ValueType The value type of the DOM tree. E.g. GenericValue > + \tparam Allocator The allocator type for allocating memory for internal representation. + + \note GenericPointer uses same encoding of ValueType. + However, Allocator of GenericPointer is independent of Allocator of Value. +*/ +template +class GenericPointer { +public: + typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value + typedef typename ValueType::Ch Ch; //!< Character type from Value + + //! A token is the basic units of internal representation. + /*! + A JSON pointer string representation "/foo/123" is parsed to two tokens: + "foo" and 123. 123 will be represented in both numeric form and string form. + They are resolved according to the actual value type (object or array). + + For token that are not numbers, or the numeric value is out of bound + (greater than limits of SizeType), they are only treated as string form + (i.e. the token's index will be equal to kPointerInvalidIndex). + + This struct is public so that user can create a Pointer without parsing and + allocation, using a special constructor. + */ + struct Token { + const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. + SizeType length; //!< Length of the name. + SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. + }; + + //!@name Constructors and destructor. + //@{ + + //! Default constructor. + GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A null-terminated, string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + */ + explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, internal::StrLen(source)); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Constructor that parses a string or URI fragment representation. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. + */ + explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source.c_str(), source.size()); + } +#endif + + //! Constructor that parses a string or URI fragment representation, with length of the source string. + /*! + \param source A string or URI fragment representation of JSON pointer. + \param length Length of source. + \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. + \note Slightly faster than the overload without length. + */ + GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + Parse(source, length); + } + + //! Constructor with user-supplied tokens. + /*! + This constructor let user supplies const array of tokens. + This prevents the parsing process and eliminates allocation. + This is preferred for memory constrained environments. + + \param tokens An constant array of tokens representing the JSON pointer. + \param tokenCount Number of tokens. + + \b Example + \code + #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } + #define INDEX(i) { #i, sizeof(#i) - 1, i } + + static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; + static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); + // Equivalent to static const Pointer p("/foo/123"); + + #undef NAME + #undef INDEX + \endcode + */ + GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs) : allocator_(rhs.allocator_), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Copy constructor. + GenericPointer(const GenericPointer& rhs, Allocator* allocator) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { + *this = rhs; + } + + //! Destructor. + ~GenericPointer() { + if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. + Allocator::Free(tokens_); + RAPIDJSON_DELETE(ownAllocator_); + } + + //! Assignment operator. + GenericPointer& operator=(const GenericPointer& rhs) { + if (this != &rhs) { + // Do not delete ownAllcator + if (nameBuffer_) + Allocator::Free(tokens_); + + tokenCount_ = rhs.tokenCount_; + parseErrorOffset_ = rhs.parseErrorOffset_; + parseErrorCode_ = rhs.parseErrorCode_; + + if (rhs.nameBuffer_) + CopyFromRaw(rhs); // Normally parsed tokens. + else { + tokens_ = rhs.tokens_; // User supplied const tokens. + nameBuffer_ = 0; + } + } + return *this; + } + + //! Swap the content of this pointer with an other. + /*! + \param other The pointer to swap with. + \note Constant complexity. + */ + GenericPointer& Swap(GenericPointer& other) RAPIDJSON_NOEXCEPT { + internal::Swap(allocator_, other.allocator_); + internal::Swap(ownAllocator_, other.ownAllocator_); + internal::Swap(nameBuffer_, other.nameBuffer_); + internal::Swap(tokens_, other.tokens_); + internal::Swap(tokenCount_, other.tokenCount_); + internal::Swap(parseErrorOffset_, other.parseErrorOffset_); + internal::Swap(parseErrorCode_, other.parseErrorCode_); + return *this; + } + + //! free-standing swap function helper + /*! + Helper function to enable support for common swap implementation pattern based on \c std::swap: + \code + void swap(MyClass& a, MyClass& b) { + using std::swap; + swap(a.pointer, b.pointer); + // ... + } + \endcode + \see Swap() + */ + friend inline void swap(GenericPointer& a, GenericPointer& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } + + //@} + + //!@name Append token + //@{ + + //! Append a token and return a new Pointer + /*! + \param token Token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Token& token, Allocator* allocator = 0) const { + GenericPointer r; + r.allocator_ = allocator; + Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); + std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); + r.tokens_[tokenCount_].name = p; + r.tokens_[tokenCount_].length = token.length; + r.tokens_[tokenCount_].index = token.index; + return r; + } + + //! Append a name token with length, and return a new Pointer + /*! + \param name Name to be appended. + \param length Length of name. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { + Token token = { name, length, kPointerInvalidIndex }; + return Append(token, allocator); + } + + //! Append a name token without length, and return a new Pointer + /*! + \param name Name (const Ch*) to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) + Append(T* name, Allocator* allocator = 0) const { + return Append(name, internal::StrLen(name), allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Append a name token, and return a new Pointer + /*! + \param name Name to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { + return Append(name.c_str(), static_cast(name.size()), allocator); + } +#endif + + //! Append a index token, and return a new Pointer + /*! + \param index Index to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(SizeType index, Allocator* allocator = 0) const { + char buffer[21]; + char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); + SizeType length = static_cast(end - buffer); + buffer[length] = '\0'; + + if (sizeof(Ch) == 1) { + Token token = { reinterpret_cast(buffer), length, index }; + return Append(token, allocator); + } + else { + Ch name[21]; + for (size_t i = 0; i <= length; i++) + name[i] = static_cast(buffer[i]); + Token token = { name, length, index }; + return Append(token, allocator); + } + } + + //! Append a token by value, and return a new Pointer + /*! + \param token token to be appended. + \param allocator Allocator for the newly return Pointer. + \return A new Pointer with appended token. + */ + GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { + if (token.IsString()) + return Append(token.GetString(), token.GetStringLength(), allocator); + else { + RAPIDJSON_ASSERT(token.IsUint64()); + RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); + return Append(static_cast(token.GetUint64()), allocator); + } + } + + //!@name Handling Parse Error + //@{ + + //! Check whether this is a valid pointer. + bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } + + //! Get the parsing error offset in code unit. + size_t GetParseErrorOffset() const { return parseErrorOffset_; } + + //! Get the parsing error code. + PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } + + //@} + + //! Get the allocator of this pointer. + Allocator& GetAllocator() { return *allocator_; } + + //!@name Tokens + //@{ + + //! Get the token array (const version only). + const Token* GetTokens() const { return tokens_; } + + //! Get the number of tokens. + size_t GetTokenCount() const { return tokenCount_; } + + //@} + + //!@name Equality/inequality operators + //@{ + + //! Equality operator. + /*! + \note When any pointers are invalid, always returns false. + */ + bool operator==(const GenericPointer& rhs) const { + if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) + return false; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index || + tokens_[i].length != rhs.tokens_[i].length || + (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) + { + return false; + } + } + + return true; + } + + //! Inequality operator. + /*! + \note When any pointers are invalid, always returns true. + */ + bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } + + //! Less than operator. + /*! + \note Invalid pointers are always greater than valid ones. + */ + bool operator<(const GenericPointer& rhs) const { + if (!IsValid()) + return false; + if (!rhs.IsValid()) + return true; + + if (tokenCount_ != rhs.tokenCount_) + return tokenCount_ < rhs.tokenCount_; + + for (size_t i = 0; i < tokenCount_; i++) { + if (tokens_[i].index != rhs.tokens_[i].index) + return tokens_[i].index < rhs.tokens_[i].index; + + if (tokens_[i].length != rhs.tokens_[i].length) + return tokens_[i].length < rhs.tokens_[i].length; + + if (int cmp = std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch) * tokens_[i].length)) + return cmp < 0; + } + + return false; + } + + //@} + + //!@name Stringify + //@{ + + //! Stringify the pointer into string representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + return Stringify(os); + } + + //! Stringify the pointer into URI fragment representation. + /*! + \tparam OutputStream Type of output stream. + \param os The output stream. + */ + template + bool StringifyUriFragment(OutputStream& os) const { + return Stringify(os); + } + + //@} + + //!@name Create value + //@{ + + //! Create a value in a subtree. + /*! + If the value is not exist, it creates all parent values and a JSON Null value. + So it always succeed and return the newly created or existing value. + + Remind that it may change types of parents according to tokens, so it + potentially removes previously stored values. For example, if a document + was an array, and "/foo" is used to create a value, then the document + will be changed to an object, and all existing array elements are lost. + + \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created (a JSON Null value), or already exists value. + */ + ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { + RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + bool exist = true; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + if (v->IsArray() && t->name[0] == '-' && t->length == 1) { + v->PushBack(ValueType().Move(), allocator); + v = &((*v)[v->Size() - 1]); + exist = false; + } + else { + if (t->index == kPointerInvalidIndex) { // must be object name + if (!v->IsObject()) + v->SetObject(); // Change to Object + } + else { // object name or array index + if (!v->IsArray() && !v->IsObject()) + v->SetArray(); // Change to Array + } + + if (v->IsArray()) { + if (t->index >= v->Size()) { + v->Reserve(t->index + 1, allocator); + while (t->index >= v->Size()) + v->PushBack(ValueType().Move(), allocator); + exist = false; + } + v = &((*v)[t->index]); + } + else { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) { + v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); + v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end + exist = false; + } + else + v = &m->value; + } + } + } + + if (alreadyExist) + *alreadyExist = exist; + + return *v; + } + + //! Creates a value in a document. + /*! + \param document A document to be resolved. + \param alreadyExist If non-null, it stores whether the resolved value is already exist. + \return The resolved newly created, or already exists value. + */ + template + ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { + return Create(document, document.GetAllocator(), alreadyExist); + } + + //@} + + //!@name Query value + //@{ + + //! Query a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. + \return Pointer to the value if it can be resolved. Otherwise null. + + \note + There are only 3 situations when a value cannot be resolved: + 1. A value in the path is not an array nor object. + 2. An object value does not contain the token. + 3. A token is out of range of an array value. + + Use unresolvedTokenIndex to retrieve the token index. + */ + ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { + RAPIDJSON_ASSERT(IsValid()); + ValueType* v = &root; + for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + break; + v = &m->value; + } + continue; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + break; + v = &((*v)[t->index]); + continue; + default: + break; + } + + // Error: unresolved token + if (unresolvedTokenIndex) + *unresolvedTokenIndex = static_cast(t - tokens_); + return 0; + } + return v; + } + + //! Query a const value in a const subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Pointer to the value if it can be resolved. Otherwise null. + */ + const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { + return Get(const_cast(root), unresolvedTokenIndex); + } + + //@} + + //!@name Query a value with default + //@{ + + //! Query a value in a subtree with default value. + /*! + Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. + So that this function always succeed. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param defaultValue Default value to be cloned if the value was not exists. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + ValueType& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); + } + + //! Query a value in a subtree with default null-terminated string. + ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + ValueType& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Query a value in a subtree with default std::basic_string. + ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { + bool alreadyExist; + ValueType& v = Create(root, allocator, &alreadyExist); + return alreadyExist ? v : v.SetString(defaultValue, allocator); + } +#endif + + //! Query a value in a subtree with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { + return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); + } + + //! Query a value in a document with default value. + template + ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //! Query a value in a document with default null-terminated string. + template + ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Query a value in a document with default std::basic_string. + template + ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } +#endif + + //! Query a value in a document with default primitive value. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + GetWithDefault(GenericDocument& document, T defaultValue) const { + return GetWithDefault(document, defaultValue, document.GetAllocator()); + } + + //@} + + //!@name Set a value + //@{ + + //! Set a value in a subtree, with move semantics. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be set. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = value; + } + + //! Set a value in a subtree, with copy semantics. + ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).CopyFrom(value, allocator); + } + + //! Set a null-terminated string in a subtree. + ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Set a std::basic_string in a subtree. + ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value, allocator).Move(); + } +#endif + + //! Set a primitive value in a subtree. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator) = ValueType(value).Move(); + } + + //! Set a value in a document, with move semantics. + template + ValueType& Set(GenericDocument& document, ValueType& value) const { + return Create(document) = value; + } + + //! Set a value in a document, with copy semantics. + template + ValueType& Set(GenericDocument& document, const ValueType& value) const { + return Create(document).CopyFrom(value, document.GetAllocator()); + } + + //! Set a null-terminated string in a document. + template + ValueType& Set(GenericDocument& document, const Ch* value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } + +#if RAPIDJSON_HAS_STDSTRING + //! Sets a std::basic_string in a document. + template + ValueType& Set(GenericDocument& document, const std::basic_string& value) const { + return Create(document) = ValueType(value, document.GetAllocator()).Move(); + } +#endif + + //! Set a primitive value in a document. + /*! + \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool + */ + template + RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) + Set(GenericDocument& document, T value) const { + return Create(document) = value; + } + + //@} + + //!@name Swap a value + //@{ + + //! Swap a value with a value in a subtree. + /*! + It creates all parents if they are not exist or types are different to the tokens. + So this function always succeeds but potentially remove existing values. + + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \param value Value to be swapped. + \param allocator Allocator for creating the values if the specified value or its parents are not exist. + \see Create() + */ + ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { + return Create(root, allocator).Swap(value); + } + + //! Swap a value with a value in a document. + template + ValueType& Swap(GenericDocument& document, ValueType& value) const { + return Create(document).Swap(value); + } + + //@} + + //! Erase a value in a subtree. + /*! + \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. + \return Whether the resolved value is found and erased. + + \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. + */ + bool Erase(ValueType& root) const { + RAPIDJSON_ASSERT(IsValid()); + if (tokenCount_ == 0) // Cannot erase the root + return false; + + ValueType* v = &root; + const Token* last = tokens_ + (tokenCount_ - 1); + for (const Token *t = tokens_; t != last; ++t) { + switch (v->GetType()) { + case kObjectType: + { + typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); + if (m == v->MemberEnd()) + return false; + v = &m->value; + } + break; + case kArrayType: + if (t->index == kPointerInvalidIndex || t->index >= v->Size()) + return false; + v = &((*v)[t->index]); + break; + default: + return false; + } + } + + switch (v->GetType()) { + case kObjectType: + return v->EraseMember(GenericStringRef(last->name, last->length)); + case kArrayType: + if (last->index == kPointerInvalidIndex || last->index >= v->Size()) + return false; + v->Erase(v->Begin() + last->index); + return true; + default: + return false; + } + } + +private: + //! Clone the content from rhs to this. + /*! + \param rhs Source pointer. + \param extraToken Extra tokens to be allocated. + \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. + \return Start of non-occupied name buffer, for storing extra names. + */ + Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { + if (!allocator_) // allocator is independently owned. + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + + size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens + for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) + nameBufferSize += t->length; + + tokenCount_ = rhs.tokenCount_ + extraToken; + tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); + nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + if (rhs.tokenCount_ > 0) { + std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); + } + if (nameBufferSize > 0) { + std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); + } + + // Adjust pointers to name buffer + std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; + for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) + t->name += diff; + + return nameBuffer_ + nameBufferSize; + } + + //! Check whether a character should be percent-encoded. + /*! + According to RFC 3986 2.3 Unreserved Characters. + \param c The character (code unit) to be tested. + */ + bool NeedPercentEncode(Ch c) const { + return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); + } + + //! Parse a JSON String or its URI fragment representation into tokens. +#ifndef __clang__ // -Wdocumentation + /*! + \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. + \param length Length of the source string. + \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. + */ +#endif + void Parse(const Ch* source, size_t length) { + RAPIDJSON_ASSERT(source != NULL); + RAPIDJSON_ASSERT(nameBuffer_ == 0); + RAPIDJSON_ASSERT(tokens_ == 0); + + // Create own allocator if user did not supply. + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + + // Count number of '/' as tokenCount + tokenCount_ = 0; + for (const Ch* s = source; s != source + length; s++) + if (*s == '/') + tokenCount_++; + + Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); + Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); + size_t i = 0; + + // Detect if it is a URI fragment + bool uriFragment = false; + if (source[i] == '#') { + uriFragment = true; + i++; + } + + if (i != length && source[i] != '/') { + parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; + goto error; + } + + while (i < length) { + RAPIDJSON_ASSERT(source[i] == '/'); + i++; // consumes '/' + + token->name = name; + bool isNumber = true; + + while (i < length && source[i] != '/') { + Ch c = source[i]; + if (uriFragment) { + // Decoding percent-encoding for URI fragment + if (c == '%') { + PercentDecodeStream is(&source[i], source + length); + GenericInsituStringStream os(name); + Ch* begin = os.PutBegin(); + if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { + parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; + goto error; + } + size_t len = os.PutEnd(begin); + i += is.Tell() - 1; + if (len == 1) + c = *name; + else { + name += len; + isNumber = false; + i++; + continue; + } + } + else if (NeedPercentEncode(c)) { + parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; + goto error; + } + } + + i++; + + // Escaping "~0" -> '~', "~1" -> '/' + if (c == '~') { + if (i < length) { + c = source[i]; + if (c == '0') c = '~'; + else if (c == '1') c = '/'; + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + i++; + } + else { + parseErrorCode_ = kPointerParseErrorInvalidEscape; + goto error; + } + } + + // First check for index: all of characters are digit + if (c < '0' || c > '9') + isNumber = false; + + *name++ = c; + } + token->length = static_cast(name - token->name); + if (token->length == 0) + isNumber = false; + *name++ = '\0'; // Null terminator + + // Second check for index: more than one digit cannot have leading zero + if (isNumber && token->length > 1 && token->name[0] == '0') + isNumber = false; + + // String to SizeType conversion + SizeType n = 0; + if (isNumber) { + for (size_t j = 0; j < token->length; j++) { + SizeType m = n * 10 + static_cast(token->name[j] - '0'); + if (m < n) { // overflow detection + isNumber = false; + break; + } + n = m; + } + } + + token->index = isNumber ? n : kPointerInvalidIndex; + token++; + } + + RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer + parseErrorCode_ = kPointerParseErrorNone; + return; + + error: + Allocator::Free(tokens_); + nameBuffer_ = 0; + tokens_ = 0; + tokenCount_ = 0; + parseErrorOffset_ = i; + return; + } + + //! Stringify to string or URI fragment representation. + /*! + \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. + \tparam OutputStream type of output stream. + \param os The output stream. + */ + template + bool Stringify(OutputStream& os) const { + RAPIDJSON_ASSERT(IsValid()); + + if (uriFragment) + os.Put('#'); + + for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { + os.Put('/'); + for (size_t j = 0; j < t->length; j++) { + Ch c = t->name[j]; + if (c == '~') { + os.Put('~'); + os.Put('0'); + } + else if (c == '/') { + os.Put('~'); + os.Put('1'); + } + else if (uriFragment && NeedPercentEncode(c)) { + // Transcode to UTF8 sequence + GenericStringStream source(&t->name[j]); + PercentEncodeStream target(os); + if (!Transcoder >().Validate(source, target)) + return false; + j += source.Tell() - 1; + } + else + os.Put(c); + } + } + return true; + } + + //! A helper stream for decoding a percent-encoded sequence into code unit. + /*! + This stream decodes %XY triplet into code unit (0-255). + If it encounters invalid characters, it sets output code unit as 0 and + mark invalid, and to be checked by IsValid(). + */ + class PercentDecodeStream { + public: + typedef typename ValueType::Ch Ch; + + //! Constructor + /*! + \param source Start of the stream + \param end Past-the-end of the stream. + */ + PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} + + Ch Take() { + if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet + valid_ = false; + return 0; + } + src_++; + Ch c = 0; + for (int j = 0; j < 2; j++) { + c = static_cast(c << 4); + Ch h = *src_; + if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); + else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); + else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); + else { + valid_ = false; + return 0; + } + src_++; + } + return c; + } + + size_t Tell() const { return static_cast(src_ - head_); } + bool IsValid() const { return valid_; } + + private: + const Ch* src_; //!< Current read position. + const Ch* head_; //!< Original head of the string. + const Ch* end_; //!< Past-the-end position. + bool valid_; //!< Whether the parsing is valid. + }; + + //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. + template + class PercentEncodeStream { + public: + PercentEncodeStream(OutputStream& os) : os_(os) {} + void Put(char c) { // UTF-8 must be byte + unsigned char u = static_cast(c); + static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; + os_.Put('%'); + os_.Put(static_cast(hexDigits[u >> 4])); + os_.Put(static_cast(hexDigits[u & 15])); + } + private: + OutputStream& os_; + }; + + Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. + Allocator* ownAllocator_; //!< Allocator owned by this Pointer. + Ch* nameBuffer_; //!< A buffer containing all names in tokens. + Token* tokens_; //!< A list of tokens. + size_t tokenCount_; //!< Number of tokens in tokens_. + size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. + PointerParseErrorCode parseErrorCode_; //!< Parsing error code. +}; + +//! GenericPointer for Value (UTF-8, default allocator). +typedef GenericPointer Pointer; + +//!@name Helper functions for GenericPointer +//@{ + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { + return pointer.Create(root, a); +} + +template +typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Create(root, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { + return pointer.Create(document); +} + +template +typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Create(document); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { + return pointer.Get(root, unresolvedTokenIndex); +} + +template +typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +template +const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { + return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { + return pointer.GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { + return pointer.GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { + return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { + return pointer.Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) +SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Set(root, value, a); +} + +// No allocator parameter + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { + return pointer.Set(document, value); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { + return pointer.Set(document, value); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { + return pointer.Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +#if RAPIDJSON_HAS_STDSTRING +template +typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { + return GenericPointer(source, N - 1).Set(document, value); +} +#endif + +template +RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) +SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { + return GenericPointer(source, N - 1).Set(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { + return pointer.Swap(root, value, a); +} + +template +typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { + return GenericPointer(source, N - 1).Swap(root, value, a); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { + return pointer.Swap(document, value); +} + +template +typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { + return GenericPointer(source, N - 1).Swap(document, value); +} + +////////////////////////////////////////////////////////////////////////////// + +template +bool EraseValueByPointer(T& root, const GenericPointer& pointer) { + return pointer.Erase(root); +} + +template +bool EraseValueByPointer(T& root, const CharType(&source)[N]) { + return GenericPointer(source, N - 1).Erase(root); +} + +//@} + +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) || defined(_MSC_VER) +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_POINTER_H_ diff --git a/src/native/external/rapidjson/prettywriter.h b/src/native/external/rapidjson/prettywriter.h new file mode 100644 index 00000000000000..45afb6949deb41 --- /dev/null +++ b/src/native/external/rapidjson/prettywriter.h @@ -0,0 +1,277 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_PRETTYWRITER_H_ +#define RAPIDJSON_PRETTYWRITER_H_ + +#include "writer.h" + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#if defined(__clang__) +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(c++98-compat) +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +//! Combination of PrettyWriter format flags. +/*! \see PrettyWriter::SetFormatOptions + */ +enum PrettyFormatOptions { + kFormatDefault = 0, //!< Default pretty formatting. + kFormatSingleLineArray = 1 //!< Format arrays on a single line. +}; + +//! Writer with indentation and spacing. +/*! + \tparam OutputStream Type of output os. + \tparam SourceEncoding Encoding of source string. + \tparam TargetEncoding Encoding of output stream. + \tparam StackAllocator Type of allocator for allocating memory of stack. +*/ +template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> +class PrettyWriter : public Writer { +public: + typedef Writer Base; + typedef typename Base::Ch Ch; + + //! Constructor + /*! \param os Output stream. + \param allocator User supplied allocator. If it is null, it will create a private one. + \param levelDepth Initial capacity of stack. + */ + explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} + + + explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : + Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + PrettyWriter(PrettyWriter&& rhs) : + Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} +#endif + + //! Set custom indentation. + /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). + \param indentCharCount Number of indent characters for each indentation level. + \note The default indentation is 4 spaces. + */ + PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { + RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); + indentChar_ = indentChar; + indentCharCount_ = indentCharCount; + return *this; + } + + //! Set pretty writer formatting options. + /*! \param options Formatting options. + */ + PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { + formatOptions_ = options; + return *this; + } + + /*! @name Implementation of Handler + \see Handler + */ + //@{ + + bool Null() { PrettyPrefix(kNullType); return Base::EndValue(Base::WriteNull()); } + bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::EndValue(Base::WriteBool(b)); } + bool Int(int i) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt(i)); } + bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint(u)); } + bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt64(i64)); } + bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint64(u64)); } + bool Double(double d) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteDouble(d)); } + + bool RawNumber(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); + (void)copy; + PrettyPrefix(kNumberType); + return Base::EndValue(Base::WriteString(str, length)); + } + + bool String(const Ch* str, SizeType length, bool copy = false) { + RAPIDJSON_ASSERT(str != 0); + (void)copy; + PrettyPrefix(kStringType); + return Base::EndValue(Base::WriteString(str, length)); + } + +#if RAPIDJSON_HAS_STDSTRING + bool String(const std::basic_string& str) { + return String(str.data(), SizeType(str.size())); + } +#endif + + bool StartObject() { + PrettyPrefix(kObjectType); + new (Base::level_stack_.template Push()) typename Base::Level(false); + return Base::WriteStartObject(); + } + + bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } + +#if RAPIDJSON_HAS_STDSTRING + bool Key(const std::basic_string& str) { + return Key(str.data(), SizeType(str.size())); + } +#endif + + bool EndObject(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object + RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object + RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value + + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::EndValue(Base::WriteEndObject()); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::Flush(); + return true; + } + + bool StartArray() { + PrettyPrefix(kArrayType); + new (Base::level_stack_.template Push()) typename Base::Level(true); + return Base::WriteStartArray(); + } + + bool EndArray(SizeType memberCount = 0) { + (void)memberCount; + RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); + RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); + bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; + + if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + bool ret = Base::EndValue(Base::WriteEndArray()); + (void)ret; + RAPIDJSON_ASSERT(ret == true); + if (Base::level_stack_.Empty()) // end of json text + Base::Flush(); + return true; + } + + //@} + + /*! @name Convenience extensions */ + //@{ + + //! Simpler but slower overload. + bool String(const Ch* str) { return String(str, internal::StrLen(str)); } + bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } + + //@} + + //! Write a raw JSON value. + /*! + For user to write a stringified JSON as a value. + + \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. + \param length Length of the json. + \param type Type of the root of json. + \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. + */ + bool RawValue(const Ch* json, size_t length, Type type) { + RAPIDJSON_ASSERT(json != 0); + PrettyPrefix(type); + return Base::EndValue(Base::WriteRawValue(json, length)); + } + +protected: + void PrettyPrefix(Type type) { + (void)type; + if (Base::level_stack_.GetSize() != 0) { // this value is not at root + typename Base::Level* level = Base::level_stack_.template Top(); + + if (level->inArray) { + if (level->valueCount > 0) { + Base::os_->Put(','); // add comma if it is not the first element in array + if (formatOptions_ & kFormatSingleLineArray) + Base::os_->Put(' '); + } + + if (!(formatOptions_ & kFormatSingleLineArray)) { + Base::os_->Put('\n'); + WriteIndent(); + } + } + else { // in object + if (level->valueCount > 0) { + if (level->valueCount % 2 == 0) { + Base::os_->Put(','); + Base::os_->Put('\n'); + } + else { + Base::os_->Put(':'); + Base::os_->Put(' '); + } + } + else + Base::os_->Put('\n'); + + if (level->valueCount % 2 == 0) + WriteIndent(); + } + if (!level->inArray && level->valueCount % 2 == 0) + RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name + level->valueCount++; + } + else { + RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. + Base::hasRoot_ = true; + } + } + + void WriteIndent() { + size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; + PutN(*Base::os_, static_cast(indentChar_), count); + } + + Ch indentChar_; + unsigned indentCharCount_; + PrettyFormatOptions formatOptions_; + +private: + // Prohibit copy constructor & assignment operator. + PrettyWriter(const PrettyWriter&); + PrettyWriter& operator=(const PrettyWriter&); +}; + +RAPIDJSON_NAMESPACE_END + +#if defined(__clang__) +RAPIDJSON_DIAG_POP +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + +#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/src/native/external/rapidjson/rapidjson.h b/src/native/external/rapidjson/rapidjson.h index 5ea69479501a3b..549936ffe06c4b 100644 --- a/src/native/external/rapidjson/rapidjson.h +++ b/src/native/external/rapidjson/rapidjson.h @@ -1,15 +1,15 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_RAPIDJSON_H_ @@ -17,7 +17,7 @@ /*!\file rapidjson.h \brief common definitions and configuration - + \see RAPIDJSON_CONFIG */ @@ -124,19 +124,6 @@ #define RAPIDJSON_NAMESPACE_END } #endif -/////////////////////////////////////////////////////////////////////////////// -// __cplusplus macro - -//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN - -#if defined(_MSC_VER) -#define RAPIDJSON_CPLUSPLUS _MSVC_LANG -#else -#define RAPIDJSON_CPLUSPLUS __cplusplus -#endif - -//!@endcond - /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_HAS_STDSTRING @@ -162,24 +149,6 @@ #include #endif // RAPIDJSON_HAS_STDSTRING -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_USE_MEMBERSMAP - -/*! \def RAPIDJSON_USE_MEMBERSMAP - \ingroup RAPIDJSON_CONFIG - \brief Enable RapidJSON support for object members handling in a \c std::multimap - - By defining this preprocessor symbol to \c 1, \ref rapidjson::GenericValue object - members are stored in a \c std::multimap for faster lookup and deletion times, a - trade off with a slightly slower insertion time and a small object allocat(or)ed - memory overhead. - - \hideinitializer -*/ -#ifndef RAPIDJSON_USE_MEMBERSMAP -#define RAPIDJSON_USE_MEMBERSMAP 0 // not by default -#endif - /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE @@ -195,7 +164,7 @@ */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else @@ -277,7 +246,7 @@ # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else -# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. +# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. # endif #endif // RAPIDJSON_ENDIAN @@ -442,7 +411,7 @@ RAPIDJSON_NAMESPACE_END // Prefer C++11 static_assert, if available #ifndef RAPIDJSON_STATIC_ASSERT -#if RAPIDJSON_CPLUSPLUS >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) #define RAPIDJSON_STATIC_ASSERT(x) \ static_assert(x, RAPIDJSON_STRINGIFY(x)) #endif // C++11 @@ -513,7 +482,7 @@ RAPIDJSON_NAMESPACE_END //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define RAPIDJSON_MULTILINEMACRO_BEGIN do { #define RAPIDJSON_MULTILINEMACRO_END \ } while((void)0, 0) @@ -521,12 +490,6 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_VERSION_CODE(x,y,z) \ (((x)*100000) + ((y)*100) + (z)) -#if defined(__has_builtin) -#define RAPIDJSON_HAS_BUILTIN(x) __has_builtin(x) -#else -#define RAPIDJSON_HAS_BUILTIN(x) 0 -#endif - /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF @@ -572,14 +535,8 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // C++11 features -#ifndef RAPIDJSON_HAS_CXX11 -#define RAPIDJSON_HAS_CXX11 (RAPIDJSON_CPLUSPLUS >= 201103L) -#endif - #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS -#if RAPIDJSON_HAS_CXX11 -#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 -#elif defined(__clang__) +#if defined(__clang__) #if __has_feature(cxx_rvalue_references) && \ (defined(_MSC_VER) || defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 @@ -596,14 +553,8 @@ RAPIDJSON_NAMESPACE_END #endif #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS -#include // std::move -#endif - #ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT -#if RAPIDJSON_HAS_CXX11 -#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 -#elif defined(__clang__) +#if defined(__clang__) #define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1900) || \ @@ -613,13 +564,11 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 #endif #endif -#ifndef RAPIDJSON_NOEXCEPT #if RAPIDJSON_HAS_CXX11_NOEXCEPT #define RAPIDJSON_NOEXCEPT noexcept #else -#define RAPIDJSON_NOEXCEPT throw() +#define RAPIDJSON_NOEXCEPT /* noexcept */ #endif // RAPIDJSON_HAS_CXX11_NOEXCEPT -#endif // no automatic detection, yet #ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS @@ -642,27 +591,6 @@ RAPIDJSON_NAMESPACE_END #endif #endif // RAPIDJSON_HAS_CXX11_RANGE_FOR -/////////////////////////////////////////////////////////////////////////////// -// C++17 features - -#ifndef RAPIDJSON_HAS_CXX17 -#define RAPIDJSON_HAS_CXX17 (RAPIDJSON_CPLUSPLUS >= 201703L) -#endif - -#if RAPIDJSON_HAS_CXX17 -# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[fallthrough]] -#elif defined(__has_cpp_attribute) -# if __has_cpp_attribute(clang::fallthrough) -# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[clang::fallthrough]] -# elif __has_cpp_attribute(fallthrough) -# define RAPIDJSON_DELIBERATE_FALLTHROUGH __attribute__((fallthrough)) -# else -# define RAPIDJSON_DELIBERATE_FALLTHROUGH -# endif -#else -# define RAPIDJSON_DELIBERATE_FALLTHROUGH -#endif - //!@endcond //! Assertion (in non-throwing contexts). @@ -681,29 +609,16 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_NOEXCEPT_ASSERT #ifdef RAPIDJSON_ASSERT_THROWS -#include -#define RAPIDJSON_NOEXCEPT_ASSERT(x) assert(x) +#if RAPIDJSON_HAS_CXX11_NOEXCEPT +#define RAPIDJSON_NOEXCEPT_ASSERT(x) +#else +#define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) +#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT #else #define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) #endif // RAPIDJSON_ASSERT_THROWS #endif // RAPIDJSON_NOEXCEPT_ASSERT -/////////////////////////////////////////////////////////////////////////////// -// malloc/realloc/free - -#ifndef RAPIDJSON_MALLOC -///! customization point for global \c malloc -#define RAPIDJSON_MALLOC(size) std::malloc(size) -#endif -#ifndef RAPIDJSON_REALLOC -///! customization point for global \c realloc -#define RAPIDJSON_REALLOC(ptr, new_size) std::realloc(ptr, new_size) -#endif -#ifndef RAPIDJSON_FREE -///! customization point for global \c free -#define RAPIDJSON_FREE(ptr) std::free(ptr) -#endif - /////////////////////////////////////////////////////////////////////////////// // new/delete @@ -731,7 +646,7 @@ enum Type { kFalseType = 1, //!< false kTrueType = 2, //!< true kObjectType = 3, //!< object - kArrayType = 4, //!< array + kArrayType = 4, //!< array kStringType = 5, //!< string kNumberType = 6 //!< number }; diff --git a/src/native/external/rapidjson/reader.h b/src/native/external/rapidjson/reader.h index 55546601e29bdf..44a6bcd30cf248 100644 --- a/src/native/external/rapidjson/reader.h +++ b/src/native/external/rapidjson/reader.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -20,7 +20,6 @@ #include "allocators.h" #include "stream.h" #include "encodedstream.h" -#include "internal/clzll.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strtod.h" @@ -154,7 +153,6 @@ enum ParseFlag { kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. - kParseEscapedApostropheFlag = 512, //!< Allow escaped apostrophe in strings. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; @@ -445,16 +443,16 @@ inline const char *SkipWhitespace_SIMD(const char* p) { x = vmvnq_u8(x); // Negate x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract - uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract if (low == 0) { if (high != 0) { - uint32_t lz = internal::clzll(high); + int lz =__builtin_clzll(high);; return p + 8 + (lz >> 3); } } else { - uint32_t lz = internal::clzll(low); + int lz = __builtin_clzll(low);; return p + (lz >> 3); } } @@ -481,16 +479,16 @@ inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { x = vmvnq_u8(x); // Negate x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract - uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract if (low == 0) { if (high != 0) { - uint32_t lz = internal::clzll(high); + int lz = __builtin_clzll(high); return p + 8 + (lz >> 3); } } else { - uint32_t lz = internal::clzll(low); + int lz = __builtin_clzll(low); return p + (lz >> 3); } } @@ -992,7 +990,7 @@ class GenericReader { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '/', + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1015,31 +1013,19 @@ class GenericReader { is.Take(); os.Put(static_cast(escape[static_cast(e)])); } - else if ((parseFlags & kParseEscapedApostropheFlag) && RAPIDJSON_LIKELY(e == '\'')) { // Allow escaped apostrophe - is.Take(); - os.Put('\''); - } else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode is.Take(); unsigned codepoint = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDFFF)) { - // high surrogate, check if followed by valid low surrogate - if (RAPIDJSON_LIKELY(codepoint <= 0xDBFF)) { - // Handle UTF-16 surrogate pair - if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - unsigned codepoint2 = ParseHex4(is, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; - } - // single low surrogate - else - { + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { + // Handle UTF-16 surrogate pair + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - } + unsigned codepoint2 = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; } TEncoding::Encode(os, codepoint); } @@ -1258,19 +1244,19 @@ class GenericReader { x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract - uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract SizeType length = 0; bool escaped = false; if (low == 0) { if (high != 0) { - uint32_t lz = internal::clzll(high); + unsigned lz = (unsigned)__builtin_clzll(high);; length = 8 + (lz >> 3); escaped = true; } } else { - uint32_t lz = internal::clzll(low); + unsigned lz = (unsigned)__builtin_clzll(low);; length = lz >> 3; escaped = true; } @@ -1328,19 +1314,19 @@ class GenericReader { x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract - uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract SizeType length = 0; bool escaped = false; if (low == 0) { if (high != 0) { - uint32_t lz = internal::clzll(high); + unsigned lz = (unsigned)__builtin_clzll(high); length = 8 + (lz >> 3); escaped = true; } } else { - uint32_t lz = internal::clzll(low); + unsigned lz = (unsigned)__builtin_clzll(low); length = lz >> 3; escaped = true; } @@ -1384,17 +1370,17 @@ class GenericReader { x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract - uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract if (low == 0) { if (high != 0) { - uint32_t lz = internal::clzll(high); + int lz = __builtin_clzll(high); p += 8 + (lz >> 3); break; } } else { - uint32_t lz = internal::clzll(low); + int lz = __builtin_clzll(low); p += lz >> 3; break; } @@ -1404,11 +1390,11 @@ class GenericReader { } #endif // RAPIDJSON_NEON - template + template class NumberStream; - template - class NumberStream { + template + class NumberStream { public: typedef typename InputStream::Ch Ch; @@ -1417,11 +1403,11 @@ class GenericReader { RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char) {} + RAPIDJSON_FORCEINLINE void Push(char) {} size_t Tell() { return is.Tell(); } size_t Length() { return 0; } - const StackCharacter* Pop() { return 0; } + const char* Pop() { return 0; } protected: NumberStream& operator=(const NumberStream&); @@ -1429,47 +1415,45 @@ class GenericReader { InputStream& is; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s), stackStream(reader.stack_) {} + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put(static_cast(Base::is.Peek())); + stackStream.Put(static_cast(Base::is.Peek())); return Base::is.Take(); } - RAPIDJSON_FORCEINLINE void Push(StackCharacter c) { + RAPIDJSON_FORCEINLINE void Push(char c) { stackStream.Put(c); } size_t Length() { return stackStream.Length(); } - const StackCharacter* Pop() { + const char* Pop() { stackStream.Put('\0'); return stackStream.Pop(); } private: - StackStream stackStream; + StackStream stackStream; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s) {} + NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; template void ParseNumber(InputStream& is, Handler& handler) { - typedef typename internal::SelectIf, typename TargetEncoding::Ch, char>::Type NumberCharacter; - internal::StreamLocalCopy copy(is); - NumberStream(s.Length()); - GenericStringStream > srcStream(s.Pop()); + StringStream srcStream(s.Pop()); StackStream dstStream(stack_); while (numCharsToCopy--) { - Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); } dstStream.Put('\0'); const typename TargetEncoding::Ch* str = dstStream.Pop(); @@ -1707,7 +1691,7 @@ class GenericReader { } else { size_t length = s.Length(); - const NumberCharacter* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. if (useDouble) { int p = exp + expFrac; diff --git a/src/native/external/rapidjson/schema.h b/src/native/external/rapidjson/schema.h new file mode 100644 index 00000000000000..26ae9474806340 --- /dev/null +++ b/src/native/external/rapidjson/schema.h @@ -0,0 +1,2497 @@ +// Tencent is pleased to support the open source community by making RapidJSON available-> +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License-> You may obtain a copy of the License at +// +// http://opensource->org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied-> See the License for the +// specific language governing permissions and limitations under the License-> + +#ifndef RAPIDJSON_SCHEMA_H_ +#define RAPIDJSON_SCHEMA_H_ + +#include "document.h" +#include "pointer.h" +#include "stringbuffer.h" +#include // abs, floor + +#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 +#endif + +#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) +#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 +#else +#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX +#include "internal/regex.h" +#elif RAPIDJSON_SCHEMA_USE_STDREGEX +#include +#endif + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX +#define RAPIDJSON_SCHEMA_HAS_REGEX 1 +#else +#define RAPIDJSON_SCHEMA_HAS_REGEX 0 +#endif + +#ifndef RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_VERBOSE 0 +#endif + +#if RAPIDJSON_SCHEMA_VERBOSE +#include "stringbuffer.h" +#endif + +RAPIDJSON_DIAG_PUSH + +#if defined(__GNUC__) +RAPIDJSON_DIAG_OFF(effc++) +#endif + +#ifdef __clang__ +RAPIDJSON_DIAG_OFF(weak-vtables) +RAPIDJSON_DIAG_OFF(exit-time-destructors) +RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) +RAPIDJSON_DIAG_OFF(variadic-macros) +#elif defined(_MSC_VER) +RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated +#endif + +RAPIDJSON_NAMESPACE_BEGIN + +/////////////////////////////////////////////////////////////////////////////// +// Verbose Utilities + +#if RAPIDJSON_SCHEMA_VERBOSE + +namespace internal { + +inline void PrintInvalidKeyword(const char* keyword) { + printf("Fail keyword: %s\n", keyword); +} + +inline void PrintInvalidKeyword(const wchar_t* keyword) { + wprintf(L"Fail keyword: %ls\n", keyword); +} + +inline void PrintInvalidDocument(const char* document) { + printf("Fail document: %s\n\n", document); +} + +inline void PrintInvalidDocument(const wchar_t* document) { + wprintf(L"Fail document: %ls\n\n", document); +} + +inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { + printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); +} + +inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { + wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); +} + +} // namespace internal + +#endif // RAPIDJSON_SCHEMA_VERBOSE + +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_INVALID_KEYWORD_RETURN + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) +#else +#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) +#endif + +#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + context.invalidKeyword = keyword.GetString();\ + RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ + return false;\ +RAPIDJSON_MULTILINEMACRO_END + +/////////////////////////////////////////////////////////////////////////////// +// Forward declarations + +template +class GenericSchemaDocument; + +namespace internal { + +template +class Schema; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaValidator + +class ISchemaValidator { +public: + virtual ~ISchemaValidator() {} + virtual bool IsValid() const = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// ISchemaStateFactory + +template +class ISchemaStateFactory { +public: + virtual ~ISchemaStateFactory() {} + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; + virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; + virtual void* CreateHasher() = 0; + virtual uint64_t GetHashCode(void* hasher) = 0; + virtual void DestroryHasher(void* hasher) = 0; + virtual void* MallocState(size_t size) = 0; + virtual void FreeState(void* p) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// IValidationErrorHandler + +template +class IValidationErrorHandler { +public: + typedef typename SchemaType::Ch Ch; + typedef typename SchemaType::SValue SValue; + + virtual ~IValidationErrorHandler() {} + + virtual void NotMultipleOf(int64_t actual, const SValue& expected) = 0; + virtual void NotMultipleOf(uint64_t actual, const SValue& expected) = 0; + virtual void NotMultipleOf(double actual, const SValue& expected) = 0; + virtual void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void AboveMaximum(double actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) = 0; + virtual void BelowMinimum(double actual, const SValue& expected, bool exclusive) = 0; + + virtual void TooLong(const Ch* str, SizeType length, SizeType expected) = 0; + virtual void TooShort(const Ch* str, SizeType length, SizeType expected) = 0; + virtual void DoesNotMatch(const Ch* str, SizeType length) = 0; + + virtual void DisallowedItem(SizeType index) = 0; + virtual void TooFewItems(SizeType actualCount, SizeType expectedCount) = 0; + virtual void TooManyItems(SizeType actualCount, SizeType expectedCount) = 0; + virtual void DuplicateItems(SizeType index1, SizeType index2) = 0; + + virtual void TooManyProperties(SizeType actualCount, SizeType expectedCount) = 0; + virtual void TooFewProperties(SizeType actualCount, SizeType expectedCount) = 0; + virtual void StartMissingProperties() = 0; + virtual void AddMissingProperty(const SValue& name) = 0; + virtual bool EndMissingProperties() = 0; + virtual void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void DisallowedProperty(const Ch* name, SizeType length) = 0; + + virtual void StartDependencyErrors() = 0; + virtual void StartMissingDependentProperties() = 0; + virtual void AddMissingDependentProperty(const SValue& targetName) = 0; + virtual void EndMissingDependentProperties(const SValue& sourceName) = 0; + virtual void AddDependencySchemaError(const SValue& souceName, ISchemaValidator* subvalidator) = 0; + virtual bool EndDependencyErrors() = 0; + + virtual void DisallowedValue() = 0; + virtual void StartDisallowedType() = 0; + virtual void AddExpectedType(const typename SchemaType::ValueType& expectedType) = 0; + virtual void EndDisallowedType(const typename SchemaType::ValueType& actualType) = 0; + virtual void NotAllOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void NoneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void NotOneOf(ISchemaValidator** subvalidators, SizeType count) = 0; + virtual void Disallowed() = 0; +}; + + +/////////////////////////////////////////////////////////////////////////////// +// Hasher + +// For comparison of compound value +template +class Hasher { +public: + typedef typename Encoding::Ch Ch; + + Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} + + bool Null() { return WriteType(kNullType); } + bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } + bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } + bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } + bool Double(double d) { + Number n; + if (d < 0) n.u.i = static_cast(d); + else n.u.u = static_cast(d); + n.d = d; + return WriteNumber(n); + } + + bool RawNumber(const Ch* str, SizeType len, bool) { + WriteBuffer(kNumberType, str, len * sizeof(Ch)); + return true; + } + + bool String(const Ch* str, SizeType len, bool) { + WriteBuffer(kStringType, str, len * sizeof(Ch)); + return true; + } + + bool StartObject() { return true; } + bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } + bool EndObject(SizeType memberCount) { + uint64_t h = Hash(0, kObjectType); + uint64_t* kv = stack_.template Pop(memberCount * 2); + for (SizeType i = 0; i < memberCount; i++) + h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive + *stack_.template Push() = h; + return true; + } + + bool StartArray() { return true; } + bool EndArray(SizeType elementCount) { + uint64_t h = Hash(0, kArrayType); + uint64_t* e = stack_.template Pop(elementCount); + for (SizeType i = 0; i < elementCount; i++) + h = Hash(h, e[i]); // Use hash to achieve element order sensitive + *stack_.template Push() = h; + return true; + } + + bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } + + uint64_t GetHashCode() const { + RAPIDJSON_ASSERT(IsValid()); + return *stack_.template Top(); + } + +private: + static const size_t kDefaultSize = 256; + struct Number { + union U { + uint64_t u; + int64_t i; + }u; + double d; + }; + + bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } + + bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } + + bool WriteBuffer(Type type, const void* data, size_t len) { + // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ + uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); + const unsigned char* d = static_cast(data); + for (size_t i = 0; i < len; i++) + h = Hash(h, d[i]); + *stack_.template Push() = h; + return true; + } + + static uint64_t Hash(uint64_t h, uint64_t d) { + static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); + h ^= d; + h *= kPrime; + return h; + } + + Stack stack_; +}; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidationContext + +template +struct SchemaValidationContext { + typedef Schema SchemaType; + typedef ISchemaStateFactory SchemaValidatorFactoryType; + typedef IValidationErrorHandler ErrorHandlerType; + typedef typename SchemaType::ValueType ValueType; + typedef typename ValueType::Ch Ch; + + enum PatternValidatorType { + kPatternValidatorOnly, + kPatternValidatorWithProperty, + kPatternValidatorWithAdditionalProperty + }; + + SchemaValidationContext(SchemaValidatorFactoryType& f, ErrorHandlerType& eh, const SchemaType* s) : + factory(f), + error_handler(eh), + schema(s), + valueSchema(), + invalidKeyword(), + hasher(), + arrayElementHashCodes(), + validators(), + validatorCount(), + patternPropertiesValidators(), + patternPropertiesValidatorCount(), + patternPropertiesSchemas(), + patternPropertiesSchemaCount(), + valuePatternValidatorType(kPatternValidatorOnly), + propertyExist(), + inArray(false), + valueUniqueness(false), + arrayUniqueness(false) + { + } + + ~SchemaValidationContext() { + if (hasher) + factory.DestroryHasher(hasher); + if (validators) { + for (SizeType i = 0; i < validatorCount; i++) + factory.DestroySchemaValidator(validators[i]); + factory.FreeState(validators); + } + if (patternPropertiesValidators) { + for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) + factory.DestroySchemaValidator(patternPropertiesValidators[i]); + factory.FreeState(patternPropertiesValidators); + } + if (patternPropertiesSchemas) + factory.FreeState(patternPropertiesSchemas); + if (propertyExist) + factory.FreeState(propertyExist); + } + + SchemaValidatorFactoryType& factory; + ErrorHandlerType& error_handler; + const SchemaType* schema; + const SchemaType* valueSchema; + const Ch* invalidKeyword; + void* hasher; // Only validator access + void* arrayElementHashCodes; // Only validator access this + ISchemaValidator** validators; + SizeType validatorCount; + ISchemaValidator** patternPropertiesValidators; + SizeType patternPropertiesValidatorCount; + const SchemaType** patternPropertiesSchemas; + SizeType patternPropertiesSchemaCount; + PatternValidatorType valuePatternValidatorType; + PatternValidatorType objectPatternValidatorType; + SizeType arrayElementIndex; + bool* propertyExist; + bool inArray; + bool valueUniqueness; + bool arrayUniqueness; +}; + +/////////////////////////////////////////////////////////////////////////////// +// Schema + +template +class Schema { +public: + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename SchemaDocumentType::AllocatorType AllocatorType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef SchemaValidationContext Context; + typedef Schema SchemaType; + typedef GenericValue SValue; + typedef IValidationErrorHandler ErrorHandler; + friend class GenericSchemaDocument; + + Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : + allocator_(allocator), + uri_(schemaDocument->GetURI(), *allocator), + pointer_(p, allocator), + typeless_(schemaDocument->GetTypeless()), + enum_(), + enumCount_(), + not_(), + type_((1 << kTotalSchemaType) - 1), // typeless + validatorCount_(), + notValidatorIndex_(), + properties_(), + additionalPropertiesSchema_(), + patternProperties_(), + patternPropertyCount_(), + propertyCount_(), + minProperties_(), + maxProperties_(SizeType(~0)), + additionalProperties_(true), + hasDependencies_(), + hasRequired_(), + hasSchemaDependencies_(), + additionalItemsSchema_(), + itemsList_(), + itemsTuple_(), + itemsTupleCount_(), + minItems_(), + maxItems_(SizeType(~0)), + additionalItems_(true), + uniqueItems_(false), + pattern_(), + minLength_(0), + maxLength_(~SizeType(0)), + exclusiveMinimum_(false), + exclusiveMaximum_(false), + defaultValueLength_(0) + { + typedef typename SchemaDocumentType::ValueType ValueType; + typedef typename ValueType::ConstValueIterator ConstValueIterator; + typedef typename ValueType::ConstMemberIterator ConstMemberIterator; + + if (!value.IsObject()) + return; + + if (const ValueType* v = GetMember(value, GetTypeString())) { + type_ = 0; + if (v->IsString()) + AddType(*v); + else if (v->IsArray()) + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) + AddType(*itr); + } + + if (const ValueType* v = GetMember(value, GetEnumString())) + if (v->IsArray() && v->Size() > 0) { + enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { + typedef Hasher > EnumHasherType; + char buffer[256u + 24]; + MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); + EnumHasherType h(&hasherAllocator, 256); + itr->Accept(h); + enum_[enumCount_++] = h.GetHashCode(); + } + } + + if (schemaDocument) { + AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); + AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); + AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); + } + + if (const ValueType* v = GetMember(value, GetNotString())) { + schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); + notValidatorIndex_ = validatorCount_; + validatorCount_++; + } + + // Object + + const ValueType* properties = GetMember(value, GetPropertiesString()); + const ValueType* required = GetMember(value, GetRequiredString()); + const ValueType* dependencies = GetMember(value, GetDependenciesString()); + { + // Gather properties from properties/required/dependencies + SValue allProperties(kArrayType); + + if (properties && properties->IsObject()) + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) + AddUniqueElement(allProperties, itr->name); + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) + AddUniqueElement(allProperties, *itr); + + if (dependencies && dependencies->IsObject()) + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + AddUniqueElement(allProperties, itr->name); + if (itr->value.IsArray()) + for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) + if (i->IsString()) + AddUniqueElement(allProperties, *i); + } + + if (allProperties.Size() > 0) { + propertyCount_ = allProperties.Size(); + properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); + for (SizeType i = 0; i < propertyCount_; i++) { + new (&properties_[i]) Property(); + properties_[i].name = allProperties[i]; + properties_[i].schema = typeless_; + } + } + } + + if (properties && properties->IsObject()) { + PointerType q = p.Append(GetPropertiesString(), allocator_); + for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { + SizeType index; + if (FindPropertyIndex(itr->name, &index)) + schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); + } + } + + if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { + PointerType q = p.Append(GetPatternPropertiesString(), allocator_); + patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); + patternPropertyCount_ = 0; + + for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { + new (&patternProperties_[patternPropertyCount_]) PatternProperty(); + patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); + schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); + patternPropertyCount_++; + } + } + + if (required && required->IsArray()) + for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) + if (itr->IsString()) { + SizeType index; + if (FindPropertyIndex(*itr, &index)) { + properties_[index].required = true; + hasRequired_ = true; + } + } + + if (dependencies && dependencies->IsObject()) { + PointerType q = p.Append(GetDependenciesString(), allocator_); + hasDependencies_ = true; + for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { + SizeType sourceIndex; + if (FindPropertyIndex(itr->name, &sourceIndex)) { + if (itr->value.IsArray()) { + properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); + std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); + for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { + SizeType targetIndex; + if (FindPropertyIndex(*targetItr, &targetIndex)) + properties_[sourceIndex].dependencies[targetIndex] = true; + } + } + else if (itr->value.IsObject()) { + hasSchemaDependencies_ = true; + schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); + properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; + validatorCount_++; + } + } + } + } + + if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { + if (v->IsBool()) + additionalProperties_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); + } + + AssignIfExist(minProperties_, value, GetMinPropertiesString()); + AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); + + // Array + if (const ValueType* v = GetMember(value, GetItemsString())) { + PointerType q = p.Append(GetItemsString(), allocator_); + if (v->IsObject()) // List validation + schemaDocument->CreateSchema(&itemsList_, q, *v, document); + else if (v->IsArray()) { // Tuple validation + itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); + SizeType index = 0; + for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) + schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); + } + } + + AssignIfExist(minItems_, value, GetMinItemsString()); + AssignIfExist(maxItems_, value, GetMaxItemsString()); + + if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { + if (v->IsBool()) + additionalItems_ = v->GetBool(); + else if (v->IsObject()) + schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); + } + + AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); + + // String + AssignIfExist(minLength_, value, GetMinLengthString()); + AssignIfExist(maxLength_, value, GetMaxLengthString()); + + if (const ValueType* v = GetMember(value, GetPatternString())) + pattern_ = CreatePattern(*v); + + // Number + if (const ValueType* v = GetMember(value, GetMinimumString())) + if (v->IsNumber()) + minimum_.CopyFrom(*v, *allocator_); + + if (const ValueType* v = GetMember(value, GetMaximumString())) + if (v->IsNumber()) + maximum_.CopyFrom(*v, *allocator_); + + AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); + AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); + + if (const ValueType* v = GetMember(value, GetMultipleOfString())) + if (v->IsNumber() && v->GetDouble() > 0.0) + multipleOf_.CopyFrom(*v, *allocator_); + + // Default + if (const ValueType* v = GetMember(value, GetDefaultValueString())) + if (v->IsString()) + defaultValueLength_ = v->GetStringLength(); + + } + + ~Schema() { + AllocatorType::Free(enum_); + if (properties_) { + for (SizeType i = 0; i < propertyCount_; i++) + properties_[i].~Property(); + AllocatorType::Free(properties_); + } + if (patternProperties_) { + for (SizeType i = 0; i < patternPropertyCount_; i++) + patternProperties_[i].~PatternProperty(); + AllocatorType::Free(patternProperties_); + } + AllocatorType::Free(itemsTuple_); +#if RAPIDJSON_SCHEMA_HAS_REGEX + if (pattern_) { + pattern_->~RegexType(); + AllocatorType::Free(pattern_); + } +#endif + } + + const SValue& GetURI() const { + return uri_; + } + + const PointerType& GetPointer() const { + return pointer_; + } + + bool BeginValue(Context& context) const { + if (context.inArray) { + if (uniqueItems_) + context.valueUniqueness = true; + + if (itemsList_) + context.valueSchema = itemsList_; + else if (itemsTuple_) { + if (context.arrayElementIndex < itemsTupleCount_) + context.valueSchema = itemsTuple_[context.arrayElementIndex]; + else if (additionalItemsSchema_) + context.valueSchema = additionalItemsSchema_; + else if (additionalItems_) + context.valueSchema = typeless_; + else { + context.error_handler.DisallowedItem(context.arrayElementIndex); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); + } + } + else + context.valueSchema = typeless_; + + context.arrayElementIndex++; + } + return true; + } + + RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { + if (context.patternPropertiesValidatorCount > 0) { + bool otherValid = false; + SizeType count = context.patternPropertiesValidatorCount; + if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) + otherValid = context.patternPropertiesValidators[--count]->IsValid(); + + bool patternValid = true; + for (SizeType i = 0; i < count; i++) + if (!context.patternPropertiesValidators[i]->IsValid()) { + patternValid = false; + break; + } + + if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { + if (!patternValid) { + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + } + else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { + if (!patternValid || !otherValid) { + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + } + else if (!patternValid && !otherValid) { // kPatternValidatorWithAdditionalProperty) + context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); + } + } + + if (enum_) { + const uint64_t h = context.factory.GetHashCode(context.hasher); + for (SizeType i = 0; i < enumCount_; i++) + if (enum_[i] == h) + goto foundEnum; + context.error_handler.DisallowedValue(); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); + foundEnum:; + } + + if (allOf_.schemas) + for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) + if (!context.validators[i]->IsValid()) { + context.error_handler.NotAllOf(&context.validators[allOf_.begin], allOf_.count); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); + } + + if (anyOf_.schemas) { + for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) + if (context.validators[i]->IsValid()) + goto foundAny; + context.error_handler.NoneOf(&context.validators[anyOf_.begin], anyOf_.count); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); + foundAny:; + } + + if (oneOf_.schemas) { + bool oneValid = false; + for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) + if (context.validators[i]->IsValid()) { + if (oneValid) { + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } else + oneValid = true; + } + if (!oneValid) { + context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); + } + } + + if (not_ && context.validators[notValidatorIndex_]->IsValid()) { + context.error_handler.Disallowed(); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); + } + + return true; + } + + bool Null(Context& context) const { + if (!(type_ & (1 << kNullSchemaType))) { + DisallowedType(context, GetNullString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + return CreateParallelValidator(context); + } + + bool Bool(Context& context, bool) const { + if (!(type_ & (1 << kBooleanSchemaType))) { + DisallowedType(context, GetBooleanString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + return CreateParallelValidator(context); + } + + bool Int(Context& context, int i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint(Context& context, unsigned u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Int64(Context& context, int64_t i) const { + if (!CheckInt(context, i)) + return false; + return CreateParallelValidator(context); + } + + bool Uint64(Context& context, uint64_t u) const { + if (!CheckUint(context, u)) + return false; + return CreateParallelValidator(context); + } + + bool Double(Context& context, double d) const { + if (!(type_ & (1 << kNumberSchemaType))) { + DisallowedType(context, GetNumberString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) + return false; + + if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) + return false; + + if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) + return false; + + return CreateParallelValidator(context); + } + + bool String(Context& context, const Ch* str, SizeType length, bool) const { + if (!(type_ & (1 << kStringSchemaType))) { + DisallowedType(context, GetStringString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (minLength_ != 0 || maxLength_ != SizeType(~0)) { + SizeType count; + if (internal::CountStringCodePoint(str, length, &count)) { + if (count < minLength_) { + context.error_handler.TooShort(str, length, minLength_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); + } + if (count > maxLength_) { + context.error_handler.TooLong(str, length, maxLength_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); + } + } + } + + if (pattern_ && !IsPatternMatch(pattern_, str, length)) { + context.error_handler.DoesNotMatch(str, length); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); + } + + return CreateParallelValidator(context); + } + + bool StartObject(Context& context) const { + if (!(type_ & (1 << kObjectSchemaType))) { + DisallowedType(context, GetObjectString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (hasDependencies_ || hasRequired_) { + context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); + std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); + } + + if (patternProperties_) { // pre-allocate schema array + SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType + context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); + context.patternPropertiesSchemaCount = 0; + std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); + } + + return CreateParallelValidator(context); + } + + bool Key(Context& context, const Ch* str, SizeType len, bool) const { + if (patternProperties_) { + context.patternPropertiesSchemaCount = 0; + for (SizeType i = 0; i < patternPropertyCount_; i++) + if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; + context.valueSchema = typeless_; + } + } + + SizeType index; + if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { + if (context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; + } + else + context.valueSchema = properties_[index].schema; + + if (context.propertyExist) + context.propertyExist[index] = true; + + return true; + } + + if (additionalPropertiesSchema_) { + if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { + context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; + context.valueSchema = typeless_; + context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; + } + else + context.valueSchema = additionalPropertiesSchema_; + return true; + } + else if (additionalProperties_) { + context.valueSchema = typeless_; + return true; + } + + if (context.patternPropertiesSchemaCount == 0) { // patternProperties are not additional properties + context.error_handler.DisallowedProperty(str, len); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); + } + + return true; + } + + bool EndObject(Context& context, SizeType memberCount) const { + if (hasRequired_) { + context.error_handler.StartMissingProperties(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].required && !context.propertyExist[index]) + if (properties_[index].schema->defaultValueLength_ == 0 ) + context.error_handler.AddMissingProperty(properties_[index].name); + if (context.error_handler.EndMissingProperties()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); + } + + if (memberCount < minProperties_) { + context.error_handler.TooFewProperties(memberCount, minProperties_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); + } + + if (memberCount > maxProperties_) { + context.error_handler.TooManyProperties(memberCount, maxProperties_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); + } + + if (hasDependencies_) { + context.error_handler.StartDependencyErrors(); + for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) { + const Property& source = properties_[sourceIndex]; + if (context.propertyExist[sourceIndex]) { + if (source.dependencies) { + context.error_handler.StartMissingDependentProperties(); + for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) + if (source.dependencies[targetIndex] && !context.propertyExist[targetIndex]) + context.error_handler.AddMissingDependentProperty(properties_[targetIndex].name); + context.error_handler.EndMissingDependentProperties(source.name); + } + else if (source.dependenciesSchema) { + ISchemaValidator* dependenciesValidator = context.validators[source.dependenciesValidatorIndex]; + if (!dependenciesValidator->IsValid()) + context.error_handler.AddDependencySchemaError(source.name, dependenciesValidator); + } + } + } + if (context.error_handler.EndDependencyErrors()) + RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); + } + + return true; + } + + bool StartArray(Context& context) const { + if (!(type_ & (1 << kArraySchemaType))) { + DisallowedType(context, GetArrayString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + context.arrayElementIndex = 0; + context.inArray = true; + + return CreateParallelValidator(context); + } + + bool EndArray(Context& context, SizeType elementCount) const { + context.inArray = false; + + if (elementCount < minItems_) { + context.error_handler.TooFewItems(elementCount, minItems_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); + } + + if (elementCount > maxItems_) { + context.error_handler.TooManyItems(elementCount, maxItems_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); + } + + return true; + } + + // Generate functions for string literal according to Ch +#define RAPIDJSON_STRING_(name, ...) \ + static const ValueType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ + return v;\ + } + + RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') + RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') + RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') + RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') + RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') + RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') + RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') + RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') + RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') + RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') + RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') + RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') + RAPIDJSON_STRING_(Not, 'n', 'o', 't') + RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') + RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') + RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') + RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') + RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') + RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') + RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') + RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') + RAPIDJSON_STRING_(DefaultValue, 'd', 'e', 'f', 'a', 'u', 'l', 't') + +#undef RAPIDJSON_STRING_ + +private: + enum SchemaValueType { + kNullSchemaType, + kBooleanSchemaType, + kObjectSchemaType, + kArraySchemaType, + kStringSchemaType, + kNumberSchemaType, + kIntegerSchemaType, + kTotalSchemaType + }; + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + typedef internal::GenericRegex RegexType; +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + typedef std::basic_regex RegexType; +#else + typedef char RegexType; +#endif + + struct SchemaArray { + SchemaArray() : schemas(), count() {} + ~SchemaArray() { AllocatorType::Free(schemas); } + const SchemaType** schemas; + SizeType begin; // begin index of context.validators + SizeType count; + }; + + template + void AddUniqueElement(V1& a, const V2& v) { + for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) + if (*itr == v) + return; + V1 c(v, *allocator_); + a.PushBack(c, *allocator_); + } + + static const ValueType* GetMember(const ValueType& value, const ValueType& name) { + typename ValueType::ConstMemberIterator itr = value.FindMember(name); + return itr != value.MemberEnd() ? &(itr->value) : 0; + } + + static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsBool()) + out = v->GetBool(); + } + + static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { + if (const ValueType* v = GetMember(value, name)) + if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) + out = static_cast(v->GetUint64()); + } + + void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { + if (const ValueType* v = GetMember(value, name)) { + if (v->IsArray() && v->Size() > 0) { + PointerType q = p.Append(name, allocator_); + out.count = v->Size(); + out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); + memset(out.schemas, 0, sizeof(Schema*)* out.count); + for (SizeType i = 0; i < out.count; i++) + schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); + out.begin = validatorCount_; + validatorCount_ += out.count; + } + } + } + +#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), allocator_); + if (!r->IsValid()) { + r->~RegexType(); + AllocatorType::Free(r); + r = 0; + } + return r; + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { + GenericRegexSearch rs(*pattern); + return rs.Search(str); + } +#elif RAPIDJSON_SCHEMA_USE_STDREGEX + template + RegexType* CreatePattern(const ValueType& value) { + if (value.IsString()) { + RegexType *r = static_cast(allocator_->Malloc(sizeof(RegexType))); + try { + return new (r) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); + } + catch (const std::regex_error&) { + AllocatorType::Free(r); + } + } + return 0; + } + + static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { + std::match_results r; + return std::regex_search(str, str + length, r, *pattern); + } +#else + template + RegexType* CreatePattern(const ValueType&) { return 0; } + + static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } +#endif // RAPIDJSON_SCHEMA_USE_STDREGEX + + void AddType(const ValueType& type) { + if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; + else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; + else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; + else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; + else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; + else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; + else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); + } + + bool CreateParallelValidator(Context& context) const { + if (enum_ || context.arrayUniqueness) + context.hasher = context.factory.CreateHasher(); + + if (validatorCount_) { + RAPIDJSON_ASSERT(context.validators == 0); + context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); + context.validatorCount = validatorCount_; + + if (allOf_.schemas) + CreateSchemaValidators(context, allOf_); + + if (anyOf_.schemas) + CreateSchemaValidators(context, anyOf_); + + if (oneOf_.schemas) + CreateSchemaValidators(context, oneOf_); + + if (not_) + context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); + + if (hasSchemaDependencies_) { + for (SizeType i = 0; i < propertyCount_; i++) + if (properties_[i].dependenciesSchema) + context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); + } + } + + return true; + } + + void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { + for (SizeType i = 0; i < schemas.count; i++) + context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); + } + + // O(n) + bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { + SizeType len = name.GetStringLength(); + const Ch* str = name.GetString(); + for (SizeType index = 0; index < propertyCount_; index++) + if (properties_[index].name.GetStringLength() == len && + (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) + { + *outIndex = index; + return true; + } + return false; + } + + bool CheckInt(Context& context, int64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { + DisallowedType(context, GetIntegerString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (!minimum_.IsNull()) { + if (minimum_.IsInt64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + } + else if (minimum_.IsUint64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() + } + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsInt64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + } + else if (maximum_.IsUint64()) { } + /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) { + context.error_handler.NotMultipleOf(i, multipleOf_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckUint(Context& context, uint64_t i) const { + if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { + DisallowedType(context, GetIntegerString()); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); + } + + if (!minimum_.IsNull()) { + if (minimum_.IsUint64()) { + if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) { + context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + } + else if (minimum_.IsInt64()) + /* do nothing */; // i >= 0 > minimum.Getint64() + else if (!CheckDoubleMinimum(context, static_cast(i))) + return false; + } + + if (!maximum_.IsNull()) { + if (maximum_.IsUint64()) { + if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + } + else if (maximum_.IsInt64()) { + context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ + } + else if (!CheckDoubleMaximum(context, static_cast(i))) + return false; + } + + if (!multipleOf_.IsNull()) { + if (multipleOf_.IsUint64()) { + if (i % multipleOf_.GetUint64() != 0) { + context.error_handler.NotMultipleOf(i, multipleOf_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + } + else if (!CheckDoubleMultipleOf(context, static_cast(i))) + return false; + } + + return true; + } + + bool CheckDoubleMinimum(Context& context, double d) const { + if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) { + context.error_handler.BelowMinimum(d, minimum_, exclusiveMinimum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); + } + return true; + } + + bool CheckDoubleMaximum(Context& context, double d) const { + if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) { + context.error_handler.AboveMaximum(d, maximum_, exclusiveMaximum_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); + } + return true; + } + + bool CheckDoubleMultipleOf(Context& context, double d) const { + double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); + double q = std::floor(a / b); + double r = a - q * b; + if (r > 0.0) { + context.error_handler.NotMultipleOf(d, multipleOf_); + RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); + } + return true; + } + + void DisallowedType(Context& context, const ValueType& actualType) const { + ErrorHandler& eh = context.error_handler; + eh.StartDisallowedType(); + + if (type_ & (1 << kNullSchemaType)) eh.AddExpectedType(GetNullString()); + if (type_ & (1 << kBooleanSchemaType)) eh.AddExpectedType(GetBooleanString()); + if (type_ & (1 << kObjectSchemaType)) eh.AddExpectedType(GetObjectString()); + if (type_ & (1 << kArraySchemaType)) eh.AddExpectedType(GetArrayString()); + if (type_ & (1 << kStringSchemaType)) eh.AddExpectedType(GetStringString()); + + if (type_ & (1 << kNumberSchemaType)) eh.AddExpectedType(GetNumberString()); + else if (type_ & (1 << kIntegerSchemaType)) eh.AddExpectedType(GetIntegerString()); + + eh.EndDisallowedType(actualType); + } + + struct Property { + Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} + ~Property() { AllocatorType::Free(dependencies); } + SValue name; + const SchemaType* schema; + const SchemaType* dependenciesSchema; + SizeType dependenciesValidatorIndex; + bool* dependencies; + bool required; + }; + + struct PatternProperty { + PatternProperty() : schema(), pattern() {} + ~PatternProperty() { + if (pattern) { + pattern->~RegexType(); + AllocatorType::Free(pattern); + } + } + const SchemaType* schema; + RegexType* pattern; + }; + + AllocatorType* allocator_; + SValue uri_; + PointerType pointer_; + const SchemaType* typeless_; + uint64_t* enum_; + SizeType enumCount_; + SchemaArray allOf_; + SchemaArray anyOf_; + SchemaArray oneOf_; + const SchemaType* not_; + unsigned type_; // bitmask of kSchemaType + SizeType validatorCount_; + SizeType notValidatorIndex_; + + Property* properties_; + const SchemaType* additionalPropertiesSchema_; + PatternProperty* patternProperties_; + SizeType patternPropertyCount_; + SizeType propertyCount_; + SizeType minProperties_; + SizeType maxProperties_; + bool additionalProperties_; + bool hasDependencies_; + bool hasRequired_; + bool hasSchemaDependencies_; + + const SchemaType* additionalItemsSchema_; + const SchemaType* itemsList_; + const SchemaType** itemsTuple_; + SizeType itemsTupleCount_; + SizeType minItems_; + SizeType maxItems_; + bool additionalItems_; + bool uniqueItems_; + + RegexType* pattern_; + SizeType minLength_; + SizeType maxLength_; + + SValue minimum_; + SValue maximum_; + SValue multipleOf_; + bool exclusiveMinimum_; + bool exclusiveMaximum_; + + SizeType defaultValueLength_; +}; + +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + *documentStack.template Push() = '/'; + char buffer[21]; + size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); + for (size_t i = 0; i < length; i++) + *documentStack.template Push() = static_cast(buffer[i]); + } +}; + +// Partial specialized version for char to prevent buffer copying. +template +struct TokenHelper { + RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { + if (sizeof(SizeType) == 4) { + char *buffer = documentStack.template Push(1 + 10); // '/' + uint + *buffer++ = '/'; + const char* end = internal::u32toa(index, buffer); + documentStack.template Pop(static_cast(10 - (end - buffer))); + } + else { + char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 + *buffer++ = '/'; + const char* end = internal::u64toa(index, buffer); + documentStack.template Pop(static_cast(20 - (end - buffer))); + } + } +}; + +} // namespace internal + +/////////////////////////////////////////////////////////////////////////////// +// IGenericRemoteSchemaDocumentProvider + +template +class IGenericRemoteSchemaDocumentProvider { +public: + typedef typename SchemaDocumentType::Ch Ch; + + virtual ~IGenericRemoteSchemaDocumentProvider() {} + virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; +}; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaDocument + +//! JSON schema document. +/*! + A JSON schema document is a compiled version of a JSON schema. + It is basically a tree of internal::Schema. + + \note This is an immutable class (i.e. its instance cannot be modified after construction). + \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. + \tparam Allocator Allocator type for allocating memory of this document. +*/ +template +class GenericSchemaDocument { +public: + typedef ValueT ValueType; + typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; + typedef Allocator AllocatorType; + typedef typename ValueType::EncodingType EncodingType; + typedef typename EncodingType::Ch Ch; + typedef internal::Schema SchemaType; + typedef GenericPointer PointerType; + typedef GenericValue URIType; + friend class internal::Schema; + template + friend class GenericSchemaValidator; + + //! Constructor. + /*! + Compile a JSON document into schema document. + + \param document A JSON document as source. + \param uri The base URI of this schema document for purposes of violation reporting. + \param uriLength Length of \c name, in code points. + \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. + \param allocator An optional allocator instance for allocating memory. Can be null. + */ + explicit GenericSchemaDocument(const ValueType& document, const Ch* uri = 0, SizeType uriLength = 0, + IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : + remoteProvider_(remoteProvider), + allocator_(allocator), + ownAllocator_(), + root_(), + typeless_(), + schemaMap_(allocator, kInitialSchemaMapSize), + schemaRef_(allocator, kInitialSchemaRefSize) + { + if (!allocator_) + ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); + + Ch noUri[1] = {0}; + uri_.SetString(uri ? uri : noUri, uriLength, *allocator_); + + typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); + new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), allocator_); + + // Generate root schema, it will call CreateSchema() to create sub-schemas, + // And call AddRefSchema() if there are $ref. + CreateSchemaRecursive(&root_, PointerType(), document, document); + + // Resolve $ref + while (!schemaRef_.Empty()) { + SchemaRefEntry* refEntry = schemaRef_.template Pop(1); + if (const SchemaType* s = GetSchema(refEntry->target)) { + if (refEntry->schema) + *refEntry->schema = s; + + // Create entry in map if not exist + if (!GetSchema(refEntry->source)) { + new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); + } + } + else if (refEntry->schema) + *refEntry->schema = typeless_; + + refEntry->~SchemaRefEntry(); + } + + RAPIDJSON_ASSERT(root_ != 0); + + schemaRef_.ShrinkToFit(); // Deallocate all memory for ref + } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : + remoteProvider_(rhs.remoteProvider_), + allocator_(rhs.allocator_), + ownAllocator_(rhs.ownAllocator_), + root_(rhs.root_), + typeless_(rhs.typeless_), + schemaMap_(std::move(rhs.schemaMap_)), + schemaRef_(std::move(rhs.schemaRef_)), + uri_(std::move(rhs.uri_)) + { + rhs.remoteProvider_ = 0; + rhs.allocator_ = 0; + rhs.ownAllocator_ = 0; + rhs.typeless_ = 0; + } +#endif + + //! Destructor + ~GenericSchemaDocument() { + while (!schemaMap_.Empty()) + schemaMap_.template Pop(1)->~SchemaEntry(); + + if (typeless_) { + typeless_->~SchemaType(); + Allocator::Free(typeless_); + } + + RAPIDJSON_DELETE(ownAllocator_); + } + + const URIType& GetURI() const { return uri_; } + + //! Get the root schema. + const SchemaType& GetRoot() const { return *root_; } + +private: + //! Prohibit copying + GenericSchemaDocument(const GenericSchemaDocument&); + //! Prohibit assignment + GenericSchemaDocument& operator=(const GenericSchemaDocument&); + + struct SchemaRefEntry { + SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} + PointerType source; + PointerType target; + const SchemaType** schema; + }; + + struct SchemaEntry { + SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} + ~SchemaEntry() { + if (owned) { + schema->~SchemaType(); + Allocator::Free(schema); + } + } + PointerType pointer; + SchemaType* schema; + bool owned; + }; + + void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + if (schema) + *schema = typeless_; + + if (v.GetType() == kObjectType) { + const SchemaType* s = GetSchema(pointer); + if (!s) + CreateSchema(schema, pointer, v, document); + + for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) + CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); + } + else if (v.GetType() == kArrayType) + for (SizeType i = 0; i < v.Size(); i++) + CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); + } + + void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { + RAPIDJSON_ASSERT(pointer.IsValid()); + if (v.IsObject()) { + if (!HandleRefSchema(pointer, schema, v, document)) { + SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); + new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); + if (schema) + *schema = s; + } + } + } + + bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { + static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; + static const ValueType kRefValue(kRefString, 4); + + typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); + if (itr == v.MemberEnd()) + return false; + + if (itr->value.IsString()) { + SizeType len = itr->value.GetStringLength(); + if (len > 0) { + const Ch* s = itr->value.GetString(); + SizeType i = 0; + while (i < len && s[i] != '#') // Find the first # + i++; + + if (i > 0) { // Remote reference, resolve immediately + if (remoteProvider_) { + if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { + if (schema) + *schema = sc; + new (schemaMap_.template Push()) SchemaEntry(source, const_cast(sc), false, allocator_); + return true; + } + } + } + } + } + else if (s[i] == '#') { // Local reference, defer resolution + PointerType pointer(&s[i], len - i, allocator_); + if (pointer.IsValid()) { + if (const ValueType* nv = pointer.Get(document)) + if (HandleRefSchema(source, schema, *nv, document)) + return true; + + new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); + return true; + } + } + } + } + return false; + } + + const SchemaType* GetSchema(const PointerType& pointer) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (pointer == target->pointer) + return target->schema; + return 0; + } + + PointerType GetPointer(const SchemaType* schema) const { + for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) + if (schema == target->schema) + return target->pointer; + return PointerType(); + } + + const SchemaType* GetTypeless() const { return typeless_; } + + static const size_t kInitialSchemaMapSize = 64; + static const size_t kInitialSchemaRefSize = 64; + + IRemoteSchemaDocumentProviderType* remoteProvider_; + Allocator *allocator_; + Allocator *ownAllocator_; + const SchemaType* root_; //!< Root schema. + SchemaType* typeless_; + internal::Stack schemaMap_; // Stores created Pointer -> Schemas + internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref + URIType uri_; +}; + +//! GenericSchemaDocument using Value type. +typedef GenericSchemaDocument SchemaDocument; +//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. +typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; + +/////////////////////////////////////////////////////////////////////////////// +// GenericSchemaValidator + +//! JSON Schema Validator. +/*! + A SAX style JSON schema validator. + It uses a \c GenericSchemaDocument to validate SAX events. + It delegates the incoming SAX events to an output handler. + The default output handler does nothing. + It can be reused multiple times by calling \c Reset(). + + \tparam SchemaDocumentType Type of schema document. + \tparam OutputHandler Type of output handler. Default handler does nothing. + \tparam StateAllocator Allocator for storing the internal validation states. +*/ +template < + typename SchemaDocumentType, + typename OutputHandler = BaseReaderHandler, + typename StateAllocator = CrtAllocator> +class GenericSchemaValidator : + public internal::ISchemaStateFactory, + public internal::ISchemaValidator, + public internal::IValidationErrorHandler +{ +public: + typedef typename SchemaDocumentType::SchemaType SchemaType; + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename SchemaType::EncodingType EncodingType; + typedef typename SchemaType::SValue SValue; + typedef typename EncodingType::Ch Ch; + typedef GenericStringRef StringRefType; + typedef GenericValue ValueType; + + //! Constructor without output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + error_(kObjectType), + currentError_(), + missingDependents_(), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Constructor with output handler. + /*! + \param schemaDocument The schema document to conform to. + \param allocator Optional allocator for storing internal validation states. + \param schemaStackCapacity Optional initial capacity of schema path stack. + \param documentStackCapacity Optional initial capacity of document path stack. + */ + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + OutputHandler& outputHandler, + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(schemaDocument.GetRoot()), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(&outputHandler), + error_(kObjectType), + currentError_(), + missingDependents_(), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(0) +#endif + { + } + + //! Destructor. + ~GenericSchemaValidator() { + Reset(); + RAPIDJSON_DELETE(ownStateAllocator_); + } + + //! Reset the internal states. + void Reset() { + while (!schemaStack_.Empty()) + PopSchema(); + documentStack_.Clear(); + error_.SetObject(); + currentError_.SetNull(); + missingDependents_.SetNull(); + valid_ = true; + } + + //! Checks whether the current state is valid. + // Implementation of ISchemaValidator + virtual bool IsValid() const { return valid_; } + + //! Gets the error object. + ValueType& GetError() { return error_; } + const ValueType& GetError() const { return error_; } + + //! Gets the JSON pointer pointed to the invalid schema. + PointerType GetInvalidSchemaPointer() const { + return schemaStack_.Empty() ? PointerType() : CurrentSchema().GetPointer(); + } + + //! Gets the keyword of invalid schema. + const Ch* GetInvalidSchemaKeyword() const { + return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; + } + + //! Gets the JSON pointer pointed to the invalid value. + PointerType GetInvalidDocumentPointer() const { + if (documentStack_.Empty()) { + return PointerType(); + } + else { + return PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); + } + } + + void NotMultipleOf(int64_t actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void NotMultipleOf(uint64_t actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void NotMultipleOf(double actual, const SValue& expected) { + AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); + } + void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void AboveMaximum(double actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMaximumString : 0); + } + void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + void BelowMinimum(double actual, const SValue& expected, bool exclusive) { + AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, + exclusive ? &SchemaType::GetExclusiveMinimumString : 0); + } + + void TooLong(const Ch* str, SizeType length, SizeType expected) { + AddNumberError(SchemaType::GetMaxLengthString(), + ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); + } + void TooShort(const Ch* str, SizeType length, SizeType expected) { + AddNumberError(SchemaType::GetMinLengthString(), + ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); + } + void DoesNotMatch(const Ch* str, SizeType length) { + currentError_.SetObject(); + currentError_.AddMember(GetActualString(), ValueType(str, length, GetStateAllocator()).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetPatternString()); + } + + void DisallowedItem(SizeType index) { + currentError_.SetObject(); + currentError_.AddMember(GetDisallowedString(), ValueType(index).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetAdditionalItemsString(), true); + } + void TooFewItems(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMinItemsString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void TooManyItems(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMaxItemsString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void DuplicateItems(SizeType index1, SizeType index2) { + ValueType duplicates(kArrayType); + duplicates.PushBack(index1, GetStateAllocator()); + duplicates.PushBack(index2, GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetDuplicatesString(), duplicates, GetStateAllocator()); + AddCurrentError(SchemaType::GetUniqueItemsString(), true); + } + + void TooManyProperties(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMaxPropertiesString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void TooFewProperties(SizeType actualCount, SizeType expectedCount) { + AddNumberError(SchemaType::GetMinPropertiesString(), + ValueType(actualCount).Move(), SValue(expectedCount).Move()); + } + void StartMissingProperties() { + currentError_.SetArray(); + } + void AddMissingProperty(const SValue& name) { + currentError_.PushBack(ValueType(name, GetStateAllocator()).Move(), GetStateAllocator()); + } + bool EndMissingProperties() { + if (currentError_.Empty()) + return false; + ValueType error(kObjectType); + error.AddMember(GetMissingString(), currentError_, GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetRequiredString()); + return true; + } + void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) { + for (SizeType i = 0; i < count; ++i) + MergeError(static_cast(subvalidators[i])->GetError()); + } + void DisallowedProperty(const Ch* name, SizeType length) { + currentError_.SetObject(); + currentError_.AddMember(GetDisallowedString(), ValueType(name, length, GetStateAllocator()).Move(), GetStateAllocator()); + AddCurrentError(SchemaType::GetAdditionalPropertiesString(), true); + } + + void StartDependencyErrors() { + currentError_.SetObject(); + } + void StartMissingDependentProperties() { + missingDependents_.SetArray(); + } + void AddMissingDependentProperty(const SValue& targetName) { + missingDependents_.PushBack(ValueType(targetName, GetStateAllocator()).Move(), GetStateAllocator()); + } + void EndMissingDependentProperties(const SValue& sourceName) { + if (!missingDependents_.Empty()) + currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), + missingDependents_, GetStateAllocator()); + } + void AddDependencySchemaError(const SValue& sourceName, ISchemaValidator* subvalidator) { + currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), + static_cast(subvalidator)->GetError(), GetStateAllocator()); + } + bool EndDependencyErrors() { + if (currentError_.ObjectEmpty()) + return false; + ValueType error(kObjectType); + error.AddMember(GetErrorsString(), currentError_, GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetDependenciesString()); + return true; + } + + void DisallowedValue() { + currentError_.SetObject(); + AddCurrentError(SchemaType::GetEnumString()); + } + void StartDisallowedType() { + currentError_.SetArray(); + } + void AddExpectedType(const typename SchemaType::ValueType& expectedType) { + currentError_.PushBack(ValueType(expectedType, GetStateAllocator()).Move(), GetStateAllocator()); + } + void EndDisallowedType(const typename SchemaType::ValueType& actualType) { + ValueType error(kObjectType); + error.AddMember(GetExpectedString(), currentError_, GetStateAllocator()); + error.AddMember(GetActualString(), ValueType(actualType, GetStateAllocator()).Move(), GetStateAllocator()); + currentError_ = error; + AddCurrentError(SchemaType::GetTypeString()); + } + void NotAllOf(ISchemaValidator** subvalidators, SizeType count) { + for (SizeType i = 0; i < count; ++i) { + MergeError(static_cast(subvalidators[i])->GetError()); + } + } + void NoneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(SchemaType::GetAnyOfString(), subvalidators, count); + } + void NotOneOf(ISchemaValidator** subvalidators, SizeType count) { + AddErrorArray(SchemaType::GetOneOfString(), subvalidators, count); + } + void Disallowed() { + currentError_.SetObject(); + AddCurrentError(SchemaType::GetNotString()); + } + +#define RAPIDJSON_STRING_(name, ...) \ + static const StringRefType& Get##name##String() {\ + static const Ch s[] = { __VA_ARGS__, '\0' };\ + static const StringRefType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1)); \ + return v;\ + } + + RAPIDJSON_STRING_(InstanceRef, 'i', 'n', 's', 't', 'a', 'n', 'c', 'e', 'R', 'e', 'f') + RAPIDJSON_STRING_(SchemaRef, 's', 'c', 'h', 'e', 'm', 'a', 'R', 'e', 'f') + RAPIDJSON_STRING_(Expected, 'e', 'x', 'p', 'e', 'c', 't', 'e', 'd') + RAPIDJSON_STRING_(Actual, 'a', 'c', 't', 'u', 'a', 'l') + RAPIDJSON_STRING_(Disallowed, 'd', 'i', 's', 'a', 'l', 'l', 'o', 'w', 'e', 'd') + RAPIDJSON_STRING_(Missing, 'm', 'i', 's', 's', 'i', 'n', 'g') + RAPIDJSON_STRING_(Errors, 'e', 'r', 'r', 'o', 'r', 's') + RAPIDJSON_STRING_(Duplicates, 'd', 'u', 'p', 'l', 'i', 'c', 'a', 't', 'e', 's') + +#undef RAPIDJSON_STRING_ + +#if RAPIDJSON_SCHEMA_VERBOSE +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ +RAPIDJSON_MULTILINEMACRO_BEGIN\ + *documentStack_.template Push() = '\0';\ + documentStack_.template Pop(1);\ + internal::PrintInvalidDocument(documentStack_.template Bottom());\ +RAPIDJSON_MULTILINEMACRO_END +#else +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() +#endif + +#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ + if (!valid_) return false; \ + if (!BeginValue() || !CurrentSchema().method arg1) {\ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ + return valid_ = false;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ + for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ + if (context->hasher)\ + static_cast(context->hasher)->method arg2;\ + if (context->validators)\ + for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ + static_cast(context->validators[i_])->method arg2;\ + if (context->patternPropertiesValidators)\ + for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ + static_cast(context->patternPropertiesValidators[i_])->method arg2;\ + } + +#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ + return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) + +#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ + RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ + RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) + + bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext()), ( )); } + bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } + bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } + bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } + bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } + bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } + bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } + bool RawNumber(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + bool String(const Ch* str, SizeType length, bool copy) + { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } + + bool StartObject() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); + return valid_ = !outputHandler_ || outputHandler_->StartObject(); + } + + bool Key(const Ch* str, SizeType len, bool copy) { + if (!valid_) return false; + AppendToken(str, len); + if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); + return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); + } + + bool EndObject(SizeType memberCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); + if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); + } + + bool StartArray() { + RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); + return valid_ = !outputHandler_ || outputHandler_->StartArray(); + } + + bool EndArray(SizeType elementCount) { + if (!valid_) return false; + RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); + if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; + RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); + } + +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ +#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ +#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ +#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ + + // Implementation of ISchemaStateFactory + virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { + return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, documentStack_.template Bottom(), documentStack_.GetSize(), +#if RAPIDJSON_SCHEMA_VERBOSE + depth_ + 1, +#endif + &GetStateAllocator()); + } + + virtual void DestroySchemaValidator(ISchemaValidator* validator) { + GenericSchemaValidator* v = static_cast(validator); + v->~GenericSchemaValidator(); + StateAllocator::Free(v); + } + + virtual void* CreateHasher() { + return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); + } + + virtual uint64_t GetHashCode(void* hasher) { + return static_cast(hasher)->GetHashCode(); + } + + virtual void DestroryHasher(void* hasher) { + HasherType* h = static_cast(hasher); + h->~HasherType(); + StateAllocator::Free(h); + } + + virtual void* MallocState(size_t size) { + return GetStateAllocator().Malloc(size); + } + + virtual void FreeState(void* p) { + StateAllocator::Free(p); + } + +private: + typedef typename SchemaType::Context Context; + typedef GenericValue, StateAllocator> HashCodeArray; + typedef internal::Hasher HasherType; + + GenericSchemaValidator( + const SchemaDocumentType& schemaDocument, + const SchemaType& root, + const char* basePath, size_t basePathSize, +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth, +#endif + StateAllocator* allocator = 0, + size_t schemaStackCapacity = kDefaultSchemaStackCapacity, + size_t documentStackCapacity = kDefaultDocumentStackCapacity) + : + schemaDocument_(&schemaDocument), + root_(root), + stateAllocator_(allocator), + ownStateAllocator_(0), + schemaStack_(allocator, schemaStackCapacity), + documentStack_(allocator, documentStackCapacity), + outputHandler_(0), + error_(kObjectType), + currentError_(), + missingDependents_(), + valid_(true) +#if RAPIDJSON_SCHEMA_VERBOSE + , depth_(depth) +#endif + { + if (basePath && basePathSize) + memcpy(documentStack_.template Push(basePathSize), basePath, basePathSize); + } + + StateAllocator& GetStateAllocator() { + if (!stateAllocator_) + stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator)(); + return *stateAllocator_; + } + + bool BeginValue() { + if (schemaStack_.Empty()) + PushSchema(root_); + else { + if (CurrentContext().inArray) + internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); + + if (!CurrentSchema().BeginValue(CurrentContext())) + return false; + + SizeType count = CurrentContext().patternPropertiesSchemaCount; + const SchemaType** sa = CurrentContext().patternPropertiesSchemas; + typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; + bool valueUniqueness = CurrentContext().valueUniqueness; + RAPIDJSON_ASSERT(CurrentContext().valueSchema); + PushSchema(*CurrentContext().valueSchema); + + if (count > 0) { + CurrentContext().objectPatternValidatorType = patternValidatorType; + ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; + SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; + va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); + for (SizeType i = 0; i < count; i++) + va[validatorCount++] = CreateSchemaValidator(*sa[i]); + } + + CurrentContext().arrayUniqueness = valueUniqueness; + } + return true; + } + + bool EndValue() { + if (!CurrentSchema().EndValue(CurrentContext())) + return false; + +#if RAPIDJSON_SCHEMA_VERBOSE + GenericStringBuffer sb; + schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); + + *documentStack_.template Push() = '\0'; + documentStack_.template Pop(1); + internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); +#endif + + uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; + + PopSchema(); + + if (!schemaStack_.Empty()) { + Context& context = CurrentContext(); + if (context.valueUniqueness) { + HashCodeArray* a = static_cast(context.arrayElementHashCodes); + if (!a) + CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); + for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) + if (itr->GetUint64() == h) { + DuplicateItems(static_cast(itr - a->Begin()), a->Size()); + RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); + } + a->PushBack(h, GetStateAllocator()); + } + } + + // Remove the last token of document pointer + while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') + ; + + return true; + } + + void AppendToken(const Ch* str, SizeType len) { + documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters + *documentStack_.template PushUnsafe() = '/'; + for (SizeType i = 0; i < len; i++) { + if (str[i] == '~') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '0'; + } + else if (str[i] == '/') { + *documentStack_.template PushUnsafe() = '~'; + *documentStack_.template PushUnsafe() = '1'; + } + else + *documentStack_.template PushUnsafe() = str[i]; + } + } + + RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, *this, &schema); } + + RAPIDJSON_FORCEINLINE void PopSchema() { + Context* c = schemaStack_.template Pop(1); + if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { + a->~HashCodeArray(); + StateAllocator::Free(a); + } + c->~Context(); + } + + void AddErrorLocation(ValueType& result, bool parent) { + GenericStringBuffer sb; + PointerType instancePointer = GetInvalidDocumentPointer(); + ((parent && instancePointer.GetTokenCount() > 0) + ? PointerType(instancePointer.GetTokens(), instancePointer.GetTokenCount() - 1) + : instancePointer).StringifyUriFragment(sb); + ValueType instanceRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), + GetStateAllocator()); + result.AddMember(GetInstanceRefString(), instanceRef, GetStateAllocator()); + sb.Clear(); + memcpy(sb.Push(CurrentSchema().GetURI().GetStringLength()), + CurrentSchema().GetURI().GetString(), + CurrentSchema().GetURI().GetStringLength() * sizeof(Ch)); + GetInvalidSchemaPointer().StringifyUriFragment(sb); + ValueType schemaRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), + GetStateAllocator()); + result.AddMember(GetSchemaRefString(), schemaRef, GetStateAllocator()); + } + + void AddError(ValueType& keyword, ValueType& error) { + typename ValueType::MemberIterator member = error_.FindMember(keyword); + if (member == error_.MemberEnd()) + error_.AddMember(keyword, error, GetStateAllocator()); + else { + if (member->value.IsObject()) { + ValueType errors(kArrayType); + errors.PushBack(member->value, GetStateAllocator()); + member->value = errors; + } + member->value.PushBack(error, GetStateAllocator()); + } + } + + void AddCurrentError(const typename SchemaType::ValueType& keyword, bool parent = false) { + AddErrorLocation(currentError_, parent); + AddError(ValueType(keyword, GetStateAllocator(), false).Move(), currentError_); + } + + void MergeError(ValueType& other) { + for (typename ValueType::MemberIterator it = other.MemberBegin(), end = other.MemberEnd(); it != end; ++it) { + AddError(it->name, it->value); + } + } + + void AddNumberError(const typename SchemaType::ValueType& keyword, ValueType& actual, const SValue& expected, + const typename SchemaType::ValueType& (*exclusive)() = 0) { + currentError_.SetObject(); + currentError_.AddMember(GetActualString(), actual, GetStateAllocator()); + currentError_.AddMember(GetExpectedString(), ValueType(expected, GetStateAllocator()).Move(), GetStateAllocator()); + if (exclusive) + currentError_.AddMember(ValueType(exclusive(), GetStateAllocator()).Move(), true, GetStateAllocator()); + AddCurrentError(keyword); + } + + void AddErrorArray(const typename SchemaType::ValueType& keyword, + ISchemaValidator** subvalidators, SizeType count) { + ValueType errors(kArrayType); + for (SizeType i = 0; i < count; ++i) + errors.PushBack(static_cast(subvalidators[i])->GetError(), GetStateAllocator()); + currentError_.SetObject(); + currentError_.AddMember(GetErrorsString(), errors, GetStateAllocator()); + AddCurrentError(keyword); + } + + const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } + Context& CurrentContext() { return *schemaStack_.template Top(); } + const Context& CurrentContext() const { return *schemaStack_.template Top(); } + + static const size_t kDefaultSchemaStackCapacity = 1024; + static const size_t kDefaultDocumentStackCapacity = 256; + const SchemaDocumentType* schemaDocument_; + const SchemaType& root_; + StateAllocator* stateAllocator_; + StateAllocator* ownStateAllocator_; + internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) + internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) + OutputHandler* outputHandler_; + ValueType error_; + ValueType currentError_; + ValueType missingDependents_; + bool valid_; +#if RAPIDJSON_SCHEMA_VERBOSE + unsigned depth_; +#endif +}; + +typedef GenericSchemaValidator SchemaValidator; + +/////////////////////////////////////////////////////////////////////////////// +// SchemaValidatingReader + +//! A helper class for parsing with validation. +/*! + This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). + + \tparam parseFlags Combination of \ref ParseFlag. + \tparam InputStream Type of input stream, implementing Stream concept. + \tparam SourceEncoding Encoding of the input stream. + \tparam SchemaDocumentType Type of schema document. + \tparam StackAllocator Allocator type for stack. +*/ +template < + unsigned parseFlags, + typename InputStream, + typename SourceEncoding, + typename SchemaDocumentType = SchemaDocument, + typename StackAllocator = CrtAllocator> +class SchemaValidatingReader { +public: + typedef typename SchemaDocumentType::PointerType PointerType; + typedef typename InputStream::Ch Ch; + typedef GenericValue ValueType; + + //! Constructor + /*! + \param is Input stream. + \param sd Schema document. + */ + SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), error_(kObjectType), isValid_(true) {} + + template + bool operator()(Handler& handler) { + GenericReader reader; + GenericSchemaValidator validator(sd_, handler); + parseResult_ = reader.template Parse(is_, validator); + + isValid_ = validator.IsValid(); + if (isValid_) { + invalidSchemaPointer_ = PointerType(); + invalidSchemaKeyword_ = 0; + invalidDocumentPointer_ = PointerType(); + error_.SetObject(); + } + else { + invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); + invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); + invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); + error_.CopyFrom(validator.GetError(), allocator_); + } + + return parseResult_; + } + + const ParseResult& GetParseResult() const { return parseResult_; } + bool IsValid() const { return isValid_; } + const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } + const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } + const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } + const ValueType& GetError() const { return error_; } + +private: + InputStream& is_; + const SchemaDocumentType& sd_; + + ParseResult parseResult_; + PointerType invalidSchemaPointer_; + const Ch* invalidSchemaKeyword_; + PointerType invalidDocumentPointer_; + StackAllocator allocator_; + ValueType error_; + bool isValid_; +}; + +RAPIDJSON_NAMESPACE_END +RAPIDJSON_DIAG_POP + +#endif // RAPIDJSON_SCHEMA_H_ diff --git a/src/native/external/rapidjson/stream.h b/src/native/external/rapidjson/stream.h index 1fd70915c5472a..7f2643e4814257 100644 --- a/src/native/external/rapidjson/stream.h +++ b/src/native/external/rapidjson/stream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/stringbuffer.h b/src/native/external/rapidjson/stringbuffer.h index 82ad3ca6bbfe8a..4e38b82c3d9849 100644 --- a/src/native/external/rapidjson/stringbuffer.h +++ b/src/native/external/rapidjson/stringbuffer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/writer.h b/src/native/external/rapidjson/writer.h index 632e02ce74a555..6f5b6903467ada 100644 --- a/src/native/external/rapidjson/writer.h +++ b/src/native/external/rapidjson/writer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -16,7 +16,6 @@ #define RAPIDJSON_WRITER_H_ #include "stream.h" -#include "internal/clzll.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strfunc.h" @@ -67,7 +66,6 @@ enum WriteFlag { kWriteNoFlags = 0, //!< No flags are set. kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. - kWriteNanAndInfNullFlag = 4, //!< Allow writing of Infinity, -Infinity and NaN as null. kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS }; @@ -228,7 +226,7 @@ class Writer { return Key(str.data(), SizeType(str.size())); } #endif - + bool EndObject(SizeType memberCount = 0) { (void)memberCount; RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object @@ -284,8 +282,6 @@ class Writer { os_->Flush(); } - static const size_t kDefaultLevelDepth = 32; - protected: //! Information for each nested level struct Level { @@ -294,6 +290,8 @@ class Writer { bool inArray; //!< true if in array, otherwise in object }; + static const size_t kDefaultLevelDepth = 32; + bool WriteNull() { PutReserve(*os_, 4); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; @@ -349,13 +347,8 @@ class Writer { bool WriteDouble(double d) { if (internal::Double(d).IsNanOrInf()) { - if (!(writeFlags & kWriteNanAndInfFlag) && !(writeFlags & kWriteNanAndInfNullFlag)) + if (!(writeFlags & kWriteNanAndInfFlag)) return false; - if (writeFlags & kWriteNanAndInfNullFlag) { - PutReserve(*os_, 4); - PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); - return true; - } if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); @@ -554,11 +547,6 @@ inline bool Writer::WriteDouble(double d) { // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) return false; - if (kWriteDefaultFlags & kWriteNanAndInfNullFlag) { - PutReserve(*os_, 4); - PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); - return true; - } if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); @@ -680,19 +668,19 @@ inline bool Writer::ScanWriteUnescapedString(StringStream& is, siz x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract - uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract + uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract + uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract SizeType len = 0; bool escaped = false; if (low == 0) { if (high != 0) { - uint32_t lz = internal::clzll(high); + unsigned lz = (unsigned)__builtin_clzll(high); len = 8 + (lz >> 3); escaped = true; } } else { - uint32_t lz = internal::clzll(low); + unsigned lz = (unsigned)__builtin_clzll(low); len = lz >> 3; escaped = true; } diff --git a/src/native/external/zlib-version.txt b/src/native/external/zlib-version.txt index 5da9869df33f59..fcac66cc4645f4 100644 --- a/src/native/external/zlib-version.txt +++ b/src/native/external/zlib-version.txt @@ -1,9 +1,17 @@ -v1.3.1 -(51b7f2abdade71cd9bb0e7a373ef2610ec6f9daf) +v1.2.13 +(04f42ceca40f73e2978b50e93806c2a18c1281fc) -https://github.com/madler/zlib/releases/tag/v1.3.1 +https://github.com/madler/zlib/releases/tag/v1.2.13 We have removed zlib.3.pdf from our local copy, as it is a binary file which is not needed for our compilation. +We have also cherry-picked into our local copy: + +- https://github.com/madler/zlib/commit/e554695638228b846d49657f31eeff0ca4680e8a + + This patch only affects memLevel 9 compression. .NET doesn't currently use this + memLevel, but we'll take this patch out of an abundance of caution just in case + we enable this functionality in a future release. + We have also applied the custom patches under the patches/zlib folder. diff --git a/src/native/external/zlib/CMakeLists.txt b/src/native/external/zlib/CMakeLists.txt index 15ceebe787e7d9..b412dc7feb732a 100644 --- a/src/native/external/zlib/CMakeLists.txt +++ b/src/native/external/zlib/CMakeLists.txt @@ -1,11 +1,9 @@ -cmake_minimum_required(VERSION 2.4.4...3.15.0) +cmake_minimum_required(VERSION 2.4.4) set(CMAKE_ALLOW_LOOSE_LOOP_CONSTRUCTS ON) project(zlib C) -set(VERSION "1.3.1") - -option(ZLIB_BUILD_EXAMPLES "Enable Zlib Examples" ON) +set(VERSION "1.2.13") set(INSTALL_BIN_DIR "${CMAKE_INSTALL_PREFIX}/bin" CACHE PATH "Installation directory for executables") set(INSTALL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib" CACHE PATH "Installation directory for libraries") @@ -150,9 +148,7 @@ if(MINGW) endif(MINGW) add_library(zlib SHARED ${ZLIB_SRCS} ${ZLIB_DLL_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -target_include_directories(zlib PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) add_library(zlibstatic STATIC ${ZLIB_SRCS} ${ZLIB_PUBLIC_HDRS} ${ZLIB_PRIVATE_HDRS}) -target_include_directories(zlibstatic PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(zlib PROPERTIES DEFINE_SYMBOL ZLIB_DLL) set_target_properties(zlib PROPERTIES SOVERSION 1) @@ -170,7 +166,7 @@ endif() if(UNIX) # On unix-like platforms the library is almost always called libz set_target_properties(zlib zlibstatic PROPERTIES OUTPUT_NAME z) - if(NOT APPLE AND NOT(CMAKE_SYSTEM_NAME STREQUAL AIX)) + if(NOT APPLE) set_target_properties(zlib PROPERTIES LINK_FLAGS "-Wl,--version-script,\"${CMAKE_CURRENT_SOURCE_DIR}/zlib.map\"") endif() elseif(BUILD_SHARED_LIBS AND WIN32) @@ -197,22 +193,21 @@ endif() #============================================================================ # Example binaries #============================================================================ -if(ZLIB_BUILD_EXAMPLES) - add_executable(example test/example.c) - target_link_libraries(example zlib) - add_test(example example) - - add_executable(minigzip test/minigzip.c) - target_link_libraries(minigzip zlib) - - if(HAVE_OFF64_T) - add_executable(example64 test/example.c) - target_link_libraries(example64 zlib) - set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") - add_test(example64 example64) - - add_executable(minigzip64 test/minigzip.c) - target_link_libraries(minigzip64 zlib) - set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") - endif() + +add_executable(example test/example.c) +target_link_libraries(example zlib) +add_test(example example) + +add_executable(minigzip test/minigzip.c) +target_link_libraries(minigzip zlib) + +if(HAVE_OFF64_T) + add_executable(example64 test/example.c) + target_link_libraries(example64 zlib) + set_target_properties(example64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") + add_test(example64 example64) + + add_executable(minigzip64 test/minigzip.c) + target_link_libraries(minigzip64 zlib) + set_target_properties(minigzip64 PROPERTIES COMPILE_FLAGS "-D_FILE_OFFSET_BITS=64") endif() diff --git a/src/native/external/zlib/ChangeLog b/src/native/external/zlib/ChangeLog index b801a1031ec0f5..457526bc6a51f5 100644 --- a/src/native/external/zlib/ChangeLog +++ b/src/native/external/zlib/ChangeLog @@ -1,34 +1,6 @@ ChangeLog file for zlib -Changes in 1.3.1 (22 Jan 2024) -- Reject overflows of zip header fields in minizip -- Fix bug in inflateSync() for data held in bit buffer -- Add LIT_MEM define to use more memory for a small deflate speedup -- Fix decision on the emission of Zip64 end records in minizip -- Add bounds checking to ERR_MSG() macro, used by zError() -- Neutralize zip file traversal attacks in miniunz -- Fix a bug in ZLIB_DEBUG compiles in check_match() -- Various portability and appearance improvements - -Changes in 1.3 (18 Aug 2023) -- Remove K&R function definitions and zlib2ansi -- Fix bug in deflateBound() for level 0 and memLevel 9 -- Fix bug when gzungetc() is used immediately after gzopen() -- Fix bug when using gzflush() with a very small buffer -- Fix crash when gzsetparams() attempted for transparent write -- Fix test/example.c to work with FORCE_STORED -- Rewrite of zran in examples (see zran.c version history) -- Fix minizip to allow it to open an empty zip file -- Fix reading disk number start on zip64 files in minizip -- Fix logic error in minizip argument processing -- Add minizip testing to Makefile -- Read multiple bytes instead of byte-by-byte in minizip unzip.c -- Add memory sanitizer to configure (--memory) -- Various portability improvements -- Various documentation improvements -- Various spelling and typo corrections - Changes in 1.2.13 (13 Oct 2022) - Fix configure issue that discarded provided CC definition - Correct incorrect inputs provided to the CRC functions @@ -1473,7 +1445,7 @@ Changes in 0.99 (27 Jan 96) - fix typo in Make_vms.com (f$trnlnm -> f$getsyi) - in fcalloc, normalize pointer if size > 65520 bytes - don't use special fcalloc for 32 bit Borland C++ -- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc. +- use STDC instead of __GO32__ to avoid redeclaring exit, calloc, etc... - use Z_BINARY instead of BINARY - document that gzclose after gzdopen will close the file - allow "a" as mode in gzopen diff --git a/src/native/external/zlib/FAQ b/src/native/external/zlib/FAQ index 92f5d3e29fab46..99b7cf92e45497 100644 --- a/src/native/external/zlib/FAQ +++ b/src/native/external/zlib/FAQ @@ -4,7 +4,7 @@ If your question is not there, please check the zlib home page http://zlib.net/ which may have more recent information. -The latest zlib FAQ is at http://zlib.net/zlib_faq.html +The lastest zlib FAQ is at http://zlib.net/zlib_faq.html 1. Is zlib Y2K-compliant? @@ -14,7 +14,8 @@ The latest zlib FAQ is at http://zlib.net/zlib_faq.html 2. Where can I get a Windows DLL version? The zlib sources can be compiled without change to produce a DLL. See the - file win32/DLL_FAQ.txt in the zlib distribution. + file win32/DLL_FAQ.txt in the zlib distribution. Pointers to the + precompiled DLL are found in the zlib web site at http://zlib.net/ . 3. Where can I get a Visual Basic interface to zlib? diff --git a/src/native/external/zlib/Makefile.in b/src/native/external/zlib/Makefile.in index cb8b00a9b078fb..7d2713f4c574a9 100644 --- a/src/native/external/zlib/Makefile.in +++ b/src/native/external/zlib/Makefile.in @@ -1,5 +1,5 @@ # Makefile for zlib -# Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler +# Copyright (C) 1995-2017 Jean-loup Gailly, Mark Adler # For conditions of distribution and use, see copyright notice in zlib.h # To compile and test, type: @@ -22,13 +22,13 @@ CFLAGS=-O SFLAGS=-O LDFLAGS= -TEST_LIBS=-L. libz.a +TEST_LDFLAGS=$(LDFLAGS) -L. libz.a LDSHARED=$(CC) CPP=$(CC) -E STATICLIB=libz.a SHAREDLIB=libz.so -SHAREDLIBV=libz.so.1.3.1 +SHAREDLIBV=libz.so.1.2.13 SHAREDLIBM=libz.so.1 LIBS=$(STATICLIB) $(SHAREDLIBV) @@ -282,10 +282,10 @@ placebo $(SHAREDLIBV): $(PIC_OBJS) libz.a -@rmdir objs example$(EXE): example.o $(STATICLIB) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ example.o $(TEST_LIBS) + $(CC) $(CFLAGS) -o $@ example.o $(TEST_LDFLAGS) minigzip$(EXE): minigzip.o $(STATICLIB) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ minigzip.o $(TEST_LIBS) + $(CC) $(CFLAGS) -o $@ minigzip.o $(TEST_LDFLAGS) examplesh$(EXE): example.o $(SHAREDLIBV) $(CC) $(CFLAGS) -o $@ example.o $(LDFLAGS) -L. $(SHAREDLIBV) @@ -294,10 +294,10 @@ minigzipsh$(EXE): minigzip.o $(SHAREDLIBV) $(CC) $(CFLAGS) -o $@ minigzip.o $(LDFLAGS) -L. $(SHAREDLIBV) example64$(EXE): example64.o $(STATICLIB) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ example64.o $(TEST_LIBS) + $(CC) $(CFLAGS) -o $@ example64.o $(TEST_LDFLAGS) minigzip64$(EXE): minigzip64.o $(STATICLIB) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ minigzip64.o $(TEST_LIBS) + $(CC) $(CFLAGS) -o $@ minigzip64.o $(TEST_LDFLAGS) install-libs: $(LIBS) -@if [ ! -d $(DESTDIR)$(exec_prefix) ]; then mkdir -p $(DESTDIR)$(exec_prefix); fi @@ -359,14 +359,8 @@ zconf.h.cmakein: $(SRCDIR)zconf.h.in zconf: $(SRCDIR)zconf.h.in cp -p $(SRCDIR)zconf.h.in zconf.h -minizip-test: static - cd contrib/minizip && { CC="$(CC)" CFLAGS="$(CFLAGS)" $(MAKE) test ; cd ../.. ; } - -minizip-clean: - cd contrib/minizip && { $(MAKE) clean ; cd ../.. ; } - mostlyclean: clean -clean: minizip-clean +clean: rm -f *.o *.lo *~ \ example$(EXE) minigzip$(EXE) examplesh$(EXE) minigzipsh$(EXE) \ example64$(EXE) minigzip64$(EXE) \ diff --git a/src/native/external/zlib/README b/src/native/external/zlib/README index c5f917540b6fd2..ba34d1894a9b4a 100644 --- a/src/native/external/zlib/README +++ b/src/native/external/zlib/README @@ -1,6 +1,6 @@ ZLIB DATA COMPRESSION LIBRARY -zlib 1.3.1 is a general purpose data compression library. All the code is +zlib 1.2.13 is a general purpose data compression library. All the code is thread safe. The data format used by the zlib library is described by RFCs (Request for Comments) 1950 to 1952 in the files http://tools.ietf.org/html/rfc1950 (zlib format), rfc1951 (deflate format) and @@ -29,17 +29,18 @@ PLEASE read the zlib FAQ http://zlib.net/zlib_faq.html before asking for help. Mark Nelson wrote an article about zlib for the Jan. 1997 issue of Dr. Dobb's Journal; a copy of the article is available at -https://marknelson.us/posts/1997/01/01/zlib-engine.html . +http://marknelson.us/1997/01/01/zlib-engine/ . -The changes made in version 1.3.1 are documented in the file ChangeLog. +The changes made in version 1.2.13 are documented in the file ChangeLog. Unsupported third party contributions are provided in directory contrib/ . -zlib is available in Java using the java.util.zip package. Follow the API -Documentation link at: https://docs.oracle.com/search/?q=java.util.zip . +zlib is available in Java using the java.util.zip package, documented at +http://java.sun.com/developer/technicalArticles/Programming/compression/ . -A Perl interface to zlib and bzip2 written by Paul Marquess -can be found at https://github.com/pmqs/IO-Compress . +A Perl interface to zlib written by Paul Marquess is available +at CPAN (Comprehensive Perl Archive Network) sites, including +http://search.cpan.org/~pmqs/IO-Compress-Zlib/ . A Python interface to zlib written by A.M. Kuchling is available in Python 1.5 and later versions, see @@ -63,7 +64,7 @@ Notes for some targets: - zlib doesn't work with gcc 2.6.3 on a DEC 3000/300LX under OSF/1 2.1 it works when compiled with cc. -- On Digital Unix 4.0D (formerly OSF/1) on AlphaServer, the cc option -std1 is +- On Digital Unix 4.0D (formely OSF/1) on AlphaServer, the cc option -std1 is necessary to get gzprintf working correctly. This is done by configure. - zlib doesn't work on HP-UX 9.05 with some versions of /bin/cc. It works with @@ -83,7 +84,7 @@ Acknowledgments: Copyright notice: - (C) 1995-2024 Jean-loup Gailly and Mark Adler + (C) 1995-2022 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/src/native/external/zlib/adler32.c b/src/native/external/zlib/adler32.c index 04b81d29bad169..d0be4380a39c9c 100644 --- a/src/native/external/zlib/adler32.c +++ b/src/native/external/zlib/adler32.c @@ -7,6 +7,8 @@ #include "zutil.h" +local uLong adler32_combine_ OF((uLong adler1, uLong adler2, z_off64_t len2)); + #define BASE 65521U /* largest prime smaller than 65536 */ #define NMAX 5552 /* NMAX is the largest n such that 255n(n+1)/2 + (n+1)(BASE-1) <= 2^32-1 */ @@ -58,7 +60,11 @@ #endif /* ========================================================================= */ -uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) { +uLong ZEXPORT adler32_z(adler, buf, len) + uLong adler; + const Bytef *buf; + z_size_t len; +{ unsigned long sum2; unsigned n; @@ -125,12 +131,20 @@ uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, z_size_t len) { } /* ========================================================================= */ -uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len) { +uLong ZEXPORT adler32(adler, buf, len) + uLong adler; + const Bytef *buf; + uInt len; +{ return adler32_z(adler, buf, len); } /* ========================================================================= */ -local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2) { +local uLong adler32_combine_(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ unsigned long sum1; unsigned long sum2; unsigned rem; @@ -155,10 +169,18 @@ local uLong adler32_combine_(uLong adler1, uLong adler2, z_off64_t len2) { } /* ========================================================================= */ -uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, z_off_t len2) { +uLong ZEXPORT adler32_combine(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off_t len2; +{ return adler32_combine_(adler1, adler2, len2); } -uLong ZEXPORT adler32_combine64(uLong adler1, uLong adler2, z_off64_t len2) { +uLong ZEXPORT adler32_combine64(adler1, adler2, len2) + uLong adler1; + uLong adler2; + z_off64_t len2; +{ return adler32_combine_(adler1, adler2, len2); } diff --git a/src/native/external/zlib/compress.c b/src/native/external/zlib/compress.c index f43bacf7ab9707..2ad5326c14ec04 100644 --- a/src/native/external/zlib/compress.c +++ b/src/native/external/zlib/compress.c @@ -19,8 +19,13 @@ memory, Z_BUF_ERROR if there was not enough room in the output buffer, Z_STREAM_ERROR if the level parameter is invalid. */ -int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong sourceLen, int level) { +int ZEXPORT compress2(dest, destLen, source, sourceLen, level) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; + int level; +{ z_stream stream; int err; const uInt max = (uInt)-1; @@ -60,8 +65,12 @@ int ZEXPORT compress2(Bytef *dest, uLongf *destLen, const Bytef *source, /* =========================================================================== */ -int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong sourceLen) { +int ZEXPORT compress(dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ return compress2(dest, destLen, source, sourceLen, Z_DEFAULT_COMPRESSION); } @@ -69,7 +78,9 @@ int ZEXPORT compress(Bytef *dest, uLongf *destLen, const Bytef *source, If the default memLevel or windowBits for deflateInit() is changed, then this function needs to be updated. */ -uLong ZEXPORT compressBound(uLong sourceLen) { +uLong ZEXPORT compressBound(sourceLen) + uLong sourceLen; +{ return sourceLen + (sourceLen >> 12) + (sourceLen >> 14) + (sourceLen >> 25) + 13; } diff --git a/src/native/external/zlib/configure b/src/native/external/zlib/configure index c55098afc4ae52..fa4d5daaba99f4 100644 --- a/src/native/external/zlib/configure +++ b/src/native/external/zlib/configure @@ -25,7 +25,7 @@ if test $SRCDIR = "."; then ZINCOUT="-I." SRCDIR="" else - ZINC='-I. -include zconf.h' + ZINC='-include zconf.h' ZINCOUT='-I. -I$(SRCDIR)' SRCDIR="$SRCDIR/" fi @@ -44,8 +44,9 @@ STATICLIB=libz.a # extract zlib version numbers from zlib.h VER=`sed -n -e '/VERSION "/s/.*"\(.*\)".*/\1/p' < ${SRCDIR}zlib.h` -VER3=`echo ${VER}|sed -n -e 's/\([0-9]\{1,\}\(\\.[0-9]\{1,\}\)\{1,2\}\).*/\1/p'` -VER1=`echo ${VER}|sed -n -e 's/\([0-9]\{1,\}\)\\..*/\1/p'` +VER3=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\\.[0-9]*\).*/\1/p' < ${SRCDIR}zlib.h` +VER2=`sed -n -e '/VERSION "/s/.*"\([0-9]*\\.[0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` +VER1=`sed -n -e '/VERSION "/s/.*"\([0-9]*\)\\..*/\1/p' < ${SRCDIR}zlib.h` # establish commands for library building if "${CROSS_PREFIX}ar" --version >/dev/null 2>/dev/null || test $? -lt 126; then @@ -89,8 +90,7 @@ build64=0 gcc=0 warn=0 debug=0 -address=0 -memory=0 +sanitize=0 old_cc="$CC" old_cflags="$CFLAGS" OBJC='$(OBJZ) $(OBJG)' @@ -102,7 +102,7 @@ leave() if test "$*" != "0"; then echo "** $0 aborting." | tee -a configure.log fi - rm -rf $test.[co] $test $test$shared_ext $test.gcno $test.dSYM ./--version + rm -f $test.[co] $test $test$shared_ext $test.gcno ./--version echo -------------------- >> configure.log echo >> configure.log echo >> configure.log @@ -141,9 +141,7 @@ case "$1" in -c* | --const) zconst=1; shift ;; -w* | --warn) warn=1; shift ;; -d* | --debug) debug=1; shift ;; - --sanitize) address=1; shift ;; - --address) address=1; shift ;; - --memory) memory=1; shift ;; + --sanitize) sanitize=1; shift ;; *) echo "unknown option: $1" | tee -a configure.log echo "$0 --help for help" | tee -a configure.log @@ -213,11 +211,8 @@ if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then CFLAGS="${CFLAGS} -Wall -Wextra" fi fi - if test $address -eq 1; then - CFLAGS="${CFLAGS} -g -fsanitize=address -fno-omit-frame-pointer" - fi - if test $memory -eq 1; then - CFLAGS="${CFLAGS} -g -fsanitize=memory -fno-omit-frame-pointer" + if test $sanitize -eq 1; then + CFLAGS="${CFLAGS} -g -fsanitize=address" fi if test $debug -eq 1; then CFLAGS="${CFLAGS} -DZLIB_DEBUG" @@ -265,9 +260,7 @@ if test "$gcc" -eq 1 && ($cc -c $test.c) >> configure.log 2>&1; then SHAREDLIBV=libz.$VER$shared_ext SHAREDLIBM=libz.$VER1$shared_ext LDSHARED=${LDSHARED-"$cc -dynamiclib -install_name $libdir/$SHAREDLIBM -compatibility_version $VER1 -current_version $VER3"} - if "${CROSS_PREFIX}libtool" -V 2>&1 | grep Apple > /dev/null; then - AR="${CROSS_PREFIX}libtool" - elif libtool -V 2>&1 | grep Apple > /dev/null; then + if libtool -V 2>&1 | grep Apple > /dev/null; then AR="libtool" else AR="/usr/bin/libtool" @@ -442,7 +435,7 @@ EOF if test $shared -eq 1; then echo Checking for shared library support... | tee -a configure.log # we must test in two steps (cc then ld), required at least on SunOS 4.x - if try $CC -c $SFLAGS $test.c && + if try $CC -w -c $SFLAGS $test.c && try $LDSHARED $SFLAGS -o $test$shared_ext $test.o; then echo Building shared library $SHAREDLIBV with $CC. | tee -a configure.log elif test -z "$old_cc" -a -z "$old_cflags"; then @@ -867,7 +860,7 @@ echo prefix = $prefix >> configure.log echo sharedlibdir = $sharedlibdir >> configure.log echo uname = $uname >> configure.log -# update Makefile with the configure results +# udpate Makefile with the configure results sed < ${SRCDIR}Makefile.in " /^CC *=/s#=.*#=$CC# /^CFLAGS *=/s#=.*#=$CFLAGS# diff --git a/src/native/external/zlib/crc32.c b/src/native/external/zlib/crc32.c index 6c38f5c04c6a87..f8357b083f7636 100644 --- a/src/native/external/zlib/crc32.c +++ b/src/native/external/zlib/crc32.c @@ -103,6 +103,19 @@ # define ARMCRC32 #endif +/* Local functions. */ +local z_crc_t multmodp OF((z_crc_t a, z_crc_t b)); +local z_crc_t x2nmodp OF((z_off64_t n, unsigned k)); + +#if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) + local z_word_t byte_swap OF((z_word_t word)); +#endif + +#if defined(W) && !defined(ARMCRC32) + local z_crc_t crc_word OF((z_word_t data)); + local z_word_t crc_word_big OF((z_word_t data)); +#endif + #if defined(W) && (!defined(ARMCRC32) || defined(DYNAMIC_CRC_TABLE)) /* Swap the bytes in a z_word_t to convert between little and big endian. Any @@ -110,7 +123,9 @@ instruction, if one is available. This assumes that word_t is either 32 bits or 64 bits. */ -local z_word_t byte_swap(z_word_t word) { +local z_word_t byte_swap(word) + z_word_t word; +{ # if W == 8 return (word & 0xff00000000000000) >> 56 | @@ -131,77 +146,24 @@ local z_word_t byte_swap(z_word_t word) { } #endif -#ifdef DYNAMIC_CRC_TABLE -/* ========================================================================= - * Table of powers of x for combining CRC-32s, filled in by make_crc_table() - * below. - */ - local z_crc_t FAR x2n_table[32]; -#else -/* ========================================================================= - * Tables for byte-wise and braided CRC-32 calculations, and a table of powers - * of x for combining CRC-32s, all made by make_crc_table(). - */ -# include "crc32.h" -#endif - /* CRC polynomial. */ #define POLY 0xedb88320 /* p(x) reflected, with x^32 implied */ -/* - Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, - reflected. For speed, this requires that a not be zero. - */ -local z_crc_t multmodp(z_crc_t a, z_crc_t b) { - z_crc_t m, p; - - m = (z_crc_t)1 << 31; - p = 0; - for (;;) { - if (a & m) { - p ^= b; - if ((a & (m - 1)) == 0) - break; - } - m >>= 1; - b = b & 1 ? (b >> 1) ^ POLY : b >> 1; - } - return p; -} - -/* - Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been - initialized. - */ -local z_crc_t x2nmodp(z_off64_t n, unsigned k) { - z_crc_t p; - - p = (z_crc_t)1 << 31; /* x^0 == 1 */ - while (n) { - if (n & 1) - p = multmodp(x2n_table[k & 31], p); - n >>= 1; - k++; - } - return p; -} - #ifdef DYNAMIC_CRC_TABLE -/* ========================================================================= - * Build the tables for byte-wise and braided CRC-32 calculations, and a table - * of powers of x for combining CRC-32s. - */ + local z_crc_t FAR crc_table[256]; +local z_crc_t FAR x2n_table[32]; +local void make_crc_table OF((void)); #ifdef W local z_word_t FAR crc_big_table[256]; local z_crc_t FAR crc_braid_table[W][256]; local z_word_t FAR crc_braid_big_table[W][256]; - local void braid(z_crc_t [][256], z_word_t [][256], int, int); + local void braid OF((z_crc_t [][256], z_word_t [][256], int, int)); #endif #ifdef MAKECRCH - local void write_table(FILE *, const z_crc_t FAR *, int); - local void write_table32hi(FILE *, const z_word_t FAR *, int); - local void write_table64(FILE *, const z_word_t FAR *, int); + local void write_table OF((FILE *, const z_crc_t FAR *, int)); + local void write_table32hi OF((FILE *, const z_word_t FAR *, int)); + local void write_table64 OF((FILE *, const z_word_t FAR *, int)); #endif /* MAKECRCH */ /* @@ -214,6 +176,7 @@ local z_crc_t FAR crc_table[256]; /* Definition of once functionality. */ typedef struct once_s once_t; +local void once OF((once_t *, void (*)(void))); /* Check for the availability of atomics. */ #if defined(__STDC__) && __STDC_VERSION__ >= 201112L && \ @@ -233,7 +196,10 @@ struct once_s { invoke once() at the same time. The state must be a once_t initialized with ONCE_INIT. */ -local void once(once_t *state, void (*init)(void)) { +local void once(state, init) + once_t *state; + void (*init)(void); +{ if (!atomic_load(&state->done)) { if (atomic_flag_test_and_set(&state->begun)) while (!atomic_load(&state->done)) @@ -256,7 +222,10 @@ struct once_s { /* Test and set. Alas, not atomic, but tries to minimize the period of vulnerability. */ -local int test_and_set(int volatile *flag) { +local int test_and_set OF((int volatile *)); +local int test_and_set(flag) + int volatile *flag; +{ int was; was = *flag; @@ -265,7 +234,10 @@ local int test_and_set(int volatile *flag) { } /* Run the provided init() function once. This is not thread-safe. */ -local void once(once_t *state, void (*init)(void)) { +local void once(state, init) + once_t *state; + void (*init)(void); +{ if (!state->done) { if (test_and_set(&state->begun)) while (!state->done) @@ -307,7 +279,8 @@ local once_t made = ONCE_INIT; combinations of CRC register values and incoming bytes. */ -local void make_crc_table(void) { +local void make_crc_table() +{ unsigned i, j, n; z_crc_t p; @@ -474,7 +447,11 @@ local void make_crc_table(void) { Write the 32-bit values in table[0..k-1] to out, five per line in hexadecimal separated by commas. */ -local void write_table(FILE *out, const z_crc_t FAR *table, int k) { +local void write_table(out, table, k) + FILE *out; + const z_crc_t FAR *table; + int k; +{ int n; for (n = 0; n < k; n++) @@ -487,7 +464,11 @@ local void write_table(FILE *out, const z_crc_t FAR *table, int k) { Write the high 32-bits of each value in table[0..k-1] to out, five per line in hexadecimal separated by commas. */ -local void write_table32hi(FILE *out, const z_word_t FAR *table, int k) { +local void write_table32hi(out, table, k) +FILE *out; +const z_word_t FAR *table; +int k; +{ int n; for (n = 0; n < k; n++) @@ -503,7 +484,11 @@ local void write_table32hi(FILE *out, const z_word_t FAR *table, int k) { bits. If not, then the type cast and format string can be adjusted accordingly. */ -local void write_table64(FILE *out, const z_word_t FAR *table, int k) { +local void write_table64(out, table, k) + FILE *out; + const z_word_t FAR *table; + int k; +{ int n; for (n = 0; n < k; n++) @@ -513,7 +498,8 @@ local void write_table64(FILE *out, const z_word_t FAR *table, int k) { } /* Actually do the deed. */ -int main(void) { +int main() +{ make_crc_table(); return 0; } @@ -525,7 +511,12 @@ int main(void) { Generate the little and big-endian braid tables for the given n and z_word_t size w. Each array must have room for w blocks of 256 elements. */ -local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) { +local void braid(ltl, big, n, w) + z_crc_t ltl[][256]; + z_word_t big[][256]; + int n; + int w; +{ int k; z_crc_t i, p, q; for (k = 0; k < w; k++) { @@ -540,13 +531,69 @@ local void braid(z_crc_t ltl[][256], z_word_t big[][256], int n, int w) { } #endif +#else /* !DYNAMIC_CRC_TABLE */ +/* ======================================================================== + * Tables for byte-wise and braided CRC-32 calculations, and a table of powers + * of x for combining CRC-32s, all made by make_crc_table(). + */ +#include "crc32.h" #endif /* DYNAMIC_CRC_TABLE */ +/* ======================================================================== + * Routines used for CRC calculation. Some are also required for the table + * generation above. + */ + +/* + Return a(x) multiplied by b(x) modulo p(x), where p(x) is the CRC polynomial, + reflected. For speed, this requires that a not be zero. + */ +local z_crc_t multmodp(a, b) + z_crc_t a; + z_crc_t b; +{ + z_crc_t m, p; + + m = (z_crc_t)1 << 31; + p = 0; + for (;;) { + if (a & m) { + p ^= b; + if ((a & (m - 1)) == 0) + break; + } + m >>= 1; + b = b & 1 ? (b >> 1) ^ POLY : b >> 1; + } + return p; +} + +/* + Return x^(n * 2^k) modulo p(x). Requires that x2n_table[] has been + initialized. + */ +local z_crc_t x2nmodp(n, k) + z_off64_t n; + unsigned k; +{ + z_crc_t p; + + p = (z_crc_t)1 << 31; /* x^0 == 1 */ + while (n) { + if (n & 1) + p = multmodp(x2n_table[k & 31], p); + n >>= 1; + k++; + } + return p; +} + /* ========================================================================= * This function can be used by asm versions of crc32(), and to force the * generation of the CRC tables in a threaded application. */ -const z_crc_t FAR * ZEXPORT get_crc_table(void) { +const z_crc_t FAR * ZEXPORT get_crc_table() +{ #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ @@ -572,8 +619,11 @@ const z_crc_t FAR * ZEXPORT get_crc_table(void) { #define Z_BATCH_ZEROS 0xa10d3d0c /* computed from Z_BATCH = 3990 */ #define Z_BATCH_MIN 800 /* fewest words in a final batch */ -unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, - z_size_t len) { +unsigned long ZEXPORT crc32_z(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + z_size_t len; +{ z_crc_t val; z_word_t crc1, crc2; const z_word_t *word; @@ -673,14 +723,18 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, least-significant byte of the word as the first byte of data, without any pre or post conditioning. This is used to combine the CRCs of each braid. */ -local z_crc_t crc_word(z_word_t data) { +local z_crc_t crc_word(data) + z_word_t data; +{ int k; for (k = 0; k < W; k++) data = (data >> 8) ^ crc_table[data & 0xff]; return (z_crc_t)data; } -local z_word_t crc_word_big(z_word_t data) { +local z_word_t crc_word_big(data) + z_word_t data; +{ int k; for (k = 0; k < W; k++) data = (data << 8) ^ @@ -691,8 +745,11 @@ local z_word_t crc_word_big(z_word_t data) { #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, - z_size_t len) { +unsigned long ZEXPORT crc32_z(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + z_size_t len; +{ /* Return initial CRC, if requested. */ if (buf == Z_NULL) return 0; @@ -724,8 +781,8 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, words = (z_word_t const *)buf; /* Do endian check at execution time instead of compile time, since ARM - processors can change the endianness at execution time. If the - compiler knows what the endianness will be, it can optimize out the + processors can change the endianess at execution time. If the + compiler knows what the endianess will be, it can optimize out the check and the unused branch. */ endian = 1; if (*(unsigned char *)&endian) { @@ -1012,13 +1069,20 @@ unsigned long ZEXPORT crc32_z(unsigned long crc, const unsigned char FAR *buf, #endif /* ========================================================================= */ -unsigned long ZEXPORT crc32(unsigned long crc, const unsigned char FAR *buf, - uInt len) { +unsigned long ZEXPORT crc32(crc, buf, len) + unsigned long crc; + const unsigned char FAR *buf; + uInt len; +{ return crc32_z(crc, buf, len); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) { +uLong ZEXPORT crc32_combine64(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off64_t len2; +{ #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ @@ -1026,12 +1090,18 @@ uLong ZEXPORT crc32_combine64(uLong crc1, uLong crc2, z_off64_t len2) { } /* ========================================================================= */ -uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2) { +uLong ZEXPORT crc32_combine(crc1, crc2, len2) + uLong crc1; + uLong crc2; + z_off_t len2; +{ return crc32_combine64(crc1, crc2, (z_off64_t)len2); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) { +uLong ZEXPORT crc32_combine_gen64(len2) + z_off64_t len2; +{ #ifdef DYNAMIC_CRC_TABLE once(&made, make_crc_table); #endif /* DYNAMIC_CRC_TABLE */ @@ -1039,11 +1109,17 @@ uLong ZEXPORT crc32_combine_gen64(z_off64_t len2) { } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_gen(z_off_t len2) { +uLong ZEXPORT crc32_combine_gen(len2) + z_off_t len2; +{ return crc32_combine_gen64((z_off64_t)len2); } /* ========================================================================= */ -uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op) { +uLong ZEXPORT crc32_combine_op(crc1, crc2, op) + uLong crc1; + uLong crc2; + uLong op; +{ return multmodp(op, crc1) ^ (crc2 & 0xffffffff); } diff --git a/src/native/external/zlib/deflate.c b/src/native/external/zlib/deflate.c index ca2fc59a1b54a8..b763663975458c 100644 --- a/src/native/external/zlib/deflate.c +++ b/src/native/external/zlib/deflate.c @@ -1,5 +1,5 @@ /* deflate.c -- compress data using the deflation algorithm - * Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler + * Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -52,7 +52,7 @@ #include "deflate.h" const char deflate_copyright[] = - " deflate 1.3.1 Copyright 1995-2024 Jean-loup Gailly and Mark Adler "; + " deflate 1.2.13 Copyright 1995-2022 Jean-loup Gailly and Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -60,6 +60,9 @@ const char deflate_copyright[] = copyright string in the executable of your product. */ +/* =========================================================================== + * Function prototypes. + */ typedef enum { need_more, /* block not completed, need more input or more output */ block_done, /* block flush performed */ @@ -67,16 +70,29 @@ typedef enum { finish_done /* finish done, accept no more input or output */ } block_state; -typedef block_state (*compress_func)(deflate_state *s, int flush); +typedef block_state (*compress_func) OF((deflate_state *s, int flush)); /* Compression function. Returns the block state after the call. */ -local block_state deflate_stored(deflate_state *s, int flush); -local block_state deflate_fast(deflate_state *s, int flush); +local int deflateStateCheck OF((z_streamp strm)); +local void slide_hash OF((deflate_state *s)); +local void fill_window OF((deflate_state *s)); +local block_state deflate_stored OF((deflate_state *s, int flush)); +local block_state deflate_fast OF((deflate_state *s, int flush)); #ifndef FASTEST -local block_state deflate_slow(deflate_state *s, int flush); +local block_state deflate_slow OF((deflate_state *s, int flush)); +#endif +local block_state deflate_rle OF((deflate_state *s, int flush)); +local block_state deflate_huff OF((deflate_state *s, int flush)); +local void lm_init OF((deflate_state *s)); +local void putShortMSB OF((deflate_state *s, uInt b)); +local void flush_pending OF((z_streamp strm)); +local unsigned read_buf OF((z_streamp strm, Bytef *buf, unsigned size)); +local uInt longest_match OF((deflate_state *s, IPos cur_match)); + +#ifdef ZLIB_DEBUG +local void check_match OF((deflate_state *s, IPos start, IPos match, + int length)); #endif -local block_state deflate_rle(deflate_state *s, int flush); -local block_state deflate_huff(deflate_state *s, int flush); /* =========================================================================== * Local data @@ -179,12 +195,9 @@ local const config configuration_table[10] = { * bit values at the expense of memory usage). We slide even when level == 0 to * keep the hash table consistent if we switch back to level > 0 later. */ -#if defined(__has_feature) -# if __has_feature(memory_sanitizer) - __attribute__((no_sanitize("memory"))) -# endif -#endif -local void slide_hash(deflate_state *s) { +local void slide_hash(s) + deflate_state *s; +{ unsigned n, m; Posf *p; uInt wsize = s->w_size; @@ -208,177 +221,30 @@ local void slide_hash(deflate_state *s) { #endif } -/* =========================================================================== - * Read a new buffer from the current input stream, update the adler32 - * and total number of bytes read. All deflate() input goes through - * this function so some applications may wish to modify it to avoid - * allocating a large strm->next_in buffer and copying from it. - * (See also flush_pending()). - */ -local unsigned read_buf(z_streamp strm, Bytef *buf, unsigned size) { - unsigned len = strm->avail_in; - - if (len > size) len = size; - if (len == 0) return 0; - - strm->avail_in -= len; - - zmemcpy(buf, strm->next_in, len); - if (strm->state->wrap == 1) { - strm->adler = adler32(strm->adler, buf, len); - } -#ifdef GZIP - else if (strm->state->wrap == 2) { - strm->adler = crc32(strm->adler, buf, len); - } -#endif - strm->next_in += len; - strm->total_in += len; - - return len; -} - -/* =========================================================================== - * Fill the window when the lookahead becomes insufficient. - * Updates strstart and lookahead. - * - * IN assertion: lookahead < MIN_LOOKAHEAD - * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD - * At least one byte has been read, or avail_in == 0; reads are - * performed for at least two bytes (required for the zip translate_eol - * option -- not supported here). - */ -local void fill_window(deflate_state *s) { - unsigned n; - unsigned more; /* Amount of free space at the end of the window. */ - uInt wsize = s->w_size; - - Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); - - do { - more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); - - /* Deal with !@#$% 64K limit: */ - if (sizeof(int) <= 2) { - if (more == 0 && s->strstart == 0 && s->lookahead == 0) { - more = wsize; - - } else if (more == (unsigned)(-1)) { - /* Very unlikely, but possible on 16 bit machine if - * strstart == 0 && lookahead == 1 (input done a byte at time) - */ - more--; - } - } - - /* If the window is almost full and there is insufficient lookahead, - * move the upper half to the lower one to make room in the upper half. - */ - if (s->strstart >= wsize + MAX_DIST(s)) { - - zmemcpy(s->window, s->window + wsize, (unsigned)wsize - more); - s->match_start -= wsize; - s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ - s->block_start -= (long) wsize; - if (s->insert > s->strstart) - s->insert = s->strstart; - slide_hash(s); - more += wsize; - } - if (s->strm->avail_in == 0) break; - - /* If there was no sliding: - * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && - * more == window_size - lookahead - strstart - * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) - * => more >= window_size - 2*WSIZE + 2 - * In the BIG_MEM or MMAP case (not yet supported), - * window_size == input_size + MIN_LOOKAHEAD && - * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. - * Otherwise, window_size == 2*WSIZE so more >= 2. - * If there was sliding, more >= WSIZE. So in all cases, more >= 2. - */ - Assert(more >= 2, "more < 2"); - - n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); - s->lookahead += n; - - /* Initialize the hash value now that we have some input: */ - if (s->lookahead + s->insert >= MIN_MATCH) { - uInt str = s->strstart - s->insert; - s->ins_h = s->window[str]; - UPDATE_HASH(s, s->ins_h, s->window[str + 1]); -#if MIN_MATCH != 3 - Call UPDATE_HASH() MIN_MATCH-3 more times -#endif - while (s->insert) { - UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); -#ifndef FASTEST - s->prev[str & s->w_mask] = s->head[s->ins_h]; -#endif - s->head[s->ins_h] = (Pos)str; - str++; - s->insert--; - if (s->lookahead + s->insert < MIN_MATCH) - break; - } - } - /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, - * but this is not important since only literal bytes will be emitted. - */ - - } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); - - /* If the WIN_INIT bytes after the end of the current data have never been - * written, then zero those bytes in order to avoid memory check reports of - * the use of uninitialized (or uninitialised as Julian writes) bytes by - * the longest match routines. Update the high water mark for the next - * time through here. WIN_INIT is set to MAX_MATCH since the longest match - * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. - */ - if (s->high_water < s->window_size) { - ulg curr = s->strstart + (ulg)(s->lookahead); - ulg init; - - if (s->high_water < curr) { - /* Previous high water mark below current data -- zero WIN_INIT - * bytes or up to end of window, whichever is less. - */ - init = s->window_size - curr; - if (init > WIN_INIT) - init = WIN_INIT; - zmemzero(s->window + curr, (unsigned)init); - s->high_water = curr + init; - } - else if (s->high_water < (ulg)curr + WIN_INIT) { - /* High water mark at or above current data, but below current data - * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up - * to end of window, whichever is less. - */ - init = (ulg)curr + WIN_INIT - s->high_water; - if (init > s->window_size - s->high_water) - init = s->window_size - s->high_water; - zmemzero(s->window + s->high_water, (unsigned)init); - s->high_water += init; - } - } - - Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, - "not enough room for search"); -} - /* ========================================================================= */ -int ZEXPORT deflateInit_(z_streamp strm, int level, const char *version, - int stream_size) { +int ZEXPORT deflateInit_(strm, level, version, stream_size) + z_streamp strm; + int level; + const char *version; + int stream_size; +{ return deflateInit2_(strm, level, Z_DEFLATED, MAX_WBITS, DEF_MEM_LEVEL, Z_DEFAULT_STRATEGY, version, stream_size); /* To do: ignore strm->next_in if we use it as window */ } /* ========================================================================= */ -int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, - int windowBits, int memLevel, int strategy, - const char *version, int stream_size) { +int ZEXPORT deflateInit2_(strm, level, method, windowBits, memLevel, strategy, + version, stream_size) + z_streamp strm; + int level; + int method; + int windowBits; + int memLevel; + int strategy; + const char *version; + int stream_size; +{ deflate_state *s; int wrap = 1; static const char my_version[] = ZLIB_VERSION; @@ -493,7 +359,7 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, * symbols from which it is being constructed. */ - s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, LIT_BUFS); + s->pending_buf = (uchf *) ZALLOC(strm, s->lit_bufsize, 4); s->pending_buf_size = (ulg)s->lit_bufsize * 4; if (s->window == Z_NULL || s->prev == Z_NULL || s->head == Z_NULL || @@ -503,14 +369,8 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, deflateEnd (strm); return Z_MEM_ERROR; } -#ifdef LIT_MEM - s->d_buf = (ushf *)(s->pending_buf + (s->lit_bufsize << 1)); - s->l_buf = s->pending_buf + (s->lit_bufsize << 2); - s->sym_end = s->lit_bufsize - 1; -#else s->sym_buf = s->pending_buf + s->lit_bufsize; s->sym_end = (s->lit_bufsize - 1) * 3; -#endif /* We avoid equality with lit_bufsize*3 because of wraparound at 64K * on 16 bit machines and because stored blocks are restricted to * 64K-1 bytes. @@ -526,7 +386,9 @@ int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, /* ========================================================================= * Check for a valid deflate stream state. Return 0 if ok, 1 if not. */ -local int deflateStateCheck(z_streamp strm) { +local int deflateStateCheck(strm) + z_streamp strm; +{ deflate_state *s; if (strm == Z_NULL || strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) @@ -547,8 +409,11 @@ local int deflateStateCheck(z_streamp strm) { } /* ========================================================================= */ -int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef *dictionary, - uInt dictLength) { +int ZEXPORT deflateSetDictionary(strm, dictionary, dictLength) + z_streamp strm; + const Bytef *dictionary; + uInt dictLength; +{ deflate_state *s; uInt str, n; int wrap; @@ -613,8 +478,11 @@ int ZEXPORT deflateSetDictionary(z_streamp strm, const Bytef *dictionary, } /* ========================================================================= */ -int ZEXPORT deflateGetDictionary(z_streamp strm, Bytef *dictionary, - uInt *dictLength) { +int ZEXPORT deflateGetDictionary(strm, dictionary, dictLength) + z_streamp strm; + Bytef *dictionary; + uInt *dictLength; +{ deflate_state *s; uInt len; @@ -632,7 +500,9 @@ int ZEXPORT deflateGetDictionary(z_streamp strm, Bytef *dictionary, } /* ========================================================================= */ -int ZEXPORT deflateResetKeep(z_streamp strm) { +int ZEXPORT deflateResetKeep(strm) + z_streamp strm; +{ deflate_state *s; if (deflateStateCheck(strm)) { @@ -667,32 +537,10 @@ int ZEXPORT deflateResetKeep(z_streamp strm) { return Z_OK; } -/* =========================================================================== - * Initialize the "longest match" routines for a new zlib stream - */ -local void lm_init(deflate_state *s) { - s->window_size = (ulg)2L*s->w_size; - - CLEAR_HASH(s); - - /* Set the default configuration parameters: - */ - s->max_lazy_match = configuration_table[s->level].max_lazy; - s->good_match = configuration_table[s->level].good_length; - s->nice_match = configuration_table[s->level].nice_length; - s->max_chain_length = configuration_table[s->level].max_chain; - - s->strstart = 0; - s->block_start = 0L; - s->lookahead = 0; - s->insert = 0; - s->match_length = s->prev_length = MIN_MATCH-1; - s->match_available = 0; - s->ins_h = 0; -} - /* ========================================================================= */ -int ZEXPORT deflateReset(z_streamp strm) { +int ZEXPORT deflateReset(strm) + z_streamp strm; +{ int ret; ret = deflateResetKeep(strm); @@ -702,7 +550,10 @@ int ZEXPORT deflateReset(z_streamp strm) { } /* ========================================================================= */ -int ZEXPORT deflateSetHeader(z_streamp strm, gz_headerp head) { +int ZEXPORT deflateSetHeader(strm, head) + z_streamp strm; + gz_headerp head; +{ if (deflateStateCheck(strm) || strm->state->wrap != 2) return Z_STREAM_ERROR; strm->state->gzhead = head; @@ -710,7 +561,11 @@ int ZEXPORT deflateSetHeader(z_streamp strm, gz_headerp head) { } /* ========================================================================= */ -int ZEXPORT deflatePending(z_streamp strm, unsigned *pending, int *bits) { +int ZEXPORT deflatePending(strm, pending, bits) + unsigned *pending; + int *bits; + z_streamp strm; +{ if (deflateStateCheck(strm)) return Z_STREAM_ERROR; if (pending != Z_NULL) *pending = strm->state->pending; @@ -720,21 +575,19 @@ int ZEXPORT deflatePending(z_streamp strm, unsigned *pending, int *bits) { } /* ========================================================================= */ -int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) { +int ZEXPORT deflatePrime(strm, bits, value) + z_streamp strm; + int bits; + int value; +{ deflate_state *s; int put; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; s = strm->state; -#ifdef LIT_MEM - if (bits < 0 || bits > 16 || - (uchf *)s->d_buf < s->pending_out + ((Buf_size + 7) >> 3)) - return Z_BUF_ERROR; -#else if (bits < 0 || bits > 16 || s->sym_buf < s->pending_out + ((Buf_size + 7) >> 3)) return Z_BUF_ERROR; -#endif do { put = Buf_size - s->bi_valid; if (put > bits) @@ -749,7 +602,11 @@ int ZEXPORT deflatePrime(z_streamp strm, int bits, int value) { } /* ========================================================================= */ -int ZEXPORT deflateParams(z_streamp strm, int level, int strategy) { +int ZEXPORT deflateParams(strm, level, strategy) + z_streamp strm; + int level; + int strategy; +{ deflate_state *s; compress_func func; @@ -794,8 +651,13 @@ int ZEXPORT deflateParams(z_streamp strm, int level, int strategy) { } /* ========================================================================= */ -int ZEXPORT deflateTune(z_streamp strm, int good_length, int max_lazy, - int nice_length, int max_chain) { +int ZEXPORT deflateTune(strm, good_length, max_lazy, nice_length, max_chain) + z_streamp strm; + int good_length; + int max_lazy; + int nice_length; + int max_chain; +{ deflate_state *s; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -831,7 +693,10 @@ int ZEXPORT deflateTune(z_streamp strm, int good_length, int max_lazy, * * Shifts are used to approximate divisions, for speed. */ -uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) { +uLong ZEXPORT deflateBound(strm, sourceLen) + z_streamp strm; + uLong sourceLen; +{ deflate_state *s; uLong fixedlen, storelen, wraplen; @@ -901,7 +766,10 @@ uLong ZEXPORT deflateBound(z_streamp strm, uLong sourceLen) { * IN assertion: the stream state is correct and there is enough room in * pending_buf. */ -local void putShortMSB(deflate_state *s, uInt b) { +local void putShortMSB(s, b) + deflate_state *s; + uInt b; +{ put_byte(s, (Byte)(b >> 8)); put_byte(s, (Byte)(b & 0xff)); } @@ -912,7 +780,9 @@ local void putShortMSB(deflate_state *s, uInt b) { * applications may wish to modify it to avoid allocating a large * strm->next_out buffer and copying into it. (See also read_buf()). */ -local void flush_pending(z_streamp strm) { +local void flush_pending(strm) + z_streamp strm; +{ unsigned len; deflate_state *s = strm->state; @@ -943,7 +813,10 @@ local void flush_pending(z_streamp strm) { } while (0) /* ========================================================================= */ -int ZEXPORT deflate(z_streamp strm, int flush) { +int ZEXPORT deflate(strm, flush) + z_streamp strm; + int flush; +{ int old_flush; /* value of flush param for previous deflate call */ deflate_state *s; @@ -1255,7 +1128,9 @@ int ZEXPORT deflate(z_streamp strm, int flush) { } /* ========================================================================= */ -int ZEXPORT deflateEnd(z_streamp strm) { +int ZEXPORT deflateEnd(strm) + z_streamp strm; +{ int status; if (deflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1279,10 +1154,11 @@ int ZEXPORT deflateEnd(z_streamp strm) { * To simplify the source, this is not supported for 16-bit MSDOS (which * doesn't have enough memory anyway to duplicate compression states). */ -int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { +int ZEXPORT deflateCopy(dest, source) + z_streamp dest; + z_streamp source; +{ #ifdef MAXSEG_64K - (void)dest; - (void)source; return Z_STREAM_ERROR; #else deflate_state *ds; @@ -1306,7 +1182,7 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { ds->window = (Bytef *) ZALLOC(dest, ds->w_size, 2*sizeof(Byte)); ds->prev = (Posf *) ZALLOC(dest, ds->w_size, sizeof(Pos)); ds->head = (Posf *) ZALLOC(dest, ds->hash_size, sizeof(Pos)); - ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, LIT_BUFS); + ds->pending_buf = (uchf *) ZALLOC(dest, ds->lit_bufsize, 4); if (ds->window == Z_NULL || ds->prev == Z_NULL || ds->head == Z_NULL || ds->pending_buf == Z_NULL) { @@ -1317,15 +1193,10 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { zmemcpy(ds->window, ss->window, ds->w_size * 2 * sizeof(Byte)); zmemcpy((voidpf)ds->prev, (voidpf)ss->prev, ds->w_size * sizeof(Pos)); zmemcpy((voidpf)ds->head, (voidpf)ss->head, ds->hash_size * sizeof(Pos)); - zmemcpy(ds->pending_buf, ss->pending_buf, ds->lit_bufsize * LIT_BUFS); + zmemcpy(ds->pending_buf, ss->pending_buf, (uInt)ds->pending_buf_size); ds->pending_out = ds->pending_buf + (ss->pending_out - ss->pending_buf); -#ifdef LIT_MEM - ds->d_buf = (ushf *)(ds->pending_buf + (ds->lit_bufsize << 1)); - ds->l_buf = ds->pending_buf + (ds->lit_bufsize << 2); -#else ds->sym_buf = ds->pending_buf + ds->lit_bufsize; -#endif ds->l_desc.dyn_tree = ds->dyn_ltree; ds->d_desc.dyn_tree = ds->dyn_dtree; @@ -1335,6 +1206,66 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { #endif /* MAXSEG_64K */ } +/* =========================================================================== + * Read a new buffer from the current input stream, update the adler32 + * and total number of bytes read. All deflate() input goes through + * this function so some applications may wish to modify it to avoid + * allocating a large strm->next_in buffer and copying from it. + * (See also flush_pending()). + */ +local unsigned read_buf(strm, buf, size) + z_streamp strm; + Bytef *buf; + unsigned size; +{ + unsigned len = strm->avail_in; + + if (len > size) len = size; + if (len == 0) return 0; + + strm->avail_in -= len; + + zmemcpy(buf, strm->next_in, len); + if (strm->state->wrap == 1) { + strm->adler = adler32(strm->adler, buf, len); + } +#ifdef GZIP + else if (strm->state->wrap == 2) { + strm->adler = crc32(strm->adler, buf, len); + } +#endif + strm->next_in += len; + strm->total_in += len; + + return len; +} + +/* =========================================================================== + * Initialize the "longest match" routines for a new zlib stream + */ +local void lm_init(s) + deflate_state *s; +{ + s->window_size = (ulg)2L*s->w_size; + + CLEAR_HASH(s); + + /* Set the default configuration parameters: + */ + s->max_lazy_match = configuration_table[s->level].max_lazy; + s->good_match = configuration_table[s->level].good_length; + s->nice_match = configuration_table[s->level].nice_length; + s->max_chain_length = configuration_table[s->level].max_chain; + + s->strstart = 0; + s->block_start = 0L; + s->lookahead = 0; + s->insert = 0; + s->match_length = s->prev_length = MIN_MATCH-1; + s->match_available = 0; + s->ins_h = 0; +} + #ifndef FASTEST /* =========================================================================== * Set match_start to the longest match starting at the given string and @@ -1345,7 +1276,10 @@ int ZEXPORT deflateCopy(z_streamp dest, z_streamp source) { * string (strstart) and its distance is <= MAX_DIST, and prev_length >= 1 * OUT assertion: the match length is not greater than s->lookahead. */ -local uInt longest_match(deflate_state *s, IPos cur_match) { +local uInt longest_match(s, cur_match) + deflate_state *s; + IPos cur_match; /* current match */ +{ unsigned chain_length = s->max_chain_length;/* max hash chain length */ register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ @@ -1493,7 +1427,10 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { /* --------------------------------------------------------------------------- * Optimized version for FASTEST only */ -local uInt longest_match(deflate_state *s, IPos cur_match) { +local uInt longest_match(s, cur_match) + deflate_state *s; + IPos cur_match; /* current match */ +{ register Bytef *scan = s->window + s->strstart; /* current string */ register Bytef *match; /* matched string */ register int len; /* length of current match */ @@ -1554,23 +1491,19 @@ local uInt longest_match(deflate_state *s, IPos cur_match) { /* =========================================================================== * Check that the match at match_start is indeed a match. */ -local void check_match(deflate_state *s, IPos start, IPos match, int length) { +local void check_match(s, start, match, length) + deflate_state *s; + IPos start, match; + int length; +{ /* check that the match is indeed a match */ - Bytef *back = s->window + (int)match, *here = s->window + start; - IPos len = length; - if (match == (IPos)-1) { - /* match starts one byte before the current window -- just compare the - subsequent length-1 bytes */ - back++; - here++; - len--; - } - if (zmemcmp(back, here, len) != EQUAL) { - fprintf(stderr, " start %u, match %d, length %d\n", - start, (int)match, length); + if (zmemcmp(s->window + match, + s->window + start, length) != EQUAL) { + fprintf(stderr, " start %u, match %u, length %d\n", + start, match, length); do { - fprintf(stderr, "(%02x %02x)", *back++, *here++); - } while (--len != 0); + fprintf(stderr, "%c%c", s->window[match++], s->window[start++]); + } while (--length != 0); z_error("invalid match"); } if (z_verbose > 1) { @@ -1582,6 +1515,137 @@ local void check_match(deflate_state *s, IPos start, IPos match, int length) { # define check_match(s, start, match, length) #endif /* ZLIB_DEBUG */ +/* =========================================================================== + * Fill the window when the lookahead becomes insufficient. + * Updates strstart and lookahead. + * + * IN assertion: lookahead < MIN_LOOKAHEAD + * OUT assertions: strstart <= window_size-MIN_LOOKAHEAD + * At least one byte has been read, or avail_in == 0; reads are + * performed for at least two bytes (required for the zip translate_eol + * option -- not supported here). + */ +local void fill_window(s) + deflate_state *s; +{ + unsigned n; + unsigned more; /* Amount of free space at the end of the window. */ + uInt wsize = s->w_size; + + Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead"); + + do { + more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart); + + /* Deal with !@#$% 64K limit: */ + if (sizeof(int) <= 2) { + if (more == 0 && s->strstart == 0 && s->lookahead == 0) { + more = wsize; + + } else if (more == (unsigned)(-1)) { + /* Very unlikely, but possible on 16 bit machine if + * strstart == 0 && lookahead == 1 (input done a byte at time) + */ + more--; + } + } + + /* If the window is almost full and there is insufficient lookahead, + * move the upper half to the lower one to make room in the upper half. + */ + if (s->strstart >= wsize + MAX_DIST(s)) { + + zmemcpy(s->window, s->window + wsize, (unsigned)wsize - more); + s->match_start -= wsize; + s->strstart -= wsize; /* we now have strstart >= MAX_DIST */ + s->block_start -= (long) wsize; + if (s->insert > s->strstart) + s->insert = s->strstart; + slide_hash(s); + more += wsize; + } + if (s->strm->avail_in == 0) break; + + /* If there was no sliding: + * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 && + * more == window_size - lookahead - strstart + * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1) + * => more >= window_size - 2*WSIZE + 2 + * In the BIG_MEM or MMAP case (not yet supported), + * window_size == input_size + MIN_LOOKAHEAD && + * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD. + * Otherwise, window_size == 2*WSIZE so more >= 2. + * If there was sliding, more >= WSIZE. So in all cases, more >= 2. + */ + Assert(more >= 2, "more < 2"); + + n = read_buf(s->strm, s->window + s->strstart + s->lookahead, more); + s->lookahead += n; + + /* Initialize the hash value now that we have some input: */ + if (s->lookahead + s->insert >= MIN_MATCH) { + uInt str = s->strstart - s->insert; + s->ins_h = s->window[str]; + UPDATE_HASH(s, s->ins_h, s->window[str + 1]); +#if MIN_MATCH != 3 + Call UPDATE_HASH() MIN_MATCH-3 more times +#endif + while (s->insert) { + UPDATE_HASH(s, s->ins_h, s->window[str + MIN_MATCH-1]); +#ifndef FASTEST + s->prev[str & s->w_mask] = s->head[s->ins_h]; +#endif + s->head[s->ins_h] = (Pos)str; + str++; + s->insert--; + if (s->lookahead + s->insert < MIN_MATCH) + break; + } + } + /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage, + * but this is not important since only literal bytes will be emitted. + */ + + } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0); + + /* If the WIN_INIT bytes after the end of the current data have never been + * written, then zero those bytes in order to avoid memory check reports of + * the use of uninitialized (or uninitialised as Julian writes) bytes by + * the longest match routines. Update the high water mark for the next + * time through here. WIN_INIT is set to MAX_MATCH since the longest match + * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead. + */ + if (s->high_water < s->window_size) { + ulg curr = s->strstart + (ulg)(s->lookahead); + ulg init; + + if (s->high_water < curr) { + /* Previous high water mark below current data -- zero WIN_INIT + * bytes or up to end of window, whichever is less. + */ + init = s->window_size - curr; + if (init > WIN_INIT) + init = WIN_INIT; + zmemzero(s->window + curr, (unsigned)init); + s->high_water = curr + init; + } + else if (s->high_water < (ulg)curr + WIN_INIT) { + /* High water mark at or above current data, but below current data + * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up + * to end of window, whichever is less. + */ + init = (ulg)curr + WIN_INIT - s->high_water; + if (init > s->window_size - s->high_water) + init = s->window_size - s->high_water; + zmemzero(s->window + s->high_water, (unsigned)init); + s->high_water += init; + } + } + + Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD, + "not enough room for search"); +} + /* =========================================================================== * Flush the current block, with given end-of-file flag. * IN assertion: strstart is set to the end of the current match. @@ -1624,7 +1688,10 @@ local void check_match(deflate_state *s, IPos start, IPos match, int length) { * copied. It is most efficient with large input and output buffers, which * maximizes the opportunities to have a single copy from next_in to next_out. */ -local block_state deflate_stored(deflate_state *s, int flush) { +local block_state deflate_stored(s, flush) + deflate_state *s; + int flush; +{ /* Smallest worthy block size when not flushing or finishing. By default * this is 32K. This can be as small as 507 bytes for memLevel == 1. For * large input and output buffers, the stored block size will be larger. @@ -1808,7 +1875,10 @@ local block_state deflate_stored(deflate_state *s, int flush) { * new strings in the dictionary only for unmatched strings or for short * matches. It is used only for the fast compression options. */ -local block_state deflate_fast(deflate_state *s, int flush) { +local block_state deflate_fast(s, flush) + deflate_state *s; + int flush; +{ IPos hash_head; /* head of the hash chain */ int bflush; /* set if current block must be flushed */ @@ -1907,7 +1977,10 @@ local block_state deflate_fast(deflate_state *s, int flush) { * evaluation for matches: a match is finally adopted only if there is * no better match at the next window position. */ -local block_state deflate_slow(deflate_state *s, int flush) { +local block_state deflate_slow(s, flush) + deflate_state *s; + int flush; +{ IPos hash_head; /* head of hash chain */ int bflush; /* set if current block must be flushed */ @@ -2035,7 +2108,10 @@ local block_state deflate_slow(deflate_state *s, int flush) { * one. Do not maintain a hash table. (It will be regenerated if this run of * deflate switches away from Z_RLE.) */ -local block_state deflate_rle(deflate_state *s, int flush) { +local block_state deflate_rle(s, flush) + deflate_state *s; + int flush; +{ int bflush; /* set if current block must be flushed */ uInt prev; /* byte at distance one to match */ Bytef *scan, *strend; /* scan goes up to strend for length of run */ @@ -2106,7 +2182,10 @@ local block_state deflate_rle(deflate_state *s, int flush) { * For Z_HUFFMAN_ONLY, do not look for matches. Do not maintain a hash table. * (It will be regenerated if this run of deflate switches away from Huffman.) */ -local block_state deflate_huff(deflate_state *s, int flush) { +local block_state deflate_huff(s, flush) + deflate_state *s; + int flush; +{ int bflush; /* set if current block must be flushed */ for (;;) { diff --git a/src/native/external/zlib/deflate.h b/src/native/external/zlib/deflate.h index 300c6ada62b82e..1a06cd5f25d107 100644 --- a/src/native/external/zlib/deflate.h +++ b/src/native/external/zlib/deflate.h @@ -1,5 +1,5 @@ /* deflate.h -- internal compression state - * Copyright (C) 1995-2024 Jean-loup Gailly + * Copyright (C) 1995-2018 Jean-loup Gailly * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -23,10 +23,6 @@ # define GZIP #endif -/* define LIT_MEM to slightly increase the speed of deflate (order 1% to 2%) at - the cost of a larger memory footprint */ -/* #define LIT_MEM */ - /* =========================================================================== * Internal compression state. */ @@ -221,14 +217,7 @@ typedef struct internal_state { /* Depth of each subtree used as tie breaker for trees of equal frequency */ -#ifdef LIT_MEM -# define LIT_BUFS 5 - ushf *d_buf; /* buffer for distances */ - uchf *l_buf; /* buffer for literals/lengths */ -#else -# define LIT_BUFS 4 uchf *sym_buf; /* buffer for distances and literals/lengths */ -#endif uInt lit_bufsize; /* Size of match buffer for literals/lengths. There are 4 reasons for @@ -250,7 +239,7 @@ typedef struct internal_state { * - I can't count above 4 */ - uInt sym_next; /* running index in symbol buffer */ + uInt sym_next; /* running index in sym_buf */ uInt sym_end; /* symbol table full when sym_next reaches this */ ulg opt_len; /* bit length of current block with optimal trees */ @@ -302,14 +291,14 @@ typedef struct internal_state { memory checker errors from longest match routines */ /* in trees.c */ -void ZLIB_INTERNAL _tr_init(deflate_state *s); -int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc); -void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, - ulg stored_len, int last); -void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s); -void ZLIB_INTERNAL _tr_align(deflate_state *s); -void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, - ulg stored_len, int last); +void ZLIB_INTERNAL _tr_init OF((deflate_state *s)); +int ZLIB_INTERNAL _tr_tally OF((deflate_state *s, unsigned dist, unsigned lc)); +void ZLIB_INTERNAL _tr_flush_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); +void ZLIB_INTERNAL _tr_flush_bits OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_align OF((deflate_state *s)); +void ZLIB_INTERNAL _tr_stored_block OF((deflate_state *s, charf *buf, + ulg stored_len, int last)); #define d_code(dist) \ ((dist) < 256 ? _dist_code[dist] : _dist_code[256+((dist)>>7)]) @@ -329,25 +318,6 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, extern const uch ZLIB_INTERNAL _dist_code[]; #endif -#ifdef LIT_MEM -# define _tr_tally_lit(s, c, flush) \ - { uch cc = (c); \ - s->d_buf[s->sym_next] = 0; \ - s->l_buf[s->sym_next++] = cc; \ - s->dyn_ltree[cc].Freq++; \ - flush = (s->sym_next == s->sym_end); \ - } -# define _tr_tally_dist(s, distance, length, flush) \ - { uch len = (uch)(length); \ - ush dist = (ush)(distance); \ - s->d_buf[s->sym_next] = dist; \ - s->l_buf[s->sym_next++] = len; \ - dist--; \ - s->dyn_ltree[_length_code[len]+LITERALS+1].Freq++; \ - s->dyn_dtree[d_code(dist)].Freq++; \ - flush = (s->sym_next == s->sym_end); \ - } -#else # define _tr_tally_lit(s, c, flush) \ { uch cc = (c); \ s->sym_buf[s->sym_next++] = 0; \ @@ -367,7 +337,6 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, s->dyn_dtree[d_code(dist)].Freq++; \ flush = (s->sym_next == s->sym_end); \ } -#endif #else # define _tr_tally_lit(s, c, flush) flush = _tr_tally(s, 0, c) # define _tr_tally_dist(s, distance, length, flush) \ diff --git a/src/native/external/zlib/gzclose.c b/src/native/external/zlib/gzclose.c index 48d6a86f04b6ea..caeb99a3177f47 100644 --- a/src/native/external/zlib/gzclose.c +++ b/src/native/external/zlib/gzclose.c @@ -8,7 +8,9 @@ /* gzclose() is in a separate file so that it is linked in only if it is used. That way the other gzclose functions can be used instead to avoid linking in unneeded compression or decompression routines. */ -int ZEXPORT gzclose(gzFile file) { +int ZEXPORT gzclose(file) + gzFile file; +{ #ifndef NO_GZCOMPRESS gz_statep state; diff --git a/src/native/external/zlib/gzguts.h b/src/native/external/zlib/gzguts.h index eba72085bb756b..57faf37165a354 100644 --- a/src/native/external/zlib/gzguts.h +++ b/src/native/external/zlib/gzguts.h @@ -1,5 +1,5 @@ /* gzguts.h -- zlib internal header definitions for gz* operations - * Copyright (C) 2004-2024 Mark Adler + * Copyright (C) 2004-2019 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -7,8 +7,9 @@ # ifndef _LARGEFILE_SOURCE # define _LARGEFILE_SOURCE 1 # endif -# undef _FILE_OFFSET_BITS -# undef _TIME_BITS +# ifdef _FILE_OFFSET_BITS +# undef _FILE_OFFSET_BITS +# endif #endif #ifdef HAVE_HIDDEN @@ -118,8 +119,8 @@ /* gz* functions always use library allocation functions */ #ifndef STDC - extern voidp malloc(uInt size); - extern void free(voidpf ptr); + extern voidp malloc OF((uInt size)); + extern void free OF((voidpf ptr)); #endif /* get errno and strerror definition */ @@ -137,10 +138,10 @@ /* provide prototypes for these when building zlib without LFS */ #if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 - ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); - ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int); - ZEXTERN z_off64_t ZEXPORT gztell64(gzFile); - ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile); + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); #endif /* default memLevel */ @@ -202,13 +203,17 @@ typedef struct { typedef gz_state FAR *gz_statep; /* shared functions */ -void ZLIB_INTERNAL gz_error(gz_statep, int, const char *); +void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); #if defined UNDER_CE -char ZLIB_INTERNAL *gz_strwinerror(DWORD error); +char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); #endif /* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t value -- needed when comparing unsigned to z_off64_t, which is signed (possible z_off64_t types off_t, off64_t, and long are all signed) */ -unsigned ZLIB_INTERNAL gz_intmax(void); -#define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#ifdef INT_MAX +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) +#else +unsigned ZLIB_INTERNAL gz_intmax OF((void)); +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#endif diff --git a/src/native/external/zlib/gzlib.c b/src/native/external/zlib/gzlib.c index 983153cc8e4965..55da46a453fd18 100644 --- a/src/native/external/zlib/gzlib.c +++ b/src/native/external/zlib/gzlib.c @@ -1,5 +1,5 @@ /* gzlib.c -- zlib functions common to reading and writing gzip files - * Copyright (C) 2004-2024 Mark Adler + * Copyright (C) 2004-2019 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -15,6 +15,10 @@ #endif #endif +/* Local functions */ +local void gz_reset OF((gz_statep)); +local gzFile gz_open OF((const void *, int, const char *)); + #if defined UNDER_CE /* Map the Windows error number in ERROR to a locale-dependent error message @@ -26,7 +30,9 @@ The gz_strwinerror function does not change the current setting of GetLastError. */ -char ZLIB_INTERNAL *gz_strwinerror(DWORD error) { +char ZLIB_INTERNAL *gz_strwinerror(error) + DWORD error; +{ static char buf[1024]; wchar_t *msgbuf; @@ -66,7 +72,9 @@ char ZLIB_INTERNAL *gz_strwinerror(DWORD error) { #endif /* UNDER_CE */ /* Reset gzip file state */ -local void gz_reset(gz_statep state) { +local void gz_reset(state) + gz_statep state; +{ state->x.have = 0; /* no output data available */ if (state->mode == GZ_READ) { /* for reading ... */ state->eof = 0; /* not at end of file */ @@ -82,7 +90,11 @@ local void gz_reset(gz_statep state) { } /* Open a gzip file either by name or file descriptor. */ -local gzFile gz_open(const void *path, int fd, const char *mode) { +local gzFile gz_open(path, fd, mode) + const void *path; + int fd; + const char *mode; +{ gz_statep state; z_size_t len; int oflag; @@ -257,17 +269,26 @@ local gzFile gz_open(const void *path, int fd, const char *mode) { } /* -- see zlib.h -- */ -gzFile ZEXPORT gzopen(const char *path, const char *mode) { +gzFile ZEXPORT gzopen(path, mode) + const char *path; + const char *mode; +{ return gz_open(path, -1, mode); } /* -- see zlib.h -- */ -gzFile ZEXPORT gzopen64(const char *path, const char *mode) { +gzFile ZEXPORT gzopen64(path, mode) + const char *path; + const char *mode; +{ return gz_open(path, -1, mode); } /* -- see zlib.h -- */ -gzFile ZEXPORT gzdopen(int fd, const char *mode) { +gzFile ZEXPORT gzdopen(fd, mode) + int fd; + const char *mode; +{ char *path; /* identifier for error messages */ gzFile gz; @@ -285,13 +306,19 @@ gzFile ZEXPORT gzdopen(int fd, const char *mode) { /* -- see zlib.h -- */ #ifdef WIDECHAR -gzFile ZEXPORT gzopen_w(const wchar_t *path, const char *mode) { +gzFile ZEXPORT gzopen_w(path, mode) + const wchar_t *path; + const char *mode; +{ return gz_open(path, -2, mode); } #endif /* -- see zlib.h -- */ -int ZEXPORT gzbuffer(gzFile file, unsigned size) { +int ZEXPORT gzbuffer(file, size) + gzFile file; + unsigned size; +{ gz_statep state; /* get internal structure and check integrity */ @@ -308,14 +335,16 @@ int ZEXPORT gzbuffer(gzFile file, unsigned size) { /* check and set requested size */ if ((size << 1) < size) return -1; /* need to be able to double it */ - if (size < 8) - size = 8; /* needed to behave well with flushing */ + if (size < 2) + size = 2; /* need two bytes to check magic header */ state->want = size; return 0; } /* -- see zlib.h -- */ -int ZEXPORT gzrewind(gzFile file) { +int ZEXPORT gzrewind(file) + gzFile file; +{ gz_statep state; /* get internal structure */ @@ -336,7 +365,11 @@ int ZEXPORT gzrewind(gzFile file) { } /* -- see zlib.h -- */ -z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) { +z_off64_t ZEXPORT gzseek64(file, offset, whence) + gzFile file; + z_off64_t offset; + int whence; +{ unsigned n; z_off64_t ret; gz_statep state; @@ -409,7 +442,11 @@ z_off64_t ZEXPORT gzseek64(gzFile file, z_off64_t offset, int whence) { } /* -- see zlib.h -- */ -z_off_t ZEXPORT gzseek(gzFile file, z_off_t offset, int whence) { +z_off_t ZEXPORT gzseek(file, offset, whence) + gzFile file; + z_off_t offset; + int whence; +{ z_off64_t ret; ret = gzseek64(file, (z_off64_t)offset, whence); @@ -417,7 +454,9 @@ z_off_t ZEXPORT gzseek(gzFile file, z_off_t offset, int whence) { } /* -- see zlib.h -- */ -z_off64_t ZEXPORT gztell64(gzFile file) { +z_off64_t ZEXPORT gztell64(file) + gzFile file; +{ gz_statep state; /* get internal structure and check integrity */ @@ -432,7 +471,9 @@ z_off64_t ZEXPORT gztell64(gzFile file) { } /* -- see zlib.h -- */ -z_off_t ZEXPORT gztell(gzFile file) { +z_off_t ZEXPORT gztell(file) + gzFile file; +{ z_off64_t ret; ret = gztell64(file); @@ -440,7 +481,9 @@ z_off_t ZEXPORT gztell(gzFile file) { } /* -- see zlib.h -- */ -z_off64_t ZEXPORT gzoffset64(gzFile file) { +z_off64_t ZEXPORT gzoffset64(file) + gzFile file; +{ z_off64_t offset; gz_statep state; @@ -461,7 +504,9 @@ z_off64_t ZEXPORT gzoffset64(gzFile file) { } /* -- see zlib.h -- */ -z_off_t ZEXPORT gzoffset(gzFile file) { +z_off_t ZEXPORT gzoffset(file) + gzFile file; +{ z_off64_t ret; ret = gzoffset64(file); @@ -469,7 +514,9 @@ z_off_t ZEXPORT gzoffset(gzFile file) { } /* -- see zlib.h -- */ -int ZEXPORT gzeof(gzFile file) { +int ZEXPORT gzeof(file) + gzFile file; +{ gz_statep state; /* get internal structure and check integrity */ @@ -484,7 +531,10 @@ int ZEXPORT gzeof(gzFile file) { } /* -- see zlib.h -- */ -const char * ZEXPORT gzerror(gzFile file, int *errnum) { +const char * ZEXPORT gzerror(file, errnum) + gzFile file; + int *errnum; +{ gz_statep state; /* get internal structure and check integrity */ @@ -502,7 +552,9 @@ const char * ZEXPORT gzerror(gzFile file, int *errnum) { } /* -- see zlib.h -- */ -void ZEXPORT gzclearerr(gzFile file) { +void ZEXPORT gzclearerr(file) + gzFile file; +{ gz_statep state; /* get internal structure and check integrity */ @@ -526,7 +578,11 @@ void ZEXPORT gzclearerr(gzFile file) { memory). Simply save the error message as a static string. If there is an allocation failure constructing the error message, then convert the error to out of memory. */ -void ZLIB_INTERNAL gz_error(gz_statep state, int err, const char *msg) { +void ZLIB_INTERNAL gz_error(state, err, msg) + gz_statep state; + int err; + const char *msg; +{ /* free previously allocated message and clear */ if (state->msg != NULL) { if (state->err != Z_MEM_ERROR) @@ -563,20 +619,21 @@ void ZLIB_INTERNAL gz_error(gz_statep state, int err, const char *msg) { #endif } +#ifndef INT_MAX /* portably return maximum value for an int (when limits.h presumed not available) -- we need to do this to cover cases where 2's complement not used, since C standard permits 1's complement and sign-bit representations, otherwise we could just use ((unsigned)-1) >> 1 */ -unsigned ZLIB_INTERNAL gz_intmax(void) { -#ifdef INT_MAX - return INT_MAX; -#else - unsigned p = 1, q; +unsigned ZLIB_INTERNAL gz_intmax() +{ + unsigned p, q; + + p = 1; do { q = p; p <<= 1; p++; } while (p > q); return q >> 1; -#endif } +#endif diff --git a/src/native/external/zlib/gzread.c b/src/native/external/zlib/gzread.c index 4168cbc88752e1..dd77381596cbc0 100644 --- a/src/native/external/zlib/gzread.c +++ b/src/native/external/zlib/gzread.c @@ -5,12 +5,25 @@ #include "gzguts.h" +/* Local functions */ +local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); +local int gz_avail OF((gz_statep)); +local int gz_look OF((gz_statep)); +local int gz_decomp OF((gz_statep)); +local int gz_fetch OF((gz_statep)); +local int gz_skip OF((gz_statep, z_off64_t)); +local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); + /* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from state->fd, and update state->eof, state->err, and state->msg as appropriate. This function needs to loop on read(), since read() is not guaranteed to read the number of bytes requested, depending on the type of descriptor. */ -local int gz_load(gz_statep state, unsigned char *buf, unsigned len, - unsigned *have) { +local int gz_load(state, buf, len, have) + gz_statep state; + unsigned char *buf; + unsigned len; + unsigned *have; +{ int ret; unsigned get, max = ((unsigned)-1 >> 2) + 1; @@ -40,7 +53,9 @@ local int gz_load(gz_statep state, unsigned char *buf, unsigned len, If strm->avail_in != 0, then the current data is moved to the beginning of the input buffer, and then the remainder of the buffer is loaded with the available data from the input file. */ -local int gz_avail(gz_statep state) { +local int gz_avail(state) + gz_statep state; +{ unsigned got; z_streamp strm = &(state->strm); @@ -73,7 +88,9 @@ local int gz_avail(gz_statep state) { case, all further file reads will be directly to either the output buffer or a user buffer. If decompressing, the inflate state will be initialized. gz_look() will return 0 on success or -1 on failure. */ -local int gz_look(gz_statep state) { +local int gz_look(state) + gz_statep state; +{ z_streamp strm = &(state->strm); /* allocate read buffers and inflate memory */ @@ -153,7 +170,9 @@ local int gz_look(gz_statep state) { data. If the gzip stream completes, state->how is reset to LOOK to look for the next gzip stream or raw data, once state->x.have is depleted. Returns 0 on success, -1 on failure. */ -local int gz_decomp(gz_statep state) { +local int gz_decomp(state) + gz_statep state; +{ int ret = Z_OK; unsigned had; z_streamp strm = &(state->strm); @@ -205,7 +224,9 @@ local int gz_decomp(gz_statep state) { looked for to determine whether to copy or decompress. Returns -1 on error, otherwise 0. gz_fetch() will leave state->how as COPY or GZIP unless the end of the input file has been reached and all data has been processed. */ -local int gz_fetch(gz_statep state) { +local int gz_fetch(state) + gz_statep state; +{ z_streamp strm = &(state->strm); do { @@ -233,7 +254,10 @@ local int gz_fetch(gz_statep state) { } /* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ -local int gz_skip(gz_statep state, z_off64_t len) { +local int gz_skip(state, len) + gz_statep state; + z_off64_t len; +{ unsigned n; /* skip over len bytes or reach end-of-file, whichever comes first */ @@ -265,7 +289,11 @@ local int gz_skip(gz_statep state, z_off64_t len) { input. Return the number of bytes read. If zero is returned, either the end of file was reached, or there was an error. state->err must be consulted in that case to determine which. */ -local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { +local z_size_t gz_read(state, buf, len) + gz_statep state; + voidp buf; + z_size_t len; +{ z_size_t got; unsigned n; @@ -342,7 +370,11 @@ local z_size_t gz_read(gz_statep state, voidp buf, z_size_t len) { } /* -- see zlib.h -- */ -int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ gz_statep state; /* get internal structure */ @@ -374,7 +406,12 @@ int ZEXPORT gzread(gzFile file, voidp buf, unsigned len) { } /* -- see zlib.h -- */ -z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, gzFile file) { +z_size_t ZEXPORT gzfread(buf, size, nitems, file) + voidp buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ z_size_t len; gz_statep state; @@ -405,7 +442,9 @@ z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, gzFile file) #else # undef gzgetc #endif -int ZEXPORT gzgetc(gzFile file) { +int ZEXPORT gzgetc(file) + gzFile file; +{ unsigned char buf[1]; gz_statep state; @@ -430,12 +469,17 @@ int ZEXPORT gzgetc(gzFile file) { return gz_read(state, buf, 1) < 1 ? -1 : buf[0]; } -int ZEXPORT gzgetc_(gzFile file) { +int ZEXPORT gzgetc_(file) +gzFile file; +{ return gzgetc(file); } /* -- see zlib.h -- */ -int ZEXPORT gzungetc(int c, gzFile file) { +int ZEXPORT gzungetc(c, file) + int c; + gzFile file; +{ gz_statep state; /* get internal structure */ @@ -443,10 +487,6 @@ int ZEXPORT gzungetc(int c, gzFile file) { return -1; state = (gz_statep)file; - /* in case this was just opened, set up the input buffer */ - if (state->mode == GZ_READ && state->how == LOOK && state->x.have == 0) - (void)gz_look(state); - /* check that we're reading and that there's no (serious) error */ if (state->mode != GZ_READ || (state->err != Z_OK && state->err != Z_BUF_ERROR)) @@ -496,7 +536,11 @@ int ZEXPORT gzungetc(int c, gzFile file) { } /* -- see zlib.h -- */ -char * ZEXPORT gzgets(gzFile file, char *buf, int len) { +char * ZEXPORT gzgets(file, buf, len) + gzFile file; + char *buf; + int len; +{ unsigned left, n; char *str; unsigned char *eol; @@ -556,7 +600,9 @@ char * ZEXPORT gzgets(gzFile file, char *buf, int len) { } /* -- see zlib.h -- */ -int ZEXPORT gzdirect(gzFile file) { +int ZEXPORT gzdirect(file) + gzFile file; +{ gz_statep state; /* get internal structure */ @@ -574,7 +620,9 @@ int ZEXPORT gzdirect(gzFile file) { } /* -- see zlib.h -- */ -int ZEXPORT gzclose_r(gzFile file) { +int ZEXPORT gzclose_r(file) + gzFile file; +{ int ret, err; gz_statep state; diff --git a/src/native/external/zlib/gzwrite.c b/src/native/external/zlib/gzwrite.c index 435b4621b5349f..eb8a0e5893ff6a 100644 --- a/src/native/external/zlib/gzwrite.c +++ b/src/native/external/zlib/gzwrite.c @@ -5,10 +5,18 @@ #include "gzguts.h" +/* Local functions */ +local int gz_init OF((gz_statep)); +local int gz_comp OF((gz_statep, int)); +local int gz_zero OF((gz_statep, z_off64_t)); +local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); + /* Initialize state for writing a gzip file. Mark initialization by setting state->size to non-zero. Return -1 on a memory allocation failure, or 0 on success. */ -local int gz_init(gz_statep state) { +local int gz_init(state) + gz_statep state; +{ int ret; z_streamp strm = &(state->strm); @@ -62,7 +70,10 @@ local int gz_init(gz_statep state) { deflate() flush value. If flush is Z_FINISH, then the deflate() state is reset to start a new gzip stream. If gz->direct is true, then simply write to the output file without compressing, and ignore flush. */ -local int gz_comp(gz_statep state, int flush) { +local int gz_comp(state, flush) + gz_statep state; + int flush; +{ int ret, writ; unsigned have, put, max = ((unsigned)-1 >> 2) + 1; z_streamp strm = &(state->strm); @@ -140,7 +151,10 @@ local int gz_comp(gz_statep state, int flush) { /* Compress len zeros to output. Return -1 on a write error or memory allocation failure by gz_comp(), or 0 on success. */ -local int gz_zero(gz_statep state, z_off64_t len) { +local int gz_zero(state, len) + gz_statep state; + z_off64_t len; +{ int first; unsigned n; z_streamp strm = &(state->strm); @@ -170,7 +184,11 @@ local int gz_zero(gz_statep state, z_off64_t len) { /* Write len bytes from buf to file. Return the number of bytes written. If the returned value is less than len, then there was an error. */ -local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { +local z_size_t gz_write(state, buf, len) + gz_statep state; + voidpc buf; + z_size_t len; +{ z_size_t put = len; /* if len is zero, avoid unnecessary operations */ @@ -234,7 +252,11 @@ local z_size_t gz_write(gz_statep state, voidpc buf, z_size_t len) { } /* -- see zlib.h -- */ -int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ gz_statep state; /* get internal structure */ @@ -258,8 +280,12 @@ int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len) { } /* -- see zlib.h -- */ -z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, z_size_t nitems, - gzFile file) { +z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) + voidpc buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ z_size_t len; gz_statep state; @@ -284,7 +310,10 @@ z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, z_size_t nitems, } /* -- see zlib.h -- */ -int ZEXPORT gzputc(gzFile file, int c) { +int ZEXPORT gzputc(file, c) + gzFile file; + int c; +{ unsigned have; unsigned char buf[1]; gz_statep state; @@ -329,7 +358,10 @@ int ZEXPORT gzputc(gzFile file, int c) { } /* -- see zlib.h -- */ -int ZEXPORT gzputs(gzFile file, const char *s) { +int ZEXPORT gzputs(file, s) + gzFile file; + const char *s; +{ z_size_t len, put; gz_statep state; @@ -356,7 +388,8 @@ int ZEXPORT gzputs(gzFile file, const char *s) { #include /* -- see zlib.h -- */ -int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) +{ int len; unsigned left; char *next; @@ -427,7 +460,8 @@ int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) { return len; } -int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) { +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) +{ va_list va; int ret; @@ -440,10 +474,13 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) { #else /* !STDC && !Z_HAVE_STDARG_H */ /* -- see zlib.h -- */ -int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3, - int a4, int a5, int a6, int a7, int a8, int a9, int a10, - int a11, int a12, int a13, int a14, int a15, int a16, - int a17, int a18, int a19, int a20) { +int ZEXPORTVA gzprintf(file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) + gzFile file; + const char *format; + int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; +{ unsigned len, left; char *next; gz_statep state; @@ -525,7 +562,10 @@ int ZEXPORTVA gzprintf(gzFile file, const char *format, int a1, int a2, int a3, #endif /* -- see zlib.h -- */ -int ZEXPORT gzflush(gzFile file, int flush) { +int ZEXPORT gzflush(file, flush) + gzFile file; + int flush; +{ gz_statep state; /* get internal structure */ @@ -554,7 +594,11 @@ int ZEXPORT gzflush(gzFile file, int flush) { } /* -- see zlib.h -- */ -int ZEXPORT gzsetparams(gzFile file, int level, int strategy) { +int ZEXPORT gzsetparams(file, level, strategy) + gzFile file; + int level; + int strategy; +{ gz_statep state; z_streamp strm; @@ -565,7 +609,7 @@ int ZEXPORT gzsetparams(gzFile file, int level, int strategy) { strm = &(state->strm); /* check that we're writing and that there's no error */ - if (state->mode != GZ_WRITE || state->err != Z_OK || state->direct) + if (state->mode != GZ_WRITE || state->err != Z_OK) return Z_STREAM_ERROR; /* if no change is requested, then do nothing */ @@ -592,7 +636,9 @@ int ZEXPORT gzsetparams(gzFile file, int level, int strategy) { } /* -- see zlib.h -- */ -int ZEXPORT gzclose_w(gzFile file) { +int ZEXPORT gzclose_w(file) + gzFile file; +{ int ret = Z_OK; gz_statep state; diff --git a/src/native/external/zlib/infback.c b/src/native/external/zlib/infback.c index e7b25b307a3072..babeaf1806f98f 100644 --- a/src/native/external/zlib/infback.c +++ b/src/native/external/zlib/infback.c @@ -15,6 +15,9 @@ #include "inflate.h" #include "inffast.h" +/* function prototypes */ +local void fixedtables OF((struct inflate_state FAR *state)); + /* strm provides memory allocation functions in zalloc and zfree, or Z_NULL to use the library memory allocation functions. @@ -22,9 +25,13 @@ windowBits is in the range 8..15, and window is a user-supplied window and output buffer that is 2**windowBits bytes. */ -int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, - unsigned char FAR *window, const char *version, - int stream_size) { +int ZEXPORT inflateBackInit_(strm, windowBits, window, version, stream_size) +z_streamp strm; +int windowBits; +unsigned char FAR *window; +const char *version; +int stream_size; +{ struct inflate_state FAR *state; if (version == Z_NULL || version[0] != ZLIB_VERSION[0] || @@ -73,7 +80,9 @@ int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ -local void fixedtables(struct inflate_state FAR *state) { +local void fixedtables(state) +struct inflate_state FAR *state; +{ #ifdef BUILDFIXED static int virgin = 1; static code *lenfix, *distfix; @@ -239,8 +248,13 @@ local void fixedtables(struct inflate_state FAR *state) { inflateBack() can also return Z_STREAM_ERROR if the input parameters are not correct, i.e. strm is Z_NULL or the state was not initialized. */ -int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, - out_func out, void FAR *out_desc) { +int ZEXPORT inflateBack(strm, in, in_desc, out, out_desc) +z_streamp strm; +in_func in; +void FAR *in_desc; +out_func out; +void FAR *out_desc; +{ struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ @@ -618,7 +632,9 @@ int ZEXPORT inflateBack(z_streamp strm, in_func in, void FAR *in_desc, return ret; } -int ZEXPORT inflateBackEnd(z_streamp strm) { +int ZEXPORT inflateBackEnd(strm) +z_streamp strm; +{ if (strm == Z_NULL || strm->state == Z_NULL || strm->zfree == (free_func)0) return Z_STREAM_ERROR; ZFREE(strm, strm->state); diff --git a/src/native/external/zlib/inffast.c b/src/native/external/zlib/inffast.c index 9354676e786ee7..1fec7f363fa665 100644 --- a/src/native/external/zlib/inffast.c +++ b/src/native/external/zlib/inffast.c @@ -47,7 +47,10 @@ requires strm->avail_out >= 258 for each loop to avoid checking for output space. */ -void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start) { +void ZLIB_INTERNAL inflate_fast(strm, start) +z_streamp strm; +unsigned start; /* inflate()'s starting value for strm->avail_out */ +{ struct inflate_state FAR *state; z_const unsigned char FAR *in; /* local strm->next_in */ z_const unsigned char FAR *last; /* have enough input while in < last */ diff --git a/src/native/external/zlib/inffast.h b/src/native/external/zlib/inffast.h index 49c6d156c5c652..e5c1aa4ca8cd52 100644 --- a/src/native/external/zlib/inffast.h +++ b/src/native/external/zlib/inffast.h @@ -8,4 +8,4 @@ subject to change. Applications should only use zlib.h. */ -void ZLIB_INTERNAL inflate_fast(z_streamp strm, unsigned start); +void ZLIB_INTERNAL inflate_fast OF((z_streamp strm, unsigned start)); diff --git a/src/native/external/zlib/inflate.c b/src/native/external/zlib/inflate.c index 94ecff015a9be7..8acbef44e993ba 100644 --- a/src/native/external/zlib/inflate.c +++ b/src/native/external/zlib/inflate.c @@ -91,7 +91,20 @@ # endif #endif -local int inflateStateCheck(z_streamp strm) { +/* function prototypes */ +local int inflateStateCheck OF((z_streamp strm)); +local void fixedtables OF((struct inflate_state FAR *state)); +local int updatewindow OF((z_streamp strm, const unsigned char FAR *end, + unsigned copy)); +#ifdef BUILDFIXED + void makefixed OF((void)); +#endif +local unsigned syncsearch OF((unsigned FAR *have, const unsigned char FAR *buf, + unsigned len)); + +local int inflateStateCheck(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (strm == Z_NULL || strm->zalloc == (alloc_func)0 || strm->zfree == (free_func)0) @@ -103,7 +116,9 @@ local int inflateStateCheck(z_streamp strm) { return 0; } -int ZEXPORT inflateResetKeep(z_streamp strm) { +int ZEXPORT inflateResetKeep(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -127,7 +142,9 @@ int ZEXPORT inflateResetKeep(z_streamp strm) { return Z_OK; } -int ZEXPORT inflateReset(z_streamp strm) { +int ZEXPORT inflateReset(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -138,7 +155,10 @@ int ZEXPORT inflateReset(z_streamp strm) { return inflateResetKeep(strm); } -int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { +int ZEXPORT inflateReset2(strm, windowBits) +z_streamp strm; +int windowBits; +{ int wrap; struct inflate_state FAR *state; @@ -175,8 +195,12 @@ int ZEXPORT inflateReset2(z_streamp strm, int windowBits) { return inflateReset(strm); } -int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, - const char *version, int stream_size) { +int ZEXPORT inflateInit2_(strm, windowBits, version, stream_size) +z_streamp strm; +int windowBits; +const char *version; +int stream_size; +{ int ret; struct inflate_state FAR *state; @@ -215,17 +239,22 @@ int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, return ret; } -int ZEXPORT inflateInit_(z_streamp strm, const char *version, - int stream_size) { +int ZEXPORT inflateInit_(strm, version, stream_size) +z_streamp strm; +const char *version; +int stream_size; +{ return inflateInit2_(strm, DEF_WBITS, version, stream_size); } -int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { +int ZEXPORT inflatePrime(strm, bits, value) +z_streamp strm; +int bits; +int value; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; - if (bits == 0) - return Z_OK; state = (struct inflate_state FAR *)strm->state; if (bits < 0) { state->hold = 0; @@ -249,7 +278,9 @@ int ZEXPORT inflatePrime(z_streamp strm, int bits, int value) { used for threaded applications, since the rewriting of the tables and virgin may not be thread-safe. */ -local void fixedtables(struct inflate_state FAR *state) { +local void fixedtables(state) +struct inflate_state FAR *state; +{ #ifdef BUILDFIXED static int virgin = 1; static code *lenfix, *distfix; @@ -311,7 +342,7 @@ local void fixedtables(struct inflate_state FAR *state) { a.out > inffixed.h */ -void makefixed(void) +void makefixed() { unsigned low, size; struct inflate_state state; @@ -365,7 +396,11 @@ void makefixed(void) output will fall in the output data, making match copies simpler and faster. The advantage may be dependent on the size of the processor's data caches. */ -local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) { +local int updatewindow(strm, end, copy) +z_streamp strm; +const Bytef *end; +unsigned copy; +{ struct inflate_state FAR *state; unsigned dist; @@ -587,7 +622,10 @@ local int updatewindow(z_streamp strm, const Bytef *end, unsigned copy) { will return Z_BUF_ERROR if it has not reached the end of the stream. */ -int ZEXPORT inflate(z_streamp strm, int flush) { +int ZEXPORT inflate(strm, flush) +z_streamp strm; +int flush; +{ struct inflate_state FAR *state; z_const unsigned char FAR *next; /* next input */ unsigned char FAR *put; /* next output */ @@ -1263,7 +1301,9 @@ int ZEXPORT inflate(z_streamp strm, int flush) { return ret; } -int ZEXPORT inflateEnd(z_streamp strm) { +int ZEXPORT inflateEnd(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1275,8 +1315,11 @@ int ZEXPORT inflateEnd(z_streamp strm) { return Z_OK; } -int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary, - uInt *dictLength) { +int ZEXPORT inflateGetDictionary(strm, dictionary, dictLength) +z_streamp strm; +Bytef *dictionary; +uInt *dictLength; +{ struct inflate_state FAR *state; /* check state */ @@ -1295,8 +1338,11 @@ int ZEXPORT inflateGetDictionary(z_streamp strm, Bytef *dictionary, return Z_OK; } -int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary, - uInt dictLength) { +int ZEXPORT inflateSetDictionary(strm, dictionary, dictLength) +z_streamp strm; +const Bytef *dictionary; +uInt dictLength; +{ struct inflate_state FAR *state; unsigned long dictid; int ret; @@ -1327,7 +1373,10 @@ int ZEXPORT inflateSetDictionary(z_streamp strm, const Bytef *dictionary, return Z_OK; } -int ZEXPORT inflateGetHeader(z_streamp strm, gz_headerp head) { +int ZEXPORT inflateGetHeader(strm, head) +z_streamp strm; +gz_headerp head; +{ struct inflate_state FAR *state; /* check state */ @@ -1352,8 +1401,11 @@ int ZEXPORT inflateGetHeader(z_streamp strm, gz_headerp head) { called again with more data and the *have state. *have is initialized to zero for the first call. */ -local unsigned syncsearch(unsigned FAR *have, const unsigned char FAR *buf, - unsigned len) { +local unsigned syncsearch(have, buf, len) +unsigned FAR *have; +const unsigned char FAR *buf; +unsigned len; +{ unsigned got; unsigned next; @@ -1372,7 +1424,9 @@ local unsigned syncsearch(unsigned FAR *have, const unsigned char FAR *buf, return next; } -int ZEXPORT inflateSync(z_streamp strm) { +int ZEXPORT inflateSync(strm) +z_streamp strm; +{ unsigned len; /* number of bytes to look at or looked at */ int flags; /* temporary to save header status */ unsigned long in, out; /* temporary to save total_in and total_out */ @@ -1387,7 +1441,7 @@ int ZEXPORT inflateSync(z_streamp strm) { /* if first time, start search in bit buffer */ if (state->mode != SYNC) { state->mode = SYNC; - state->hold >>= state->bits & 7; + state->hold <<= state->bits & 7; state->bits -= state->bits & 7; len = 0; while (state->bits >= 8) { @@ -1428,7 +1482,9 @@ int ZEXPORT inflateSync(z_streamp strm) { block. When decompressing, PPP checks that at the end of input packet, inflate is waiting for these length bytes. */ -int ZEXPORT inflateSyncPoint(z_streamp strm) { +int ZEXPORT inflateSyncPoint(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1436,7 +1492,10 @@ int ZEXPORT inflateSyncPoint(z_streamp strm) { return state->mode == STORED && state->bits == 0; } -int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { +int ZEXPORT inflateCopy(dest, source) +z_streamp dest; +z_streamp source; +{ struct inflate_state FAR *state; struct inflate_state FAR *copy; unsigned char FAR *window; @@ -1480,7 +1539,10 @@ int ZEXPORT inflateCopy(z_streamp dest, z_streamp source) { return Z_OK; } -int ZEXPORT inflateUndermine(z_streamp strm, int subvert) { +int ZEXPORT inflateUndermine(strm, subvert) +z_streamp strm; +int subvert; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1495,7 +1557,10 @@ int ZEXPORT inflateUndermine(z_streamp strm, int subvert) { #endif } -int ZEXPORT inflateValidate(z_streamp strm, int check) { +int ZEXPORT inflateValidate(strm, check) +z_streamp strm; +int check; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return Z_STREAM_ERROR; @@ -1507,7 +1572,9 @@ int ZEXPORT inflateValidate(z_streamp strm, int check) { return Z_OK; } -long ZEXPORT inflateMark(z_streamp strm) { +long ZEXPORT inflateMark(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) @@ -1518,7 +1585,9 @@ long ZEXPORT inflateMark(z_streamp strm) { (state->mode == MATCH ? state->was - state->length : 0)); } -unsigned long ZEXPORT inflateCodesUsed(z_streamp strm) { +unsigned long ZEXPORT inflateCodesUsed(strm) +z_streamp strm; +{ struct inflate_state FAR *state; if (inflateStateCheck(strm)) return (unsigned long)-1; state = (struct inflate_state FAR *)strm->state; diff --git a/src/native/external/zlib/inftrees.c b/src/native/external/zlib/inftrees.c index 98cfe164458c40..57d2793bec931f 100644 --- a/src/native/external/zlib/inftrees.c +++ b/src/native/external/zlib/inftrees.c @@ -1,5 +1,5 @@ /* inftrees.c -- generate Huffman trees for efficient decoding - * Copyright (C) 1995-2024 Mark Adler + * Copyright (C) 1995-2022 Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -9,7 +9,7 @@ #define MAXBITS 15 const char inflate_copyright[] = - " inflate 1.3.1 Copyright 1995-2024 Mark Adler "; + " inflate 1.2.13 Copyright 1995-2022 Mark Adler "; /* If you use the zlib library in a product, an acknowledgment is welcome in the documentation of your product. If for some reason you cannot @@ -29,9 +29,14 @@ const char inflate_copyright[] = table index bits. It will differ if the request is greater than the longest code or if it is less than the shortest code. */ -int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, - unsigned codes, code FAR * FAR *table, - unsigned FAR *bits, unsigned short FAR *work) { +int ZLIB_INTERNAL inflate_table(type, lens, codes, table, bits, work) +codetype type; +unsigned short FAR *lens; +unsigned codes; +code FAR * FAR *table; +unsigned FAR *bits; +unsigned short FAR *work; +{ unsigned len; /* a code's length in bits */ unsigned sym; /* index of code symbols */ unsigned min, max; /* minimum and maximum code lengths */ @@ -57,7 +62,7 @@ int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0}; static const unsigned short lext[31] = { /* Length codes 257..285 extra */ 16, 16, 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 18, 18, 18, 18, - 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 203, 77}; + 19, 19, 19, 19, 20, 20, 20, 20, 21, 21, 21, 21, 16, 194, 65}; static const unsigned short dbase[32] = { /* Distance codes 0..29 base */ 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, diff --git a/src/native/external/zlib/inftrees.h b/src/native/external/zlib/inftrees.h index 396f74b5da798a..f53665311c1624 100644 --- a/src/native/external/zlib/inftrees.h +++ b/src/native/external/zlib/inftrees.h @@ -41,8 +41,8 @@ typedef struct { examples/enough.c found in the zlib distribution. The arguments to that program are the number of symbols, the initial root table size, and the maximum bit length of a code. "enough 286 9 15" for literal/length codes - returns 852, and "enough 30 6 15" for distance codes returns 592. The - initial root table size (9 or 6) is found in the fifth argument of the + returns returns 852, and "enough 30 6 15" for distance codes returns 592. + The initial root table size (9 or 6) is found in the fifth argument of the inflate_table() calls in inflate.c and infback.c. If the root table size is changed, then these maximum sizes would be need to be recalculated and updated. */ @@ -57,6 +57,6 @@ typedef enum { DISTS } codetype; -int ZLIB_INTERNAL inflate_table(codetype type, unsigned short FAR *lens, - unsigned codes, code FAR * FAR *table, - unsigned FAR *bits, unsigned short FAR *work); +int ZLIB_INTERNAL inflate_table OF((codetype type, unsigned short FAR *lens, + unsigned codes, code FAR * FAR *table, + unsigned FAR *bits, unsigned short FAR *work)); diff --git a/src/native/external/zlib/treebuild.xml b/src/native/external/zlib/treebuild.xml index 930b00be4a853a..0017a45d3c5cbf 100644 --- a/src/native/external/zlib/treebuild.xml +++ b/src/native/external/zlib/treebuild.xml @@ -1,6 +1,6 @@ - - + + zip compression library diff --git a/src/native/external/zlib/trees.c b/src/native/external/zlib/trees.c index 979ae4100a02e0..8a3eec559e55bc 100644 --- a/src/native/external/zlib/trees.c +++ b/src/native/external/zlib/trees.c @@ -1,5 +1,5 @@ /* trees.c -- output deflated data using Huffman coding - * Copyright (C) 1995-2024 Jean-loup Gailly + * Copyright (C) 1995-2021 Jean-loup Gailly * detect_data_type() function provided freely by Cosmin Truta, 2006 * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -122,116 +122,39 @@ struct static_tree_desc_s { int max_length; /* max bit length for the codes */ }; -#ifdef NO_INIT_GLOBAL_POINTERS -# define TCONST -#else -# define TCONST const -#endif - -local TCONST static_tree_desc static_l_desc = +local const static_tree_desc static_l_desc = {static_ltree, extra_lbits, LITERALS+1, L_CODES, MAX_BITS}; -local TCONST static_tree_desc static_d_desc = +local const static_tree_desc static_d_desc = {static_dtree, extra_dbits, 0, D_CODES, MAX_BITS}; -local TCONST static_tree_desc static_bl_desc = +local const static_tree_desc static_bl_desc = {(const ct_data *)0, extra_blbits, 0, BL_CODES, MAX_BL_BITS}; /* =========================================================================== - * Output a short LSB first on the stream. - * IN assertion: there is enough room in pendingBuf. - */ -#define put_short(s, w) { \ - put_byte(s, (uch)((w) & 0xff)); \ - put_byte(s, (uch)((ush)(w) >> 8)); \ -} - -/* =========================================================================== - * Reverse the first len bits of a code, using straightforward code (a faster - * method would use a table) - * IN assertion: 1 <= len <= 15 + * Local (static) routines in this file. */ -local unsigned bi_reverse(unsigned code, int len) { - register unsigned res = 0; - do { - res |= code & 1; - code >>= 1, res <<= 1; - } while (--len > 0); - return res >> 1; -} -/* =========================================================================== - * Flush the bit buffer, keeping at most 7 bits in it. - */ -local void bi_flush(deflate_state *s) { - if (s->bi_valid == 16) { - put_short(s, s->bi_buf); - s->bi_buf = 0; - s->bi_valid = 0; - } else if (s->bi_valid >= 8) { - put_byte(s, (Byte)s->bi_buf); - s->bi_buf >>= 8; - s->bi_valid -= 8; - } -} - -/* =========================================================================== - * Flush the bit buffer and align the output on a byte boundary - */ -local void bi_windup(deflate_state *s) { - if (s->bi_valid > 8) { - put_short(s, s->bi_buf); - } else if (s->bi_valid > 0) { - put_byte(s, (Byte)s->bi_buf); - } - s->bi_buf = 0; - s->bi_valid = 0; -#ifdef ZLIB_DEBUG - s->bits_sent = (s->bits_sent + 7) & ~7; -#endif -} - -/* =========================================================================== - * Generate the codes for a given tree and bit counts (which need not be - * optimal). - * IN assertion: the array bl_count contains the bit length statistics for - * the given tree and the field len is set for all tree elements. - * OUT assertion: the field code is set for all tree elements of non - * zero code length. - */ -local void gen_codes(ct_data *tree, int max_code, ushf *bl_count) { - ush next_code[MAX_BITS+1]; /* next code value for each bit length */ - unsigned code = 0; /* running code value */ - int bits; /* bit index */ - int n; /* code index */ - - /* The distribution counts are first used to generate the code values - * without bit reversal. - */ - for (bits = 1; bits <= MAX_BITS; bits++) { - code = (code + bl_count[bits - 1]) << 1; - next_code[bits] = (ush)code; - } - /* Check that the bit counts in bl_count are consistent. The last code - * must be all ones. - */ - Assert (code + bl_count[MAX_BITS] - 1 == (1 << MAX_BITS) - 1, - "inconsistent bit counts"); - Tracev((stderr,"\ngen_codes: max_code %d ", max_code)); - - for (n = 0; n <= max_code; n++) { - int len = tree[n].Len; - if (len == 0) continue; - /* Now reverse the bits */ - tree[n].Code = (ush)bi_reverse(next_code[len]++, len); - - Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", - n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len] - 1)); - } -} +local void tr_static_init OF((void)); +local void init_block OF((deflate_state *s)); +local void pqdownheap OF((deflate_state *s, ct_data *tree, int k)); +local void gen_bitlen OF((deflate_state *s, tree_desc *desc)); +local void gen_codes OF((ct_data *tree, int max_code, ushf *bl_count)); +local void build_tree OF((deflate_state *s, tree_desc *desc)); +local void scan_tree OF((deflate_state *s, ct_data *tree, int max_code)); +local void send_tree OF((deflate_state *s, ct_data *tree, int max_code)); +local int build_bl_tree OF((deflate_state *s)); +local void send_all_trees OF((deflate_state *s, int lcodes, int dcodes, + int blcodes)); +local void compress_block OF((deflate_state *s, const ct_data *ltree, + const ct_data *dtree)); +local int detect_data_type OF((deflate_state *s)); +local unsigned bi_reverse OF((unsigned code, int len)); +local void bi_windup OF((deflate_state *s)); +local void bi_flush OF((deflate_state *s)); #ifdef GEN_TREES_H -local void gen_trees_header(void); +local void gen_trees_header OF((void)); #endif #ifndef ZLIB_DEBUG @@ -244,12 +167,27 @@ local void gen_trees_header(void); send_bits(s, tree[c].Code, tree[c].Len); } #endif +/* =========================================================================== + * Output a short LSB first on the stream. + * IN assertion: there is enough room in pendingBuf. + */ +#define put_short(s, w) { \ + put_byte(s, (uch)((w) & 0xff)); \ + put_byte(s, (uch)((ush)(w) >> 8)); \ +} + /* =========================================================================== * Send a value on a given number of bits. * IN assertion: length <= 16 and value fits in length bits. */ #ifdef ZLIB_DEBUG -local void send_bits(deflate_state *s, int value, int length) { +local void send_bits OF((deflate_state *s, int value, int length)); + +local void send_bits(s, value, length) + deflate_state *s; + int value; /* value to send */ + int length; /* number of bits */ +{ Tracevv((stderr," l %2d v %4x ", length, value)); Assert(length > 0 && length <= 15, "invalid length"); s->bits_sent += (ulg)length; @@ -291,7 +229,8 @@ local void send_bits(deflate_state *s, int value, int length) { /* =========================================================================== * Initialize the various 'constant' tables. */ -local void tr_static_init(void) { +local void tr_static_init() +{ #if defined(GEN_TREES_H) || !defined(STDC) static int static_init_done = 0; int n; /* iterates over tree elements */ @@ -384,7 +323,8 @@ local void tr_static_init(void) { ((i) == (last)? "\n};\n\n" : \ ((i) % (width) == (width) - 1 ? ",\n" : ", ")) -void gen_trees_header(void) { +void gen_trees_header() +{ FILE *header = fopen("trees.h", "w"); int i; @@ -433,26 +373,12 @@ void gen_trees_header(void) { } #endif /* GEN_TREES_H */ -/* =========================================================================== - * Initialize a new block. - */ -local void init_block(deflate_state *s) { - int n; /* iterates over tree elements */ - - /* Initialize the trees. */ - for (n = 0; n < L_CODES; n++) s->dyn_ltree[n].Freq = 0; - for (n = 0; n < D_CODES; n++) s->dyn_dtree[n].Freq = 0; - for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0; - - s->dyn_ltree[END_BLOCK].Freq = 1; - s->opt_len = s->static_len = 0L; - s->sym_next = s->matches = 0; -} - /* =========================================================================== * Initialize the tree data structures for a new zlib stream. */ -void ZLIB_INTERNAL _tr_init(deflate_state *s) { +void ZLIB_INTERNAL _tr_init(s) + deflate_state *s; +{ tr_static_init(); s->l_desc.dyn_tree = s->dyn_ltree; @@ -475,6 +401,24 @@ void ZLIB_INTERNAL _tr_init(deflate_state *s) { init_block(s); } +/* =========================================================================== + * Initialize a new block. + */ +local void init_block(s) + deflate_state *s; +{ + int n; /* iterates over tree elements */ + + /* Initialize the trees. */ + for (n = 0; n < L_CODES; n++) s->dyn_ltree[n].Freq = 0; + for (n = 0; n < D_CODES; n++) s->dyn_dtree[n].Freq = 0; + for (n = 0; n < BL_CODES; n++) s->bl_tree[n].Freq = 0; + + s->dyn_ltree[END_BLOCK].Freq = 1; + s->opt_len = s->static_len = 0L; + s->sym_next = s->matches = 0; +} + #define SMALLEST 1 /* Index within the heap array of least frequent node in the Huffman tree */ @@ -504,7 +448,11 @@ void ZLIB_INTERNAL _tr_init(deflate_state *s) { * when the heap property is re-established (each father smaller than its * two sons). */ -local void pqdownheap(deflate_state *s, ct_data *tree, int k) { +local void pqdownheap(s, tree, k) + deflate_state *s; + ct_data *tree; /* the tree to restore */ + int k; /* node to move down */ +{ int v = s->heap[k]; int j = k << 1; /* left son of k */ while (j <= s->heap_len) { @@ -535,7 +483,10 @@ local void pqdownheap(deflate_state *s, ct_data *tree, int k) { * The length opt_len is updated; static_len is also updated if stree is * not null. */ -local void gen_bitlen(deflate_state *s, tree_desc *desc) { +local void gen_bitlen(s, desc) + deflate_state *s; + tree_desc *desc; /* the tree descriptor */ +{ ct_data *tree = desc->dyn_tree; int max_code = desc->max_code; const ct_data *stree = desc->stat_desc->static_tree; @@ -610,9 +561,48 @@ local void gen_bitlen(deflate_state *s, tree_desc *desc) { } } -#ifdef DUMP_BL_TREE -# include -#endif +/* =========================================================================== + * Generate the codes for a given tree and bit counts (which need not be + * optimal). + * IN assertion: the array bl_count contains the bit length statistics for + * the given tree and the field len is set for all tree elements. + * OUT assertion: the field code is set for all tree elements of non + * zero code length. + */ +local void gen_codes(tree, max_code, bl_count) + ct_data *tree; /* the tree to decorate */ + int max_code; /* largest code with non zero frequency */ + ushf *bl_count; /* number of codes at each bit length */ +{ + ush next_code[MAX_BITS+1]; /* next code value for each bit length */ + unsigned code = 0; /* running code value */ + int bits; /* bit index */ + int n; /* code index */ + + /* The distribution counts are first used to generate the code values + * without bit reversal. + */ + for (bits = 1; bits <= MAX_BITS; bits++) { + code = (code + bl_count[bits - 1]) << 1; + next_code[bits] = (ush)code; + } + /* Check that the bit counts in bl_count are consistent. The last code + * must be all ones. + */ + Assert (code + bl_count[MAX_BITS] - 1 == (1 << MAX_BITS) - 1, + "inconsistent bit counts"); + Tracev((stderr,"\ngen_codes: max_code %d ", max_code)); + + for (n = 0; n <= max_code; n++) { + int len = tree[n].Len; + if (len == 0) continue; + /* Now reverse the bits */ + tree[n].Code = (ush)bi_reverse(next_code[len]++, len); + + Tracecv(tree != static_ltree, (stderr,"\nn %3d %c l %2d c %4x (%x) ", + n, (isgraph(n) ? n : ' '), len, tree[n].Code, next_code[len] - 1)); + } +} /* =========================================================================== * Construct one Huffman tree and assigns the code bit strings and lengths. @@ -622,7 +612,10 @@ local void gen_bitlen(deflate_state *s, tree_desc *desc) { * and corresponding code. The length opt_len is updated; static_len is * also updated if stree is not null. The field max_code is set. */ -local void build_tree(deflate_state *s, tree_desc *desc) { +local void build_tree(s, desc) + deflate_state *s; + tree_desc *desc; /* the tree descriptor */ +{ ct_data *tree = desc->dyn_tree; const ct_data *stree = desc->stat_desc->static_tree; int elems = desc->stat_desc->elems; @@ -707,7 +700,11 @@ local void build_tree(deflate_state *s, tree_desc *desc) { * Scan a literal or distance tree to determine the frequencies of the codes * in the bit length tree. */ -local void scan_tree(deflate_state *s, ct_data *tree, int max_code) { +local void scan_tree(s, tree, max_code) + deflate_state *s; + ct_data *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ int curlen; /* length of current code */ @@ -748,7 +745,11 @@ local void scan_tree(deflate_state *s, ct_data *tree, int max_code) { * Send a literal or distance tree in compressed form, using the codes in * bl_tree. */ -local void send_tree(deflate_state *s, ct_data *tree, int max_code) { +local void send_tree(s, tree, max_code) + deflate_state *s; + ct_data *tree; /* the tree to be scanned */ + int max_code; /* and its largest code of non zero frequency */ +{ int n; /* iterates over all tree elements */ int prevlen = -1; /* last emitted length */ int curlen; /* length of current code */ @@ -795,7 +796,9 @@ local void send_tree(deflate_state *s, ct_data *tree, int max_code) { * Construct the Huffman tree for the bit lengths and return the index in * bl_order of the last bit length code to send. */ -local int build_bl_tree(deflate_state *s) { +local int build_bl_tree(s) + deflate_state *s; +{ int max_blindex; /* index of last bit length code of non zero freq */ /* Determine the bit length frequencies for literal and distance trees */ @@ -828,8 +831,10 @@ local int build_bl_tree(deflate_state *s) { * lengths of the bit length codes, the literal tree and the distance tree. * IN assertion: lcodes >= 257, dcodes >= 1, blcodes >= 4. */ -local void send_all_trees(deflate_state *s, int lcodes, int dcodes, - int blcodes) { +local void send_all_trees(s, lcodes, dcodes, blcodes) + deflate_state *s; + int lcodes, dcodes, blcodes; /* number of codes for each tree */ +{ int rank; /* index in bl_order */ Assert (lcodes >= 257 && dcodes >= 1 && blcodes >= 4, "not enough codes"); @@ -855,8 +860,12 @@ local void send_all_trees(deflate_state *s, int lcodes, int dcodes, /* =========================================================================== * Send a stored block */ -void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, - ulg stored_len, int last) { +void ZLIB_INTERNAL _tr_stored_block(s, buf, stored_len, last) + deflate_state *s; + charf *buf; /* input block */ + ulg stored_len; /* length of input block */ + int last; /* one if this is the last block for a file */ +{ send_bits(s, (STORED_BLOCK<<1) + last, 3); /* send block type */ bi_windup(s); /* align on byte boundary */ put_short(s, (ush)stored_len); @@ -875,7 +884,9 @@ void ZLIB_INTERNAL _tr_stored_block(deflate_state *s, charf *buf, /* =========================================================================== * Flush the bits in the bit buffer to pending output (leaves at most 7 bits) */ -void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s) { +void ZLIB_INTERNAL _tr_flush_bits(s) + deflate_state *s; +{ bi_flush(s); } @@ -883,7 +894,9 @@ void ZLIB_INTERNAL _tr_flush_bits(deflate_state *s) { * Send one empty static block to give enough lookahead for inflate. * This takes 10 bits, of which 7 may remain in the bit buffer. */ -void ZLIB_INTERNAL _tr_align(deflate_state *s) { +void ZLIB_INTERNAL _tr_align(s) + deflate_state *s; +{ send_bits(s, STATIC_TREES<<1, 3); send_code(s, END_BLOCK, static_ltree); #ifdef ZLIB_DEBUG @@ -892,108 +905,16 @@ void ZLIB_INTERNAL _tr_align(deflate_state *s) { bi_flush(s); } -/* =========================================================================== - * Send the block data compressed using the given Huffman trees - */ -local void compress_block(deflate_state *s, const ct_data *ltree, - const ct_data *dtree) { - unsigned dist; /* distance of matched string */ - int lc; /* match length or unmatched char (if dist == 0) */ - unsigned sx = 0; /* running index in symbol buffers */ - unsigned code; /* the code to send */ - int extra; /* number of extra bits to send */ - - if (s->sym_next != 0) do { -#ifdef LIT_MEM - dist = s->d_buf[sx]; - lc = s->l_buf[sx++]; -#else - dist = s->sym_buf[sx++] & 0xff; - dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; - lc = s->sym_buf[sx++]; -#endif - if (dist == 0) { - send_code(s, lc, ltree); /* send a literal byte */ - Tracecv(isgraph(lc), (stderr," '%c' ", lc)); - } else { - /* Here, lc is the match length - MIN_MATCH */ - code = _length_code[lc]; - send_code(s, code + LITERALS + 1, ltree); /* send length code */ - extra = extra_lbits[code]; - if (extra != 0) { - lc -= base_length[code]; - send_bits(s, lc, extra); /* send the extra length bits */ - } - dist--; /* dist is now the match distance - 1 */ - code = d_code(dist); - Assert (code < D_CODES, "bad d_code"); - - send_code(s, code, dtree); /* send the distance code */ - extra = extra_dbits[code]; - if (extra != 0) { - dist -= (unsigned)base_dist[code]; - send_bits(s, dist, extra); /* send the extra distance bits */ - } - } /* literal or match pair ? */ - - /* Check for no overlay of pending_buf on needed symbols */ -#ifdef LIT_MEM - Assert(s->pending < 2 * (s->lit_bufsize + sx), "pendingBuf overflow"); -#else - Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); -#endif - - } while (sx < s->sym_next); - - send_code(s, END_BLOCK, ltree); -} - -/* =========================================================================== - * Check if the data type is TEXT or BINARY, using the following algorithm: - * - TEXT if the two conditions below are satisfied: - * a) There are no non-portable control characters belonging to the - * "block list" (0..6, 14..25, 28..31). - * b) There is at least one printable character belonging to the - * "allow list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). - * - BINARY otherwise. - * - The following partially-portable control characters form a - * "gray list" that is ignored in this detection algorithm: - * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). - * IN assertion: the fields Freq of dyn_ltree are set. - */ -local int detect_data_type(deflate_state *s) { - /* block_mask is the bit mask of block-listed bytes - * set bits 0..6, 14..25, and 28..31 - * 0xf3ffc07f = binary 11110011111111111100000001111111 - */ - unsigned long block_mask = 0xf3ffc07fUL; - int n; - - /* Check for non-textual ("block-listed") bytes. */ - for (n = 0; n <= 31; n++, block_mask >>= 1) - if ((block_mask & 1) && (s->dyn_ltree[n].Freq != 0)) - return Z_BINARY; - - /* Check for textual ("allow-listed") bytes. */ - if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 - || s->dyn_ltree[13].Freq != 0) - return Z_TEXT; - for (n = 32; n < LITERALS; n++) - if (s->dyn_ltree[n].Freq != 0) - return Z_TEXT; - - /* There are no "block-listed" or "allow-listed" bytes: - * this stream either is empty or has tolerated ("gray-listed") bytes only. - */ - return Z_BINARY; -} - /* =========================================================================== * Determine the best encoding for the current block: dynamic trees, static * trees or store, and write out the encoded block. */ -void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, - ulg stored_len, int last) { +void ZLIB_INTERNAL _tr_flush_block(s, buf, stored_len, last) + deflate_state *s; + charf *buf; /* input block, or NULL if too old */ + ulg stored_len; /* length of input block */ + int last; /* one if this is the last block for a file */ +{ ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ int max_blindex = 0; /* index of last bit length code of non zero freq */ @@ -1090,15 +1011,14 @@ void ZLIB_INTERNAL _tr_flush_block(deflate_state *s, charf *buf, * Save the match info and tally the frequency counts. Return true if * the current block must be flushed. */ -int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc) { -#ifdef LIT_MEM - s->d_buf[s->sym_next] = (ush)dist; - s->l_buf[s->sym_next++] = (uch)lc; -#else +int ZLIB_INTERNAL _tr_tally(s, dist, lc) + deflate_state *s; + unsigned dist; /* distance of matched string */ + unsigned lc; /* match length - MIN_MATCH or unmatched char (dist==0) */ +{ s->sym_buf[s->sym_next++] = (uch)dist; s->sym_buf[s->sym_next++] = (uch)(dist >> 8); s->sym_buf[s->sym_next++] = (uch)lc; -#endif if (dist == 0) { /* lc is the unmatched char */ s->dyn_ltree[lc].Freq++; @@ -1115,3 +1035,147 @@ int ZLIB_INTERNAL _tr_tally(deflate_state *s, unsigned dist, unsigned lc) { } return (s->sym_next == s->sym_end); } + +/* =========================================================================== + * Send the block data compressed using the given Huffman trees + */ +local void compress_block(s, ltree, dtree) + deflate_state *s; + const ct_data *ltree; /* literal tree */ + const ct_data *dtree; /* distance tree */ +{ + unsigned dist; /* distance of matched string */ + int lc; /* match length or unmatched char (if dist == 0) */ + unsigned sx = 0; /* running index in sym_buf */ + unsigned code; /* the code to send */ + int extra; /* number of extra bits to send */ + + if (s->sym_next != 0) do { + dist = s->sym_buf[sx++] & 0xff; + dist += (unsigned)(s->sym_buf[sx++] & 0xff) << 8; + lc = s->sym_buf[sx++]; + if (dist == 0) { + send_code(s, lc, ltree); /* send a literal byte */ + Tracecv(isgraph(lc), (stderr," '%c' ", lc)); + } else { + /* Here, lc is the match length - MIN_MATCH */ + code = _length_code[lc]; + send_code(s, code + LITERALS + 1, ltree); /* send length code */ + extra = extra_lbits[code]; + if (extra != 0) { + lc -= base_length[code]; + send_bits(s, lc, extra); /* send the extra length bits */ + } + dist--; /* dist is now the match distance - 1 */ + code = d_code(dist); + Assert (code < D_CODES, "bad d_code"); + + send_code(s, code, dtree); /* send the distance code */ + extra = extra_dbits[code]; + if (extra != 0) { + dist -= (unsigned)base_dist[code]; + send_bits(s, dist, extra); /* send the extra distance bits */ + } + } /* literal or match pair ? */ + + /* Check that the overlay between pending_buf and sym_buf is ok: */ + Assert(s->pending < s->lit_bufsize + sx, "pendingBuf overflow"); + + } while (sx < s->sym_next); + + send_code(s, END_BLOCK, ltree); +} + +/* =========================================================================== + * Check if the data type is TEXT or BINARY, using the following algorithm: + * - TEXT if the two conditions below are satisfied: + * a) There are no non-portable control characters belonging to the + * "block list" (0..6, 14..25, 28..31). + * b) There is at least one printable character belonging to the + * "allow list" (9 {TAB}, 10 {LF}, 13 {CR}, 32..255). + * - BINARY otherwise. + * - The following partially-portable control characters form a + * "gray list" that is ignored in this detection algorithm: + * (7 {BEL}, 8 {BS}, 11 {VT}, 12 {FF}, 26 {SUB}, 27 {ESC}). + * IN assertion: the fields Freq of dyn_ltree are set. + */ +local int detect_data_type(s) + deflate_state *s; +{ + /* block_mask is the bit mask of block-listed bytes + * set bits 0..6, 14..25, and 28..31 + * 0xf3ffc07f = binary 11110011111111111100000001111111 + */ + unsigned long block_mask = 0xf3ffc07fUL; + int n; + + /* Check for non-textual ("block-listed") bytes. */ + for (n = 0; n <= 31; n++, block_mask >>= 1) + if ((block_mask & 1) && (s->dyn_ltree[n].Freq != 0)) + return Z_BINARY; + + /* Check for textual ("allow-listed") bytes. */ + if (s->dyn_ltree[9].Freq != 0 || s->dyn_ltree[10].Freq != 0 + || s->dyn_ltree[13].Freq != 0) + return Z_TEXT; + for (n = 32; n < LITERALS; n++) + if (s->dyn_ltree[n].Freq != 0) + return Z_TEXT; + + /* There are no "block-listed" or "allow-listed" bytes: + * this stream either is empty or has tolerated ("gray-listed") bytes only. + */ + return Z_BINARY; +} + +/* =========================================================================== + * Reverse the first len bits of a code, using straightforward code (a faster + * method would use a table) + * IN assertion: 1 <= len <= 15 + */ +local unsigned bi_reverse(code, len) + unsigned code; /* the value to invert */ + int len; /* its bit length */ +{ + register unsigned res = 0; + do { + res |= code & 1; + code >>= 1, res <<= 1; + } while (--len > 0); + return res >> 1; +} + +/* =========================================================================== + * Flush the bit buffer, keeping at most 7 bits in it. + */ +local void bi_flush(s) + deflate_state *s; +{ + if (s->bi_valid == 16) { + put_short(s, s->bi_buf); + s->bi_buf = 0; + s->bi_valid = 0; + } else if (s->bi_valid >= 8) { + put_byte(s, (Byte)s->bi_buf); + s->bi_buf >>= 8; + s->bi_valid -= 8; + } +} + +/* =========================================================================== + * Flush the bit buffer and align the output on a byte boundary + */ +local void bi_windup(s) + deflate_state *s; +{ + if (s->bi_valid > 8) { + put_short(s, s->bi_buf); + } else if (s->bi_valid > 0) { + put_byte(s, (Byte)s->bi_buf); + } + s->bi_buf = 0; + s->bi_valid = 0; +#ifdef ZLIB_DEBUG + s->bits_sent = (s->bits_sent + 7) & ~7; +#endif +} diff --git a/src/native/external/zlib/uncompr.c b/src/native/external/zlib/uncompr.c index 5e256663b4511c..f9532f46c1a69f 100644 --- a/src/native/external/zlib/uncompr.c +++ b/src/native/external/zlib/uncompr.c @@ -24,8 +24,12 @@ Z_DATA_ERROR if the input data was corrupted, including if the input data is an incomplete zlib stream. */ -int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong *sourceLen) { +int ZEXPORT uncompress2(dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong *sourceLen; +{ z_stream stream; int err; const uInt max = (uInt)-1; @@ -79,7 +83,11 @@ int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, const Bytef *source, err; } -int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, const Bytef *source, - uLong sourceLen) { +int ZEXPORT uncompress(dest, destLen, source, sourceLen) + Bytef *dest; + uLongf *destLen; + const Bytef *source; + uLong sourceLen; +{ return uncompress2(dest, destLen, source, &sourceLen); } diff --git a/src/native/external/zlib/zconf.h b/src/native/external/zlib/zconf.h index 62adc8d8431f2f..bf977d3e70adef 100644 --- a/src/native/external/zlib/zconf.h +++ b/src/native/external/zlib/zconf.h @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -241,11 +241,7 @@ #endif #ifdef Z_SOLO -# ifdef _WIN64 - typedef unsigned long long z_size_t; -# else - typedef unsigned long z_size_t; -# endif + typedef unsigned long z_size_t; #else # define z_longlong long long # if defined(NO_SIZE_T) @@ -300,6 +296,14 @@ # endif #endif +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -516,7 +520,7 @@ typedef uLong FAR uLongf; #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# if defined(_WIN32) && !defined(__GNUC__) +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) # define z_off64_t __int64 # else # define z_off64_t z_off_t diff --git a/src/native/external/zlib/zconf.h.cmakein b/src/native/external/zlib/zconf.h.cmakein index 0abe3bc9d8fa4e..247ba2461dd09e 100644 --- a/src/native/external/zlib/zconf.h.cmakein +++ b/src/native/external/zlib/zconf.h.cmakein @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -243,11 +243,7 @@ #endif #ifdef Z_SOLO -# ifdef _WIN64 - typedef unsigned long long z_size_t; -# else - typedef unsigned long z_size_t; -# endif + typedef unsigned long z_size_t; #else # define z_longlong long long # if defined(NO_SIZE_T) @@ -302,6 +298,14 @@ # endif #endif +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -518,7 +522,7 @@ typedef uLong FAR uLongf; #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# if defined(_WIN32) && !defined(__GNUC__) +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) # define z_off64_t __int64 # else # define z_off64_t z_off_t diff --git a/src/native/external/zlib/zconf.h.in b/src/native/external/zlib/zconf.h.in index 62adc8d8431f2f..bf977d3e70adef 100644 --- a/src/native/external/zlib/zconf.h.in +++ b/src/native/external/zlib/zconf.h.in @@ -1,5 +1,5 @@ /* zconf.h -- configuration of the zlib compression library - * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -241,11 +241,7 @@ #endif #ifdef Z_SOLO -# ifdef _WIN64 - typedef unsigned long long z_size_t; -# else - typedef unsigned long z_size_t; -# endif + typedef unsigned long z_size_t; #else # define z_longlong long long # if defined(NO_SIZE_T) @@ -300,6 +296,14 @@ # endif #endif +#ifndef Z_ARG /* function prototypes for stdarg */ +# if defined(STDC) || defined(Z_HAVE_STDARG_H) +# define Z_ARG(args) args +# else +# define Z_ARG(args) () +# endif +#endif + /* The following definitions for FAR are needed only for MSDOS mixed * model programming (small or medium model with some far allocations). * This was tested only with MSC; for other MSDOS compilers you may have @@ -516,7 +520,7 @@ typedef uLong FAR uLongf; #if !defined(_WIN32) && defined(Z_LARGE64) # define z_off64_t off64_t #else -# if defined(_WIN32) && !defined(__GNUC__) +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) # define z_off64_t __int64 # else # define z_off64_t z_off_t diff --git a/src/native/external/zlib/zlib.3 b/src/native/external/zlib/zlib.3 index c716020ea9c420..6f6e91404dff19 100644 --- a/src/native/external/zlib/zlib.3 +++ b/src/native/external/zlib/zlib.3 @@ -1,4 +1,4 @@ -.TH ZLIB 3 "22 Jan 2024" +.TH ZLIB 3 "13 Oct 2022" .SH NAME zlib \- compression/decompression library .SH SYNOPSIS @@ -105,9 +105,9 @@ before asking for help. Send questions and/or comments to zlib@gzip.org, or (for the Windows DLL version) to Gilles Vollant (info@winimage.com). .SH AUTHORS AND LICENSE -Version 1.3.1 +Version 1.2.13 .LP -Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler +Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler .LP This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages diff --git a/src/native/external/zlib/zlib.h b/src/native/external/zlib/zlib.h index 8d4b932eaf6a0f..953cb5012dc203 100644 --- a/src/native/external/zlib/zlib.h +++ b/src/native/external/zlib/zlib.h @@ -1,7 +1,7 @@ /* zlib.h -- interface of the 'zlib' general purpose compression library - version 1.3.1, January 22nd, 2024 + version 1.2.13, October 13th, 2022 - Copyright (C) 1995-2024 Jean-loup Gailly and Mark Adler + Copyright (C) 1995-2022 Jean-loup Gailly and Mark Adler This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages @@ -37,11 +37,11 @@ extern "C" { #endif -#define ZLIB_VERSION "1.3.1" -#define ZLIB_VERNUM 0x1310 +#define ZLIB_VERSION "1.2.13" +#define ZLIB_VERNUM 0x12d0 #define ZLIB_VER_MAJOR 1 -#define ZLIB_VER_MINOR 3 -#define ZLIB_VER_REVISION 1 +#define ZLIB_VER_MINOR 2 +#define ZLIB_VER_REVISION 13 #define ZLIB_VER_SUBREVISION 0 /* @@ -78,8 +78,8 @@ extern "C" { even in the case of corrupted input. */ -typedef voidpf (*alloc_func)(voidpf opaque, uInt items, uInt size); -typedef void (*free_func)(voidpf opaque, voidpf address); +typedef voidpf (*alloc_func) OF((voidpf opaque, uInt items, uInt size)); +typedef void (*free_func) OF((voidpf opaque, voidpf address)); struct internal_state; @@ -217,7 +217,7 @@ typedef gz_header FAR *gz_headerp; /* basic functions */ -ZEXTERN const char * ZEXPORT zlibVersion(void); +ZEXTERN const char * ZEXPORT zlibVersion OF((void)); /* The application can compare zlibVersion and ZLIB_VERSION for consistency. If the first character differs, the library code actually used is not compatible with the zlib.h header file used by the application. This check @@ -225,12 +225,12 @@ ZEXTERN const char * ZEXPORT zlibVersion(void); */ /* -ZEXTERN int ZEXPORT deflateInit(z_streamp strm, int level); +ZEXTERN int ZEXPORT deflateInit OF((z_streamp strm, int level)); Initializes the internal stream state for compression. The fields zalloc, zfree and opaque must be initialized before by the caller. If zalloc and zfree are set to Z_NULL, deflateInit updates them to use default - allocation functions. total_in, total_out, adler, and msg are initialized. + allocation functions. The compression level must be Z_DEFAULT_COMPRESSION, or between 0 and 9: 1 gives best speed, 9 gives best compression, 0 gives no compression at all @@ -247,7 +247,7 @@ ZEXTERN int ZEXPORT deflateInit(z_streamp strm, int level); */ -ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush); +ZEXTERN int ZEXPORT deflate OF((z_streamp strm, int flush)); /* deflate compresses as much data as possible, and stops when the input buffer becomes empty or the output buffer becomes full. It may introduce @@ -320,8 +320,8 @@ ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush); with the same value of the flush parameter and more output space (updated avail_out), until the flush is complete (deflate returns with non-zero avail_out). In the case of a Z_FULL_FLUSH or Z_SYNC_FLUSH, make sure that - avail_out is greater than six when the flush marker begins, in order to avoid - repeated flush markers upon calling deflate() again when avail_out == 0. + avail_out is greater than six to avoid repeated flush markers due to + avail_out == 0 on return. If the parameter flush is set to Z_FINISH, pending input is processed, pending output is flushed and deflate returns with Z_STREAM_END if there was @@ -360,7 +360,7 @@ ZEXTERN int ZEXPORT deflate(z_streamp strm, int flush); */ -ZEXTERN int ZEXPORT deflateEnd(z_streamp strm); +ZEXTERN int ZEXPORT deflateEnd OF((z_streamp strm)); /* All dynamically allocated data structures for this stream are freed. This function discards any unprocessed input and does not flush any pending @@ -375,7 +375,7 @@ ZEXTERN int ZEXPORT deflateEnd(z_streamp strm); /* -ZEXTERN int ZEXPORT inflateInit(z_streamp strm); +ZEXTERN int ZEXPORT inflateInit OF((z_streamp strm)); Initializes the internal stream state for decompression. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized before by @@ -383,8 +383,7 @@ ZEXTERN int ZEXPORT inflateInit(z_streamp strm); read or consumed. The allocation of a sliding window will be deferred to the first call of inflate (if the decompression does not complete on the first call). If zalloc and zfree are set to Z_NULL, inflateInit updates - them to use default allocation functions. total_in, total_out, adler, and - msg are initialized. + them to use default allocation functions. inflateInit returns Z_OK if success, Z_MEM_ERROR if there was not enough memory, Z_VERSION_ERROR if the zlib library version is incompatible with the @@ -398,7 +397,7 @@ ZEXTERN int ZEXPORT inflateInit(z_streamp strm); */ -ZEXTERN int ZEXPORT inflate(z_streamp strm, int flush); +ZEXTERN int ZEXPORT inflate OF((z_streamp strm, int flush)); /* inflate decompresses as much data as possible, and stops when the input buffer becomes empty or the output buffer becomes full. It may introduce @@ -518,7 +517,7 @@ ZEXTERN int ZEXPORT inflate(z_streamp strm, int flush); */ -ZEXTERN int ZEXPORT inflateEnd(z_streamp strm); +ZEXTERN int ZEXPORT inflateEnd OF((z_streamp strm)); /* All dynamically allocated data structures for this stream are freed. This function discards any unprocessed input and does not flush any pending @@ -536,12 +535,12 @@ ZEXTERN int ZEXPORT inflateEnd(z_streamp strm); */ /* -ZEXTERN int ZEXPORT deflateInit2(z_streamp strm, - int level, - int method, - int windowBits, - int memLevel, - int strategy); +ZEXTERN int ZEXPORT deflateInit2 OF((z_streamp strm, + int level, + int method, + int windowBits, + int memLevel, + int strategy)); This is another version of deflateInit with more compression options. The fields zalloc, zfree and opaque must be initialized before by the caller. @@ -608,9 +607,9 @@ ZEXTERN int ZEXPORT deflateInit2(z_streamp strm, compression: this will be done by deflate(). */ -ZEXTERN int ZEXPORT deflateSetDictionary(z_streamp strm, - const Bytef *dictionary, - uInt dictLength); +ZEXTERN int ZEXPORT deflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); /* Initializes the compression dictionary from the given byte sequence without producing any compressed output. When using the zlib format, this @@ -652,9 +651,9 @@ ZEXTERN int ZEXPORT deflateSetDictionary(z_streamp strm, not perform any compression: this will be done by deflate(). */ -ZEXTERN int ZEXPORT deflateGetDictionary(z_streamp strm, - Bytef *dictionary, - uInt *dictLength); +ZEXTERN int ZEXPORT deflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); /* Returns the sliding dictionary being maintained by deflate. dictLength is set to the number of bytes in the dictionary, and that many bytes are copied @@ -674,8 +673,8 @@ ZEXTERN int ZEXPORT deflateGetDictionary(z_streamp strm, stream state is inconsistent. */ -ZEXTERN int ZEXPORT deflateCopy(z_streamp dest, - z_streamp source); +ZEXTERN int ZEXPORT deflateCopy OF((z_streamp dest, + z_streamp source)); /* Sets the destination stream as a complete copy of the source stream. @@ -692,20 +691,20 @@ ZEXTERN int ZEXPORT deflateCopy(z_streamp dest, destination. */ -ZEXTERN int ZEXPORT deflateReset(z_streamp strm); +ZEXTERN int ZEXPORT deflateReset OF((z_streamp strm)); /* This function is equivalent to deflateEnd followed by deflateInit, but does not free and reallocate the internal compression state. The stream will leave the compression level and any other attributes that may have been - set unchanged. total_in, total_out, adler, and msg are initialized. + set unchanged. deflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). */ -ZEXTERN int ZEXPORT deflateParams(z_streamp strm, - int level, - int strategy); +ZEXTERN int ZEXPORT deflateParams OF((z_streamp strm, + int level, + int strategy)); /* Dynamically update the compression level and compression strategy. The interpretation of level and strategy is as in deflateInit2(). This can be @@ -730,7 +729,7 @@ ZEXTERN int ZEXPORT deflateParams(z_streamp strm, Then no more input data should be provided before the deflateParams() call. If this is done, the old level and strategy will be applied to the data compressed before deflateParams(), and the new level and strategy will be - applied to the data compressed after deflateParams(). + applied to the the data compressed after deflateParams(). deflateParams returns Z_OK on success, Z_STREAM_ERROR if the source stream state was inconsistent or if a parameter was invalid, or Z_BUF_ERROR if @@ -741,11 +740,11 @@ ZEXTERN int ZEXPORT deflateParams(z_streamp strm, retried with more output space. */ -ZEXTERN int ZEXPORT deflateTune(z_streamp strm, - int good_length, - int max_lazy, - int nice_length, - int max_chain); +ZEXTERN int ZEXPORT deflateTune OF((z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain)); /* Fine tune deflate's internal compression parameters. This should only be used by someone who understands the algorithm used by zlib's deflate for @@ -758,8 +757,8 @@ ZEXTERN int ZEXPORT deflateTune(z_streamp strm, returns Z_OK on success, or Z_STREAM_ERROR for an invalid deflate stream. */ -ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, - uLong sourceLen); +ZEXTERN uLong ZEXPORT deflateBound OF((z_streamp strm, + uLong sourceLen)); /* deflateBound() returns an upper bound on the compressed size after deflation of sourceLen bytes. It must be called after deflateInit() or @@ -773,9 +772,9 @@ ZEXTERN uLong ZEXPORT deflateBound(z_streamp strm, than Z_FINISH or Z_NO_FLUSH are used. */ -ZEXTERN int ZEXPORT deflatePending(z_streamp strm, - unsigned *pending, - int *bits); +ZEXTERN int ZEXPORT deflatePending OF((z_streamp strm, + unsigned *pending, + int *bits)); /* deflatePending() returns the number of bytes and bits of output that have been generated, but not yet provided in the available output. The bytes not @@ -788,9 +787,9 @@ ZEXTERN int ZEXPORT deflatePending(z_streamp strm, stream state was inconsistent. */ -ZEXTERN int ZEXPORT deflatePrime(z_streamp strm, - int bits, - int value); +ZEXTERN int ZEXPORT deflatePrime OF((z_streamp strm, + int bits, + int value)); /* deflatePrime() inserts bits in the deflate output stream. The intent is that this function is used to start off the deflate output with the bits @@ -805,8 +804,8 @@ ZEXTERN int ZEXPORT deflatePrime(z_streamp strm, source stream state was inconsistent. */ -ZEXTERN int ZEXPORT deflateSetHeader(z_streamp strm, - gz_headerp head); +ZEXTERN int ZEXPORT deflateSetHeader OF((z_streamp strm, + gz_headerp head)); /* deflateSetHeader() provides gzip header information for when a gzip stream is requested by deflateInit2(). deflateSetHeader() may be called @@ -822,17 +821,16 @@ ZEXTERN int ZEXPORT deflateSetHeader(z_streamp strm, gzip file" and give up. If deflateSetHeader is not used, the default gzip header has text false, - the time set to zero, and os set to the current operating system, with no - extra, name, or comment fields. The gzip header is returned to the default - state by deflateReset(). + the time set to zero, and os set to 255, with no extra, name, or comment + fields. The gzip header is returned to the default state by deflateReset(). deflateSetHeader returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent. */ /* -ZEXTERN int ZEXPORT inflateInit2(z_streamp strm, - int windowBits); +ZEXTERN int ZEXPORT inflateInit2 OF((z_streamp strm, + int windowBits)); This is another version of inflateInit with an extra parameter. The fields next_in, avail_in, zalloc, zfree and opaque must be initialized @@ -885,9 +883,9 @@ ZEXTERN int ZEXPORT inflateInit2(z_streamp strm, deferred until inflate() is called. */ -ZEXTERN int ZEXPORT inflateSetDictionary(z_streamp strm, - const Bytef *dictionary, - uInt dictLength); +ZEXTERN int ZEXPORT inflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)); /* Initializes the decompression dictionary from the given uncompressed byte sequence. This function must be called immediately after a call of inflate, @@ -908,9 +906,9 @@ ZEXTERN int ZEXPORT inflateSetDictionary(z_streamp strm, inflate(). */ -ZEXTERN int ZEXPORT inflateGetDictionary(z_streamp strm, - Bytef *dictionary, - uInt *dictLength); +ZEXTERN int ZEXPORT inflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)); /* Returns the sliding dictionary being maintained by inflate. dictLength is set to the number of bytes in the dictionary, and that many bytes are copied @@ -923,7 +921,7 @@ ZEXTERN int ZEXPORT inflateGetDictionary(z_streamp strm, stream state is inconsistent. */ -ZEXTERN int ZEXPORT inflateSync(z_streamp strm); +ZEXTERN int ZEXPORT inflateSync OF((z_streamp strm)); /* Skips invalid compressed data until a possible full flush point (see above for the description of deflate with Z_FULL_FLUSH) can be found, or until all @@ -936,14 +934,14 @@ ZEXTERN int ZEXPORT inflateSync(z_streamp strm); inflateSync returns Z_OK if a possible full flush point has been found, Z_BUF_ERROR if no more input was provided, Z_DATA_ERROR if no flush point has been found, or Z_STREAM_ERROR if the stream structure was inconsistent. - In the success case, the application may save the current value of total_in - which indicates where valid compressed data was found. In the error case, - the application may repeatedly call inflateSync, providing more input each - time, until success or end of the input data. + In the success case, the application may save the current current value of + total_in which indicates where valid compressed data was found. In the + error case, the application may repeatedly call inflateSync, providing more + input each time, until success or end of the input data. */ -ZEXTERN int ZEXPORT inflateCopy(z_streamp dest, - z_streamp source); +ZEXTERN int ZEXPORT inflateCopy OF((z_streamp dest, + z_streamp source)); /* Sets the destination stream as a complete copy of the source stream. @@ -958,19 +956,18 @@ ZEXTERN int ZEXPORT inflateCopy(z_streamp dest, destination. */ -ZEXTERN int ZEXPORT inflateReset(z_streamp strm); +ZEXTERN int ZEXPORT inflateReset OF((z_streamp strm)); /* This function is equivalent to inflateEnd followed by inflateInit, but does not free and reallocate the internal decompression state. The stream will keep attributes that may have been set by inflateInit2. - total_in, total_out, adler, and msg are initialized. inflateReset returns Z_OK if success, or Z_STREAM_ERROR if the source stream state was inconsistent (such as zalloc or state being Z_NULL). */ -ZEXTERN int ZEXPORT inflateReset2(z_streamp strm, - int windowBits); +ZEXTERN int ZEXPORT inflateReset2 OF((z_streamp strm, + int windowBits)); /* This function is the same as inflateReset, but it also permits changing the wrap and window size requests. The windowBits parameter is interpreted @@ -983,9 +980,9 @@ ZEXTERN int ZEXPORT inflateReset2(z_streamp strm, the windowBits parameter is invalid. */ -ZEXTERN int ZEXPORT inflatePrime(z_streamp strm, - int bits, - int value); +ZEXTERN int ZEXPORT inflatePrime OF((z_streamp strm, + int bits, + int value)); /* This function inserts bits in the inflate input stream. The intent is that this function is used to start inflating at a bit position in the @@ -1004,7 +1001,7 @@ ZEXTERN int ZEXPORT inflatePrime(z_streamp strm, stream state was inconsistent. */ -ZEXTERN long ZEXPORT inflateMark(z_streamp strm); +ZEXTERN long ZEXPORT inflateMark OF((z_streamp strm)); /* This function returns two values, one in the lower 16 bits of the return value, and the other in the remaining upper bits, obtained by shifting the @@ -1032,8 +1029,8 @@ ZEXTERN long ZEXPORT inflateMark(z_streamp strm); source stream state was inconsistent. */ -ZEXTERN int ZEXPORT inflateGetHeader(z_streamp strm, - gz_headerp head); +ZEXTERN int ZEXPORT inflateGetHeader OF((z_streamp strm, + gz_headerp head)); /* inflateGetHeader() requests that gzip header information be stored in the provided gz_header structure. inflateGetHeader() may be called after @@ -1073,8 +1070,8 @@ ZEXTERN int ZEXPORT inflateGetHeader(z_streamp strm, */ /* -ZEXTERN int ZEXPORT inflateBackInit(z_streamp strm, int windowBits, - unsigned char FAR *window); +ZEXTERN int ZEXPORT inflateBackInit OF((z_streamp strm, int windowBits, + unsigned char FAR *window)); Initialize the internal stream state for decompression using inflateBack() calls. The fields zalloc, zfree and opaque in strm must be initialized @@ -1094,13 +1091,13 @@ ZEXTERN int ZEXPORT inflateBackInit(z_streamp strm, int windowBits, the version of the header file. */ -typedef unsigned (*in_func)(void FAR *, - z_const unsigned char FAR * FAR *); -typedef int (*out_func)(void FAR *, unsigned char FAR *, unsigned); +typedef unsigned (*in_func) OF((void FAR *, + z_const unsigned char FAR * FAR *)); +typedef int (*out_func) OF((void FAR *, unsigned char FAR *, unsigned)); -ZEXTERN int ZEXPORT inflateBack(z_streamp strm, - in_func in, void FAR *in_desc, - out_func out, void FAR *out_desc); +ZEXTERN int ZEXPORT inflateBack OF((z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc)); /* inflateBack() does a raw inflate with a single call using a call-back interface for input and output. This is potentially more efficient than @@ -1168,7 +1165,7 @@ ZEXTERN int ZEXPORT inflateBack(z_streamp strm, cannot return Z_OK. */ -ZEXTERN int ZEXPORT inflateBackEnd(z_streamp strm); +ZEXTERN int ZEXPORT inflateBackEnd OF((z_streamp strm)); /* All memory allocated by inflateBackInit() is freed. @@ -1176,7 +1173,7 @@ ZEXTERN int ZEXPORT inflateBackEnd(z_streamp strm); state was inconsistent. */ -ZEXTERN uLong ZEXPORT zlibCompileFlags(void); +ZEXTERN uLong ZEXPORT zlibCompileFlags OF((void)); /* Return flags indicating compile-time options. Type sizes, two bits each, 00 = 16 bits, 01 = 32, 10 = 64, 11 = other: @@ -1229,8 +1226,8 @@ ZEXTERN uLong ZEXPORT zlibCompileFlags(void); you need special options. */ -ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen); +ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); /* Compresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size @@ -1244,9 +1241,9 @@ ZEXTERN int ZEXPORT compress(Bytef *dest, uLongf *destLen, buffer. */ -ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen, - int level); +ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level)); /* Compresses the source buffer into the destination buffer. The level parameter has the same meaning as in deflateInit. sourceLen is the byte @@ -1260,15 +1257,15 @@ ZEXTERN int ZEXPORT compress2(Bytef *dest, uLongf *destLen, Z_STREAM_ERROR if the level parameter is invalid. */ -ZEXTERN uLong ZEXPORT compressBound(uLong sourceLen); +ZEXTERN uLong ZEXPORT compressBound OF((uLong sourceLen)); /* compressBound() returns an upper bound on the compressed size after compress() or compress2() on sourceLen bytes. It would be used before a compress() or compress2() call to allocate the destination buffer. */ -ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen); +ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)); /* Decompresses the source buffer into the destination buffer. sourceLen is the byte length of the source buffer. Upon entry, destLen is the total size @@ -1285,8 +1282,8 @@ ZEXTERN int ZEXPORT uncompress(Bytef *dest, uLongf *destLen, buffer with the uncompressed data up to that point. */ -ZEXTERN int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, - const Bytef *source, uLong *sourceLen); +ZEXTERN int ZEXPORT uncompress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong *sourceLen)); /* Same as uncompress, except that sourceLen is a pointer, where the length of the source is *sourceLen. On return, *sourceLen is the number of @@ -1305,7 +1302,7 @@ ZEXTERN int ZEXPORT uncompress2(Bytef *dest, uLongf *destLen, typedef struct gzFile_s *gzFile; /* semi-opaque gzip file descriptor */ /* -ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode); +ZEXTERN gzFile ZEXPORT gzopen OF((const char *path, const char *mode)); Open the gzip (.gz) file at path for reading and decompressing, or compressing and writing. The mode parameter is as in fopen ("rb" or "wb") @@ -1342,7 +1339,7 @@ ZEXTERN gzFile ZEXPORT gzopen(const char *path, const char *mode); file could not be opened. */ -ZEXTERN gzFile ZEXPORT gzdopen(int fd, const char *mode); +ZEXTERN gzFile ZEXPORT gzdopen OF((int fd, const char *mode)); /* Associate a gzFile with the file descriptor fd. File descriptors are obtained from calls like open, dup, creat, pipe or fileno (if the file has @@ -1365,7 +1362,7 @@ ZEXTERN gzFile ZEXPORT gzdopen(int fd, const char *mode); will not detect if fd is invalid (unless fd is -1). */ -ZEXTERN int ZEXPORT gzbuffer(gzFile file, unsigned size); +ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); /* Set the internal buffer size used by this library's functions for file to size. The default buffer size is 8192 bytes. This function must be called @@ -1381,7 +1378,7 @@ ZEXTERN int ZEXPORT gzbuffer(gzFile file, unsigned size); too late. */ -ZEXTERN int ZEXPORT gzsetparams(gzFile file, int level, int strategy); +ZEXTERN int ZEXPORT gzsetparams OF((gzFile file, int level, int strategy)); /* Dynamically update the compression level and strategy for file. See the description of deflateInit2 for the meaning of these parameters. Previously @@ -1392,7 +1389,7 @@ ZEXTERN int ZEXPORT gzsetparams(gzFile file, int level, int strategy); or Z_MEM_ERROR if there is a memory allocation error. */ -ZEXTERN int ZEXPORT gzread(gzFile file, voidp buf, unsigned len); +ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len)); /* Read and decompress up to len uncompressed bytes from file into buf. If the input file is not in gzip format, gzread copies the given number of @@ -1422,8 +1419,8 @@ ZEXTERN int ZEXPORT gzread(gzFile file, voidp buf, unsigned len); Z_STREAM_ERROR. */ -ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, - gzFile file); +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); /* Read and decompress up to nitems items of size size from file into buf, otherwise operating as gzread() does. This duplicates the interface of @@ -1448,14 +1445,14 @@ ZEXTERN z_size_t ZEXPORT gzfread(voidp buf, z_size_t size, z_size_t nitems, file, resetting and retrying on end-of-file, when size is not 1. */ -ZEXTERN int ZEXPORT gzwrite(gzFile file, voidpc buf, unsigned len); +ZEXTERN int ZEXPORT gzwrite OF((gzFile file, voidpc buf, unsigned len)); /* Compress and write the len uncompressed bytes at buf to file. gzwrite returns the number of uncompressed bytes written or 0 in case of error. */ -ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, - z_size_t nitems, gzFile file); +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); /* Compress and write nitems items of size size from buf to file, duplicating the interface of stdio's fwrite(), with size_t request and return types. If @@ -1468,7 +1465,7 @@ ZEXTERN z_size_t ZEXPORT gzfwrite(voidpc buf, z_size_t size, is returned, and the error state is set to Z_STREAM_ERROR. */ -ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...); +ZEXTERN int ZEXPORTVA gzprintf Z_ARG((gzFile file, const char *format, ...)); /* Convert, format, compress, and write the arguments (...) to file under control of the string format, as in fprintf. gzprintf returns the number of @@ -1483,7 +1480,7 @@ ZEXTERN int ZEXPORTVA gzprintf(gzFile file, const char *format, ...); This can be determined using zlibCompileFlags(). */ -ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s); +ZEXTERN int ZEXPORT gzputs OF((gzFile file, const char *s)); /* Compress and write the given null-terminated string s to file, excluding the terminating null character. @@ -1491,7 +1488,7 @@ ZEXTERN int ZEXPORT gzputs(gzFile file, const char *s); gzputs returns the number of characters written, or -1 in case of error. */ -ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len); +ZEXTERN char * ZEXPORT gzgets OF((gzFile file, char *buf, int len)); /* Read and decompress bytes from file into buf, until len-1 characters are read, or until a newline character is read and transferred to buf, or an @@ -1505,13 +1502,13 @@ ZEXTERN char * ZEXPORT gzgets(gzFile file, char *buf, int len); buf are indeterminate. */ -ZEXTERN int ZEXPORT gzputc(gzFile file, int c); +ZEXTERN int ZEXPORT gzputc OF((gzFile file, int c)); /* Compress and write c, converted to an unsigned char, into file. gzputc returns the value that was written, or -1 in case of error. */ -ZEXTERN int ZEXPORT gzgetc(gzFile file); +ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); /* Read and decompress one byte from file. gzgetc returns this byte or -1 in case of end of file or error. This is implemented as a macro for speed. @@ -1520,7 +1517,7 @@ ZEXTERN int ZEXPORT gzgetc(gzFile file); points to has been clobbered or not. */ -ZEXTERN int ZEXPORT gzungetc(int c, gzFile file); +ZEXTERN int ZEXPORT gzungetc OF((int c, gzFile file)); /* Push c back onto the stream for file to be read as the first character on the next read. At least one character of push-back is always allowed. @@ -1532,7 +1529,7 @@ ZEXTERN int ZEXPORT gzungetc(int c, gzFile file); gzseek() or gzrewind(). */ -ZEXTERN int ZEXPORT gzflush(gzFile file, int flush); +ZEXTERN int ZEXPORT gzflush OF((gzFile file, int flush)); /* Flush all pending output to file. The parameter flush is as in the deflate() function. The return value is the zlib error number (see function @@ -1548,8 +1545,8 @@ ZEXTERN int ZEXPORT gzflush(gzFile file, int flush); */ /* -ZEXTERN z_off_t ZEXPORT gzseek(gzFile file, - z_off_t offset, int whence); +ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile file, + z_off_t offset, int whence)); Set the starting position to offset relative to whence for the next gzread or gzwrite on file. The offset represents a number of bytes in the @@ -1567,7 +1564,7 @@ ZEXTERN z_off_t ZEXPORT gzseek(gzFile file, would be before the current position. */ -ZEXTERN int ZEXPORT gzrewind(gzFile file); +ZEXTERN int ZEXPORT gzrewind OF((gzFile file)); /* Rewind file. This function is supported only for reading. @@ -1575,7 +1572,7 @@ ZEXTERN int ZEXPORT gzrewind(gzFile file); */ /* -ZEXTERN z_off_t ZEXPORT gztell(gzFile file); +ZEXTERN z_off_t ZEXPORT gztell OF((gzFile file)); Return the starting position for the next gzread or gzwrite on file. This position represents a number of bytes in the uncompressed data stream, @@ -1586,7 +1583,7 @@ ZEXTERN z_off_t ZEXPORT gztell(gzFile file); */ /* -ZEXTERN z_off_t ZEXPORT gzoffset(gzFile file); +ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); Return the current compressed (actual) read or write offset of file. This offset includes the count of bytes that precede the gzip stream, for example @@ -1595,7 +1592,7 @@ ZEXTERN z_off_t ZEXPORT gzoffset(gzFile file); be used for a progress indicator. On error, gzoffset() returns -1. */ -ZEXTERN int ZEXPORT gzeof(gzFile file); +ZEXTERN int ZEXPORT gzeof OF((gzFile file)); /* Return true (1) if the end-of-file indicator for file has been set while reading, false (0) otherwise. Note that the end-of-file indicator is set @@ -1610,7 +1607,7 @@ ZEXTERN int ZEXPORT gzeof(gzFile file); has grown since the previous end of file was detected. */ -ZEXTERN int ZEXPORT gzdirect(gzFile file); +ZEXTERN int ZEXPORT gzdirect OF((gzFile file)); /* Return true (1) if file is being copied directly while reading, or false (0) if file is a gzip stream being decompressed. @@ -1631,7 +1628,7 @@ ZEXTERN int ZEXPORT gzdirect(gzFile file); gzip file reading and decompression, which may not be desired.) */ -ZEXTERN int ZEXPORT gzclose(gzFile file); +ZEXTERN int ZEXPORT gzclose OF((gzFile file)); /* Flush all pending output for file, if necessary, close file and deallocate the (de)compression state. Note that once file is closed, you @@ -1644,8 +1641,8 @@ ZEXTERN int ZEXPORT gzclose(gzFile file); last read ended in the middle of a gzip stream, or Z_OK on success. */ -ZEXTERN int ZEXPORT gzclose_r(gzFile file); -ZEXTERN int ZEXPORT gzclose_w(gzFile file); +ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); /* Same as gzclose(), but gzclose_r() is only for use when reading, and gzclose_w() is only for use when writing or appending. The advantage to @@ -1656,7 +1653,7 @@ ZEXTERN int ZEXPORT gzclose_w(gzFile file); zlib library. */ -ZEXTERN const char * ZEXPORT gzerror(gzFile file, int *errnum); +ZEXTERN const char * ZEXPORT gzerror OF((gzFile file, int *errnum)); /* Return the error message for the last error which occurred on file. errnum is set to zlib error number. If an error occurred in the file system @@ -1672,7 +1669,7 @@ ZEXTERN const char * ZEXPORT gzerror(gzFile file, int *errnum); functions above that do not distinguish those cases in their return values. */ -ZEXTERN void ZEXPORT gzclearerr(gzFile file); +ZEXTERN void ZEXPORT gzclearerr OF((gzFile file)); /* Clear the error and end-of-file flags for file. This is analogous to the clearerr() function in stdio. This is useful for continuing to read a gzip @@ -1689,7 +1686,7 @@ ZEXTERN void ZEXPORT gzclearerr(gzFile file); library. */ -ZEXTERN uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len); +ZEXTERN uLong ZEXPORT adler32 OF((uLong adler, const Bytef *buf, uInt len)); /* Update a running Adler-32 checksum with the bytes buf[0..len-1] and return the updated checksum. An Adler-32 value is in the range of a 32-bit @@ -1709,15 +1706,15 @@ ZEXTERN uLong ZEXPORT adler32(uLong adler, const Bytef *buf, uInt len); if (adler != original_adler) error(); */ -ZEXTERN uLong ZEXPORT adler32_z(uLong adler, const Bytef *buf, - z_size_t len); +ZEXTERN uLong ZEXPORT adler32_z OF((uLong adler, const Bytef *buf, + z_size_t len)); /* Same as adler32(), but with a size_t length. */ /* -ZEXTERN uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, - z_off_t len2); +ZEXTERN uLong ZEXPORT adler32_combine OF((uLong adler1, uLong adler2, + z_off_t len2)); Combine two Adler-32 checksums into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, Adler-32 checksums were calculated for @@ -1727,7 +1724,7 @@ ZEXTERN uLong ZEXPORT adler32_combine(uLong adler1, uLong adler2, negative, the result has no meaning or utility. */ -ZEXTERN uLong ZEXPORT crc32(uLong crc, const Bytef *buf, uInt len); +ZEXTERN uLong ZEXPORT crc32 OF((uLong crc, const Bytef *buf, uInt len)); /* Update a running CRC-32 with the bytes buf[0..len-1] and return the updated CRC-32. A CRC-32 value is in the range of a 32-bit unsigned integer. @@ -1745,30 +1742,30 @@ ZEXTERN uLong ZEXPORT crc32(uLong crc, const Bytef *buf, uInt len); if (crc != original_crc) error(); */ -ZEXTERN uLong ZEXPORT crc32_z(uLong crc, const Bytef *buf, - z_size_t len); +ZEXTERN uLong ZEXPORT crc32_z OF((uLong crc, const Bytef *buf, + z_size_t len)); /* Same as crc32(), but with a size_t length. */ /* -ZEXTERN uLong ZEXPORT crc32_combine(uLong crc1, uLong crc2, z_off_t len2); +ZEXTERN uLong ZEXPORT crc32_combine OF((uLong crc1, uLong crc2, z_off_t len2)); Combine two CRC-32 check values into one. For two sequences of bytes, seq1 and seq2 with lengths len1 and len2, CRC-32 check values were calculated for each, crc1 and crc2. crc32_combine() returns the CRC-32 check value of seq1 and seq2 concatenated, requiring only crc1, crc2, and - len2. len2 must be non-negative. + len2. */ /* -ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t len2); +ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t len2)); Return the operator corresponding to length len2, to be used with - crc32_combine_op(). len2 must be non-negative. + crc32_combine_op(). */ -ZEXTERN uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op); +ZEXTERN uLong ZEXPORT crc32_combine_op OF((uLong crc1, uLong crc2, uLong op)); /* Give the same result as crc32_combine(), using op in place of len2. op is is generated from len2 by crc32_combine_gen(). This will be faster than @@ -1781,20 +1778,20 @@ ZEXTERN uLong ZEXPORT crc32_combine_op(uLong crc1, uLong crc2, uLong op); /* deflateInit and inflateInit are macros to allow checking the zlib version * and the compiler's view of z_stream: */ -ZEXTERN int ZEXPORT deflateInit_(z_streamp strm, int level, - const char *version, int stream_size); -ZEXTERN int ZEXPORT inflateInit_(z_streamp strm, - const char *version, int stream_size); -ZEXTERN int ZEXPORT deflateInit2_(z_streamp strm, int level, int method, - int windowBits, int memLevel, - int strategy, const char *version, - int stream_size); -ZEXTERN int ZEXPORT inflateInit2_(z_streamp strm, int windowBits, - const char *version, int stream_size); -ZEXTERN int ZEXPORT inflateBackInit_(z_streamp strm, int windowBits, - unsigned char FAR *window, - const char *version, - int stream_size); +ZEXTERN int ZEXPORT deflateInit_ OF((z_streamp strm, int level, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateInit_ OF((z_streamp strm, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT deflateInit2_ OF((z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size)); +ZEXTERN int ZEXPORT inflateInit2_ OF((z_streamp strm, int windowBits, + const char *version, int stream_size)); +ZEXTERN int ZEXPORT inflateBackInit_ OF((z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size)); #ifdef Z_PREFIX_SET # define z_deflateInit(strm, level) \ deflateInit_((strm), (level), ZLIB_VERSION, (int)sizeof(z_stream)) @@ -1839,7 +1836,7 @@ struct gzFile_s { unsigned char *next; z_off64_t pos; }; -ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ +ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); /* backward compatibility */ #ifdef Z_PREFIX_SET # undef z_gzgetc # define z_gzgetc(g) \ @@ -1856,13 +1853,13 @@ ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ * without large file support, _LFS64_LARGEFILE must also be true */ #ifdef Z_LARGE64 - ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); - ZEXTERN z_off64_t ZEXPORT gzseek64(gzFile, z_off64_t, int); - ZEXTERN z_off64_t ZEXPORT gztell64(gzFile); - ZEXTERN z_off64_t ZEXPORT gzoffset64(gzFile); - ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off64_t); - ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off64_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off64_t); + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off64_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off64_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off64_t)); #endif #if !defined(ZLIB_INTERNAL) && defined(Z_WANT64) @@ -1884,50 +1881,50 @@ ZEXTERN int ZEXPORT gzgetc_(gzFile file); /* backward compatibility */ # define crc32_combine_gen crc32_combine_gen64 # endif # ifndef Z_LARGE64 - ZEXTERN gzFile ZEXPORT gzopen64(const char *, const char *); - ZEXTERN z_off_t ZEXPORT gzseek64(gzFile, z_off_t, int); - ZEXTERN z_off_t ZEXPORT gztell64(gzFile); - ZEXTERN z_off_t ZEXPORT gzoffset64(gzFile); - ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek64 OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset64 OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off_t)); # endif #else - ZEXTERN gzFile ZEXPORT gzopen(const char *, const char *); - ZEXTERN z_off_t ZEXPORT gzseek(gzFile, z_off_t, int); - ZEXTERN z_off_t ZEXPORT gztell(gzFile); - ZEXTERN z_off_t ZEXPORT gzoffset(gzFile); - ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t); + ZEXTERN gzFile ZEXPORT gzopen OF((const char *, const char *)); + ZEXTERN z_off_t ZEXPORT gzseek OF((gzFile, z_off_t, int)); + ZEXTERN z_off_t ZEXPORT gztell OF((gzFile)); + ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile)); + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t)); #endif #else /* Z_SOLO */ - ZEXTERN uLong ZEXPORT adler32_combine(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen(z_off_t); + ZEXTERN uLong ZEXPORT adler32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen OF((z_off_t)); #endif /* !Z_SOLO */ /* undocumented functions */ -ZEXTERN const char * ZEXPORT zError(int); -ZEXTERN int ZEXPORT inflateSyncPoint(z_streamp); -ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table(void); -ZEXTERN int ZEXPORT inflateUndermine(z_streamp, int); -ZEXTERN int ZEXPORT inflateValidate(z_streamp, int); -ZEXTERN unsigned long ZEXPORT inflateCodesUsed(z_streamp); -ZEXTERN int ZEXPORT inflateResetKeep(z_streamp); -ZEXTERN int ZEXPORT deflateResetKeep(z_streamp); +ZEXTERN const char * ZEXPORT zError OF((int)); +ZEXTERN int ZEXPORT inflateSyncPoint OF((z_streamp)); +ZEXTERN const z_crc_t FAR * ZEXPORT get_crc_table OF((void)); +ZEXTERN int ZEXPORT inflateUndermine OF((z_streamp, int)); +ZEXTERN int ZEXPORT inflateValidate OF((z_streamp, int)); +ZEXTERN unsigned long ZEXPORT inflateCodesUsed OF((z_streamp)); +ZEXTERN int ZEXPORT inflateResetKeep OF((z_streamp)); +ZEXTERN int ZEXPORT deflateResetKeep OF((z_streamp)); #if defined(_WIN32) && !defined(Z_SOLO) -ZEXTERN gzFile ZEXPORT gzopen_w(const wchar_t *path, - const char *mode); +ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, + const char *mode)); #endif #if defined(STDC) || defined(Z_HAVE_STDARG_H) # ifndef Z_SOLO -ZEXTERN int ZEXPORTVA gzvprintf(gzFile file, - const char *format, - va_list va); +ZEXTERN int ZEXPORTVA gzvprintf Z_ARG((gzFile file, + const char *format, + va_list va)); # endif #endif diff --git a/src/native/external/zlib/zlib2ansi b/src/native/external/zlib/zlib2ansi new file mode 100644 index 00000000000000..23b2a1d5a3ec2f --- /dev/null +++ b/src/native/external/zlib/zlib2ansi @@ -0,0 +1,152 @@ +#!/usr/bin/perl + +# Transform K&R C function definitions into ANSI equivalent. +# +# Author: Paul Marquess +# Version: 1.0 +# Date: 3 October 2006 + +# TODO +# +# Assumes no function pointer parameters. unless they are typedefed. +# Assumes no literal strings that look like function definitions +# Assumes functions start at the beginning of a line + +use strict; +use warnings; + +local $/; +$_ = <>; + +my $sp = qr{ \s* (?: /\* .*? \*/ )? \s* }x; # assume no nested comments + +my $d1 = qr{ $sp (?: [\w\*\s]+ $sp)* $sp \w+ $sp [\[\]\s]* $sp }x ; +my $decl = qr{ $sp (?: \w+ $sp )+ $d1 }xo ; +my $dList = qr{ $sp $decl (?: $sp , $d1 )* $sp ; $sp }xo ; + + +while (s/^ + ( # Start $1 + ( # Start $2 + .*? # Minimal eat content + ( ^ \w [\w\s\*]+ ) # $3 -- function name + \s* # optional whitespace + ) # $2 - Matched up to before parameter list + + \( \s* # Literal "(" + optional whitespace + ( [^\)]+ ) # $4 - one or more anythings except ")" + \s* \) # optional whitespace surrounding a Literal ")" + + ( (?: $dList )+ ) # $5 + + $sp ^ { # literal "{" at start of line + ) # Remember to $1 + //xsom + ) +{ + my $all = $1 ; + my $prefix = $2; + my $param_list = $4 ; + my $params = $5; + + StripComments($params); + StripComments($param_list); + $param_list =~ s/^\s+//; + $param_list =~ s/\s+$//; + + my $i = 0 ; + my %pList = map { $_ => $i++ } + split /\s*,\s*/, $param_list; + my $pMatch = '(\b' . join('|', keys %pList) . '\b)\W*$' ; + + my @params = split /\s*;\s*/, $params; + my @outParams = (); + foreach my $p (@params) + { + if ($p =~ /,/) + { + my @bits = split /\s*,\s*/, $p; + my $first = shift @bits; + $first =~ s/^\s*//; + push @outParams, $first; + $first =~ /^(\w+\s*)/; + my $type = $1 ; + push @outParams, map { $type . $_ } @bits; + } + else + { + $p =~ s/^\s+//; + push @outParams, $p; + } + } + + + my %tmp = map { /$pMatch/; $_ => $pList{$1} } + @outParams ; + + @outParams = map { " $_" } + sort { $tmp{$a} <=> $tmp{$b} } + @outParams ; + + print $prefix ; + print "(\n" . join(",\n", @outParams) . ")\n"; + print "{" ; + +} + +# Output any trailing code. +print ; +exit 0; + + +sub StripComments +{ + + no warnings; + + # Strip C & C++ comments + # From the perlfaq + $_[0] =~ + + s{ + /\* ## Start of /* ... */ comment + [^*]*\*+ ## Non-* followed by 1-or-more *'s + ( + [^/*][^*]*\*+ + )* ## 0-or-more things which don't start with / + ## but do end with '*' + / ## End of /* ... */ comment + + | ## OR C++ Comment + // ## Start of C++ comment // + [^\n]* ## followed by 0-or-more non end of line characters + + | ## OR various things which aren't comments: + + ( + " ## Start of " ... " string + ( + \\. ## Escaped char + | ## OR + [^"\\] ## Non "\ + )* + " ## End of " ... " string + + | ## OR + + ' ## Start of ' ... ' string + ( + \\. ## Escaped char + | ## OR + [^'\\] ## Non '\ + )* + ' ## End of ' ... ' string + + | ## OR + + . ## Anything other char + [^/"'\\]* ## Chars which doesn't start a comment, string or escape + ) + }{$2}gxs; + +} diff --git a/src/native/external/zlib/zutil.c b/src/native/external/zlib/zutil.c index b1c5d2d3c6daf5..9543ae825e3250 100644 --- a/src/native/external/zlib/zutil.c +++ b/src/native/external/zlib/zutil.c @@ -24,11 +24,13 @@ z_const char * const z_errmsg[10] = { }; -const char * ZEXPORT zlibVersion(void) { +const char * ZEXPORT zlibVersion() +{ return ZLIB_VERSION; } -uLong ZEXPORT zlibCompileFlags(void) { +uLong ZEXPORT zlibCompileFlags() +{ uLong flags; flags = 0; @@ -119,7 +121,9 @@ uLong ZEXPORT zlibCompileFlags(void) { # endif int ZLIB_INTERNAL z_verbose = verbose; -void ZLIB_INTERNAL z_error(char *m) { +void ZLIB_INTERNAL z_error(m) + char *m; +{ fprintf(stderr, "%s\n", m); exit(1); } @@ -128,7 +132,9 @@ void ZLIB_INTERNAL z_error(char *m) { /* exported to allow conversion of error code to string for compress() and * uncompress() */ -const char * ZEXPORT zError(int err) { +const char * ZEXPORT zError(err) + int err; +{ return ERR_MSG(err); } @@ -142,14 +148,22 @@ const char * ZEXPORT zError(int err) { #ifndef HAVE_MEMCPY -void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len) { +void ZLIB_INTERNAL zmemcpy(dest, source, len) + Bytef* dest; + const Bytef* source; + uInt len; +{ if (len == 0) return; do { *dest++ = *source++; /* ??? to be unrolled */ } while (--len != 0); } -int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len) { +int ZLIB_INTERNAL zmemcmp(s1, s2, len) + const Bytef* s1; + const Bytef* s2; + uInt len; +{ uInt j; for (j = 0; j < len; j++) { @@ -158,7 +172,10 @@ int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len) { return 0; } -void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len) { +void ZLIB_INTERNAL zmemzero(dest, len) + Bytef* dest; + uInt len; +{ if (len == 0) return; do { *dest++ = 0; /* ??? to be unrolled */ @@ -199,7 +216,8 @@ local ptr_table table[MAX_PTR]; * a protected system like OS/2. Use Microsoft C instead. */ -voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) +{ voidpf buf; ulg bsize = (ulg)items*size; @@ -224,7 +242,8 @@ voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { return buf; } -void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) +{ int n; (void)opaque; @@ -260,12 +279,14 @@ void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { # define _hfree hfree #endif -voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, uInt items, uInt size) { +voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, uInt items, uInt size) +{ (void)opaque; return _halloc((long)items, size); } -void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { +void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) +{ (void)opaque; _hfree(ptr); } @@ -278,18 +299,25 @@ void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { #ifndef MY_ZCALLOC /* Any system without a special alloc function */ #ifndef STDC -extern voidp malloc(uInt size); -extern voidp calloc(uInt items, uInt size); -extern void free(voidpf ptr); +extern voidp malloc OF((uInt size)); +extern voidp calloc OF((uInt items, uInt size)); +extern void free OF((voidpf ptr)); #endif -voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, unsigned size) { +voidpf ZLIB_INTERNAL zcalloc(opaque, items, size) + voidpf opaque; + unsigned items; + unsigned size; +{ (void)opaque; return sizeof(uInt) > 2 ? (voidpf)malloc(items * size) : (voidpf)calloc(items, size); } -void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr) { +void ZLIB_INTERNAL zcfree(opaque, ptr) + voidpf opaque; + voidpf ptr; +{ (void)opaque; free(ptr); } diff --git a/src/native/external/zlib/zutil.h b/src/native/external/zlib/zutil.h index 48dd7febae65ee..0bc7f4ecd1c0e5 100644 --- a/src/native/external/zlib/zutil.h +++ b/src/native/external/zlib/zutil.h @@ -1,5 +1,5 @@ /* zutil.h -- internal interface and configuration of the compression library - * Copyright (C) 1995-2024 Jean-loup Gailly, Mark Adler + * Copyright (C) 1995-2022 Jean-loup Gailly, Mark Adler * For conditions of distribution and use, see copyright notice in zlib.h */ @@ -56,7 +56,7 @@ typedef unsigned long ulg; extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* (size given to avoid silly warnings with Visual C++) */ -#define ERR_MSG(err) z_errmsg[(err) < -6 || (err) > 2 ? 9 : 2 - (err)] +#define ERR_MSG(err) z_errmsg[Z_NEED_DICT-(err)] #define ERR_RETURN(strm,err) \ return (strm->msg = ERR_MSG(err), (err)) @@ -137,8 +137,17 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # endif #endif -#if defined(MACOS) +#if defined(MACOS) || defined(TARGET_OS_MAC) # define OS_CODE 7 +# ifndef Z_SOLO +# if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include /* for fdopen */ +# else +# ifndef fdopen +# define fdopen(fd,mode) NULL /* No fdopen() */ +# endif +# endif +# endif #endif #ifdef __acorn @@ -161,6 +170,18 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define OS_CODE 19 #endif +#if defined(_BEOS_) || defined(RISCOS) +# define fdopen(fd,mode) NULL /* No fdopen() */ +#endif + +#if (defined(_MSC_VER) && (_MSC_VER > 600)) && !defined __INTERIX +# if defined(_WIN32_WCE) +# define fdopen(fd,mode) NULL /* No fdopen() */ +# else +# define fdopen(fd,type) _fdopen(fd,type) +# endif +#endif + #if defined(__BORLANDC__) && !defined(MSDOS) #pragma warn -8004 #pragma warn -8008 @@ -170,9 +191,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ /* provide prototypes for these when building zlib without LFS */ #if !defined(_WIN32) && \ (!defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0) - ZEXTERN uLong ZEXPORT adler32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine64(uLong, uLong, z_off_t); - ZEXTERN uLong ZEXPORT crc32_combine_gen64(z_off_t); + ZEXTERN uLong ZEXPORT adler32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine64 OF((uLong, uLong, z_off_t)); + ZEXTERN uLong ZEXPORT crc32_combine_gen64 OF((z_off_t)); #endif /* common defaults */ @@ -211,16 +232,16 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ # define zmemzero(dest, len) memset(dest, 0, len) # endif #else - void ZLIB_INTERNAL zmemcpy(Bytef* dest, const Bytef* source, uInt len); - int ZLIB_INTERNAL zmemcmp(const Bytef* s1, const Bytef* s2, uInt len); - void ZLIB_INTERNAL zmemzero(Bytef* dest, uInt len); + void ZLIB_INTERNAL zmemcpy OF((Bytef* dest, const Bytef* source, uInt len)); + int ZLIB_INTERNAL zmemcmp OF((const Bytef* s1, const Bytef* s2, uInt len)); + void ZLIB_INTERNAL zmemzero OF((Bytef* dest, uInt len)); #endif /* Diagnostic functions */ #ifdef ZLIB_DEBUG # include extern int ZLIB_INTERNAL z_verbose; - extern void ZLIB_INTERNAL z_error(char *m); + extern void ZLIB_INTERNAL z_error OF((char *m)); # define Assert(cond,msg) {if(!(cond)) z_error(msg);} # define Trace(x) {if (z_verbose>=0) fprintf x ;} # define Tracev(x) {if (z_verbose>0) fprintf x ;} @@ -237,9 +258,9 @@ extern z_const char * const z_errmsg[10]; /* indexed by 2-zlib_error */ #endif #ifndef Z_SOLO - voidpf ZLIB_INTERNAL zcalloc(voidpf opaque, unsigned items, - unsigned size); - void ZLIB_INTERNAL zcfree(voidpf opaque, voidpf ptr); + voidpf ZLIB_INTERNAL zcalloc OF((voidpf opaque, unsigned items, + unsigned size)); + void ZLIB_INTERNAL zcfree OF((voidpf opaque, voidpf ptr)); #endif #define ZALLOC(strm, items, size) \ diff --git a/src/native/libs/System.Security.Cryptography.Native/opensslshim.c b/src/native/libs/System.Security.Cryptography.Native/opensslshim.c index cd3e5f46b87da6..d21d6273410119 100644 --- a/src/native/libs/System.Security.Cryptography.Native/opensslshim.c +++ b/src/native/libs/System.Security.Cryptography.Native/opensslshim.c @@ -116,21 +116,8 @@ static void OpenLibraryOnce(void) DlOpen(MAKELIB("10")); } -#ifdef __FreeBSD__ - // The ports version of OpenSSL is used over base where possible - if (libssl == NULL) - { - // OpenSSL 3.0 from ports - DlOpen(MAKELIB("12")); - } - - if (libssl == NULL) - { - // OpenSSL 3.0 from base as found in FreeBSD 14.0 - DlOpen(MAKELIB("30")); - } - - // Fallbacks for OpenSSL 1.1.x + // FreeBSD uses a different suffix numbering convention. + // Current supported FreeBSD releases should use the order .11 -> .111 if (libssl == NULL) { DlOpen(MAKELIB("11")); @@ -140,8 +127,6 @@ static void OpenLibraryOnce(void) { DlOpen(MAKELIB("111")); } -#endif - } static pthread_once_t g_openLibrary = PTHREAD_ONCE_INIT; diff --git a/src/native/managed/Directory.Build.props b/src/native/managed/Directory.Build.props deleted file mode 100644 index a431ff1b38fd64..00000000000000 --- a/src/native/managed/Directory.Build.props +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/src/native/managed/Directory.Build.targets b/src/native/managed/Directory.Build.targets deleted file mode 100644 index 40dd9aec1ccbaa..00000000000000 --- a/src/native/managed/Directory.Build.targets +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/src/native/managed/README.md b/src/native/managed/README.md deleted file mode 100644 index 047d9fcc327d83..00000000000000 --- a/src/native/managed/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Native runtime component libraries using NativeAOT - -This directory contains managed libraries that will be compiled using NativeAOT and can be used in runtime components. - -## Adding a new managed library - -Add a new subdirectory to `src/native/managed` for your library with a `src`, `inc` and `test` subdirectories: - -``` console -$ mkdir -p libMyNewLibrary/src libMyNewLibrary/inc libMyNewLibrary/test -$ dotnet new classlib -n libMyNewLibrary -o libMyNewLibrary/src -``` - -In `src/native/managed/compile-native.proj`, add -`src/native/managed/libMyNewLibrary/src/libMyNewLibrary.csproj` to the `NativeLibsProjectsToBuild` -item group. - -In `src/native/managed/libMyNewLibrary/src/libMyNewLibrary.csproj`: -1. Define an item `@(InstallRuntimeComponentDestination)` that has directory names relative to `artifacts/bin///` where the shared library should be installed. It's a good idea to have at least `.`: - ```xml - - - - - ``` - -Limitations: - -* The project should be called `libXXXX` - currently the infrastructure expects a `lib` prefix on all platforms. - -* Currently only shared library output is supported. In principle static linking is possible, but the -infrastructure is not finished yet. Additionally, mixing Debug/Release configurations with static -linking will not be supported on Windows. diff --git a/src/native/managed/compile-native.proj b/src/native/managed/compile-native.proj deleted file mode 100644 index 4203835936ecbc..00000000000000 --- a/src/native/managed/compile-native.proj +++ /dev/null @@ -1,60 +0,0 @@ - - - - Release - - shared - - - $(TraversalPublishGlobalProperties);_IsPublishing=true - - - - - - - - - - - false - - false - - false - true - false - - - - - $(ROOTFS_DIR) - lld - --gcc-toolchain=$(ROOTFS_DIR)/usr - - - - - - - - - - - - - - - - - - @(SubprojectProps->'%(Identity)=%(Value)', ';') - - - - - - diff --git a/src/native/managed/native-library.props b/src/native/managed/native-library.props deleted file mode 100644 index 89a80a8005a0d0..00000000000000 --- a/src/native/managed/native-library.props +++ /dev/null @@ -1,35 +0,0 @@ - - - true - true - - false - - - - - - @rpath/$(MSBuildProjectName).dylib - - - - - - - - - - - - true - - - - - - - diff --git a/src/native/managed/native-library.targets b/src/native/managed/native-library.targets deleted file mode 100644 index e0f23322bba481..00000000000000 --- a/src/native/managed/native-library.targets +++ /dev/null @@ -1,107 +0,0 @@ - - - - - - $(OutputPath)stripped\ - $(StrippedOutputPath)$(TargetName)$(NativeBinaryExt) - .dylib.dwarf - .so.dbg - $(StrippedOutputPath)$(TargetName)$(StrippedExt) - - - - - - - true - - - - - - false - - - - - - - - - - - - - <_StripLike Condition="'$(TargetsOSX)' == 'true' or '$(TargetsAppleMobile)' == 'true'">apple - <_StripLike Condition="'$(_StripLike)' == ''">gnu - - - - - - - - - - - - - - - - - - $([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', '$(RuntimeFlavor.ToLower())', '$(TargetOS).$(TargetArchitecture).$(RuntimeConfiguration)')) - - - - <_NormalizedInstallRuntimeComponentDest Include="$([MSBuild]::NormalizeDirectory('$(FinalRuntimeComponentDestinationBase)', '%(InstallRuntimeComponentDestination.Identity)'))" /> - - - - - - - - - - - - - - - - - - - - - diff --git a/src/tasks/Common/FileCache.cs b/src/tasks/Common/FileCache.cs index 05d92bdbc2c8ee..39ecd5e70ab2d2 100644 --- a/src/tasks/Common/FileCache.cs +++ b/src/tasks/Common/FileCache.cs @@ -33,7 +33,9 @@ public FileCache(string? cacheFilePath, TaskLoggingHelper log) Enabled = true; if (File.Exists(cacheFilePath)) { - _oldCache = JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), s_jsonOptions); + _oldCache = (CompilerCache?)JsonSerializer.Deserialize(File.ReadAllText(cacheFilePath), + typeof(CompilerCache), + s_jsonOptions); } _oldCache ??= new(); diff --git a/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs b/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs index 843333ff3fcbbd..ced9becf153a77 100644 --- a/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs +++ b/src/tasks/Crossgen2Tasks/ResolveReadyToRunCompilers.cs @@ -230,9 +230,6 @@ private static bool ExtractTargetPlatformAndArchitecture(string runtimeIdentifie case "x86": architecture = Architecture.X86; break; - case "riscv64": - architecture = Architecture.RiscV64; - break; default: return false; } @@ -390,7 +387,6 @@ private static string ArchitectureToString(Architecture architecture) Architecture.X64 => "x64", Architecture.Arm => "arm", Architecture.Arm64 => "arm64", - Architecture.RiscV64 => "riscv64", _ => null }; } diff --git a/src/tasks/Microsoft.NET.WebAssembly.Webcil/Microsoft.NET.WebAssembly.Webcil.csproj b/src/tasks/Microsoft.NET.WebAssembly.Webcil/Microsoft.NET.WebAssembly.Webcil.csproj index 14ecec43bf1255..846dbc9e042aaa 100644 --- a/src/tasks/Microsoft.NET.WebAssembly.Webcil/Microsoft.NET.WebAssembly.Webcil.csproj +++ b/src/tasks/Microsoft.NET.WebAssembly.Webcil/Microsoft.NET.WebAssembly.Webcil.csproj @@ -8,8 +8,10 @@ true true true - false - true + true + + false diff --git a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs index e4aa070d88ea38..cd8535463bc349 100644 --- a/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs +++ b/src/tasks/WasmAppBuilder/PInvokeTableGenerator.cs @@ -269,19 +269,10 @@ private static bool TryIsMethodGetParametersUnsupported(MethodInfo method, [NotN return null; } - var realReturnType = method.ReturnType; - var realParameterTypes = method.GetParameters().Select(p => MapType(p.ParameterType)).ToList(); - - SignatureMapper.TypeToChar(realReturnType, Log, out bool resultIsByRef); - if (resultIsByRef) { - realReturnType = typeof(void); - realParameterTypes.Insert(0, "void *"); - } - return $$""" {{(pinvoke.WasmLinkage ? $"__attribute__((import_module(\"{EscapeLiteral(pinvoke.Module)}\"),import_name(\"{EscapeLiteral(pinvoke.EntryPoint)}\")))" : "")}} - {{(pinvoke.WasmLinkage ? "extern " : "")}}{{MapType(realReturnType)}} {{CEntryPoint(pinvoke)}} ({{string.Join(", ", realParameterTypes)}}); + {{(pinvoke.WasmLinkage ? "extern " : "")}}{{MapType(method.ReturnType)}} {{CEntryPoint(pinvoke)}} ({{string.Join(", ", method.GetParameters().Select(p => MapType(p.ParameterType)))}}); """; } diff --git a/src/tasks/WasmAppBuilder/SignatureMapper.cs b/src/tasks/WasmAppBuilder/SignatureMapper.cs index 3638e432f0ce38..f3b7f17ad017be 100644 --- a/src/tasks/WasmAppBuilder/SignatureMapper.cs +++ b/src/tasks/WasmAppBuilder/SignatureMapper.cs @@ -11,15 +11,8 @@ internal static class SignatureMapper { - internal static char? TypeToChar(Type t, LogAdapter log, out bool isByRefStruct, int depth = 0) + private static char? TypeToChar(Type t, LogAdapter log) { - isByRefStruct = false; - - if (depth > 5) { - log.Warning("WASM0064", $"Unbounded recursion detected through parameter type '{t.Name}'"); - return null; - } - char? c = null; if (t.Namespace == "System") { c = t.Name switch @@ -27,7 +20,6 @@ internal static class SignatureMapper nameof(String) => 'I', nameof(Boolean) => 'I', nameof(Char) => 'I', - nameof(SByte) => 'I', nameof(Byte) => 'I', nameof(Int16) => 'I', nameof(UInt16) => 'I', @@ -59,23 +51,19 @@ internal static class SignatureMapper c = 'I'; else if (t.IsInterface) c = 'I'; - else if (t.IsEnum) { - Type underlyingType = t.GetEnumUnderlyingType(); - c = TypeToChar(underlyingType, log, out _, ++depth); - } else if (t.IsPointer) + else if (t.IsEnum) + c = TypeToChar(t.GetEnumUnderlyingType(), log); + else if (t.IsPointer) c = 'I'; else if (PInvokeTableGenerator.IsFunctionPointer(t)) c = 'I'; else if (t.IsValueType) { var fields = t.GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); - if (fields.Length == 1) { - Type fieldType = fields[0].FieldType; - return TypeToChar(fieldType, log, out isByRefStruct, ++depth); - } else if (PInvokeTableGenerator.IsBlittable(t, log)) + if (fields.Length == 1) + return TypeToChar(fields[0].FieldType, log); + else if (PInvokeTableGenerator.IsBlittable(t, log)) c = 'I'; - - isByRefStruct = true; } else log.Warning("WASM0064", $"Unsupported parameter type '{t.Name}'"); @@ -86,20 +74,15 @@ internal static class SignatureMapper public static string? MethodToSignature(MethodInfo method, LogAdapter log) { - string? result = TypeToChar(method.ReturnType, log, out bool resultIsByRef)?.ToString(); + string? result = TypeToChar(method.ReturnType, log)?.ToString(); if (result == null) { return null; } - if (resultIsByRef) { - // WASM abi passes a result-pointer in slot 0 instead of returning struct results - result = "VI"; - } - foreach (var parameter in method.GetParameters()) { - char? parameterChar = TypeToChar(parameter.ParameterType, log, out _); + char? parameterChar = TypeToChar(parameter.ParameterType, log); if (parameterChar == null) { return null; diff --git a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs index 92d90e03278832..b775ad7cd19ebb 100644 --- a/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs +++ b/src/tasks/WorkloadBuildTasks/InstallWorkloadFromArtifacts.cs @@ -8,13 +8,10 @@ using System.IO; using System.IO.Compression; using System.Linq; -using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; -using System.Xml; using System.Xml.Linq; -using System.Xml.XPath; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; @@ -22,7 +19,7 @@ namespace Microsoft.Workload.Build.Tasks { - public partial class InstallWorkloadFromArtifacts : PatchNuGetConfig + public partial class InstallWorkloadFromArtifacts : Task { [Required, NotNull] public ITaskItem[] WorkloadIds { get; set; } = Array.Empty(); @@ -36,6 +33,12 @@ public partial class InstallWorkloadFromArtifacts : PatchNuGetConfig [Required, NotNull] public string? VersionBandForManifestPackages { get; set; } + [Required, NotNull] + public string? LocalNuGetsPath { get; set; } + + [Required, NotNull] + public string? TemplateNuGetConfigPath { get; set; } + [Required, NotNull] public string SdkWithNoWorkloadInstalledPath { get; set; } = string.Empty; @@ -44,9 +47,7 @@ public partial class InstallWorkloadFromArtifacts : PatchNuGetConfig public bool OnlyUpdateManifests { get; set; } public bool SkipTempDirectoryCleanup { get; set; } - // Should match enum values for MessageImportance - Low, Normal (default), High - public string? WorkloadInstallCommandOutputImportance { get; set; } - + private const string s_nugetInsertionTag = ""; private string AllManifestsStampPath => Path.Combine(SdkWithNoWorkloadInstalledPath, ".all-manifests.stamp"); private string _tempDir = string.Empty; private string _nugetCachePath = string.Empty; @@ -66,10 +67,6 @@ public override bool Execute() Directory.Delete(_tempDir, recursive: true); Directory.CreateDirectory(_tempDir); _nugetCachePath = Path.Combine(_tempDir, "nuget-cache"); - if (SkipTempDirectoryCleanup) - { - Log.LogMessage(MessageImportance.High, $"Using temporary directory {_tempDir} for installing workloads from artifacts."); - } try { @@ -220,12 +217,6 @@ private bool InstallPacks(InstallWorkloadRequest req, string nugetConfigContents string nugetConfigPath = Path.Combine(_tempDir, $"NuGet.{Path.GetRandomFileName()}.config"); File.WriteAllText(nugetConfigPath, nugetConfigContents); - if (string.IsNullOrEmpty(WorkloadInstallCommandOutputImportance) || - !Enum.TryParse(WorkloadInstallCommandOutputImportance, out var outputImportance)) - { - outputImportance = MessageImportance.Normal; - } - // Log.LogMessage(MessageImportance.High, $"{Environment.NewLine}** dotnet workload install {req.WorkloadId} **{Environment.NewLine}"); (int exitCode, string output) = Utils.TryRunProcess( Log, @@ -237,7 +228,7 @@ private bool InstallPacks(InstallWorkloadRequest req, string nugetConfigContents }, logStdErrAsMessage: req.IgnoreErrors, silent: false, - debugMessageImportance: outputImportance); + debugMessageImportance: MessageImportance.Normal); if (exitCode != 0) { if (req.IgnoreErrors) @@ -264,11 +255,11 @@ private bool InstallPacks(InstallWorkloadRequest req, string nugetConfigContents private string GetNuGetConfig() { - var nugetConfigPath = Path.GetTempFileName(); - PatchNuGetConfig.GetNuGetConfig(TemplateNuGetConfigPath, LocalNuGetsPath, PackageSourceNameForBuiltPackages, NuGetConfigPackageSourceMappings, nugetConfigPath); - string contents = File.ReadAllText(nugetConfigPath); - File.Delete(nugetConfigPath); - return contents; + string contents = File.ReadAllText(TemplateNuGetConfigPath); + if (!contents.Contains(s_nugetInsertionTag, StringComparison.InvariantCultureIgnoreCase)) + throw new LogAsErrorException($"Could not find {s_nugetInsertionTag} in {TemplateNuGetConfigPath}"); + + return contents.Replace(s_nugetInsertionTag, $@""); } private bool InstallWorkloadManifest(ITaskItem workloadId, string name, string version, string sdkDir, string nugetConfigContents, bool stopOnMissing) diff --git a/src/tasks/WorkloadBuildTasks/PackageInstaller.cs b/src/tasks/WorkloadBuildTasks/PackageInstaller.cs index b7a188afb15866..0d2f5db3290651 100644 --- a/src/tasks/WorkloadBuildTasks/PackageInstaller.cs +++ b/src/tasks/WorkloadBuildTasks/PackageInstaller.cs @@ -54,18 +54,9 @@ private bool InstallActual(PackageReference[] references, bool stopOnMissing) Directory.CreateDirectory(projecDir); - File.WriteAllText(Path.Combine(projecDir, "Directory.Build.props"), """ - - - - - false - false - - - -"""); + File.WriteAllText(Path.Combine(projecDir, "Directory.Build.props"), ""); + File.WriteAllText(Path.Combine(projecDir, "Directory.Packages.props"), ""); + File.WriteAllText(Path.Combine(projecDir, "Directory.Build.targets"), ""); File.WriteAllText(projectPath, GenerateProject(references)); File.WriteAllText(Path.Combine(projecDir, "nuget.config"), _nugetConfigContents); @@ -128,7 +119,7 @@ private static string GenerateProject(IEnumerable references) "); foreach (var reference in references) - projectFileBuilder.AppendLine($""); + projectFileBuilder.AppendLine($""); projectFileBuilder.Append(@" diff --git a/src/tasks/WorkloadBuildTasks/PatchNuGetConfig.cs b/src/tasks/WorkloadBuildTasks/PatchNuGetConfig.cs deleted file mode 100644 index 34dfe2c1706ee8..00000000000000 --- a/src/tasks/WorkloadBuildTasks/PatchNuGetConfig.cs +++ /dev/null @@ -1,140 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using System.Xml.Linq; -using System.Xml.XPath; -using System.Xml; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; - -#nullable enable - -namespace Microsoft.Workload.Build.Tasks; - -/* - * Used for patching a nuget.config to: - * - * 1. Add a new package source to the nuget.config - * 2. Add a new package source mapping to the nuget.config - * - * This is useful specifically the case of workload testing - */ -public class PatchNuGetConfig : Task -{ - [Required, NotNull] - public string? TemplateNuGetConfigPath { get; set; } - - [Required, NotNull] - public string? LocalNuGetsPath { get; set; } - - public string? OutputPath { get; set; } - - /* - * Value: ["*Aspire*", "Foo*"] - * This will be translated to: - * - * - * - * - * - * - * This is useful when using Central Package Management (https://learn.microsoft.com/en-us/nuget/consume-packages/central-package-management) - */ - public string[] NuGetConfigPackageSourceMappings { get; set; } = Array.Empty(); - - public string PackageSourceNameForBuiltPackages { get; set; } = "nuget-local"; - - public override bool Execute() - { - try - { - Validate(TemplateNuGetConfigPath, PackageSourceNameForBuiltPackages, OutputPath); - GetNuGetConfig(TemplateNuGetConfigPath, LocalNuGetsPath, PackageSourceNameForBuiltPackages, NuGetConfigPackageSourceMappings, OutputPath!); - Log.LogMessage(MessageImportance.Low, $"Generated patched nuget.config at {OutputPath}"); - return true; - } - catch (LogAsErrorException laee) - { - Log.LogError(laee.Message); - return false; - } - } - - private static void Validate(string? templateNuGetConfigPath, string? packageSourceNameForBuiltPackages, string? outputPath) - { - if (string.IsNullOrEmpty(templateNuGetConfigPath)) - throw new LogAsErrorException($"{nameof(templateNuGetConfigPath)} is required"); - - if (!File.Exists(templateNuGetConfigPath)) - throw new LogAsErrorException($"Cannot find {nameof(templateNuGetConfigPath)}={templateNuGetConfigPath}"); - - if (string.IsNullOrEmpty(packageSourceNameForBuiltPackages)) - throw new LogAsErrorException($"{nameof(packageSourceNameForBuiltPackages)} is required"); - - if (string.IsNullOrEmpty(outputPath)) - throw new LogAsErrorException($"{nameof(outputPath)} is required"); - - if (Directory.Exists(outputPath)) - throw new LogAsErrorException($"{nameof(outputPath)}={outputPath} is a directory, it should be a file"); - } - - public static void GetNuGetConfig(string templateNuGetConfigPath, string localNuGetsPath, string packageSourceNameForBuiltPackages, string[] nuGetConfigPackageSourceMappings, string outputPath) - { - Validate(templateNuGetConfigPath, packageSourceNameForBuiltPackages, outputPath); - - XDocument doc = XDocument.Load(templateNuGetConfigPath); - string xpath = "/configuration/packageSources"; - XElement? packageSources = doc.XPathSelectElement(xpath); - if (packageSources is null) - throw new LogAsErrorException($"Could not find {xpath} in {templateNuGetConfigPath}"); - - var newPackageSourceElement = new XElement("add", - new XAttribute("key", packageSourceNameForBuiltPackages), - new XAttribute("value", $"file://{localNuGetsPath}")); - if (packageSources.LastNode is not null) - { - packageSources.LastNode.AddAfterSelf(newPackageSourceElement); - } - else - { - packageSources.Add(newPackageSourceElement); - } - - if (nuGetConfigPackageSourceMappings.Length > 0) - { - string mappingXpath = "/configuration/packageSourceMapping"; - XElement? packageSourceMapping = doc.XPathSelectElement(mappingXpath); - if (packageSourceMapping is null) - { - if (doc.Root is null) - throw new LogAsErrorException($"Could not find root element in {templateNuGetConfigPath}"); - - packageSourceMapping = new XElement("packageSourceMapping"); - doc.Root.Add(packageSourceMapping); - } - - var newPackageSourceMappingElement = new XElement("packageSource", - new XAttribute("key", packageSourceNameForBuiltPackages), - nuGetConfigPackageSourceMappings.Select - (pattern => new XElement("package", new XAttribute("pattern", pattern)))); - if (packageSourceMapping.FirstNode is not null) - { - packageSourceMapping.FirstNode?.AddBeforeSelf(newPackageSourceMappingElement); - } - else - { - packageSourceMapping.Add(newPackageSourceMappingElement); - } - } - - using var xw = XmlWriter.Create(outputPath, new XmlWriterSettings { Indent = true, NewLineHandling = NewLineHandling.None, Encoding = Encoding.UTF8 }); - doc.WriteTo(xw); - xw.Close(); - } -} diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 6c7d4cb9fd512d..89303380d30c37 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -19,7 +19,6 @@ endif() if (CLR_CMAKE_HOST_WIN32) # 4100 - unreferenced formal parameter - # 4242 - conversion from 'type1' to 'type2', possible loss of data # 4244 - conversion from 'type1' to 'type2', possible loss of data # 4514 - unreferenced inline function has been removed # 4625 - copy constructor was implicitly defined as deleted because a base class copy constructor is inaccessible or deleted @@ -33,7 +32,7 @@ if (CLR_CMAKE_HOST_WIN32) # 5026 - move constructor was implicitly defined as deleted # 5027 - move assignment operator was implicitly defined as deleted # 5039 - pointer or reference to potentially throwing function passed to extern C function under -EHc. Undefined behavior may occur if this function throws an exception. - add_compile_options(-wd4100 -wd4242 -wd4244 -wd4514 -wd4625 -wd4626 -wd4668 -wd4710 -wd4711 -wd4774 -wd4820 -wd5025 -wd5026 -wd5027 -wd5039) + add_compile_options(-wd4100 -wd4244 -wd4514 -wd4625 -wd4626 -wd4668 -wd4710 -wd4711 -wd4774 -wd4820 -wd5025 -wd5026 -wd5027 -wd5039) set_property(DIRECTORY PROPERTY CLR_EH_OPTION /EHa) # enable C++ EH (w/ SEH exceptions) endif() diff --git a/src/tests/Common/CLRTest.CrossGen.targets b/src/tests/Common/CLRTest.CrossGen.targets index cacc3980c1a1de..ff8f875df7449a 100644 --- a/src/tests/Common/CLRTest.CrossGen.targets +++ b/src/tests/Common/CLRTest.CrossGen.targets @@ -165,6 +165,9 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then fi ReleaseLock fi + + export DOTNET_ZapRequire=$(ZapRequire) + export DOTNET_ZapRequireList=$(AssemblyName) fi ]]> @@ -336,6 +339,9 @@ if defined RunCrossGen2 ( ECHO R2RDump failed with exitcode - !R2RDumpStatus! Exit /b 1 ) + + set DOTNET_ZapRequire=$(ZapRequire) + set DOTNET_ZapRequireList=$(AssemblyName) ) ]]> diff --git a/src/tests/Common/CoreCLRTestLibrary/CoreClrConfigurationDetection.cs b/src/tests/Common/CoreCLRTestLibrary/CoreClrConfigurationDetection.cs index 8b6d893f8d9714..75014ad32484ce 100644 --- a/src/tests/Common/CoreCLRTestLibrary/CoreClrConfigurationDetection.cs +++ b/src/tests/Common/CoreCLRTestLibrary/CoreClrConfigurationDetection.cs @@ -17,17 +17,11 @@ public static class CoreClrConfigurationDetection public static bool IsJitStressRegs => !string.Equals(GetEnvironmentVariableValue("JitStressRegs"), "0", StringComparison.InvariantCulture); public static bool IsJitMinOpts => string.Equals(GetEnvironmentVariableValue("JITMinOpts"), "1", StringComparison.InvariantCulture); public static bool IsTailCallStress => string.Equals(GetEnvironmentVariableValue("TailcallStress"), "1", StringComparison.InvariantCulture); - public static bool IsDisableR2R => string.Equals(GetEnvironmentVariableValue("ReadyToRun"), "0", StringComparison.InvariantCulture); + public static bool IsZapDisable => string.Equals(GetEnvironmentVariableValue("ZapDisable"), "1", StringComparison.InvariantCulture); public static bool IsGCStress3 => CompareGCStressModeAsLower(GetEnvironmentVariableValue("GCStress"), "0x3", "3"); public static bool IsGCStressC => CompareGCStressModeAsLower(GetEnvironmentVariableValue("GCStress"), "0xC", "C"); - public static bool IsTieredCompilation => string.Equals(GetEnvironmentVariableValue("TieredCompilation", "1"), "1", StringComparison.InvariantCulture); - public static bool IsHeapVerify => string.Equals(GetEnvironmentVariableValue("HeapVerify"), "1", StringComparison.InvariantCulture); public static bool IsGCStress => !string.Equals(GetEnvironmentVariableValue("GCStress"), "0", StringComparison.InvariantCulture); - - public static bool IsAnyJitStress => IsJitStress || IsJitStressRegs || IsJitMinOpts || IsTailCallStress; - - public static bool IsAnyJitOptimizationStress => IsAnyJitStress || IsTieredCompilation; public static bool IsCheckedRuntime => AssemblyConfigurationEquals("Checked"); public static bool IsReleaseRuntime => AssemblyConfigurationEquals("Release"); @@ -35,12 +29,14 @@ public static class CoreClrConfigurationDetection public static bool IsStressTest => IsGCStress || - IsDisableR2R || - IsAnyJitStress || - IsHeapVerify; + IsZapDisable || + IsTailCallStress || + IsJitStressRegs || + IsJitStress || + IsJitMinOpts; - private static string GetEnvironmentVariableValue(string name, string defaultValue = "0") => - Environment.GetEnvironmentVariable("DOTNET_" + name) ?? Environment.GetEnvironmentVariable("COMPlus_" + name) ?? defaultValue; + private static string GetEnvironmentVariableValue(string name) => + Environment.GetEnvironmentVariable("DOTNET_" + name) ?? Environment.GetEnvironmentVariable("COMPlus_" + name) ?? "0"; private static bool AssemblyConfigurationEquals(string configuration) { @@ -58,4 +54,4 @@ private static bool CompareGCStressModeAsLower(string value, string first, strin string.Equals(value, "0xf", StringComparison.InvariantCulture) || string.Equals(value, "f", StringComparison.InvariantCulture); } -} \ No newline at end of file +} diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 5ee032e2842d61..510948877cd8b9 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -1702,27 +1702,28 @@ ("StoreSelectedScalarTest.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128_UInt16_7", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["ElementIndex"] = "7", ["ValidateResult"] = "firstOp[ElementIndex] != result"}), ("StoreSelectedScalarTest.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128_UInt32_3", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["ElementIndex"] = "3", ["ValidateResult"] = "firstOp[ElementIndex] != result"}), ("StoreSelectedScalarTest.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128_UInt64_1", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["ElementIndex"] = "1", ["ValidateResult"] = "firstOp[ElementIndex] != result"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Short", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_UInt32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Int32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Float", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Short", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_UInt32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Int32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Float", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Short", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_UInt32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Int32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Float", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // Tests disabled until mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Short", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_UInt32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Int32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x2_Float", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Short", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_UInt32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Int32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x3_Float", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Short", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_UInt32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Int32", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector64x4_Float", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), ("StoreVectorx2Test.template", new Dictionary { ["TestName"] = "StoreVector64x2SByte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreVector64x2", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "input1[i] != result[i] || input2[i] != result[OpElementCount + i]"}), ("StoreVectorx2Test.template", new Dictionary { ["TestName"] = "StoreVector64x2Byte", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreVector64x2", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "input1[i] != result[i] || input2[i] != result[OpElementCount + i]"}), ("StoreVectorx2Test.template", new Dictionary { ["TestName"] = "StoreVector64x2UShort", ["Isa"] = "AdvSimd", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreVector64x2", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector64", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "input1[i] != result[i] || input2[i] != result[OpElementCount + i]"}), @@ -2542,36 +2543,37 @@ ("StoreBinOpTest.template", new Dictionary { ["TestName"] = "StorePairNonTemporal_Vector128_UInt16", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StorePairNonTemporal", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector128", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "32", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "Helpers.Concat(firstOp, secondOp, i) != result[i]"}), ("StoreBinOpTest.template", new Dictionary { ["TestName"] = "StorePairNonTemporal_Vector128_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StorePairNonTemporal", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector128", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "32", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "Helpers.Concat(firstOp, secondOp, i) != result[i]"}), ("StoreBinOpTest.template", new Dictionary { ["TestName"] = "StorePairNonTemporal_Vector128_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StorePairNonTemporal", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector128", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "32", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "Helpers.Concat(firstOp, secondOp, i) != result[i]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Short", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Int32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Int64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Float", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Double", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "double", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "double", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Short", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Int32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Int64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Float", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Double", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "double", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "double", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Short", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Int32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Int64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Float", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), - ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Double", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "double", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "double", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // Tests disabled until mono implements these APIs. See https://github.com/dotnet/runtime/issues/93081 + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Short", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Int32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Int64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Float", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx2Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x2_Double", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "double", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "double", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Short", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Int32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Int64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Float", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx3Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x3_Double", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "double", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "double", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Short", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_UInt32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Int32", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_UInt64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Int64", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Float", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "float", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "float", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), + // ("StoreSelectedScalarx4Test.template", new Dictionary { ["TestName"] = "StoreSelectedScalar_Vector128x4_Double", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreSelectedScalar", ["Op1BaseType"] = "double", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "double", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateResult"] = "input1[index] != result[0] || input2[index] != result[1] || input3[index] != result[2] || input4[index] != result[3]"}), ("StoreVectorx2Test.template", new Dictionary { ["TestName"] = "StoreVector128x2SByte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreVector128x2", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "input1[i] != result[i] || input2[i] != result[OpElementCount + i]"}), ("StoreVectorx2Test.template", new Dictionary { ["TestName"] = "StoreVector128x2Byte", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreVector128x2", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "input1[i] != result[i] || input2[i] != result[OpElementCount + i]"}), ("StoreVectorx2Test.template", new Dictionary { ["TestName"] = "StoreVector128x2UShort", ["Isa"] = "AdvSimd.Arm64", ["LoadIsa"] = "AdvSimd", ["Method"] = "StoreVector128x2", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "16", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "input1[i] != result[i] || input2[i] != result[OpElementCount + i]"}), @@ -2885,18 +2887,9 @@ ("SecureHashTernOpTest.template", new Dictionary { ["TestName"] = "ScheduleUpdate1_Vector128_UInt32", ["Isa"] = "Sha256", ["LoadIsa"] = "AdvSimd", ["Method"] = "ScheduleUpdate1", ["RetVectorType"] = "Vector128", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector128", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector128", ["Op2BaseType"] = "UInt32", ["Op3VectorType"] = "Vector128", ["Op3BaseType"] = "UInt32", ["LargestVectorSize"] = "16", ["NextValueOp1"] = "0x00112233", ["NextValueOp2"] = "0x44556677", ["NextValueOp3"] = "0x8899AABB", ["ExpectedResult"] = "{0x248F1BDF, 0x248F1BDF, 0xB303DDBA, 0xF74821FE}"}), }; -(string templateFileName, Dictionary templateData)[] SveInputs = new [] +(string templateFileName, Dictionary templateData)[] SveInputs = Array.Empty<(string templateFileName, Dictionary templateData)>(); { - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_float", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_double", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_sbyte", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_short", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_int", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_long", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_byte", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_ushort", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_uint", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), - ("SveLoadMaskedUnOpTest.template", new Dictionary { ["TestName"] = "SveLoadVector_ulong", ["Isa"] = "Sve", ["Method"] = "LoadVector", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "8", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "firstOp[i] != result[i]"}), + //TODO-SVE: Add SVE tests }; diff --git a/src/tests/Common/XUnitWrapperGenerator/RuntimeTestModes.cs b/src/tests/Common/XUnitWrapperGenerator/RuntimeTestModes.cs index f4d60288ca6e77..e5a48a348d1573 100644 --- a/src/tests/Common/XUnitWrapperGenerator/RuntimeTestModes.cs +++ b/src/tests/Common/XUnitWrapperGenerator/RuntimeTestModes.cs @@ -22,9 +22,9 @@ public enum RuntimeTestModes JitMinOpts = 1 << 3, // DOTNET_JITMinOpts is set. TailcallStress = 1 << 4, // DOTNET_TailcallStress is set. - // DisableR2R says to not use ReadyToRun images. + // ZapDisable says to not use NGEN or ReadyToRun images. // This means we JIT everything. - DisableR2R = 1 << 5, // DOTNET_ReadyToRun=0 + ZapDisable = 1 << 5, // DOTNET_ZapDisable is set. // GCStress3 forces a GC at various locations, typically transitions // to/from the VM from managed code. @@ -33,15 +33,6 @@ public enum RuntimeTestModes // GCStressC forces a GC at every JIT-generated code instruction, // including in NGEN/ReadyToRun code. GCStressC = 1 << 7, // DOTNET_GCStress includes mode 0xC. - AnyGCStress = GCStress3 | GCStressC, // Disable when any GCStress is exercised. - // TieredCompilation is on by default, but can cause some tests to fail - // As TieredCompilation is on by default, it does not count as a stress mode for RegularRun. - TieredCompilation = 1 << 8, // DOTNET_TieredCompilation (or COMPlus_TieredCompilation) is not set to 0. - - AnyJitStress = JitStress | JitStressRegs | JitMinOpts | TailcallStress, // Disable when any JIT stress mode is exercised. - - AnyJitOptimizationStress = AnyJitStress | TieredCompilation, // Disable when any JIT non-full optimization stress mode is exercised. - - HeapVerify = 1 << 9, // DOTNET_HeapVerify (or COMPlus_HeapVerify) is set. + AnyGCStress = GCStress3 | GCStressC // Disable when any GCStress is exercised. } } diff --git a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs index 06cfdf3022c1b7..c5a94972b7098f 100644 --- a/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs +++ b/src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs @@ -806,10 +806,6 @@ private static IEnumerable GetTestMethodInfosForMethod(IMethodSymbol // If we're building tests not for Mono, we can skip handling the specifics of the SkipOnMonoAttribute. continue; } - if (filterAttribute.ConstructorArguments.Length <= 1) - { - return ImmutableArray.Empty; - } testInfos = DecorateWithSkipOnPlatform(testInfos, (int)filterAttribute.ConstructorArguments[1].Value!, options); break; case "Xunit.SkipOnPlatformAttribute": @@ -896,10 +892,6 @@ private static ImmutableArray DecorateWithSkipOnCoreClrConfiguration( { conditions.Add($"{ConditionClass}.IsStressTest"); } - if (skippedTestModes.HasFlag(Xunit.RuntimeTestModes.DisableR2R)) - { - conditions.Add($"!{ConditionClass}.IsDisableR2R"); - } if (skippedTestModes.HasFlag(Xunit.RuntimeTestModes.JitStress)) { conditions.Add($"!{ConditionClass}.IsJitStress"); @@ -916,13 +908,9 @@ private static ImmutableArray DecorateWithSkipOnCoreClrConfiguration( { conditions.Add($"!{ConditionClass}.IsTailcallStress"); } - if (skippedTestModes.HasFlag(Xunit.RuntimeTestModes.TieredCompilation)) - { - conditions.Add($"!{ConditionClass}.IsTieredCompilation"); - } - if (skippedTestModes.HasFlag(Xunit.RuntimeTestModes.HeapVerify)) + if (skippedTestModes.HasFlag(Xunit.RuntimeTestModes.ZapDisable)) { - conditions.Add($"!{ConditionClass}.IsHeapVerify"); + conditions.Add($"!{ConditionClass}.IsZapDisable"); } if (skippedTestModes.HasFlag(Xunit.RuntimeTestModes.AnyGCStress)) diff --git a/src/tests/Common/testenvironment.proj b/src/tests/Common/testenvironment.proj index e82110af21a8f7..d1bc1d1a94c427 100644 --- a/src/tests/Common/testenvironment.proj +++ b/src/tests/Common/testenvironment.proj @@ -53,6 +53,7 @@ DOTNET_JitStressRegs; DOTNET_TailcallStress; DOTNET_ReadyToRun; + DOTNET_ZapDisable; DOTNET_TC_OnStackReplacement; DOTNET_TC_QuickJitForLoops; DOTNET_TC_OnStackReplacement_InitialCounter; @@ -80,7 +81,7 @@ RunningIlasmRoundTrip; DOTNET_JitSynthesizeCounts; DOTNET_JitCheckSynthesizedCounts; - DOTNET_JitRLCSEGreedy; + DOTNET_JitEnableCrossBlockLocalAssertionProp @@ -206,11 +207,11 @@ - + - - - + + + @@ -223,6 +224,7 @@ + @@ -238,7 +240,6 @@ - diff --git a/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombine1.csproj b/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombine1.csproj index 9ba31e81100970..62c497a0992226 100644 --- a/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombine1.csproj +++ b/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombine1.csproj @@ -1,5 +1,7 @@ + + true true 1 diff --git a/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombineImpl.csproj b/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombineImpl.csproj index bcdc37ff1fd6b5..df8e80338fad09 100644 --- a/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombineImpl.csproj +++ b/src/tests/CoreMangLib/system/delegate/delegate/DelegateCombineImpl.csproj @@ -1,5 +1,7 @@ + + true true 1 diff --git a/src/tests/CoreMangLib/system/delegate/delegate/DelegateEquals1.csproj b/src/tests/CoreMangLib/system/delegate/delegate/DelegateEquals1.csproj index 675f0977c04bf5..2d8fbf191cbb54 100644 --- a/src/tests/CoreMangLib/system/delegate/delegate/DelegateEquals1.csproj +++ b/src/tests/CoreMangLib/system/delegate/delegate/DelegateEquals1.csproj @@ -1,5 +1,7 @@ + + true true 1 diff --git a/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetHashCode1.csproj b/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetHashCode1.csproj index 93d280cc34678b..a597f02daeda12 100644 --- a/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetHashCode1.csproj +++ b/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetHashCode1.csproj @@ -1,5 +1,7 @@ + + true true 1 diff --git a/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetInvocationList1.csproj b/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetInvocationList1.csproj new file mode 100644 index 00000000000000..e82724aaace7fe --- /dev/null +++ b/src/tests/CoreMangLib/system/delegate/delegate/DelegateGetInvocationList1.csproj @@ -0,0 +1,14 @@ + + + + true + true + 1 + + + + + + + + diff --git a/src/tests/CoreMangLib/system/delegate/delegate/DelegateRemove.csproj b/src/tests/CoreMangLib/system/delegate/delegate/DelegateRemove.csproj index fccdb6634bb1b5..0217c4ad8d879a 100644 --- a/src/tests/CoreMangLib/system/delegate/delegate/DelegateRemove.csproj +++ b/src/tests/CoreMangLib/system/delegate/delegate/DelegateRemove.csproj @@ -1,5 +1,7 @@ + + true true 1 diff --git a/src/tests/CoreMangLib/system/delegate/delegate/delegateRemoveImpl.csproj b/src/tests/CoreMangLib/system/delegate/delegate/delegateRemoveImpl.csproj index 8ef2b3c7819d32..ab802b989cda73 100644 --- a/src/tests/CoreMangLib/system/delegate/delegate/delegateRemoveImpl.csproj +++ b/src/tests/CoreMangLib/system/delegate/delegate/delegateRemoveImpl.csproj @@ -1,5 +1,7 @@ + + true true 1 diff --git a/src/tests/CoreMangLib/system/delegate/delegate/delegategetinvocationlist1.cs b/src/tests/CoreMangLib/system/delegate/delegate/delegategetinvocationlist1.cs new file mode 100644 index 00000000000000..d0c8f343273623 --- /dev/null +++ b/src/tests/CoreMangLib/system/delegate/delegate/delegategetinvocationlist1.cs @@ -0,0 +1,230 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; +using Xunit; +//test case for delegate GetInvocationList method. +namespace DelegateTest +{ + delegate bool booldelegate(); + public class DelegateGetInvocationList + { + + booldelegate starkWork; + + [Fact] + public static int TestEntryPoint() + { + DelegateGetInvocationList delegateGetInvocationList = new DelegateGetInvocationList(); + + TestLibrary.TestFramework.BeginTestCase("DelegateGetInvocationList"); + + if (delegateGetInvocationList.RunTests()) + { + TestLibrary.TestFramework.EndTestCase(); + TestLibrary.TestFramework.LogInformation("PASS"); + return 100; + + } + else + { + TestLibrary.TestFramework.EndTestCase(); + TestLibrary.TestFramework.LogInformation("FAIL"); + return 0; + } + } + + public bool RunTests() + { + bool retVal = true; + + TestLibrary.TestFramework.LogInformation("[Positive]"); + retVal = PosTest1() && retVal; + retVal = PosTest2() && retVal; + retVal = PosTest3() && retVal; + retVal = PosTest4() && retVal; + return retVal; + } + + // Returns true if the expected result is right + // Returns false if the expected result is wrong + public bool PosTest1() + { + bool retVal = true; + + TestLibrary.TestFramework.BeginScenario("PosTest1: Call GetInvocationList against a delegate with one function"); + try + { + DelegateGetInvocationList delctor = new DelegateGetInvocationList(); + booldelegate dStartWork_Bool = new booldelegate(new TestClass().StartWork_Bool); + delctor.starkWork = dStartWork_Bool; + Delegate[] invocationList = delctor.starkWork.GetInvocationList(); + if (invocationList.Length != 1) + { + TestLibrary.TestFramework.LogError("001", "Call GetInvocationList against a delegate with one function returns wrong result: " + invocationList.Length); + retVal = false; + } + if (!delctor.starkWork.GetInvocationList()[0].Equals(dStartWork_Bool)) + { + TestLibrary.TestFramework.LogError("002", " GetInvocationList return error method "); + retVal = false; + } + delctor.starkWork(); + } + catch (Exception e) + { + TestLibrary.TestFramework.LogError("003", "Unexpected exception: " + e); + retVal = false; + } + + return retVal; + } + // Returns true if the expected result is right + // Returns false if the expected result is wrong + public bool PosTest2() + { + bool retVal = true; + + TestLibrary.TestFramework.BeginScenario("PosTest2: Call GetInvocationList against a delegate with muti different functions "); + try + { + DelegateGetInvocationList delctor = new DelegateGetInvocationList(); + booldelegate bStartWork_Bool = new booldelegate(new TestClass().StartWork_Bool); + booldelegate bWorking_Bool = new booldelegate(new TestClass().Working_Bool); + booldelegate bCompleted_Bool = new booldelegate(new TestClass().Completed_Bool); + + delctor.starkWork += bStartWork_Bool; + delctor.starkWork += bWorking_Bool; + delctor.starkWork += bCompleted_Bool; + Delegate[] invocationList = delctor.starkWork.GetInvocationList(); + if (invocationList.Length != 3) + { + TestLibrary.TestFramework.LogError("004", "Call GetInvocationList against a delegate with one function returns wrong result: " + invocationList.Length); + retVal = false; + } + if (!delctor.starkWork.GetInvocationList()[0].Equals(bStartWork_Bool) + || !delctor.starkWork.GetInvocationList()[1].Equals(bWorking_Bool) + || !delctor.starkWork.GetInvocationList()[2].Equals(bCompleted_Bool)) + { + TestLibrary.TestFramework.LogError("005", " GetInvocationList return error method "); + retVal = false; + } + delctor.starkWork(); + } + catch (Exception e) + { + TestLibrary.TestFramework.LogError("006", "Unexpected exception: " + e); + retVal = false; + } + + return retVal; + } + // Returns true if the expected result is right + // Returns false if the expected result is wrong + public bool PosTest3() + { + bool retVal = true; + + TestLibrary.TestFramework.BeginScenario("PosTest3: Call GetInvocationList against a delegate with muti functions ,some is null"); + try + { + DelegateGetInvocationList delctor = new DelegateGetInvocationList(); + booldelegate bStartWork_Bool = new booldelegate(new TestClass().StartWork_Bool); + booldelegate bWorking_Bool = new booldelegate(new TestClass().Working_Bool); + booldelegate bCompleted_Bool = new booldelegate(new TestClass().Completed_Bool); + + delctor.starkWork += bStartWork_Bool; + delctor.starkWork += null; + delctor.starkWork += bWorking_Bool; + delctor.starkWork += bCompleted_Bool; + Delegate[] invocationList = delctor.starkWork.GetInvocationList(); + if (invocationList.Length != 3) + { + TestLibrary.TestFramework.LogError("007", "Call GetInvocationList against a delegate with one function returns wrong result: " + invocationList.Length); + retVal = false; + } + if (!delctor.starkWork.GetInvocationList()[0].Equals(bStartWork_Bool) + || !delctor.starkWork.GetInvocationList()[1].Equals(bWorking_Bool) + || !delctor.starkWork.GetInvocationList()[2].Equals(bCompleted_Bool)) + { + TestLibrary.TestFramework.LogError("008", " GetInvocationList return error method "); + retVal = false; + } + delctor.starkWork(); + } + catch (Exception e) + { + TestLibrary.TestFramework.LogError("009", "Unexpected exception: " + e); + retVal = false; + } + + return retVal; + } + + // Returns true if the expected result is right + // Returns false if the expected result is wrong + public bool PosTest4() + { + bool retVal = true; + + TestLibrary.TestFramework.BeginScenario("PosTest4: Call GetInvocationList against a delegate with muti functions ,some of these are the same"); + try + { + DelegateGetInvocationList delctor = new DelegateGetInvocationList(); + booldelegate bStartWork_Bool = new booldelegate(new TestClass().StartWork_Bool); + booldelegate bWorking_Bool = new booldelegate(new TestClass().Working_Bool); + booldelegate bCompleted_Bool = new booldelegate(new TestClass().Completed_Bool); + + delctor.starkWork += bStartWork_Bool; + delctor.starkWork += bStartWork_Bool; + delctor.starkWork += bWorking_Bool; + delctor.starkWork += bCompleted_Bool; + Delegate[] invocationList = delctor.starkWork.GetInvocationList(); + if (invocationList.Length != 4) + { + TestLibrary.TestFramework.LogError("010", "Call GetInvocationList against a delegate with one function returns wrong result: " + invocationList.Length); + retVal = false; + } + if (!delctor.starkWork.GetInvocationList()[0].Equals(bStartWork_Bool) + || !delctor.starkWork.GetInvocationList()[1].Equals(bStartWork_Bool) + || !delctor.starkWork.GetInvocationList()[2].Equals(bWorking_Bool) + || !delctor.starkWork.GetInvocationList()[3].Equals(bCompleted_Bool)) + { + TestLibrary.TestFramework.LogError("011", " GetInvocationList return error method "); + retVal = false; + } + delctor.starkWork(); + } + catch (Exception e) + { + TestLibrary.TestFramework.LogError("012", "Unexpected exception: " + e); + retVal = false; + } + + return retVal; + } + + } + //create testclass for providing test method and test target. + class TestClass + { + public bool StartWork_Bool() + { + TestLibrary.TestFramework.LogInformation("StartWork_Bool method is running ."); + return true; + } + public bool Working_Bool() + { + TestLibrary.TestFramework.LogInformation("Working_Bool method is running ."); + return true; + } + public bool Completed_Bool() + { + TestLibrary.TestFramework.LogInformation("Completed_Bool method is running ."); + return true; + } + } + + +} diff --git a/src/tests/Directory.Build.props b/src/tests/Directory.Build.props index fbdf9eb9cd280a..60badfac3c1a38 100644 --- a/src/tests/Directory.Build.props +++ b/src/tests/Directory.Build.props @@ -163,6 +163,11 @@ true + + + 2 + + diff --git a/src/tests/Interop/CMakeLists.txt b/src/tests/Interop/CMakeLists.txt index 248f1cad12eb60..80958de80bea5c 100644 --- a/src/tests/Interop/CMakeLists.txt +++ b/src/tests/Interop/CMakeLists.txt @@ -105,6 +105,4 @@ if(CLR_CMAKE_TARGET_APPLE) add_subdirectory(Swift/SwiftErrorHandling) add_subdirectory(Swift/SwiftSelfContext) add_subdirectory(Swift/SwiftInvalidCallConv) - add_subdirectory(Swift/SwiftAbiStress) - add_subdirectory(Swift/SwiftRetAbiStress) endif() diff --git a/src/tests/Interop/COM/Dynamic/BasicTest.cs b/src/tests/Interop/COM/Dynamic/BasicTest.cs index 4ec5d6fbbdcccd..1e6fa7f5604e99 100644 --- a/src/tests/Interop/COM/Dynamic/BasicTest.cs +++ b/src/tests/Interop/COM/Dynamic/BasicTest.cs @@ -423,9 +423,6 @@ private void Null() obj.String_Property = null; Assert.Equal(string.Empty, obj.String_Property); - - obj.Dispatch_Property = new DispatchWrapper(null); - Assert.Null(obj.Dispatch_Property); } private void StringWrapper(string toWrap, string expected) diff --git a/src/tests/Interop/MarshalAPI/FunctionPointer/FunctionPointer.cs b/src/tests/Interop/MarshalAPI/FunctionPointer/FunctionPointer.cs index 7b0fd902a7bdfe..766a37efd00cf0 100644 --- a/src/tests/Interop/MarshalAPI/FunctionPointer/FunctionPointer.cs +++ b/src/tests/Interop/MarshalAPI/FunctionPointer/FunctionPointer.cs @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.IO; -using System.Reflection; using System.Runtime.InteropServices; using Xunit; @@ -21,13 +19,15 @@ static class FunctionPointerNative [DllImport(nameof(FunctionPointerNative))] static unsafe extern void FillOutPtr(IntPtr* p); - [DllImport(nameof(FunctionPointerNative))] - static unsafe extern void FillOutIntParameter(out IntPtr p); + [DllImport(nameof(FunctionPointerNative))] + static unsafe extern void FillOutIntParameter(out IntPtr p); } delegate void VoidDelegate(); [Fact] + + [ActiveIssue("https://github.com/dotnet/runtimelab/issues/164", typeof(TestLibrary.Utilities), nameof(TestLibrary.Utilities.IsNativeAot))] public static void RunGetDelForFcnPtrTest() { Console.WriteLine($"Running {nameof(RunGetDelForFcnPtrTest)}..."); @@ -40,10 +40,6 @@ public static void RunGetDelForFcnPtrTest() VoidDelegate del = (VoidDelegate)Marshal.GetDelegateForFunctionPointer(fcnptr, typeof(VoidDelegate)); Assert.Equal(md.Target, del.Target); Assert.Equal(md.Method, del.Method); - - VoidDelegate del2 = (VoidDelegate)Marshal.GetDelegateForFunctionPointer(fcnptr, typeof(VoidDelegate)); - Assert.Equal(del, del2); - Assert.Equal(del.GetHashCode(), del2.GetHashCode()); } // Native FcnPtr -> Delegate @@ -53,10 +49,6 @@ public static void RunGetDelForFcnPtrTest() Assert.Null(del.Target); Assert.Equal("Invoke", del.Method.Name); - VoidDelegate del2 = (VoidDelegate)Marshal.GetDelegateForFunctionPointer(fcnptr, typeof(VoidDelegate));; - Assert.Equal(del, del2); - Assert.Equal(del.GetHashCode(), del2.GetHashCode()); - // Round trip of a native function pointer is never legal for a non-concrete Delegate type Assert.Throws(() => { diff --git a/src/tests/Interop/MarshalAPI/FunctionPointer/GenericFunctionPointer.cs b/src/tests/Interop/MarshalAPI/FunctionPointer/GenericFunctionPointer.cs deleted file mode 100644 index da2fc75d913883..00000000000000 --- a/src/tests/Interop/MarshalAPI/FunctionPointer/GenericFunctionPointer.cs +++ /dev/null @@ -1,126 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.IO; -using System.Reflection; -using System.Runtime.InteropServices; -using Xunit; - -public partial class FunctionPtr -{ - public static bool CanRunGenericFunctionPointerTest => !TestLibrary.Utilities.IsMonoRuntime; - public static bool CanRunInvalidGenericFunctionPointerTest => !TestLibrary.Utilities.IsNativeAot && !TestLibrary.Utilities.IsMonoRuntime; - - [UnmanagedCallersOnly] - static int UnmanagedExportedFunction(float arg) - { - return Convert.ToInt32(arg); - } - - [UnmanagedCallersOnly] - static BlittableGeneric UnmanagedExportedFunctionBlittableGenericInt(float arg) - { - return new() { X = Convert.ToInt32(arg) }; - } - - [UnmanagedCallersOnly] - static BlittableGeneric UnmanagedExportedFunctionBlittableGenericString(float arg) - { - return new() { X = Convert.ToInt32(arg) }; - } - - [UnmanagedCallersOnly] - static unsafe void UnmanagedExportedFunctionRefInt(int* pval, float arg) - { - *pval = Convert.ToInt32(arg); - } - - class GenericCaller - { - internal static unsafe T GenericCalli(void* fnptr, U arg) - { - return ((delegate* unmanaged)fnptr)(arg); - } - - internal static unsafe BlittableGeneric WrappedGenericCalli(void* fnptr, U arg) - { - return ((delegate* unmanaged>)fnptr)(arg); - } - - internal static unsafe void NonGenericCalli(void* fnptr, ref int val, float arg) - { - ((delegate* unmanaged)fnptr)(ref val, arg); - } - } - - struct BlittableGeneric - { - public int X; - } - - [ConditionalTheory(nameof(CanRunGenericFunctionPointerTest))] - [InlineData(0f)] - [InlineData(1f)] - [InlineData(-1f)] - [InlineData(42f)] - [InlineData(60f)] - public static void RunGenericFunctionPointerTest(float inVal) - { - Console.WriteLine($"Running {nameof(RunGenericFunctionPointerTest)}..."); - int outVar = 0; - int expectedValue = Convert.ToInt32(inVal); - - Console.WriteLine("Testing GenericCalli with int as the return type"); - unsafe - { - outVar = GenericCaller.GenericCalli((delegate* unmanaged)&UnmanagedExportedFunction, inVal); - } - Assert.Equal(expectedValue, outVar); - - outVar = 0; - Console.WriteLine("Testing GenericCalli with BlittableGeneric as the return type"); - unsafe - { - outVar = GenericCaller.WrappedGenericCalli((delegate* unmanaged>)&UnmanagedExportedFunctionBlittableGenericInt, inVal).X; - } - Assert.Equal(expectedValue, outVar); - - outVar = 0; - Console.WriteLine("Testing GenericCalli with BlittableGeneric as the return type"); - unsafe - { - outVar = GenericCaller.WrappedGenericCalli((delegate* unmanaged>)&UnmanagedExportedFunctionBlittableGenericString, inVal).X; - } - Assert.Equal(expectedValue, outVar); - - outVar = 0; - Console.WriteLine("Testing non-GenericCalli with non-blittable argument in a generic caller"); - unsafe - { - GenericCaller.NonGenericCalli((delegate* unmanaged)&UnmanagedExportedFunctionRefInt, ref outVar, inVal); - } - Assert.Equal(expectedValue, outVar); - } - - [ConditionalFact(nameof(CanRunInvalidGenericFunctionPointerTest))] - public static void RunInvalidGenericFunctionPointerTest() - { - Console.WriteLine($"Running {nameof(RunInvalidGenericFunctionPointerTest)}..."); - unsafe - { - nint fnptr = (nint)(delegate* unmanaged)&ReturnParameter; - Console.WriteLine("Testing GenericCalli with string as the parameter type"); - Assert.Throws(() => GenericCaller.GenericCalli((delegate* unmanaged)fnptr, "test")); - Console.WriteLine("Testing GenericCalli with string as the return type"); - Assert.Throws(() => GenericCaller.GenericCalli((delegate* unmanaged)fnptr, "test")); - Console.WriteLine("Testing GenericCalli with string as both the parameter and return type"); - Assert.Throws(() => GenericCaller.GenericCalli((delegate* unmanaged)fnptr, "test")); - } - } - - [UnmanagedCallersOnly] - static unsafe nint* ReturnParameter(nint* p) - { - return p; - } -} diff --git a/src/tests/Interop/Swift/SwiftAbiStress/CMakeLists.txt b/src/tests/Interop/Swift/SwiftAbiStress/CMakeLists.txt deleted file mode 100644 index 6170fe83678460..00000000000000 --- a/src/tests/Interop/Swift/SwiftAbiStress/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -project(SwiftAbiStress) -include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") - -set(SOURCE SwiftAbiStress) - -if (NOT SWIFT_COMPILER_TARGET AND CLR_CMAKE_TARGET_OSX) - set(SWIFT_PLATFORM "macosx") - set(SWIFT_PLATFORM_SUFFIX "") - set(SWIFT_DEPLOYMENT_TARGET ${CMAKE_OSX_DEPLOYMENT_TARGET}) - set(SWIFT_COMPILER_TARGET "${CMAKE_OSX_ARCHITECTURES}-apple-${SWIFT_PLATFORM}${SWIFT_DEPLOYMENT_TARGET}${SWIFT_PLATFORM_SUFFIX}") -endif() - -add_custom_target(${SOURCE} ALL - COMMAND xcrun swiftc -target ${SWIFT_COMPILER_TARGET} -emit-library ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}.swift -o ${CMAKE_CURRENT_BINARY_DIR}/lib${SOURCE}.dylib - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}.swift - COMMENT "Generating ${SOURCE} library" -) - -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib${SOURCE}.dylib - DESTINATION bin -) diff --git a/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.cs b/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.cs deleted file mode 100644 index 7e23c8d290d75e..00000000000000 --- a/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.cs +++ /dev/null @@ -1,6683 +0,0 @@ -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.InteropServices.Swift; -using Xunit; - -public class SwiftAbiStress -{ - private const string SwiftLib = "libSwiftAbiStress.dylib"; - - [StructLayout(LayoutKind.Sequential, Size = 14)] - struct F0_S0 - { - public double F0; - public uint F1; - public ushort F2; - - public F0_S0(double f0, uint f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F0_S1 - { - public ulong F0; - - public F0_S1(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F0_S2 - { - public float F0; - - public F0_S2(float f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc02a02a12a22a32a42a52a62a7Sis5Int16V_s5Int32Vs6UInt64Vs6UInt16VAA5F0_S0VAA0R3_S1Vs5UInt8VAA0R3_S2VtF")] - private static extern nint SwiftFunc0(short a0, int a1, ulong a2, ushort a3, F0_S0 a4, F0_S1 a5, byte a6, F0_S2 a7); - - [Fact] - public static void TestSwiftFunc0() - { - Console.Write("Running SwiftFunc0: "); - long result = SwiftFunc0(-23758, 148652722, 3833542748216839160, 21987, new F0_S0(3425626963407448, 989224444, 55562), new F0_S1(1751696348434043356), 14, new F0_S2(1047842)); - Assert.Equal(-5199645484972017144, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct F1_S0 - { - public long F0; - public double F1; - public sbyte F2; - public int F3; - public ushort F4; - - public F1_S0(long f0, double f1, sbyte f2, int f3, ushort f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F1_S1 - { - public byte F0; - - public F1_S1(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F1_S2 - { - public short F0; - - public F1_S2(short f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc12a02a12a22a3SiAA5F1_S0V_s5UInt8VAA0J3_S1VAA0J3_S2VtF")] - private static extern nint SwiftFunc1(F1_S0 a0, byte a1, F1_S1 a2, F1_S2 a3); - - [Fact] - public static void TestSwiftFunc1() - { - Console.Write("Running SwiftFunc1: "); - long result = SwiftFunc1(new F1_S0(6106136698885217102, 6195715435808, 121, 676336729, 51621), 121, new F1_S1(101), new F1_S2(-11974)); - Assert.Equal(-5789188411070459345, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F2_S0 - { - public nint F0; - public nuint F1; - - public F2_S0(nint f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct F2_S1 - { - public long F0; - public int F1; - public short F2; - public long F3; - public ushort F4; - - public F2_S1(long f0, int f1, short f2, long f3, ushort f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F2_S2_S0_S0 - { - public nint F0; - - public F2_S2_S0_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F2_S2_S0 - { - public F2_S2_S0_S0 F0; - - public F2_S2_S0(F2_S2_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F2_S2 - { - public F2_S2_S0 F0; - - public F2_S2(F2_S2_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F2_S3 - { - public byte F0; - - public F2_S3(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F2_S4 - { - public int F0; - public nuint F1; - - public F2_S4(int f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F2_S5 - { - public float F0; - - public F2_S5(float f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc22a02a12a22a32a42a52a62a72a82a93a103a113a123a13Sis5Int64V_s5Int16Vs5Int32VAA5F2_S0Vs5UInt8VAvA0W3_S1VAA0W3_S2Vs6UInt16VSfAA0W3_S3VAA0W3_S4VAA0W3_S5VARtF")] - private static extern nint SwiftFunc2(long a0, short a1, int a2, F2_S0 a3, byte a4, int a5, F2_S1 a6, F2_S2 a7, ushort a8, float a9, F2_S3 a10, F2_S4 a11, F2_S5 a12, long a13); - - [Fact] - public static void TestSwiftFunc2() - { - Console.Write("Running SwiftFunc2: "); - long result = SwiftFunc2(1467471118999515177, -1109, 1443466834, new F2_S0(unchecked((nint)8641951469425609828), unchecked((nuint)3263825339460718643)), 6, 42857709, new F2_S1(6855376760105631967, 2087467091, 25810, 2495195821026007124, 62146), new F2_S2(new F2_S2_S0(new F2_S2_S0_S0(unchecked((nint)561009218247569242)))), 46110, 7547287, new F2_S3(34), new F2_S4(203178131, unchecked((nuint)8676866947888134131)), new F2_S5(7890213), 5623254678629817168); - Assert.Equal(-1831688667491861211, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F3_S0_S0 - { - public nint F0; - public uint F1; - - public F3_S0_S0(nint f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F3_S0 - { - public sbyte F0; - public F3_S0_S0 F1; - public uint F2; - - public F3_S0(sbyte f0, F3_S0_S0 f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F3_S1 - { - public long F0; - public float F1; - - public F3_S1(long f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F3_S2 - { - public float F0; - - public F3_S2(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F3_S3 - { - public byte F0; - public nint F1; - - public F3_S3(byte f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 14)] - struct F3_S4 - { - public nuint F0; - public float F1; - public ushort F2; - - public F3_S4(nuint f0, float f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F3_S5 - { - public uint F0; - public long F1; - - public F3_S5(uint f0, long f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 3)] - struct F3_S6_S0 - { - public short F0; - public byte F1; - - public F3_S6_S0(short f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 5)] - struct F3_S6 - { - public F3_S6_S0 F0; - public sbyte F1; - public byte F2; - - public F3_S6(F3_S6_S0 f0, sbyte f1, byte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F3_S7 - { - public ulong F0; - - public F3_S7(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc32a02a12a22a32a42a52a62a72a82a93a103a113a123a13S2i_AA5F3_S0VAA0T3_S1VSdSiAA0T3_S2VAA0T3_S3VAA0T3_S4VAA0T3_S5Vs6UInt16Vs5Int32VAA0T3_S6VSiAA0T3_S7VtF")] - private static extern nint SwiftFunc3(nint a0, F3_S0 a1, F3_S1 a2, double a3, nint a4, F3_S2 a5, F3_S3 a6, F3_S4 a7, F3_S5 a8, ushort a9, int a10, F3_S6 a11, nint a12, F3_S7 a13); - - [Fact] - public static void TestSwiftFunc3() - { - Console.Write("Running SwiftFunc3: "); - long result = SwiftFunc3(unchecked((nint)3764414362291906102), new F3_S0(23, new F3_S0_S0(unchecked((nint)3007367655161186204), 549733154), 38928730), new F3_S1(338326426991485790, 7517271), 4025506815523052, unchecked((nint)431338169919855088), new F3_S2(7888763), new F3_S3(57, unchecked((nint)8933588466514096604)), new F3_S4(unchecked((nuint)7769316271655125502), 1663231, 27333), new F3_S5(887161443, 4368322322535461551), 32477, 948591564, new F3_S6(new F3_S6_S0(7033, 124), 67, 221), unchecked((nint)6195032215974632640), new F3_S7(4076570630190469380)); - Assert.Equal(-8840537967093155898, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct F4_S0 - { - public ushort F0; - public short F1; - public short F2; - - public F4_S0(ushort f0, short f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F4_S1_S0 - { - public uint F0; - - public F4_S1_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F4_S1 - { - public F4_S1_S0 F0; - public float F1; - - public F4_S1(F4_S1_S0 f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F4_S2_S0 - { - public nint F0; - - public F4_S2_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F4_S2 - { - public F4_S2_S0 F0; - public nint F1; - - public F4_S2(F4_S2_S0 f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F4_S3 - { - public ulong F0; - public ulong F1; - public long F2; - - public F4_S3(ulong f0, ulong f1, long f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc42a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a15S2i_AA5F4_S0VSus6UInt64Vs4Int8VSdAA0V3_S1Vs5UInt8Vs5Int32Vs6UInt32VAvA0V3_S2Vs5Int16VSiAA0V3_S3VA4_tF")] - private static extern nint SwiftFunc4(nint a0, F4_S0 a1, nuint a2, ulong a3, sbyte a4, double a5, F4_S1 a6, byte a7, int a8, uint a9, ulong a10, F4_S2 a11, short a12, nint a13, F4_S3 a14, uint a15); - - [Fact] - public static void TestSwiftFunc4() - { - Console.Write("Running SwiftFunc4: "); - long result = SwiftFunc4(unchecked((nint)7962207922494873063), new F4_S0(16887, 11193, 20997), unchecked((nuint)938043702598629976), 8692646626431098135, -16, 1244033228990732, new F4_S1(new F4_S1_S0(274421021), 7037264), 154, 1187166500, 1096514224, 7283010216047805604, new F4_S2(new F4_S2_S0(unchecked((nint)3285810526807361976)), unchecked((nint)2934841899954168407)), 3384, unchecked((nint)4857017836321530071), new F4_S3(9030480386017125399, 5466901523025762626, 3430278619936831574), 234522698); - Assert.Equal(5366279618472372586, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F5_S0 - { - public nuint F0; - - public F5_S0(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc52a02a12a22a3SiSu_s6UInt64Vs5UInt8VAA5F5_S0VtF")] - private static extern nint SwiftFunc5(nuint a0, ulong a1, byte a2, F5_S0 a3); - - [Fact] - public static void TestSwiftFunc5() - { - Console.Write("Running SwiftFunc5: "); - long result = SwiftFunc5(unchecked((nuint)425569624776371773), 8077063517132296390, 126, new F5_S0(unchecked((nuint)8032431538406335990))); - Assert.Equal(5832440388901373477, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 17)] - struct F6_S0 - { - public int F0; - public nint F1; - public byte F2; - - public F6_S0(int f0, nint f1, byte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F6_S1 - { - public nint F0; - public float F1; - - public F6_S1(nint f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F6_S2_S0 - { - public double F0; - - public F6_S2_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F6_S2 - { - public F6_S2_S0 F0; - public ushort F1; - - public F6_S2(F6_S2_S0 f0, ushort f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F6_S3 - { - public double F0; - public double F1; - public ulong F2; - - public F6_S3(double f0, double f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F6_S4 - { - public sbyte F0; - - public F6_S4(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F6_S5 - { - public short F0; - - public F6_S5(short f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc62a02a12a22a32a42a52a62a72a82a93a103a113a123a133a14Sis5Int64V_AA5F6_S0VAA0V3_S1VSus5UInt8Vs5Int32VAA0V3_S2VSfs5Int16VAA0V3_S3Vs6UInt16VSds6UInt32VAA0V3_S4VAA0V3_S5VtF")] - private static extern nint SwiftFunc6(long a0, F6_S0 a1, F6_S1 a2, nuint a3, byte a4, int a5, F6_S2 a6, float a7, short a8, F6_S3 a9, ushort a10, double a11, uint a12, F6_S4 a13, F6_S5 a14); - - [Fact] - public static void TestSwiftFunc6() - { - Console.Write("Running SwiftFunc6: "); - long result = SwiftFunc6(7742402881449217499, new F6_S0(158138445, unchecked((nint)4280990415451108676), 220), new F6_S1(unchecked((nint)7698928046973811162), 478730), unchecked((nuint)7348396082620937303), 76, 638113630, new F6_S2(new F6_S2_S0(55341051405503), 61378), 8235930, -20241, new F6_S3(318363825012010, 3586735152618866, 6630554942616673404), 46432, 744827194985602, 1973021571, new F6_S4(103), new F6_S5(-5345)); - Assert.Equal(-8871753131984133391, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F7_S0 - { - public short F0; - public nint F1; - - public F7_S0(short f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F7_S1 - { - public byte F0; - - public F7_S1(byte f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc72a02a12a22a32a42a5Sis5Int64V_Sis5UInt8VAA5F7_S0VAA0N3_S1Vs6UInt32VtF")] - private static extern nint SwiftFunc7(long a0, nint a1, byte a2, F7_S0 a3, F7_S1 a4, uint a5); - - [Fact] - public static void TestSwiftFunc7() - { - Console.Write("Running SwiftFunc7: "); - long result = SwiftFunc7(6953928391541094904, unchecked((nint)2531714261502554653), 224, new F7_S0(14482, unchecked((nint)4704842847707480837)), new F7_S1(148), 659764805); - Assert.Equal(5963731324167739917, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F8_S0 - { - public int F0; - - public F8_S0(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc82a02a12a22a32a42a5Sis6UInt16V_SuAJs6UInt64VAA5F8_S0VALtF")] - private static extern nint SwiftFunc8(ushort a0, nuint a1, ushort a2, ulong a3, F8_S0 a4, ulong a5); - - [Fact] - public static void TestSwiftFunc8() - { - Console.Write("Running SwiftFunc8: "); - long result = SwiftFunc8(48505, unchecked((nuint)8758330817072549915), 7130, 4163773298933598697, new F8_S0(1934119180), 2843311260726166700); - Assert.Equal(1919194302322813426, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F9_S0 - { - public double F0; - - public F9_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F9_S1 - { - public int F0; - - public F9_S1(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress10swiftFunc92a02a12a22a32a42a5Sis5Int64V_SfAA5F9_S0Vs6UInt16VAA0M3_S1VANtF")] - private static extern nint SwiftFunc9(long a0, float a1, F9_S0 a2, ushort a3, F9_S1 a4, ushort a5); - - [Fact] - public static void TestSwiftFunc9() - { - Console.Write("Running SwiftFunc9: "); - long result = SwiftFunc9(3214937834123081267, 6846768, new F9_S0(1713527158921541), 25670, new F9_S1(1650872599), 39910); - Assert.Equal(-5878079645235476214, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F10_S0 - { - public long F0; - public uint F1; - - public F10_S0(long f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F10_S1 - { - public float F0; - public byte F1; - public nuint F2; - - public F10_S1(float f0, byte f1, nuint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F10_S2 - { - public nuint F0; - public ulong F1; - - public F10_S2(nuint f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F10_S3 - { - public float F0; - - public F10_S3(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F10_S4 - { - public long F0; - - public F10_S4(long f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc102a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a153a163a173a18Sis6UInt16V_AwA6F10_S0Vs6UInt64VSfs4Int8Vs5Int64VA_A3_Sfs5Int32VA5_A3_A_AA0Z3_S1VA3_AA0Z3_S2VAA0Z3_S3VAA0Z3_S4VtF")] - private static extern nint SwiftFunc10(ushort a0, ushort a1, F10_S0 a2, ulong a3, float a4, sbyte a5, long a6, ulong a7, long a8, float a9, int a10, int a11, long a12, ulong a13, F10_S1 a14, long a15, F10_S2 a16, F10_S3 a17, F10_S4 a18); - - [Fact] - public static void TestSwiftFunc10() - { - Console.Write("Running SwiftFunc10: "); - long result = SwiftFunc10(57914, 11968, new F10_S0(155502634291755209, 2096010440), 1373054541331378384, 2401784, -16, 9038689080810964859, 521869082023571496, 8919173990791765137, 4890513, 1113752036, 1477591037, 1463349953238439103, 7521124889381630793, new F10_S1(620783, 33, unchecked((nuint)1209731409858919135)), 1560688600815438014, new F10_S2(unchecked((nuint)2244178273746563479), 4252696983313269084), new F10_S3(6539550), new F10_S4(1264398289929487498)); - Assert.Equal(-5714135075575530569, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct F11_S0 - { - public short F0; - public sbyte F1; - public ulong F2; - public short F3; - - public F11_S0(short f0, sbyte f1, ulong f2, short f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F11_S1 - { - public nuint F0; - - public F11_S1(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F11_S2 - { - public short F0; - - public F11_S2(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F11_S3_S0 - { - public float F0; - - public F11_S3_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F11_S3 - { - public F11_S3_S0 F0; - - public F11_S3(F11_S3_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc112a02a12a22a32a42a52a62a72a82a93a103a113a12S2i_s6UInt64Vs5UInt8Vs5Int16VAA6F11_S0VAA0V3_S1Vs6UInt16VSdSis6UInt32VAA0V3_S2VAA0V3_S3Vs4Int8VtF")] - private static extern nint SwiftFunc11(nint a0, ulong a1, byte a2, short a3, F11_S0 a4, F11_S1 a5, ushort a6, double a7, nint a8, uint a9, F11_S2 a10, F11_S3 a11, sbyte a12); - - [Fact] - public static void TestSwiftFunc11() - { - Console.Write("Running SwiftFunc11: "); - long result = SwiftFunc11(unchecked((nint)6199025647502478201), 6507965430585517144, 205, -31066, new F11_S0(-8843, -2, 7915533514001114122, -3518), new F11_S1(unchecked((nuint)690496938384964820)), 10269, 3817195039757571, unchecked((nint)4394294464475321144), 1182247681, new F11_S2(22246), new F11_S3(new F11_S3_S0(3714370)), 93); - Assert.Equal(946399036611801834, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F12_S0 - { - public uint F0; - - public F12_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F12_S1 - { - public byte F0; - - public F12_S1(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F12_S2 - { - public nuint F0; - - public F12_S2(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc122a02a12a22a32a42a52a62a72a82a93a103a11Sis5UInt8V_s5Int32VAA6F12_S0Vs4Int8VAA0T3_S1VAA0T3_S2Vs6UInt32Vs5Int16VA2VA0_APtF")] - private static extern nint SwiftFunc12(byte a0, int a1, F12_S0 a2, sbyte a3, F12_S1 a4, F12_S2 a5, uint a6, short a7, sbyte a8, sbyte a9, uint a10, byte a11); - - [Fact] - public static void TestSwiftFunc12() - { - Console.Write("Running SwiftFunc12: "); - long result = SwiftFunc12(233, 123593469, new F12_S0(1950949830), -122, new F12_S1(47), new F12_S2(unchecked((nuint)2600645483988824242)), 307825058, -49, -98, -5, 1582160629, 26); - Assert.Equal(102839812138332997, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F13_S0_S0_S0 - { - public ulong F0; - - public F13_S0_S0_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F13_S0_S0 - { - public F13_S0_S0_S0 F0; - - public F13_S0_S0(F13_S0_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F13_S0 - { - public sbyte F0; - public F13_S0_S0 F1; - - public F13_S0(sbyte f0, F13_S0_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F13_S1_S0 - { - public ulong F0; - - public F13_S1_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F13_S1 - { - public F13_S1_S0 F0; - - public F13_S1(F13_S1_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc132a02a12a22a32a42a5Sis4Int8V_SdAA6F13_S0VAA0M3_S1VAJSdtF")] - private static extern nint SwiftFunc13(sbyte a0, double a1, F13_S0 a2, F13_S1 a3, sbyte a4, double a5); - - [Fact] - public static void TestSwiftFunc13() - { - Console.Write("Running SwiftFunc13: "); - long result = SwiftFunc13(-6, 2395768328620295, new F13_S0(44, new F13_S0_S0(new F13_S0_S0_S0(2383685413668225247))), new F13_S1(new F13_S1_S0(5663941717310331870)), -9, 815761320969512); - Assert.Equal(-6209025030118540066, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F14_S0 - { - public nint F0; - - public F14_S0(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc142a02a12a22a32a4Sis4Int8V_SiAA6F14_S0VSfSutF")] - private static extern nint SwiftFunc14(sbyte a0, nint a1, F14_S0 a2, float a3, nuint a4); - - [Fact] - public static void TestSwiftFunc14() - { - Console.Write("Running SwiftFunc14: "); - long result = SwiftFunc14(-78, unchecked((nint)2423976036967433837), new F14_S0(unchecked((nint)2836433146306492236)), 4916388, unchecked((nuint)7716581850692162517)); - Assert.Equal(1206847964913124869, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F15_S0 - { - public float F0; - public short F1; - public byte F2; - public long F3; - public double F4; - - public F15_S0(float f0, short f1, byte f2, long f3, double f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F15_S1_S0 - { - public sbyte F0; - - public F15_S1_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F15_S1 - { - public uint F0; - public F15_S1_S0 F1; - public nuint F2; - public int F3; - - public F15_S1(uint f0, F15_S1_S0 f1, nuint f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc152a02a12a22a32a42a52a62a7SiAA6F15_S0V_s6UInt64Vs6UInt32VSuANs5Int16VAA0N3_S1Vs5Int64VtF")] - private static extern nint SwiftFunc15(F15_S0 a0, ulong a1, uint a2, nuint a3, ulong a4, short a5, F15_S1 a6, long a7); - - [Fact] - public static void TestSwiftFunc15() - { - Console.Write("Running SwiftFunc15: "); - long result = SwiftFunc15(new F15_S0(2392622, -22089, 69, 7123929674797968229, 2951758117520631), 171173680452593621, 357397954, unchecked((nuint)6020399741996935792), 3793854189677149082, 14438, new F15_S1(1572107355, new F15_S1_S0(109), unchecked((nuint)4381395046734445050), 2038949453), 9134476964305239477); - Assert.Equal(8801999574220262235, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F16_S0_S0 - { - public double F0; - - public F16_S0_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F16_S0 - { - public nint F0; - public nint F1; - public F16_S0_S0 F2; - - public F16_S0(nint f0, nint f1, F16_S0_S0 f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F16_S1 - { - public short F0; - public ulong F1; - public uint F2; - - public F16_S1(short f0, ulong f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F16_S2 - { - public byte F0; - public ulong F1; - public float F2; - - public F16_S2(byte f0, ulong f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F16_S3 - { - public int F0; - - public F16_S3(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc162a02a12a22a32a42a52a6Sis6UInt64V_AA6F16_S0VAA0N3_S1Vs6UInt16Vs5Int16VAA0N3_S2VAA0N3_S3VtF")] - private static extern nint SwiftFunc16(ulong a0, F16_S0 a1, F16_S1 a2, ushort a3, short a4, F16_S2 a5, F16_S3 a6); - - [Fact] - public static void TestSwiftFunc16() - { - Console.Write("Running SwiftFunc16: "); - long result = SwiftFunc16(3875678837451096765, new F16_S0(unchecked((nint)4720149202348788086), unchecked((nint)7476511841079774603), new F16_S0_S0(1008066799213144)), new F16_S1(3085, 11417298712821513, 12161200), 257, 7667, new F16_S2(186, 2771425808859711833, 3778779), new F16_S3(146689072)); - Assert.Equal(2726423189537230293, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F17_S0 - { - public short F0; - - public F17_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F17_S1 - { - public long F0; - public nuint F1; - public ulong F2; - - public F17_S1(long f0, nuint f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F17_S2 - { - public sbyte F0; - - public F17_S2(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F17_S3 - { - public sbyte F0; - public uint F1; - - public F17_S3(sbyte f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F17_S4 - { - public ulong F0; - - public F17_S4(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F17_S5 - { - public long F0; - - public F17_S5(long f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc172a02a12a22a32a42a52a62a72a82a9SiAA6F17_S0V_s4Int8VAA0P3_S1VAPSuAA0P3_S2Vs5Int64VAA0P3_S3VAA0P3_S4VAA0P3_S5VtF")] - private static extern nint SwiftFunc17(F17_S0 a0, sbyte a1, F17_S1 a2, sbyte a3, nuint a4, F17_S2 a5, long a6, F17_S3 a7, F17_S4 a8, F17_S5 a9); - - [Fact] - public static void TestSwiftFunc17() - { - Console.Write("Running SwiftFunc17: "); - long result = SwiftFunc17(new F17_S0(-25916), -37, new F17_S1(927673990059785474, unchecked((nuint)4067467819275701282), 4736163781163880654), 70, unchecked((nuint)1236364146053271187), new F17_S2(54), 6452671878605914679, new F17_S3(17, 1066187627), new F17_S4(961451227454237536), new F17_S5(8720978516408944945)); - Assert.Equal(6084200789584610530, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F18_S0_S0 - { - public ushort F0; - public short F1; - - public F18_S0_S0(ushort f0, short f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F18_S0 - { - public uint F0; - public F18_S0_S0 F1; - public ushort F2; - - public F18_S0(uint f0, F18_S0_S0 f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F18_S1 - { - public nint F0; - public nint F1; - - public F18_S1(nint f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F18_S2_S0 - { - public ulong F0; - - public F18_S2_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F18_S2 - { - public ulong F0; - public long F1; - public byte F2; - public F18_S2_S0 F3; - - public F18_S2(ulong f0, long f1, byte f2, F18_S2_S0 f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc182a02a12a22a32a42a52a62a72a8Sis5UInt8V_SdAA6F18_S0VAA0P3_S1Vs6UInt16Vs5Int64Vs6UInt64VAA0P3_S2VAWtF")] - private static extern nint SwiftFunc18(byte a0, double a1, F18_S0 a2, F18_S1 a3, ushort a4, long a5, ulong a6, F18_S2 a7, ulong a8); - - [Fact] - public static void TestSwiftFunc18() - { - Console.Write("Running SwiftFunc18: "); - long result = SwiftFunc18(153, 2414022997411914, new F18_S0(795806912, new F18_S0_S0(63552, 11471), 47960), new F18_S1(unchecked((nint)6143080814824714071), unchecked((nint)2654471745636317319)), 51304, 4455723326879920366, 6215563249078191014, new F18_S2(7357905541817922655, 8124331887393558663, 146, new F18_S2_S0(8835007006958775606)), 1308697068118476706); - Assert.Equal(-1238401591549550590, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct F19_S0 - { - public nint F0; - public double F1; - public ushort F2; - - public F19_S0(nint f0, double f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc192a02a12a2SiSu_AA6F19_S0Vs5Int16VtF")] - private static extern nint SwiftFunc19(nuint a0, F19_S0 a1, short a2); - - [Fact] - public static void TestSwiftFunc19() - { - Console.Write("Running SwiftFunc19: "); - long result = SwiftFunc19(unchecked((nuint)2063900917075180131), new F19_S0(unchecked((nint)7420139040061411172), 4412763638361702, 18542), 32656); - Assert.Equal(-3737785273912016840, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F20_S0 - { - public ushort F0; - public sbyte F1; - public ulong F2; - public uint F3; - public ulong F4; - - public F20_S0(ushort f0, sbyte f1, ulong f2, uint f3, ulong f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F20_S1 - { - public long F0; - - public F20_S1(long f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc202a02a12a22a32a42a52a6Sis4Int8V_AA6F20_S0Vs6UInt64VSiAA0N3_S1Vs5UInt8Vs5Int64VtF")] - private static extern nint SwiftFunc20(sbyte a0, F20_S0 a1, ulong a2, nint a3, F20_S1 a4, byte a5, long a6); - - [Fact] - public static void TestSwiftFunc20() - { - Console.Write("Running SwiftFunc20: "); - long result = SwiftFunc20(-90, new F20_S0(13173, -56, 2350829658938201640, 1333911330, 2505424063423776138), 6738010084636609242, unchecked((nint)819908193119917708), new F20_S1(1349820395385212287), 121, 3289915405437061252); - Assert.Equal(550863197950258558, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F21_S0 - { - public uint F0; - - public F21_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F21_S1 - { - public nint F0; - public uint F1; - public byte F2; - public short F3; - - public F21_S1(nint f0, uint f1, byte f2, short f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 25)] - struct F21_S2 - { - public sbyte F0; - public ulong F1; - public long F2; - public byte F3; - - public F21_S2(sbyte f0, ulong f1, long f2, byte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F21_S3 - { - public double F0; - public nint F1; - - public F21_S3(double f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc212a02a12a22a32a42a52a62a72a82a93a103a113a12Sis6UInt64V_s4Int8VSuSdSfSiAA6F21_S0VAA0U3_S1Vs6UInt16VAA0U3_S2Vs5UInt8VAA0U3_S3Vs5Int16VtF")] - private static extern nint SwiftFunc21(ulong a0, sbyte a1, nuint a2, double a3, float a4, nint a5, F21_S0 a6, F21_S1 a7, ushort a8, F21_S2 a9, byte a10, F21_S3 a11, short a12); - - [Fact] - public static void TestSwiftFunc21() - { - Console.Write("Running SwiftFunc21: "); - long result = SwiftFunc21(5269012897287813953, -91, unchecked((nuint)1201479654570648238), 3289259914874957, 6706247, unchecked((nint)5524961485867187694), new F21_S0(1842933651), new F21_S1(unchecked((nint)3105907069529682628), 1409834375, 228, 24264), 54652, new F21_S2(-49, 3442352645827709069, 7249278047379449391, 213), 207, new F21_S3(3802489474747093, unchecked((nint)7550982300494612851)), -25738); - Assert.Equal(1242333410237260188, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F22_S0 - { - public ushort F0; - public uint F1; - public short F2; - public float F3; - - public F22_S0(ushort f0, uint f1, short f2, float f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F22_S1 - { - public ushort F0; - public sbyte F1; - public byte F2; - public nint F3; - public nint F4; - - public F22_S1(ushort f0, sbyte f1, byte f2, nint f3, nint f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F22_S2_S0 - { - public sbyte F0; - - public F22_S2_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 14)] - struct F22_S2 - { - public int F0; - public int F1; - public uint F2; - public byte F3; - public F22_S2_S0 F4; - - public F22_S2(int f0, int f1, uint f2, byte f3, F22_S2_S0 f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F22_S3 - { - public short F0; - public double F1; - public double F2; - public int F3; - - public F22_S3(short f0, double f1, double f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc222a02a12a22a32a42a52a62a7Sis4Int8V_s5Int32VAA6F22_S0VAA0P3_S1VAA0P3_S2Vs6UInt64VAA0P3_S3VSutF")] - private static extern nint SwiftFunc22(sbyte a0, int a1, F22_S0 a2, F22_S1 a3, F22_S2 a4, ulong a5, F22_S3 a6, nuint a7); - - [Fact] - public static void TestSwiftFunc22() - { - Console.Write("Running SwiftFunc22: "); - long result = SwiftFunc22(-57, 637612850, new F22_S0(39888, 420817324, 7562, 2757302), new F22_S1(61019, -94, 94, unchecked((nint)2606601177110916370), unchecked((nint)5843896711210899037)), new F22_S2(400565495, 1044629988, 1076814110, 26, new F22_S2_S0(-109)), 6520156438560424018, new F22_S3(8735, 4148868269582632, 2501928198596701, 1401343024), unchecked((nuint)5955700101477425475)); - Assert.Equal(-6205677027164766590, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct F23_S0 - { - public uint F0; - public short F1; - - public F23_S0(uint f0, short f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F23_S1 - { - public nuint F0; - public uint F1; - - public F23_S1(nuint f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 17)] - struct F23_S2 - { - public double F0; - public uint F1; - public int F2; - public byte F3; - - public F23_S2(double f0, uint f1, int f2, byte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc232a02a12a22a32a4SiAA6F23_S0V_AA0K3_S1VAA0K3_S2VSds6UInt64VtF")] - private static extern nint SwiftFunc23(F23_S0 a0, F23_S1 a1, F23_S2 a2, double a3, ulong a4); - - [Fact] - public static void TestSwiftFunc23() - { - Console.Write("Running SwiftFunc23: "); - long result = SwiftFunc23(new F23_S0(119750622, -9202), new F23_S1(unchecked((nuint)2015683423731520384), 2106419422), new F23_S2(15243057156671, 484733224, 541045687, 128), 335968113268162, 4104726345028490471); - Assert.Equal(-4893219516767457464, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F24_S0 - { - public sbyte F0; - public int F1; - - public F24_S0(sbyte f0, int f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F24_S1 - { - public sbyte F0; - - public F24_S1(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F24_S2 - { - public ushort F0; - public short F1; - public double F2; - public nuint F3; - - public F24_S2(ushort f0, short f1, double f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F24_S3 - { - public nint F0; - - public F24_S3(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc242a02a12a22a32a42a5SiAA6F24_S0V_AA0L3_S1VAA0L3_S2VAA0L3_S3VSus6UInt32VtF")] - private static extern nint SwiftFunc24(F24_S0 a0, F24_S1 a1, F24_S2 a2, F24_S3 a3, nuint a4, uint a5); - - [Fact] - public static void TestSwiftFunc24() - { - Console.Write("Running SwiftFunc24: "); - long result = SwiftFunc24(new F24_S0(-79, 1590520731), new F24_S1(-91), new F24_S2(20580, 5897, 4259258535235558, unchecked((nuint)5376883129922161134)), new F24_S3(unchecked((nint)6329816641466666679)), unchecked((nuint)749917486894435068), 588417470); - Assert.Equal(2355459289566446436, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F25_S0_S0 - { - public sbyte F0; - - public F25_S0_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F25_S0 - { - public float F0; - public F25_S0_S0 F1; - public uint F2; - - public F25_S0(float f0, F25_S0_S0 f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F25_S1 - { - public short F0; - public sbyte F1; - public float F2; - - public F25_S1(short f0, sbyte f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F25_S2 - { - public long F0; - public ushort F1; - - public F25_S2(long f0, ushort f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F25_S3 - { - public ulong F0; - - public F25_S3(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F25_S4 - { - public ushort F0; - - public F25_S4(ushort f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc252a02a12a22a32a42a52a62a72a82a93a103a113a12SiSf_AA6F25_S0Vs5Int64Vs5UInt8VAA0S3_S1VSiAA0S3_S2Vs5Int32VA_Sus6UInt64VAA0S3_S3VAA0S3_S4VtF")] - private static extern nint SwiftFunc25(float a0, F25_S0 a1, long a2, byte a3, F25_S1 a4, nint a5, F25_S2 a6, int a7, int a8, nuint a9, ulong a10, F25_S3 a11, F25_S4 a12); - - [Fact] - public static void TestSwiftFunc25() - { - Console.Write("Running SwiftFunc25: "); - long result = SwiftFunc25(7574050, new F25_S0(6812822, new F25_S0_S0(-56), 265762114), 8887316512771179060, 123, new F25_S1(-7776, 73, 1925304), unchecked((nint)6156508798007114044), new F25_S2(3356802028835066684, 63590), 1072499355, 1592861041, unchecked((nuint)7083962615260029068), 6662060345720879806, new F25_S3(3582316099656415385), new F25_S4(37071)); - Assert.Equal(3486557296564493762, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F26_S0 - { - public double F0; - - public F26_S0(double f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc262a02a12a22a32a4Sis6UInt16V_Sds5Int64VAA6F26_S0Vs5UInt8VtF")] - private static extern nint SwiftFunc26(ushort a0, double a1, long a2, F26_S0 a3, byte a4); - - [Fact] - public static void TestSwiftFunc26() - { - Console.Write("Running SwiftFunc26: "); - long result = SwiftFunc26(61060, 3605567452716741, 1495534128089493599, new F26_S0(1063426277848136), 89); - Assert.Equal(5445852553218786939, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F27_S0_S0 - { - public long F0; - - public F27_S0_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F27_S0 - { - public ushort F0; - public F27_S0_S0 F1; - public double F2; - - public F27_S0(ushort f0, F27_S0_S0 f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 13)] - struct F27_S1 - { - public nint F0; - public sbyte F1; - public short F2; - public byte F3; - - public F27_S1(nint f0, sbyte f1, short f2, byte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F27_S2 - { - public ushort F0; - - public F27_S2(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F27_S3 - { - public ulong F0; - public uint F1; - - public F27_S3(ulong f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F27_S4 - { - public byte F0; - - public F27_S4(byte f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc272a02a12a22a32a42a52a62a72a82a93a103a113a12SiAA6F27_S0V_S2ds4Int8VAsA0S3_S1Vs5Int16VAA0S3_S2VASs6UInt16VAA0S3_S3VAA0S3_S4Vs6UInt32VtF")] - private static extern nint SwiftFunc27(F27_S0 a0, double a1, double a2, sbyte a3, sbyte a4, F27_S1 a5, short a6, F27_S2 a7, sbyte a8, ushort a9, F27_S3 a10, F27_S4 a11, uint a12); - - [Fact] - public static void TestSwiftFunc27() - { - Console.Write("Running SwiftFunc27: "); - long result = SwiftFunc27(new F27_S0(7130, new F27_S0_S0(6606060428339642921), 4122923031624866), 1451662996356727, 1529297186262631, 1, 24, new F27_S1(unchecked((nint)5075979081296734546), 75, -3781, 198), -26687, new F27_S2(53456), 90, 35194, new F27_S3(6318217926100193736, 1400016900), new F27_S4(11), 628995828); - Assert.Equal(-5428774405932003643, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F28_S0 - { - public double F0; - public short F1; - public double F2; - public ulong F3; - - public F28_S0(double f0, short f1, double f2, ulong f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F28_S1 - { - public nint F0; - public uint F1; - public ulong F2; - public float F3; - - public F28_S1(nint f0, uint f1, ulong f2, float f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F28_S2 - { - public double F0; - public ulong F1; - - public F28_S2(double f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F28_S3 - { - public short F0; - public ulong F1; - public double F2; - public int F3; - - public F28_S3(short f0, ulong f1, double f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F28_S4 - { - public nint F0; - - public F28_S4(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc282a02a12a22a32a42a52a62a72a82a93a103a113a12Sis5UInt8V_s6UInt16VAA6F28_S0VAA0U3_S1VAA0U3_S2Vs6UInt64Vs5Int32Vs5Int64VSdAsA0U3_S3VAA0U3_S4VSftF")] - private static extern nint SwiftFunc28(byte a0, ushort a1, F28_S0 a2, F28_S1 a3, F28_S2 a4, ulong a5, int a6, long a7, double a8, ushort a9, F28_S3 a10, F28_S4 a11, float a12); - - [Fact] - public static void TestSwiftFunc28() - { - Console.Write("Running SwiftFunc28: "); - long result = SwiftFunc28(190, 17255, new F28_S0(3216710004509072, 9709, 4049245410019897, 6996716492380286220), new F28_S1(unchecked((nint)4097715616866617693), 539407084, 4626633991924578918, 1275504), new F28_S2(3574990895078933, 7178808315522215553), 4610456141729135855, 1303811396, 5390518172407783382, 4435699869971486, 62148, new F28_S3(22518, 4183064684428798988, 4007968538134666, 433839184), new F28_S4(unchecked((nint)4835639581253218785)), 778028); - Assert.Equal(-2948821353897526623, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F29_S0 - { - public int F0; - public float F1; - public short F2; - - public F29_S0(int f0, float f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F29_S1 - { - public short F0; - public sbyte F1; - public nuint F2; - - public F29_S1(short f0, sbyte f1, nuint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F29_S2 - { - public ushort F0; - - public F29_S2(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F29_S3 - { - public long F0; - public long F1; - - public F29_S3(long f0, long f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc292a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a15Sis4Int8V_AA6F29_S0Vs5Int32VSuAA0W3_S1Vs6UInt64VAA0W3_S2Vs5Int16Vs5Int64Vs6UInt32VA0_SiAA0W3_S3Vs5UInt8VATSdtF")] - private static extern nint SwiftFunc29(sbyte a0, F29_S0 a1, int a2, nuint a3, F29_S1 a4, ulong a5, F29_S2 a6, short a7, long a8, uint a9, ulong a10, nint a11, F29_S3 a12, byte a13, sbyte a14, double a15); - - [Fact] - public static void TestSwiftFunc29() - { - Console.Write("Running SwiftFunc29: "); - long result = SwiftFunc29(-24, new F29_S0(1975390147, 2492976, -22918), 1918385726, unchecked((nuint)4330240195518051787), new F29_S1(20662, 37, unchecked((nuint)3480511823780639511)), 2969238117130521039, new F29_S2(39829), -21356, 4236774320019789885, 650424352, 974567590062881682, unchecked((nint)4949995943007509070), new F29_S3(6288374171493526635, 797442718847899480), 23, 47, 3112540527380411); - Assert.Equal(-219723436366645712, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F30_S0 - { - public nuint F0; - public float F1; - - public F30_S0(nuint f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F30_S1 - { - public ulong F0; - public byte F1; - public double F2; - public nint F3; - - public F30_S1(ulong f0, byte f1, double f2, nint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F30_S2_S0 - { - public short F0; - public short F1; - - public F30_S2_S0(short f0, short f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F30_S2_S1 - { - public long F0; - - public F30_S2_S1(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F30_S2 - { - public F30_S2_S0 F0; - public F30_S2_S1 F1; - - public F30_S2(F30_S2_S0 f0, F30_S2_S1 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F30_S3 - { - public sbyte F0; - public byte F1; - public ulong F2; - public uint F3; - - public F30_S3(sbyte f0, byte f1, ulong f2, uint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F30_S4 - { - public ushort F0; - - public F30_S4(ushort f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc302a02a12a22a32a42a52a62a72a82a93a103a113a12Sis6UInt16V_s5Int16VAqA6F30_S0VAA0U3_S1VAA0U3_S2Vs6UInt64Vs5Int32VSuAA0U3_S3VAqA0U3_S4Vs4Int8VtF")] - private static extern nint SwiftFunc30(ushort a0, short a1, ushort a2, F30_S0 a3, F30_S1 a4, F30_S2 a5, ulong a6, int a7, nuint a8, F30_S3 a9, ushort a10, F30_S4 a11, sbyte a12); - - [Fact] - public static void TestSwiftFunc30() - { - Console.Write("Running SwiftFunc30: "); - long result = SwiftFunc30(16858, 2711, 33779, new F30_S0(unchecked((nuint)8711036551441957307), 109551), new F30_S1(5557074438983413757, 145, 1614350045039200, unchecked((nint)962570826922694431)), new F30_S2(new F30_S2_S0(-2145, 18987), new F30_S2_S1(3566641512072703431)), 4070388225227154205, 2068046267, unchecked((nuint)2683069104930642879), new F30_S3(82, 154, 4455096152847314924, 2054397471), 61158, new F30_S4(61860), -85); - Assert.Equal(-6493337704322390178, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F31_S0 - { - public nint F0; - public float F1; - public uint F2; - public nint F3; - - public F31_S0(nint f0, float f1, uint f2, nint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc312a02a12a22a3Sis5Int64V_AA6F31_S0Vs6UInt32Vs6UInt64VtF")] - private static extern nint SwiftFunc31(long a0, F31_S0 a1, uint a2, ulong a3); - - [Fact] - public static void TestSwiftFunc31() - { - Console.Write("Running SwiftFunc31: "); - long result = SwiftFunc31(854114380819209961, new F31_S0(unchecked((nint)8616284744785848913), 2817216, 1674385679, unchecked((nint)6375864278077977066)), 972945684, 1323893099763572702); - Assert.Equal(5251289581384890505, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F32_S0 - { - public short F0; - public float F1; - public long F2; - - public F32_S0(short f0, float f1, long f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F32_S1_S0 - { - public nuint F0; - - public F32_S1_S0(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F32_S1 - { - public byte F0; - public F32_S1_S0 F1; - - public F32_S1(byte f0, F32_S1_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F32_S2 - { - public uint F0; - public byte F1; - public nuint F2; - - public F32_S2(uint f0, byte f1, nuint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F32_S3_S0 - { - public nuint F0; - - public F32_S3_S0(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F32_S3 - { - public ulong F0; - public F32_S3_S0 F1; - public ulong F2; - - public F32_S3(ulong f0, F32_S3_S0 f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F32_S4 - { - public double F0; - public long F1; - public long F2; - public float F3; - - public F32_S4(double f0, long f1, long f2, float f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc322a02a12a22a32a42a52a62a72a82a93a10Sis6UInt64V_AA6F32_S0VSdAA0R3_S1VAA0R3_S2VAOSfAA0R3_S3VAA0R3_S4Vs6UInt32Vs5Int16VtF")] - private static extern nint SwiftFunc32(ulong a0, F32_S0 a1, double a2, F32_S1 a3, F32_S2 a4, ulong a5, float a6, F32_S3 a7, F32_S4 a8, uint a9, short a10); - - [Fact] - public static void TestSwiftFunc32() - { - Console.Write("Running SwiftFunc32: "); - long result = SwiftFunc32(8029377143582007729, new F32_S0(17278, 7967601, 1978436908876178048), 1789368352608636, new F32_S1(255, new F32_S1_S0(unchecked((nuint)6244652548486446415))), new F32_S2(862868498, 29, unchecked((nuint)1969242341467623483)), 5279845618693914949, 1855163, new F32_S3(6102326739757366863, new F32_S3_S0(unchecked((nuint)8768252353660722957)), 3548360060427751308), new F32_S4(4443676345125115, 9168978488997364066, 3214391615557684463, 6052142), 1797618755, 17578); - Assert.Equal(-6196943681215505326, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F33_S0 - { - public sbyte F0; - public byte F1; - - public F33_S0(sbyte f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F33_S1 - { - public ushort F0; - public byte F1; - public long F2; - - public F33_S1(ushort f0, byte f1, long f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F33_S2_S0 - { - public uint F0; - - public F33_S2_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 34)] - struct F33_S2 - { - public F33_S2_S0 F0; - public nuint F1; - public float F2; - public double F3; - public ushort F4; - - public F33_S2(F33_S2_S0 f0, nuint f1, float f2, double f3, ushort f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F33_S3 - { - public nuint F0; - - public F33_S3(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc332a02a12a22a32a42a52a62a72a82a93a10SiSf_AA6F33_S0Vs6UInt64Vs5Int64VAA0Q3_S1Vs6UInt16VSuAwA0Q3_S2VAA0Q3_S3VSitF")] - private static extern nint SwiftFunc33(float a0, F33_S0 a1, ulong a2, long a3, F33_S1 a4, ushort a5, nuint a6, ushort a7, F33_S2 a8, F33_S3 a9, nint a10); - - [Fact] - public static void TestSwiftFunc33() - { - Console.Write("Running SwiftFunc33: "); - long result = SwiftFunc33(7854986, new F33_S0(-88, 250), 5301409185013630861, 59840293674446659, new F33_S1(60084, 209, 8486520240421572730), 47187, unchecked((nuint)3062806578924156555), 27556, new F33_S2(new F33_S2_S0(2034603306), unchecked((nuint)8616790058647815090), 6520318, 4264637592867522, 45572), new F33_S3(unchecked((nuint)8100077493474466447)), unchecked((nint)4177526131236757728)); - Assert.Equal(7131040958707940402, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F34_S0 - { - public byte F0; - - public F34_S0(byte f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc342a02a12a22a32a42a5Sis5Int64V_AA6F34_S0VS2us5UInt8VSdtF")] - private static extern nint SwiftFunc34(long a0, F34_S0 a1, nuint a2, nuint a3, byte a4, double a5); - - [Fact] - public static void TestSwiftFunc34() - { - Console.Write("Running SwiftFunc34: "); - long result = SwiftFunc34(6297959268257433453, new F34_S0(152), unchecked((nuint)684867108943559069), unchecked((nuint)3028084738078866117), 52, 1123384931674176); - Assert.Equal(-7354337608853973520, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F35_S0 - { - public short F0; - - public F35_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 3)] - struct F35_S1_S0 - { - public ushort F0; - public sbyte F1; - - public F35_S1_S0(ushort f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F35_S1 - { - public long F0; - public F35_S1_S0 F1; - public float F2; - - public F35_S1(long f0, F35_S1_S0 f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F35_S2 - { - public ulong F0; - public sbyte F1; - public uint F2; - public long F3; - - public F35_S2(ulong f0, sbyte f1, uint f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 5)] - struct F35_S3_S0_S0 - { - public uint F0; - public byte F1; - - public F35_S3_S0_S0(uint f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F35_S3_S0 - { - public ushort F0; - public F35_S3_S0_S0 F1; - public double F2; - - public F35_S3_S0(ushort f0, F35_S3_S0_S0 f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F35_S3 - { - public F35_S3_S0 F0; - public uint F1; - - public F35_S3(F35_S3_S0 f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F35_S4 - { - public float F0; - - public F35_S4(float f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc352a02a12a22a32a42a52a62a72a82a93a10Sis5UInt8V_AA6F35_S0VA2oA0R3_S1Vs5Int32VAA0R3_S2VSis6UInt32VAA0R3_S3VAA0R3_S4VtF")] - private static extern nint SwiftFunc35(byte a0, F35_S0 a1, byte a2, byte a3, F35_S1 a4, int a5, F35_S2 a6, nint a7, uint a8, F35_S3 a9, F35_S4 a10); - - [Fact] - public static void TestSwiftFunc35() - { - Console.Write("Running SwiftFunc35: "); - long result = SwiftFunc35(70, new F35_S0(-3405), 57, 4, new F35_S1(1893314071875920321, new F35_S1_S0(21188, -72), 1690358), 331400152, new F35_S2(629066911115913492, 24, 1741513272, 1738852017312447556), unchecked((nint)5964912267274635634), 745754721, new F35_S3(new F35_S3_S0(12969, new F35_S3_S0_S0(1922748035, 11), 1057686301404030), 1301219882), new F35_S4(4792810)); - Assert.Equal(8413899507614185381, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 9)] - struct F36_S0 - { - public ulong F0; - public sbyte F1; - - public F36_S0(ulong f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F36_S1 - { - public long F0; - public nuint F1; - public nint F2; - public int F3; - - public F36_S1(long f0, nuint f1, nint f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F36_S2 - { - public nint F0; - - public F36_S2(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F36_S3_S0 - { - public float F0; - - public F36_S3_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F36_S3 - { - public long F0; - public sbyte F1; - public F36_S3_S0 F2; - - public F36_S3(long f0, sbyte f1, F36_S3_S0 f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F36_S4 - { - public nuint F0; - public long F1; - public double F2; - public double F3; - - public F36_S4(nuint f0, long f1, double f2, double f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F36_S5 - { - public byte F0; - public byte F1; - - public F36_S5(byte f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F36_S6 - { - public ushort F0; - - public F36_S6(ushort f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc362a02a12a22a32a42a52a62a72a82a93a103a11SiAA6F36_S0V_Sds6UInt64VAA0R3_S1VAA0R3_S2VAA0R3_S3VAA0R3_S4VSfAA0R3_S5Vs5UInt8VSdAA0R3_S6VtF")] - private static extern nint SwiftFunc36(F36_S0 a0, double a1, ulong a2, F36_S1 a3, F36_S2 a4, F36_S3 a5, F36_S4 a6, float a7, F36_S5 a8, byte a9, double a10, F36_S6 a11); - - [Fact] - public static void TestSwiftFunc36() - { - Console.Write("Running SwiftFunc36: "); - long result = SwiftFunc36(new F36_S0(6433294246214898902, -21), 3881104127408136, 2284220855453859614, new F36_S1(4439404430423666401, unchecked((nuint)6899402977735223119), unchecked((nint)5232137643577323921), 622124401), new F36_S2(unchecked((nint)2215893056133254497)), new F36_S3(929506260159009104, -122, new F36_S3_S0(1015742)), new F36_S4(unchecked((nuint)3900865090022814819), 5812191011379795103, 4189883409333787, 3777993202541206), 1483351, new F36_S5(168, 87), 242, 3899885261689271, new F36_S6(49518)); - Assert.Equal(624934575149916284, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F37_S0 - { - public int F0; - - public F37_S0(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F37_S1 - { - public uint F0; - public uint F1; - public float F2; - - public F37_S1(uint f0, uint f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F37_S2 - { - public int F0; - public uint F1; - public double F2; - public nuint F3; - - public F37_S2(int f0, uint f1, double f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F37_S3_S0 - { - public nint F0; - - public F37_S3_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F37_S3 - { - public F37_S3_S0 F0; - - public F37_S3(F37_S3_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc372a02a12a22a32a42a52a62a72a82a93a103a113a123a13S2i_s6UInt64Vs6UInt32Vs5Int32Vs4Int8Vs5UInt8VArA6F37_S0VAA0Y3_S1Vs5Int16VAA0Y3_S2VSuAA0Y3_S3VARtF")] - private static extern nint SwiftFunc37(nint a0, ulong a1, uint a2, int a3, sbyte a4, byte a5, ulong a6, F37_S0 a7, F37_S1 a8, short a9, F37_S2 a10, nuint a11, F37_S3 a12, ulong a13); - - [Fact] - public static void TestSwiftFunc37() - { - Console.Write("Running SwiftFunc37: "); - long result = SwiftFunc37(unchecked((nint)7997876577338840618), 2916693561268448247, 2045535781, 1617618895, 35, 118, 8729954385529497591, new F37_S0(1590622742), new F37_S1(1445653735, 1780802910, 6918266), -302, new F37_S2(504109544, 1827855745, 3682561033291689, unchecked((nuint)6718188397722828326)), unchecked((nuint)4901939155447291041), new F37_S3(new F37_S3_S0(unchecked((nint)7671123806949823347))), 4910913885588390838); - Assert.Equal(-3950862618349704578, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct F38_S0 - { - public ushort F0; - public short F1; - public short F2; - - public F38_S0(ushort f0, short f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F38_S1 - { - public int F0; - - public F38_S1(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F38_S2 - { - public nuint F0; - - public F38_S2(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc382a02a12a22a32a4Sis6UInt32V_s5Int32VAA6F38_S0VAA0M3_S1VAA0M3_S2VtF")] - private static extern nint SwiftFunc38(uint a0, int a1, F38_S0 a2, F38_S1 a3, F38_S2 a4); - - [Fact] - public static void TestSwiftFunc38() - { - Console.Write("Running SwiftFunc38: "); - long result = SwiftFunc38(2061218718, 320687949, new F38_S0(53989, -5186, -13102), new F38_S1(1455203558), new F38_S2(unchecked((nuint)4328826644800782496))); - Assert.Equal(1423775906233216436, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F39_S0_S0 - { - public nuint F0; - - public F39_S0_S0(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F39_S0_S1 - { - public int F0; - - public F39_S0_S1(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 36)] - struct F39_S0 - { - public nint F0; - public long F1; - public uint F2; - public F39_S0_S0 F3; - public F39_S0_S1 F4; - - public F39_S0(nint f0, long f1, uint f2, F39_S0_S0 f3, F39_S0_S1 f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F39_S1 - { - public nuint F0; - public double F1; - - public F39_S1(nuint f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc392a02a12a22a32a4SiSu_SuAA6F39_S0VAA0K3_S1VSftF")] - private static extern nint SwiftFunc39(nuint a0, nuint a1, F39_S0 a2, F39_S1 a3, float a4); - - [Fact] - public static void TestSwiftFunc39() - { - Console.Write("Running SwiftFunc39: "); - long result = SwiftFunc39(unchecked((nuint)8230747730129668979), unchecked((nuint)4736775119629579479), new F39_S0(unchecked((nint)5173491896684902537), 4915765547454462242, 1028369724, new F39_S0_S0(unchecked((nuint)8662559577682755939)), new F39_S0_S1(436709185)), new F39_S1(unchecked((nuint)3203283942912276541), 3029648293570205), 5675124); - Assert.Equal(-1722913155676633924, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc402a0Sis5Int32V_tF")] - private static extern nint SwiftFunc40(int a0); - - [Fact] - public static void TestSwiftFunc40() - { - Console.Write("Running SwiftFunc40: "); - long result = SwiftFunc40(447211275); - Assert.Equal(8279520253543879998, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F41_S0 - { - public short F0; - public float F1; - public ushort F2; - - public F41_S0(short f0, float f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F41_S1 - { - public ushort F0; - public ulong F1; - public sbyte F2; - public float F3; - public ulong F4; - - public F41_S1(ushort f0, ulong f1, sbyte f2, float f3, ulong f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F41_S2_S0_S0 - { - public short F0; - - public F41_S2_S0_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F41_S2_S0 - { - public F41_S2_S0_S0 F0; - - public F41_S2_S0(F41_S2_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 22)] - struct F41_S2 - { - public int F0; - public short F1; - public ulong F2; - public float F3; - public F41_S2_S0 F4; - - public F41_S2(int f0, short f1, ulong f2, float f3, F41_S2_S0 f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc412a02a12a22a32a42a52a62a72a8SiSf_AA6F41_S0VAA0O3_S1VAA0O3_S2Vs6UInt32VSuASSis4Int8VtF")] - private static extern nint SwiftFunc41(float a0, F41_S0 a1, F41_S1 a2, F41_S2 a3, uint a4, nuint a5, uint a6, nint a7, sbyte a8); - - [Fact] - public static void TestSwiftFunc41() - { - Console.Write("Running SwiftFunc41: "); - long result = SwiftFunc41(5984057, new F41_S0(11791, 7594, 4883), new F41_S1(61253, 4089489613092392334, -39, 4246219, 6241750146529178696), new F41_S2(2097957786, -31595, 2497631910262823657, 1845838, new F41_S2_S0(new F41_S2_S0_S0(-4594))), 2146355885, unchecked((nuint)7552603789122823169), 1034389054, unchecked((nint)5088721772774365291), -61); - Assert.Equal(-8371592578322439321, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F42_S0 - { - public uint F0; - public ulong F1; - public ulong F2; - - public F42_S0(uint f0, ulong f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F42_S1 - { - public double F0; - public double F1; - - public F42_S1(double f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F42_S2_S0 - { - public nint F0; - - public F42_S2_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F42_S2 - { - public byte F0; - public long F1; - public F42_S2_S0 F2; - public nint F3; - - public F42_S2(byte f0, long f1, F42_S2_S0 f2, nint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F42_S3_S0 - { - public short F0; - - public F42_S3_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct F42_S3 - { - public float F0; - public F42_S3_S0 F1; - - public F42_S3(float f0, F42_S3_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F42_S4 - { - public uint F0; - - public F42_S4(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F42_S5_S0 - { - public uint F0; - - public F42_S5_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F42_S5 - { - public F42_S5_S0 F0; - - public F42_S5(F42_S5_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F42_S6 - { - public nuint F0; - - public F42_S6(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc422a02a12a22a32a42a52a62a72a8SiAA6F42_S0V_AA0O3_S1Vs6UInt16VAA0O3_S2VAA0O3_S3VAA0O3_S4VAA0O3_S5VAA0O3_S6Vs5Int16VtF")] - private static extern nint SwiftFunc42(F42_S0 a0, F42_S1 a1, ushort a2, F42_S2 a3, F42_S3 a4, F42_S4 a5, F42_S5 a6, F42_S6 a7, short a8); - - [Fact] - public static void TestSwiftFunc42() - { - Console.Write("Running SwiftFunc42: "); - long result = SwiftFunc42(new F42_S0(1751713754, 1990881383827669198, 7688992749840190173), new F42_S1(2820409929234558, 403450751107933), 8553, new F42_S2(0, 4857265047176672349, new F42_S2_S0(unchecked((nint)1659771770143536426)), unchecked((nint)4175194780289529190)), new F42_S3(2068820, new F42_S3_S0(-19086)), new F42_S4(499069670), new F42_S5(new F42_S5_S0(82826892)), new F42_S6(unchecked((nuint)7728490038553858908)), -843); - Assert.Equal(-5733927999088121133, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F43_S0_S0 - { - public long F0; - - public F43_S0_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F43_S0 - { - public F43_S0_S0 F0; - - public F43_S0(F43_S0_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc432a02a12a22a32a42a52a6Sis5Int64V_s5UInt8Vs4Int8VSfAKSiAA6F43_S0VtF")] - private static extern nint SwiftFunc43(long a0, byte a1, sbyte a2, float a3, long a4, nint a5, F43_S0 a6); - - [Fact] - public static void TestSwiftFunc43() - { - Console.Write("Running SwiftFunc43: "); - long result = SwiftFunc43(4912883404842918819, 157, 103, 5202238, 1699534526741372140, unchecked((nint)5944804412045224395), new F43_S0(new F43_S0_S0(8392262032814776063))); - Assert.Equal(7967353118822572137, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F44_S0 - { - public ulong F0; - - public F44_S0(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc442a0SiAA6F44_S0V_tF")] - private static extern nint SwiftFunc44(F44_S0 a0); - - [Fact] - public static void TestSwiftFunc44() - { - Console.Write("Running SwiftFunc44: "); - long result = SwiftFunc44(new F44_S0(6701010027402704605)); - Assert.Equal(-2463268961390375024, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F45_S0 - { - public double F0; - public nint F1; - - public F45_S0(double f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F45_S1_S0 - { - public double F0; - - public F45_S1_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F45_S1_S1 - { - public float F0; - - public F45_S1_S1(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F45_S1 - { - public ushort F0; - public sbyte F1; - public F45_S1_S0 F2; - public F45_S1_S1 F3; - - public F45_S1(ushort f0, sbyte f1, F45_S1_S0 f2, F45_S1_S1 f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 14)] - struct F45_S2 - { - public ulong F0; - public float F1; - public ushort F2; - - public F45_S2(ulong f0, float f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc452a02a12a22a3SiAA6F45_S0V_AA0J3_S1VAA0J3_S2VSitF")] - private static extern nint SwiftFunc45(F45_S0 a0, F45_S1 a1, F45_S2 a2, nint a3); - - [Fact] - public static void TestSwiftFunc45() - { - Console.Write("Running SwiftFunc45: "); - long result = SwiftFunc45(new F45_S0(3026820520892803, unchecked((nint)329722294948274546)), new F45_S1(13060, 14, new F45_S1_S0(173821703534560), new F45_S1_S1(6669558)), new F45_S2(7271072737280269762, 2970569, 7063), unchecked((nint)3563249765520844925)); - Assert.Equal(6216079413995056174, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 25)] - struct F46_S0 - { - public long F0; - public byte F1; - public nuint F2; - public sbyte F3; - - public F46_S0(long f0, byte f1, nuint f2, sbyte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F46_S1 - { - public byte F0; - - public F46_S1(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F46_S2 - { - public nint F0; - - public F46_S2(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F46_S3 - { - public ulong F0; - public long F1; - - public F46_S3(ulong f0, long f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F46_S4 - { - public short F0; - public int F1; - public uint F2; - - public F46_S4(short f0, int f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F46_S5 - { - public ulong F0; - - public F46_S5(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc462a02a12a22a32a42a52a62a72a82a93a103a113a123a13SiAA6F46_S0V_AA0T3_S1Vs4Int8VSfAA0T3_S2Vs5Int16VAA0T3_S3VAZSfAA0T3_S4Vs6UInt16VSfAvA0T3_S5VtF")] - private static extern nint SwiftFunc46(F46_S0 a0, F46_S1 a1, sbyte a2, float a3, F46_S2 a4, short a5, F46_S3 a6, short a7, float a8, F46_S4 a9, ushort a10, float a11, sbyte a12, F46_S5 a13); - - [Fact] - public static void TestSwiftFunc46() - { - Console.Write("Running SwiftFunc46: "); - long result = SwiftFunc46(new F46_S0(717422391795779639, 78, unchecked((nuint)7060282015706292416), -116), new F46_S1(18), 3, 2507216, new F46_S2(unchecked((nint)4201483730092308719)), -18720, new F46_S3(2236255490462487034, 3838628161824947390), -9982, 5460360, new F46_S4(-4606, 1433117890, 835780718), 6752, 6275800, 91, new F46_S5(9211362063136377356)); - Assert.Equal(343358650074914091, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 3)] - struct F47_S0_S0 - { - public ushort F0; - public sbyte F1; - - public F47_S0_S0(ushort f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F47_S0 - { - public F47_S0_S0 F0; - public ushort F1; - public nuint F2; - public long F3; - - public F47_S0(F47_S0_S0 f0, ushort f1, nuint f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 9)] - struct F47_S1 - { - public long F0; - public byte F1; - - public F47_S1(long f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc472a02a12a22a3S2i_AA6F47_S0VAA0J3_S1Vs5Int64VtF")] - private static extern nint SwiftFunc47(nint a0, F47_S0 a1, F47_S1 a2, long a3); - - [Fact] - public static void TestSwiftFunc47() - { - Console.Write("Running SwiftFunc47: "); - long result = SwiftFunc47(unchecked((nint)4962370882457048382), new F47_S0(new F47_S0_S0(58684, -2), 23837, unchecked((nuint)2492821112189780145), 4191553673129943106), new F47_S1(3653010013906471970, 124), 4972057731925125595); - Assert.Equal(-2787387042865302571, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc482a02a12a22a32a42a52a6Sis4Int8V_s6UInt32Vs5Int16VSfSiSfAMtF")] - private static extern nint SwiftFunc48(sbyte a0, uint a1, short a2, float a3, nint a4, float a5, uint a6); - - [Fact] - public static void TestSwiftFunc48() - { - Console.Write("Running SwiftFunc48: "); - long result = SwiftFunc48(93, 1756298153, -26153, 8138154, unchecked((nint)5977260391149529061), 5377189, 1353843369); - Assert.Equal(-1595422391414550142, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F49_S0 - { - public ulong F0; - - public F49_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F49_S1_S0 - { - public short F0; - - public F49_S1_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F49_S1_S1 - { - public ushort F0; - - public F49_S1_S1(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F49_S1 - { - public F49_S1_S0 F0; - public int F1; - public F49_S1_S1 F2; - public nuint F3; - - public F49_S1(F49_S1_S0 f0, int f1, F49_S1_S1 f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F49_S2 - { - public ushort F0; - public byte F1; - public float F2; - public long F3; - - public F49_S2(ushort f0, byte f1, float f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F49_S3 - { - public int F0; - public float F1; - - public F49_S3(int f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F49_S4 - { - public uint F0; - public nint F1; - public nint F2; - - public F49_S4(uint f0, nint f1, nint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc492a02a12a22a32a42a52a62a72a82a93a103a113a123a13Sis6UInt64V_s5UInt8VAA6F49_S0VAA0V3_S1VSus6UInt32VSdAA0V3_S2VAA0V3_S3Vs4Int8VAA0V3_S4Vs5Int32VArTtF")] - private static extern nint SwiftFunc49(ulong a0, byte a1, F49_S0 a2, F49_S1 a3, nuint a4, uint a5, double a6, F49_S2 a7, F49_S3 a8, sbyte a9, F49_S4 a10, int a11, ulong a12, byte a13); - - [Fact] - public static void TestSwiftFunc49() - { - Console.Write("Running SwiftFunc49: "); - long result = SwiftFunc49(1758884505462049879, 12, new F49_S0(1193104697993232570), new F49_S1(new F49_S1_S0(-23214), 1970325915, new F49_S1_S1(20900), unchecked((nuint)8432422526033383651)), unchecked((nuint)2433203633589099643), 1858554667, 2299996688980169, new F49_S2(65085, 158, 5839721, 6998202268068265472), new F49_S3(388389487, 5466404), -56, new F49_S4(1497255814, unchecked((nint)6665924212978484968), unchecked((nint)2332855076356772912)), 2065183786, 3874235334202874682, 6); - Assert.Equal(-6839703945099631142, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F50_S0 - { - public sbyte F0; - public short F1; - public int F2; - public uint F3; - - public F50_S0(sbyte f0, short f1, int f2, uint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F50_S1 - { - public int F0; - - public F50_S1(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc502a02a12a2SiAA6F50_S0V_s5UInt8VAA0I3_S1VtF")] - private static extern nint SwiftFunc50(F50_S0 a0, byte a1, F50_S1 a2); - - [Fact] - public static void TestSwiftFunc50() - { - Console.Write("Running SwiftFunc50: "); - long result = SwiftFunc50(new F50_S0(-64, 4463, 1574267626, 1599903339), 22, new F50_S1(2042416614)); - Assert.Equal(6447602248618864959, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc512a02a12a2Sis6UInt16V_s4Int8Vs5Int16VtF")] - private static extern nint SwiftFunc51(ushort a0, sbyte a1, short a2); - - [Fact] - public static void TestSwiftFunc51() - { - Console.Write("Running SwiftFunc51: "); - long result = SwiftFunc51(44154, 95, 13522); - Assert.Equal(-2544044281448828766, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc522a02a1Sis5UInt8V_s6UInt64VtF")] - private static extern nint SwiftFunc52(byte a0, ulong a1); - - [Fact] - public static void TestSwiftFunc52() - { - Console.Write("Running SwiftFunc52: "); - long result = SwiftFunc52(249, 1201897610107180823); - Assert.Equal(6106660152306827238, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F53_S0_S0 - { - public long F0; - public nuint F1; - - public F53_S0_S0(long f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 27)] - struct F53_S0 - { - public ulong F0; - public F53_S0_S0 F1; - public short F2; - public byte F3; - - public F53_S0(ulong f0, F53_S0_S0 f1, short f2, byte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F53_S1_S0 - { - public long F0; - - public F53_S1_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F53_S1 - { - public F53_S1_S0 F0; - - public F53_S1(F53_S1_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F53_S2 - { - public byte F0; - public ulong F1; - public double F2; - - public F53_S2(byte f0, ulong f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc532a02a12a22a32a42a52a62a7SiAA6F53_S0V_Sus6UInt64VSfs6UInt32VAA0N3_S1VAA0N3_S2VAPtF")] - private static extern nint SwiftFunc53(F53_S0 a0, nuint a1, ulong a2, float a3, uint a4, F53_S1 a5, F53_S2 a6, uint a7); - - [Fact] - public static void TestSwiftFunc53() - { - Console.Write("Running SwiftFunc53: "); - long result = SwiftFunc53(new F53_S0(2962492598802212039, new F53_S0_S0(1217181921916443700, unchecked((nuint)7957002726435705223)), -18332, 65), unchecked((nuint)1996569991268125865), 2786689999092271249, 3627618, 1358803132, new F53_S1(new F53_S1_S0(6851624154761347887)), new F53_S2(12, 3669418545199894911, 3500804251230011), 1238561537); - Assert.Equal(609186359525793369, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F54_S0_S0 - { - public nint F0; - - public F54_S0_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F54_S0 - { - public F54_S0_S0 F0; - - public F54_S0(F54_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F54_S1 - { - public uint F0; - - public F54_S1(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc542a02a12a22a32a42a52a6Sis4Int8V_s5Int32Vs6UInt32VAA6F54_S0VSfs5UInt8VAA0P3_S1VtF")] - private static extern nint SwiftFunc54(sbyte a0, int a1, uint a2, F54_S0 a3, float a4, byte a5, F54_S1 a6); - - [Fact] - public static void TestSwiftFunc54() - { - Console.Write("Running SwiftFunc54: "); - long result = SwiftFunc54(56, 918504001, 1944992063, new F54_S0(new F54_S0_S0(unchecked((nint)4622400191672284422))), 7815948, 27, new F54_S1(1866972157)); - Assert.Equal(604312640974773799, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F55_S0 - { - public double F0; - - public F55_S0(double f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc552a0SiAA6F55_S0V_tF")] - private static extern nint SwiftFunc55(F55_S0 a0); - - [Fact] - public static void TestSwiftFunc55() - { - Console.Write("Running SwiftFunc55: "); - long result = SwiftFunc55(new F55_S0(2475083570077114)); - Assert.Equal(4468870103647778776, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F56_S0_S0 - { - public byte F0; - - public F56_S0_S0(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 5)] - struct F56_S0 - { - public float F0; - public F56_S0_S0 F1; - - public F56_S0(float f0, F56_S0_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F56_S1_S0 - { - public short F0; - - public F56_S1_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F56_S1 - { - public F56_S1_S0 F0; - public double F1; - public nuint F2; - public uint F3; - - public F56_S1(F56_S1_S0 f0, double f1, nuint f2, uint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F56_S2 - { - public short F0; - public short F1; - - public F56_S2(short f0, short f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F56_S3 - { - public ushort F0; - - public F56_S3(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F56_S4 - { - public nuint F0; - - public F56_S4(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc562a02a12a22a32a4SiAA6F56_S0V_AA0K3_S1VAA0K3_S2VAA0K3_S3VAA0K3_S4VtF")] - private static extern nint SwiftFunc56(F56_S0 a0, F56_S1 a1, F56_S2 a2, F56_S3 a3, F56_S4 a4); - - [Fact] - public static void TestSwiftFunc56() - { - Console.Write("Running SwiftFunc56: "); - long result = SwiftFunc56(new F56_S0(3251221, new F56_S0_S0(89)), new F56_S1(new F56_S1_S0(-1474), 3308371901004609, unchecked((nuint)3728108803958130353), 1165879205), new F56_S2(-32579, 9771), new F56_S3(42395), new F56_S4(unchecked((nuint)3303076886770130768))); - Assert.Equal(7176775198947599357, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F57_S0 - { - public sbyte F0; - public uint F1; - - public F57_S0(sbyte f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F57_S1_S0 - { - public uint F0; - - public F57_S1_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F57_S1_S1 - { - public nuint F0; - - public F57_S1_S1(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct F57_S1 - { - public F57_S1_S0 F0; - public F57_S1_S1 F1; - public short F2; - - public F57_S1(F57_S1_S0 f0, F57_S1_S1 f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F57_S2 - { - public nuint F0; - - public F57_S2(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc572a02a12a22a32a42a5Sis6UInt32V_AA6F57_S0VAA0M3_S1VSuAA0M3_S2Vs5Int16VtF")] - private static extern nint SwiftFunc57(uint a0, F57_S0 a1, F57_S1 a2, nuint a3, F57_S2 a4, short a5); - - [Fact] - public static void TestSwiftFunc57() - { - Console.Write("Running SwiftFunc57: "); - long result = SwiftFunc57(567633593, new F57_S0(-86, 696416112), new F57_S1(new F57_S1_S0(1314705768), new F57_S1_S1(unchecked((nuint)4597174980182436219)), 21486), unchecked((nuint)1438778133550518555), new F57_S2(unchecked((nuint)1802821206757818124)), 4133); - Assert.Equal(-4086487603375673584, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F58_S0 - { - public long F0; - - public F58_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct F58_S1 - { - public nuint F0; - public nint F1; - public nuint F2; - public ushort F3; - - public F58_S1(nuint f0, nint f1, nuint f2, ushort f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc582a02a12a22a32a42a52a62a72a82a93a10Sis5UInt8V_AOSiAA6F58_S0VSfs6UInt64Vs4Int8VAA0R3_S1Vs6UInt16Vs5Int64VA_tF")] - private static extern nint SwiftFunc58(byte a0, byte a1, nint a2, F58_S0 a3, float a4, ulong a5, sbyte a6, F58_S1 a7, ushort a8, long a9, long a10); - - [Fact] - public static void TestSwiftFunc58() - { - Console.Write("Running SwiftFunc58: "); - long result = SwiftFunc58(51, 253, unchecked((nint)6470303599084560885), new F58_S0(356776366673201597), 612927, 1591484822310744993, -83, new F58_S1(unchecked((nuint)8720809519112624165), unchecked((nint)5290640035451064344), unchecked((nuint)991273095809135742), 45122), 55653, 5992020387203072133, 5336758723611801952); - Assert.Equal(-9219400197360619686, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F59_S0 - { - public nuint F0; - public byte F1; - public float F2; - public nint F3; - - public F59_S0(nuint f0, byte f1, float f2, nint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F59_S1 - { - public byte F0; - public int F1; - - public F59_S1(byte f0, int f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 13)] - struct F59_S2 - { - public nint F0; - public uint F1; - public sbyte F2; - - public F59_S2(nint f0, uint f1, sbyte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F59_S3 - { - public sbyte F0; - public float F1; - public int F2; - - public F59_S3(sbyte f0, float f1, int f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F59_S4_S0 - { - public byte F0; - - public F59_S4_S0(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F59_S4 - { - public F59_S4_S0 F0; - - public F59_S4(F59_S4_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc592a02a12a22a32a42a52a62a72a82a93a103a113a123a13SiAA6F59_S0V_Sfs6UInt32VAA0T3_S1VAA0T3_S2Vs6UInt16VSfS2iS2us5Int16VAA0T3_S3VAA0T3_S4VtF")] - private static extern nint SwiftFunc59(F59_S0 a0, float a1, uint a2, F59_S1 a3, F59_S2 a4, ushort a5, float a6, nint a7, nint a8, nuint a9, nuint a10, short a11, F59_S3 a12, F59_S4 a13); - - [Fact] - public static void TestSwiftFunc59() - { - Console.Write("Running SwiftFunc59: "); - long result = SwiftFunc59(new F59_S0(unchecked((nuint)1925278801109387173), 250, 6726955, unchecked((nint)4972956627127050696)), 5574199, 1873801510, new F59_S1(124, 272974688), new F59_S2(unchecked((nint)7596794567652280845), 243527419, -47), 26413, 6450212, unchecked((nint)5453709526903953920), unchecked((nint)7927376389197462736), unchecked((nuint)780576731665989106), unchecked((nuint)7709897378564152812), 32023, new F59_S3(80, 4147780, 732950914), new F59_S4(new F59_S4_S0(4))); - Assert.Equal(6864551615695935641, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F60_S0 - { - public long F0; - - public F60_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F60_S1 - { - public uint F0; - - public F60_S1(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc602a02a12a22a32a42a52a6Sis5Int32V_s4Int8VAKs6UInt16VSfAA6F60_S0VAA0P3_S1VtF")] - private static extern nint SwiftFunc60(int a0, sbyte a1, int a2, ushort a3, float a4, F60_S0 a5, F60_S1 a6); - - [Fact] - public static void TestSwiftFunc60() - { - Console.Write("Running SwiftFunc60: "); - long result = SwiftFunc60(2069764774, -78, 1337682119, 39074, 1949913, new F60_S0(6466100081502457656), new F60_S1(762188122)); - Assert.Equal(-4208534265899748964, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 9)] - struct F61_S0 - { - public ushort F0; - public int F1; - public sbyte F2; - - public F61_S0(ushort f0, int f1, sbyte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F61_S1 - { - public double F0; - public nint F1; - - public F61_S1(double f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F61_S2 - { - public nint F0; - public sbyte F1; - public float F2; - public ushort F3; - public float F4; - - public F61_S2(nint f0, sbyte f1, float f2, ushort f3, float f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F61_S3 - { - public uint F0; - public ulong F1; - public nuint F2; - public nuint F3; - - public F61_S3(uint f0, ulong f1, nuint f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F61_S4_S0 - { - public byte F0; - public ulong F1; - - public F61_S4_S0(byte f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F61_S4 - { - public F61_S4_S0 F0; - public long F1; - - public F61_S4(F61_S4_S0 f0, long f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc612a02a12a22a32a42a52a62a72a82a9SiAA6F61_S0V_s5UInt8VSfAA0P3_S1Vs4Int8Vs5Int64VAA0P3_S2VAA0P3_S3VAA0P3_S4Vs6UInt32VtF")] - private static extern nint SwiftFunc61(F61_S0 a0, byte a1, float a2, F61_S1 a3, sbyte a4, long a5, F61_S2 a6, F61_S3 a7, F61_S4 a8, uint a9); - - [Fact] - public static void TestSwiftFunc61() - { - Console.Write("Running SwiftFunc61: "); - long result = SwiftFunc61(new F61_S0(37779, 1838776162, -93), 6, 8289829, new F61_S1(87047161428510, unchecked((nint)1184205589182482579)), -29, 6533985246090322241, new F61_S2(unchecked((nint)2633423837220013660), 79, 307426, 32687, 2612234), new F61_S3(1625158302, 1379744644931696533, unchecked((nuint)1592864959164045790), unchecked((nuint)1112656184684227017)), new F61_S4(new F61_S4_S0(196, 2188268123262546231), 2448137925649839798), 691942709); - Assert.Equal(-2463957420588616123, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F62_S0 - { - public long F0; - - public F62_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F62_S1 - { - public float F0; - - public F62_S1(float f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc622a02a12a22a3SiAA6F62_S0V_s5Int16Vs5Int32VAA0J3_S1VtF")] - private static extern nint SwiftFunc62(F62_S0 a0, short a1, int a2, F62_S1 a3); - - [Fact] - public static void TestSwiftFunc62() - { - Console.Write("Running SwiftFunc62: "); - long result = SwiftFunc62(new F62_S0(7225726265078242156), 26594, 457232718, new F62_S1(5266624)); - Assert.Equal(1111474357603006336, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F63_S0 - { - public nint F0; - - public F63_S0(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc632a0SiAA6F63_S0V_tF")] - private static extern nint SwiftFunc63(F63_S0 a0); - - [Fact] - public static void TestSwiftFunc63() - { - Console.Write("Running SwiftFunc63: "); - long result = SwiftFunc63(new F63_S0(unchecked((nint)8434688641118467652))); - Assert.Equal(6012989597022805528, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F64_S0 - { - public double F0; - public ushort F1; - public int F2; - public nint F3; - public double F4; - - public F64_S0(double f0, ushort f1, int f2, nint f3, double f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F64_S1 - { - public int F0; - public float F1; - public uint F2; - - public F64_S1(int f0, float f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc642a02a12a22a32a42a52a62a72a8SiSd_AA6F64_S0Vs5UInt8VAA0O3_S1Vs5Int32Vs6UInt64Vs4Int8VAWSftF")] - private static extern nint SwiftFunc64(double a0, F64_S0 a1, byte a2, F64_S1 a3, int a4, ulong a5, sbyte a6, sbyte a7, float a8); - - [Fact] - public static void TestSwiftFunc64() - { - Console.Write("Running SwiftFunc64: "); - long result = SwiftFunc64(1537265878737137, new F64_S0(3855732434182818, 17371, 213617860, unchecked((nint)7735022256180276511), 3812880695456163), 18, new F64_S1(484340550, 65067, 1337805733), 1841310158, 1819062569669413729, 17, -123, 4111745); - Assert.Equal(2528424114157798731, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc652a02a12a22a3SiSf_SfSuSftF")] - private static extern nint SwiftFunc65(float a0, float a1, nuint a2, float a3); - - [Fact] - public static void TestSwiftFunc65() - { - Console.Write("Running SwiftFunc65: "); - long result = SwiftFunc65(3752751, 4441416, unchecked((nuint)9195654236823676231), 1490781); - Assert.Equal(1666102926850087608, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F66_S0 - { - public long F0; - - public F66_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F66_S1_S0 - { - public ushort F0; - - public F66_S1_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F66_S1 - { - public F66_S1_S0 F0; - public float F1; - - public F66_S1(F66_S1_S0 f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 9)] - struct F66_S2 - { - public double F0; - public byte F1; - - public F66_S2(double f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F66_S3 - { - public nuint F0; - - public F66_S3(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc662a02a12a22a3SiAA6F66_S0V_AA0J3_S1VAA0J3_S2VAA0J3_S3VtF")] - private static extern nint SwiftFunc66(F66_S0 a0, F66_S1 a1, F66_S2 a2, F66_S3 a3); - - [Fact] - public static void TestSwiftFunc66() - { - Console.Write("Running SwiftFunc66: "); - long result = SwiftFunc66(new F66_S0(7984064468330042160), new F66_S1(new F66_S1_S0(61382), 2971351), new F66_S2(463407482163222, 36), new F66_S3(unchecked((nuint)2172521839193002776))); - Assert.Equal(4347440879386243204, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F67_S0 - { - public ushort F0; - - public F67_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F67_S1_S0_S0 - { - public long F0; - - public F67_S1_S0_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F67_S1_S0 - { - public F67_S1_S0_S0 F0; - - public F67_S1_S0(F67_S1_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 14)] - struct F67_S1 - { - public F67_S1_S0 F0; - public uint F1; - public short F2; - - public F67_S1(F67_S1_S0 f0, uint f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc672a02a12a22a32a42a52a62a72a82a9Sis6UInt64V_s6UInt32Vs6UInt16Vs4Int8VAA6F67_S0VAnA0T3_S1VSuANs5Int64VtF")] - private static extern nint SwiftFunc67(ulong a0, uint a1, ushort a2, sbyte a3, F67_S0 a4, ulong a5, F67_S1 a6, nuint a7, ulong a8, long a9); - - [Fact] - public static void TestSwiftFunc67() - { - Console.Write("Running SwiftFunc67: "); - long result = SwiftFunc67(8417618485778766232, 263682468, 8040, 53, new F67_S0(44582), 2312853538155696297, new F67_S1(new F67_S1_S0(new F67_S1_S0_S0(358347933181524465)), 74416027, -11715), unchecked((nuint)3013147554369331538), 8581312208688354849, 3394216999618959997); - Assert.Equal(-6725369964492065998, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F68_S0_S0_S0 - { - public ushort F0; - - public F68_S0_S0_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F68_S0_S0 - { - public F68_S0_S0_S0 F0; - - public F68_S0_S0(F68_S0_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F68_S0 - { - public F68_S0_S0 F0; - - public F68_S0(F68_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F68_S1 - { - public ulong F0; - public ushort F1; - - public F68_S1(ulong f0, ushort f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F68_S2 - { - public nuint F0; - public nint F1; - public ulong F2; - public double F3; - - public F68_S2(nuint f0, nint f1, ulong f2, double f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F68_S3 - { - public nint F0; - public uint F1; - public uint F2; - public nuint F3; - - public F68_S3(nint f0, uint f1, uint f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F68_S4 - { - public int F0; - - public F68_S4(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc682a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a15Sis6UInt16V_s5Int64Vs5Int16Vs6UInt64Vs4Int8Vs5Int32Vs5UInt8VAA6F68_S0VA4_AA6F68_S1VAxA6F68_S2VA2xA6F68_S3VAA6F68_S4VtF")] - private static extern nint SwiftFunc68(ushort a0, long a1, short a2, ulong a3, sbyte a4, int a5, byte a6, F68_S0 a7, byte a8, F68_S1 a9, short a10, F68_S2 a11, short a12, short a13, F68_S3 a14, F68_S4 a15); - - [Fact] - public static void TestSwiftFunc68() - { - Console.Write("Running SwiftFunc68: "); - long result = SwiftFunc68(39378, 1879467527992319684, 2976, 7557363126592644195, -43, 2065185911, 186, new F68_S0(new F68_S0_S0(new F68_S0_S0_S0(38882))), 147, new F68_S1(7550657789172540141, 11186), 19125, new F68_S2(unchecked((nuint)7379823447100459002), unchecked((nint)2947420338952962953), 8170543862699682458, 4004920770933570), -12770, 19448, new F68_S3(unchecked((nint)4813886599424386410), 456733470, 2124904937, unchecked((nuint)4471482098861948789)), new F68_S4(1149728467)); - Assert.Equal(7624816402828697114, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F69_S0 - { - public uint F0; - public nuint F1; - - public F69_S0(uint f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F69_S1_S0_S0 - { - public byte F0; - - public F69_S1_S0_S0(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F69_S1_S0 - { - public F69_S1_S0_S0 F0; - public sbyte F1; - - public F69_S1_S0(F69_S1_S0_S0 f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F69_S1 - { - public F69_S1_S0 F0; - public nuint F1; - public nint F2; - - public F69_S1(F69_S1_S0 f0, nuint f1, nint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 11)] - struct F69_S2 - { - public float F0; - public uint F1; - public ushort F2; - public sbyte F3; - - public F69_S2(float f0, uint f1, ushort f2, sbyte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F69_S3 - { - public byte F0; - public double F1; - - public F69_S3(byte f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F69_S4 - { - public double F0; - - public F69_S4(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F69_S5 - { - public ulong F0; - - public F69_S5(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc692a02a12a22a32a42a52a62a72a82a93a103a113a123a13SiAA6F69_S0V_AA0T3_S1VS2is6UInt16Vs5Int16VSdAA0T3_S2VAA0T3_S3VAA0T3_S4VSis5Int32VAA0T3_S5VSftF")] - private static extern nint SwiftFunc69(F69_S0 a0, F69_S1 a1, nint a2, nint a3, ushort a4, short a5, double a6, F69_S2 a7, F69_S3 a8, F69_S4 a9, nint a10, int a11, F69_S5 a12, float a13); - - [Fact] - public static void TestSwiftFunc69() - { - Console.Write("Running SwiftFunc69: "); - long result = SwiftFunc69(new F69_S0(906404083, unchecked((nuint)2807168213757166759)), new F69_S1(new F69_S1_S0(new F69_S1_S0_S0(186), 23), unchecked((nuint)8471050292345736986), unchecked((nint)8019232101297716588)), unchecked((nint)1646897491666286061), unchecked((nint)4641745789339591736), 16462, 8795, 2000104158043033, new F69_S2(5507285, 2004746552, 63158, -120), new F69_S3(205, 3126404745245894), new F69_S4(1149593901597831), unchecked((nint)7568671357281245424), 32654713, new F69_S5(9162350932434820903), 7511550); - Assert.Equal(-6877731561846031803, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F70_S0 - { - public float F0; - public long F1; - - public F70_S0(float f0, long f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct F70_S1 - { - public ushort F0; - public sbyte F1; - public short F2; - - public F70_S1(ushort f0, sbyte f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F70_S2 - { - public ushort F0; - - public F70_S2(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F70_S3 - { - public ushort F0; - - public F70_S3(ushort f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc702a02a12a22a32a42a52a62a72a82a9Sis6UInt64V_AA6F70_S0Vs6UInt16Vs4Int8VSfAA0Q3_S1VSiAA0Q3_S2VAA0Q3_S3Vs6UInt32VtF")] - private static extern nint SwiftFunc70(ulong a0, F70_S0 a1, ushort a2, sbyte a3, float a4, F70_S1 a5, nint a6, F70_S2 a7, F70_S3 a8, uint a9); - - [Fact] - public static void TestSwiftFunc70() - { - Console.Write("Running SwiftFunc70: "); - long result = SwiftFunc70(1536666996478548266, new F70_S0(7778910, 3166989107756003196), 13136, 22, 8164102, new F70_S1(26774, 89, 8871), unchecked((nint)3879856935687439957), new F70_S2(24302), new F70_S3(50084), 1197721391); - Assert.Equal(-4661551892929812411, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F71_S0 - { - public nint F0; - - public F71_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F71_S1 - { - public ulong F0; - - public F71_S1(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc712a02a12a22a32a42a5Sis5Int64V_AA6F71_S0Vs4Int8VAA0M3_S1VSfs6UInt32VtF")] - private static extern nint SwiftFunc71(long a0, F71_S0 a1, sbyte a2, F71_S1 a3, float a4, uint a5); - - [Fact] - public static void TestSwiftFunc71() - { - Console.Write("Running SwiftFunc71: "); - long result = SwiftFunc71(823408652288450499, new F71_S0(unchecked((nint)1673096114526242440)), 64, new F71_S1(1767538531468972832), 3230384, 1139683594); - Assert.Equal(1763261422424450798, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F72_S0_S0 - { - public nint F0; - public double F1; - - public F72_S0_S0(nint f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F72_S0 - { - public F72_S0_S0 F0; - public uint F1; - - public F72_S0(F72_S0_S0 f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F72_S1 - { - public nint F0; - - public F72_S1(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F72_S2 - { - public double F0; - - public F72_S2(double f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc722a02a12a2SiAA6F72_S0V_AA0I3_S1VAA0I3_S2VtF")] - private static extern nint SwiftFunc72(F72_S0 a0, F72_S1 a1, F72_S2 a2); - - [Fact] - public static void TestSwiftFunc72() - { - Console.Write("Running SwiftFunc72: "); - long result = SwiftFunc72(new F72_S0(new F72_S0_S0(unchecked((nint)42112534105392604), 2206378956781748), 13345585), new F72_S1(unchecked((nint)4236181300943972186)), new F72_S2(3246931881930745)); - Assert.Equal(5209731649169576491, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc732a0Sis5Int64V_tF")] - private static extern nint SwiftFunc73(long a0); - - [Fact] - public static void TestSwiftFunc73() - { - Console.Write("Running SwiftFunc73: "); - long result = SwiftFunc73(5717467830857180976); - Assert.Equal(4464612974464506231, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 17)] - struct F74_S0 - { - public byte F0; - public byte F1; - public double F2; - public byte F3; - - public F74_S0(byte f0, byte f1, double f2, byte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F74_S1 - { - public short F0; - public ushort F1; - public long F2; - public nuint F3; - - public F74_S1(short f0, ushort f1, long f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F74_S2 - { - public short F0; - public double F1; - public float F2; - - public F74_S2(short f0, double f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F74_S3 - { - public short F0; - - public F74_S3(short f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc742a02a12a22a32a42a52a62a72a82a9SiAA6F74_S0V_AA0P3_S1Vs5Int32VAA0P3_S2VSis5Int64Vs5Int16VArA0P3_S3Vs6UInt64VtF")] - private static extern nint SwiftFunc74(F74_S0 a0, F74_S1 a1, int a2, F74_S2 a3, nint a4, long a5, short a6, int a7, F74_S3 a8, ulong a9); - - [Fact] - public static void TestSwiftFunc74() - { - Console.Write("Running SwiftFunc74: "); - long result = SwiftFunc74(new F74_S0(126, 165, 938186833815961, 37), new F74_S1(26448, 11115, 1477034907611479508, unchecked((nuint)7258103824495664788)), 1024717487, new F74_S2(-32191, 3877433950972112, 1759541), unchecked((nint)306022299836100497), 3906031458927364257, 105, 1354045377, new F74_S3(15217), 2609577929968659839); - Assert.Equal(4852068750102322513, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F75_S0_S0_S0 - { - public short F0; - - public F75_S0_S0_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F75_S0_S0 - { - public F75_S0_S0_S0 F0; - - public F75_S0_S0(F75_S0_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F75_S0 - { - public F75_S0_S0 F0; - public double F1; - public int F2; - - public F75_S0(F75_S0_S0 f0, double f1, int f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F75_S1_S0_S0 - { - public ushort F0; - - public F75_S1_S0_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F75_S1_S0 - { - public nuint F0; - public F75_S1_S0_S0 F1; - public long F2; - - public F75_S1_S0(nuint f0, F75_S1_S0_S0 f1, long f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F75_S1 - { - public F75_S1_S0 F0; - public nint F1; - - public F75_S1(F75_S1_S0 f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F75_S2 - { - public ulong F0; - - public F75_S2(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc752a02a12a22a32a42a52a6SiAA6F75_S0V_SdSiSuSiAA0M3_S1VAA0M3_S2VtF")] - private static extern nint SwiftFunc75(F75_S0 a0, double a1, nint a2, nuint a3, nint a4, F75_S1 a5, F75_S2 a6); - - [Fact] - public static void TestSwiftFunc75() - { - Console.Write("Running SwiftFunc75: "); - long result = SwiftFunc75(new F75_S0(new F75_S0_S0(new F75_S0_S0_S0(-10229)), 989267098871942, 1700151366), 1809179048674038, unchecked((nint)8327532491216230311), unchecked((nuint)2400790938015665595), unchecked((nint)9058430068368278195), new F75_S1(new F75_S1_S0(unchecked((nuint)2568090042127844270), new F75_S1_S0_S0(56529), 7258043284683232822), unchecked((nint)2580496344876818585)), new F75_S2(2518371079686790475)); - Assert.Equal(-3602049946494757864, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F76_S0 - { - public nint F0; - - public F76_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 14)] - struct F76_S1 - { - public ulong F0; - public int F1; - public short F2; - - public F76_S1(ulong f0, int f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F76_S2 - { - public uint F0; - - public F76_S2(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F76_S3 - { - public nint F0; - - public F76_S3(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc762a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a153a163a173a183a193a20SiSd_s5Int64Vs6UInt16VS2fAA6F76_S0Vs5Int16VAA6F76_S1VAYs6UInt64VA_s5UInt8Vs4Int8VSiAYA11_A11_A3_A_AA6F76_S2VAA6F76_S3VtF")] - private static extern nint SwiftFunc76(double a0, long a1, ushort a2, float a3, float a4, F76_S0 a5, short a6, F76_S1 a7, long a8, ulong a9, ushort a10, byte a11, sbyte a12, nint a13, long a14, sbyte a15, sbyte a16, short a17, ushort a18, F76_S2 a19, F76_S3 a20); - - [Fact] - public static void TestSwiftFunc76() - { - Console.Write("Running SwiftFunc76: "); - long result = SwiftFunc76(3446176204630463, 6827398998366360089, 5999, 2160153, 1821316, new F76_S0(unchecked((nint)4235786039908553749)), -1803, new F76_S1(7640434214516127655, 1290566778, -25932), 5980518466723941005, 3543741927421110901, 27548, 183, -92, unchecked((nint)2974474557334557206), 6986327999611060205, -10, -27, -1377, 28809, new F76_S2(971874601), new F76_S3(unchecked((nint)1638507434850613054))); - Assert.Equal(1945785605876240600, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F77_S0_S0 - { - public sbyte F0; - - public F77_S0_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F77_S0 - { - public ulong F0; - public F77_S0_S0 F1; - public sbyte F2; - - public F77_S0(ulong f0, F77_S0_S0 f1, sbyte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F77_S1 - { - public ulong F0; - public nint F1; - public int F2; - - public F77_S1(ulong f0, nint f1, int f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F77_S2_S0_S0 - { - public ushort F0; - - public F77_S2_S0_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F77_S2_S0 - { - public F77_S2_S0_S0 F0; - - public F77_S2_S0(F77_S2_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct F77_S2 - { - public F77_S2_S0 F0; - public short F1; - public sbyte F2; - public byte F3; - - public F77_S2(F77_S2_S0 f0, short f1, sbyte f2, byte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct F77_S3 - { - public nint F0; - public nint F1; - public nint F2; - public short F3; - - public F77_S3(nint f0, nint f1, nint f2, short f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F77_S4 - { - public double F0; - public sbyte F1; - public uint F2; - public short F3; - public uint F4; - - public F77_S4(double f0, sbyte f1, uint f2, short f3, uint f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F77_S5 - { - public nuint F0; - - public F77_S5(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc772a02a12a22a32a42a52a62a72a82a93a10SiAA6F77_S0V_s5Int16VAA0Q3_S1Vs6UInt32VAA0Q3_S2VAA0Q3_S3VAA0Q3_S4Vs6UInt64VAA0Q3_S5Vs6UInt16VSftF")] - private static extern nint SwiftFunc77(F77_S0 a0, short a1, F77_S1 a2, uint a3, F77_S2 a4, F77_S3 a5, F77_S4 a6, ulong a7, F77_S5 a8, ushort a9, float a10); - - [Fact] - public static void TestSwiftFunc77() - { - Console.Write("Running SwiftFunc77: "); - long result = SwiftFunc77(new F77_S0(5280239821396586490, new F77_S0_S0(-88), -25), -22596, new F77_S1(7240134379191021288, unchecked((nint)7659208338594056339), 884422905), 1341388922, new F77_S2(new F77_S2_S0(new F77_S2_S0_S0(45223)), 7237, -31, 116), new F77_S3(unchecked((nint)1688714381756854732), unchecked((nint)22701789196637865), unchecked((nint)76294687751840896), -6664), new F77_S4(668345825700173, -66, 484390251, -29179, 1983850392), 2083761371968657768, new F77_S5(unchecked((nuint)8754131797018708878)), 60699, 6889813); - Assert.Equal(6252428118328671717, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F78_S0 - { - public ushort F0; - public nuint F1; - - public F78_S0(ushort f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc782a02a1SiAA6F78_S0V_s6UInt64VtF")] - private static extern nint SwiftFunc78(F78_S0 a0, ulong a1); - - [Fact] - public static void TestSwiftFunc78() - { - Console.Write("Running SwiftFunc78: "); - long result = SwiftFunc78(new F78_S0(29770, unchecked((nuint)3187763107953451651)), 8011100719593217510); - Assert.Equal(-3469054734849002121, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F79_S0 - { - public double F0; - - public F79_S0(double f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc792a02a12a22a3Sis6UInt32V_AA6F79_S0Vs5Int16VSdtF")] - private static extern nint SwiftFunc79(uint a0, F79_S0 a1, short a2, double a3); - - [Fact] - public static void TestSwiftFunc79() - { - Console.Write("Running SwiftFunc79: "); - long result = SwiftFunc79(125852033, new F79_S0(589854369615867), 32411, 2567161537252427); - Assert.Equal(6919439799927692524, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F80_S0 - { - public ulong F0; - public double F1; - - public F80_S0(ulong f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F80_S1_S0 - { - public byte F0; - - public F80_S1_S0(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 13)] - struct F80_S1 - { - public int F0; - public ushort F1; - public uint F2; - public F80_S1_S0 F3; - - public F80_S1(int f0, ushort f1, uint f2, F80_S1_S0 f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 22)] - struct F80_S2 - { - public ulong F0; - public long F1; - public uint F2; - public ushort F3; - - public F80_S2(ulong f0, long f1, uint f2, ushort f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F80_S3_S0_S0 - { - public nint F0; - public long F1; - public ulong F2; - - public F80_S3_S0_S0(nint f0, long f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct F80_S3_S0 - { - public F80_S3_S0_S0 F0; - public uint F1; - - public F80_S3_S0(F80_S3_S0_S0 f0, uint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F80_S3 - { - public F80_S3_S0 F0; - public int F1; - - public F80_S3(F80_S3_S0 f0, int f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F80_S4_S0 - { - public float F0; - - public F80_S4_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F80_S4 - { - public F80_S4_S0 F0; - - public F80_S4(F80_S4_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc802a02a12a22a32a42a52a62a72a82a93a10SiAA6F80_S0V_AA0Q3_S1Vs6UInt16Vs5Int64VAA0Q3_S2VSds6UInt64Vs5Int32VAA0Q3_S3VAA0Q3_S4Vs5UInt8VtF")] - private static extern nint SwiftFunc80(F80_S0 a0, F80_S1 a1, ushort a2, long a3, F80_S2 a4, double a5, ulong a6, int a7, F80_S3 a8, F80_S4 a9, byte a10); - - [Fact] - public static void TestSwiftFunc80() - { - Console.Write("Running SwiftFunc80: "); - long result = SwiftFunc80(new F80_S0(1355360960230091831, 1784308328429357), new F80_S1(1545826500, 60913, 1298907936, new F80_S1_S0(91)), 45929, 1430265567693421435, new F80_S2(5983675317199180530, 4061656029212457057, 1539740932, 57372), 3111292213584236, 1408283785399541904, 157768849, new F80_S3(new F80_S3_S0(new F80_S3_S0_S0(unchecked((nint)7843547046297667291), 5997146939658037534, 1422472621224237194), 579010799), 912968372), new F80_S4(new F80_S4_S0(6160826)), 91); - Assert.Equal(-8787757710984015171, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 22)] - struct F81_S0 - { - public double F0; - public ulong F1; - public uint F2; - public byte F3; - public byte F4; - - public F81_S0(double f0, ulong f1, uint f2, byte f3, byte f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F81_S1 - { - public uint F0; - - public F81_S1(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc812a02a12a22a3SiAA6F81_S0V_s5Int32VSfAA0J3_S1VtF")] - private static extern nint SwiftFunc81(F81_S0 a0, int a1, float a2, F81_S1 a3); - - [Fact] - public static void TestSwiftFunc81() - { - Console.Write("Running SwiftFunc81: "); - long result = SwiftFunc81(new F81_S0(624904807476328, 8333634025352587313, 1193792370, 12, 123), 1584141967, 2042869, new F81_S1(929252664)); - Assert.Equal(-2553305027552835633, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 17)] - struct F82_S0 - { - public int F0; - public short F1; - public ulong F2; - public sbyte F3; - - public F82_S0(int f0, short f1, ulong f2, sbyte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F82_S1_S0 - { - public long F0; - - public F82_S1_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F82_S1 - { - public nint F0; - public int F1; - public F82_S1_S0 F2; - - public F82_S1(nint f0, int f1, F82_S1_S0 f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F82_S2 - { - public nint F0; - public long F1; - public uint F2; - public ushort F3; - public long F4; - - public F82_S2(nint f0, long f1, uint f2, ushort f3, long f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F82_S3 - { - public byte F0; - - public F82_S3(byte f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc822a02a12a22a32a42a5SiAA6F82_S0V_AA0L3_S1VAA0L3_S2Vs6UInt32VSiAA0L3_S3VtF")] - private static extern nint SwiftFunc82(F82_S0 a0, F82_S1 a1, F82_S2 a2, uint a3, nint a4, F82_S3 a5); - - [Fact] - public static void TestSwiftFunc82() - { - Console.Write("Running SwiftFunc82: "); - long result = SwiftFunc82(new F82_S0(1831859482, 13125, 959732722373954890, -77), new F82_S1(unchecked((nint)7895140590879382739), 1095783280, new F82_S1_S0(5569113039995240408)), new F82_S2(unchecked((nint)1146619146691566258), 9105860583981760040, 869172650, 46264, 3390698350483049795), 64268535, unchecked((nint)3935081377884943159), new F82_S3(152)); - Assert.Equal(545035333243758818, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F83_S0_S0 - { - public byte F0; - - public F83_S0_S0(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F83_S0 - { - public F83_S0_S0 F0; - public nint F1; - public float F2; - - public F83_S0(F83_S0_S0 f0, nint f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F83_S1_S0 - { - public double F0; - - public F83_S1_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F83_S1_S1_S0 - { - public ushort F0; - - public F83_S1_S1_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F83_S1_S1 - { - public F83_S1_S1_S0 F0; - - public F83_S1_S1(F83_S1_S1_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct F83_S1 - { - public uint F0; - public F83_S1_S0 F1; - public F83_S1_S1 F2; - - public F83_S1(uint f0, F83_S1_S0 f1, F83_S1_S1 f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F83_S2 - { - public nint F0; - - public F83_S2(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc832a02a12a22a32a42a52a62a7SiSf_AA6F83_S0VAA0N3_S1Vs5Int16VSiSfAA0N3_S2Vs6UInt16VtF")] - private static extern nint SwiftFunc83(float a0, F83_S0 a1, F83_S1 a2, short a3, nint a4, float a5, F83_S2 a6, ushort a7); - - [Fact] - public static void TestSwiftFunc83() - { - Console.Write("Running SwiftFunc83: "); - long result = SwiftFunc83(215523, new F83_S0(new F83_S0_S0(156), unchecked((nint)6215307075393311297), 6861006), new F83_S1(2039967569, new F83_S1_S0(225951511203809), new F83_S1_S1(new F83_S1_S1_S0(4596))), -9234, unchecked((nint)5460548577590073953), 5802323, new F83_S2(unchecked((nint)7383303204767349238)), 26127); - Assert.Equal(-2186229543452098356, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct F84_S0 - { - public short F0; - public sbyte F1; - public ushort F2; - public long F3; - public short F4; - - public F84_S0(short f0, sbyte f1, ushort f2, long f3, short f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F84_S1 - { - public int F0; - - public F84_S1(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F84_S2_S0 - { - public byte F0; - public ulong F1; - - public F84_S2_S0(byte f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct F84_S2 - { - public nuint F0; - public F84_S2_S0 F1; - public sbyte F2; - public double F3; - - public F84_S2(nuint f0, F84_S2_S0 f1, sbyte f2, double f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F84_S3 - { - public uint F0; - - public F84_S3(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F84_S4 - { - public float F0; - - public F84_S4(float f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc842a02a12a22a32a42a52a62a72a82a93a103a113a12SiAA6F84_S0V_AA0S3_S1Vs6UInt64VAA0S3_S2Vs6UInt32VAA0S3_S3VSuAA0S3_S4VA2Us6UInt16Vs5Int16VSftF")] - private static extern nint SwiftFunc84(F84_S0 a0, F84_S1 a1, ulong a2, F84_S2 a3, uint a4, F84_S3 a5, nuint a6, F84_S4 a7, ulong a8, ulong a9, ushort a10, short a11, float a12); - - [Fact] - public static void TestSwiftFunc84() - { - Console.Write("Running SwiftFunc84: "); - long result = SwiftFunc84(new F84_S0(-4484, -42, 64729, 6703360336708764515, -523), new F84_S1(1991025572), 3784369034793798079, new F84_S2(unchecked((nuint)8950003885832387073), new F84_S2_S0(212, 2246460359298562967), 110, 694425580701573), 590396201, new F84_S3(954246473), unchecked((nuint)4968200866033916175), new F84_S4(7222444), 6840076578020772755, 257938017424612706, 10826, 12362, 5240097); - Assert.Equal(6470148389371753355, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F85_S0_S0_S0 - { - public float F0; - - public F85_S0_S0_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F85_S0_S0 - { - public int F0; - public F85_S0_S0_S0 F1; - - public F85_S0_S0(int f0, F85_S0_S0_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F85_S0 - { - public float F0; - public F85_S0_S0 F1; - public nint F2; - public long F3; - - public F85_S0(float f0, F85_S0_S0 f1, nint f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F85_S1 - { - public uint F0; - public int F1; - - public F85_S1(uint f0, int f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F85_S2 - { - public nuint F0; - - public F85_S2(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc852a02a12a22a32a42a5SiAA6F85_S0V_AA0L3_S1VAA0L3_S2Vs4Int8Vs6UInt32Vs5Int16VtF")] - private static extern nint SwiftFunc85(F85_S0 a0, F85_S1 a1, F85_S2 a2, sbyte a3, uint a4, short a5); - - [Fact] - public static void TestSwiftFunc85() - { - Console.Write("Running SwiftFunc85: "); - long result = SwiftFunc85(new F85_S0(4799349, new F85_S0_S0(1649441954, new F85_S0_S0_S0(7944727)), unchecked((nint)9152994697049435513), 7643247514693376306), new F85_S1(1545626492, 422887320), new F85_S2(unchecked((nuint)6616620791022054982)), -117, 995038971, 27513); - Assert.Equal(-8992223142373774956, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 22)] - struct F86_S0 - { - public int F0; - public long F1; - public int F2; - public ushort F3; - - public F86_S0(int f0, long f1, int f2, ushort f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F86_S1_S0 - { - public nuint F0; - - public F86_S1_S0(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 10)] - struct F86_S1 - { - public F86_S1_S0 F0; - public ushort F1; - - public F86_S1(F86_S1_S0 f0, ushort f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F86_S2 - { - public uint F0; - - public F86_S2(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F86_S3 - { - public short F0; - - public F86_S3(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F86_S4 - { - public nint F0; - - public F86_S4(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F86_S5 - { - public short F0; - - public F86_S5(short f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc862a02a12a22a32a42a52a62a72a82a9SiAA6F86_S0V_S2iSuAA0P3_S1VAA0P3_S2Vs6UInt64VAA0P3_S3VAA0P3_S4VAA0P3_S5VtF")] - private static extern nint SwiftFunc86(F86_S0 a0, nint a1, nint a2, nuint a3, F86_S1 a4, F86_S2 a5, ulong a6, F86_S3 a7, F86_S4 a8, F86_S5 a9); - - [Fact] - public static void TestSwiftFunc86() - { - Console.Write("Running SwiftFunc86: "); - long result = SwiftFunc86(new F86_S0(1811942942, 5011425012386160741, 1789481754, 51980), unchecked((nint)6881030792370586912), unchecked((nint)1013091832294910089), unchecked((nuint)7426318018252287878), new F86_S1(new F86_S1_S0(unchecked((nuint)3709534733156518030)), 31161), new F86_S2(2110662074), 1492552132987044101, new F86_S3(18839), new F86_S4(unchecked((nint)3005766501093981786)), new F86_S5(-10373)); - Assert.Equal(4527117515781509085, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F87_S0_S0 - { - public long F0; - - public F87_S0_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F87_S0 - { - public F87_S0_S0 F0; - public float F1; - public long F2; - public double F3; - - public F87_S0(F87_S0_S0 f0, float f1, long f2, double f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F87_S1 - { - public int F0; - - public F87_S1(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F87_S2_S0 - { - public ushort F0; - - public F87_S2_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F87_S2 - { - public F87_S2_S0 F0; - - public F87_S2(F87_S2_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F87_S3 - { - public int F0; - - public F87_S3(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc872a02a12a22a32a42a52a62a72a82a93a103a113a123a13Sis5Int64V_AA6F87_S0VSus5UInt8VSds5Int16Vs6UInt64VSdSfAA0U3_S1VArA0U3_S2VAA0U3_S3VSftF")] - private static extern nint SwiftFunc87(long a0, F87_S0 a1, nuint a2, byte a3, double a4, short a5, ulong a6, double a7, float a8, F87_S1 a9, long a10, F87_S2 a11, F87_S3 a12, float a13); - - [Fact] - public static void TestSwiftFunc87() - { - Console.Write("Running SwiftFunc87: "); - long result = SwiftFunc87(8841098117509422820, new F87_S0(new F87_S0_S0(2192442345186020478), 1545304, 750118731442317544, 3418050830544628), unchecked((nuint)6369165430746397674), 71, 487868533855774, -7094, 2907086057865536952, 1643866436526662, 2614039, new F87_S1(248182038), 6870063012628711946, new F87_S2(new F87_S2_S0(30623)), new F87_S3(1817616635), 3689131); - Assert.Equal(359195416647062356, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F88_S0 - { - public byte F0; - public long F1; - public ulong F2; - public nint F3; - - public F88_S0(byte f0, long f1, ulong f2, nint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F88_S1 - { - public long F0; - public byte F1; - public ushort F2; - - public F88_S1(long f0, byte f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F88_S2 - { - public uint F0; - - public F88_S2(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F88_S3_S0 - { - public nint F0; - - public F88_S3_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct F88_S3 - { - public int F0; - public F88_S3_S0 F1; - public sbyte F2; - public ushort F3; - - public F88_S3(int f0, F88_S3_S0 f1, sbyte f2, ushort f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F88_S4_S0 - { - public float F0; - - public F88_S4_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 36)] - struct F88_S4 - { - public ushort F0; - public nuint F1; - public sbyte F2; - public nint F3; - public F88_S4_S0 F4; - - public F88_S4(ushort f0, nuint f1, sbyte f2, nint f3, F88_S4_S0 f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F88_S5 - { - public float F0; - - public F88_S5(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F88_S6 - { - public uint F0; - - public F88_S6(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F88_S7_S0 - { - public nint F0; - - public F88_S7_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F88_S7 - { - public F88_S7_S0 F0; - - public F88_S7(F88_S7_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc882a02a12a22a32a42a52a62a72a82a93a10SiAA6F88_S0V_s4Int8VAA0Q3_S1Vs6UInt64VAA0Q3_S2VAA0Q3_S3VAA0Q3_S4Vs5Int16VAA0Q3_S5VAA0Q3_S6VAA0Q3_S7VtF")] - private static extern nint SwiftFunc88(F88_S0 a0, sbyte a1, F88_S1 a2, ulong a3, F88_S2 a4, F88_S3 a5, F88_S4 a6, short a7, F88_S5 a8, F88_S6 a9, F88_S7 a10); - - [Fact] - public static void TestSwiftFunc88() - { - Console.Write("Running SwiftFunc88: "); - long result = SwiftFunc88(new F88_S0(66, 2515475983225256977, 8461123965387740223, unchecked((nint)6118352888016174162)), 0, new F88_S1(2355530907227990563, 120, 33210), 2006620539850377306, new F88_S2(2040050135), new F88_S3(1424272615, new F88_S3_S0(unchecked((nint)1176474304741776688)), -37, 57192), new F88_S4(57186, unchecked((nuint)3158759263845266986), 126, unchecked((nint)2352285611293949590), new F88_S4_S0(148232)), -10009, new F88_S5(6466089), new F88_S6(552549040), new F88_S7(new F88_S7_S0(unchecked((nint)4375596076925501643)))); - Assert.Equal(-6799924240836522873, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F89_S0 - { - public byte F0; - public sbyte F1; - - public F89_S0(byte f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F89_S1 - { - public int F0; - - public F89_S1(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F89_S2 - { - public ushort F0; - - public F89_S2(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F89_S3 - { - public double F0; - public double F1; - - public F89_S3(double f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F89_S4 - { - public uint F0; - - public F89_S4(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc892a02a12a22a32a42a52a6SiAA6F89_S0V_AA0M3_S1VAA0M3_S2Vs5UInt8VAA0M3_S3VAA0M3_S4Vs5Int32VtF")] - private static extern nint SwiftFunc89(F89_S0 a0, F89_S1 a1, F89_S2 a2, byte a3, F89_S3 a4, F89_S4 a5, int a6); - - [Fact] - public static void TestSwiftFunc89() - { - Console.Write("Running SwiftFunc89: "); - long result = SwiftFunc89(new F89_S0(3, -70), new F89_S1(1399800474), new F89_S2(4503), 65, new F89_S3(2901632902048261, 1806714347370258), new F89_S4(536267264), 1925050147); - Assert.Equal(-127506756024963910, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F90_S0 - { - public ushort F0; - public nint F1; - - public F90_S0(ushort f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F90_S1_S0 - { - public nint F0; - - public F90_S1_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F90_S1 - { - public F90_S1_S0 F0; - public nuint F1; - public double F2; - - public F90_S1(F90_S1_S0 f0, nuint f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct F90_S2 - { - public ulong F0; - public nint F1; - public ushort F2; - - public F90_S2(ulong f0, nint f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F90_S3_S0 - { - public long F0; - - public F90_S3_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F90_S3 - { - public F90_S3_S0 F0; - - public F90_S3(F90_S3_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F90_S4 - { - public long F0; - - public F90_S4(long f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc902a02a12a22a32a42a52a62a7SiAA6F90_S0V_s4Int8VAA0N3_S1VAA0N3_S2VAA0N3_S3Vs6UInt32VAA0N3_S4Vs5UInt8VtF")] - private static extern nint SwiftFunc90(F90_S0 a0, sbyte a1, F90_S1 a2, F90_S2 a3, F90_S3 a4, uint a5, F90_S4 a6, byte a7); - - [Fact] - public static void TestSwiftFunc90() - { - Console.Write("Running SwiftFunc90: "); - long result = SwiftFunc90(new F90_S0(50891, unchecked((nint)3526500586501844267)), 106, new F90_S1(new F90_S1_S0(unchecked((nint)1338488761303901988)), unchecked((nuint)6173879610835810848), 2724509546394616), new F90_S2(6787849318922951518, unchecked((nint)4947656706973797515), 31166), new F90_S3(new F90_S3_S0(9145287685889642436)), 126339746, new F90_S4(7529643579107652424), 32); - Assert.Equal(3094701713551479277, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F91_S0_S0 - { - public int F0; - - public F91_S0_S0(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F91_S0 - { - public F91_S0_S0 F0; - public uint F1; - public nint F2; - - public F91_S0(F91_S0_S0 f0, uint f1, nint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc912a02a1SiAA6F91_S0V_s5UInt8VtF")] - private static extern nint SwiftFunc91(F91_S0 a0, byte a1); - - [Fact] - public static void TestSwiftFunc91() - { - Console.Write("Running SwiftFunc91: "); - long result = SwiftFunc91(new F91_S0(new F91_S0_S0(1253970930), 1885655301, unchecked((nint)148902531378116685)), 122); - Assert.Equal(887289976736078648, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F92_S0 - { - public ushort F0; - public ushort F1; - - public F92_S0(ushort f0, ushort f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F92_S1 - { - public ulong F0; - - public F92_S1(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct F92_S2 - { - public ulong F0; - public ulong F1; - - public F92_S2(ulong f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F92_S3 - { - public nuint F0; - - public F92_S3(nuint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc922a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a153a163a173a183a19Sis5Int16V_s6UInt64VSus5Int64VAA6F92_S0VA0_Sds5UInt8Vs4Int8Vs6UInt32VA6_AA6F92_S1VA8_SfAZA4_s5Int32VA8_AA6F92_S2VAA6F92_S3VtF")] - private static extern nint SwiftFunc92(short a0, ulong a1, nuint a2, long a3, F92_S0 a4, long a5, double a6, byte a7, sbyte a8, uint a9, sbyte a10, F92_S1 a11, uint a12, float a13, ulong a14, byte a15, int a16, uint a17, F92_S2 a18, F92_S3 a19); - - [Fact] - public static void TestSwiftFunc92() - { - Console.Write("Running SwiftFunc92: "); - long result = SwiftFunc92(21276, 3146876064491681609, unchecked((nuint)3037098519528577447), 9061597632723103558, new F92_S0(4967, 61949), 4798856485492542774, 4305543426365472, 182, -21, 270986478, -37, new F92_S1(7527241857214360309), 1301049439, 6192745, 8959151295191616689, 19, 1578403390, 633901437, new F92_S2(4396088615663569948, 4797465448959123058), new F92_S3(unchecked((nuint)7386458829492133332))); - Assert.Equal(-7871787038267731510, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F93_S0 - { - public int F0; - public nuint F1; - public double F2; - - public F93_S0(int f0, nuint f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F93_S1 - { - public uint F0; - - public F93_S1(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F93_S2 - { - public double F0; - - public F93_S2(double f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc932a02a12a22a32a4SiAA6F93_S0V_SiAA0K3_S1VSdAA0K3_S2VtF")] - private static extern nint SwiftFunc93(F93_S0 a0, nint a1, F93_S1 a2, double a3, F93_S2 a4); - - [Fact] - public static void TestSwiftFunc93() - { - Console.Write("Running SwiftFunc93: "); - long result = SwiftFunc93(new F93_S0(982459422, unchecked((nuint)1427174739694078549), 2736620007792094), unchecked((nint)5873331022463084971), new F93_S1(1169579606), 2110866269939297, new F93_S2(2364749142642625)); - Assert.Equal(432632740260631481, result); - Console.WriteLine("OK"); - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc942a02a12a22a3Sis6UInt64V_s5Int32VAHs5Int64VtF")] - private static extern nint SwiftFunc94(ulong a0, int a1, ulong a2, long a3); - - [Fact] - public static void TestSwiftFunc94() - { - Console.Write("Running SwiftFunc94: "); - long result = SwiftFunc94(2878691982818555531, 580037131, 3143309402030542876, 3739683344990129550); - Assert.Equal(-330124951832302022, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F95_S0 - { - public long F0; - - public F95_S0(long f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc952a02a1SiAA6F95_S0V_s5Int64VtF")] - private static extern nint SwiftFunc95(F95_S0 a0, long a1); - - [Fact] - public static void TestSwiftFunc95() - { - Console.Write("Running SwiftFunc95: "); - long result = SwiftFunc95(new F95_S0(7113705515120682426), 2532424238121218748); - Assert.Equal(-5365348133343237200, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S0_S0 - { - public nint F0; - - public F96_S0_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F96_S0 - { - public ulong F0; - public double F1; - public double F2; - public F96_S0_S0 F3; - - public F96_S0(ulong f0, double f1, double f2, F96_S0_S0 f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S1_S0_S0 - { - public double F0; - - public F96_S1_S0_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S1_S0 - { - public F96_S1_S0_S0 F0; - - public F96_S1_S0(F96_S1_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S1 - { - public F96_S1_S0 F0; - - public F96_S1(F96_S1_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S2 - { - public byte F0; - public float F1; - - public F96_S2(byte f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct F96_S3 - { - public ushort F0; - - public F96_S3(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S4 - { - public nint F0; - - public F96_S4(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F96_S5_S0 - { - public byte F0; - - public F96_S5_S0(byte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct F96_S5 - { - public F96_S5_S0 F0; - - public F96_S5(F96_S5_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct F96_S6 - { - public ulong F0; - - public F96_S6(ulong f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc962a02a12a22a32a42a52a62a72a82a93a103a113a123a133a143a153a163a173a18Sis6UInt16V_AA6F96_S0VAA0Z3_S1VAA0Z3_S2VAWs6UInt64VSis5Int32Vs5Int16VSuAA0Z3_S3VA7_Sis4Int8VA5_s6UInt32VAA0Z3_S4VAA0Z3_S5VAA0Z3_S6VtF")] - private static extern nint SwiftFunc96(ushort a0, F96_S0 a1, F96_S1 a2, F96_S2 a3, ushort a4, ulong a5, nint a6, int a7, short a8, nuint a9, F96_S3 a10, short a11, nint a12, sbyte a13, int a14, uint a15, F96_S4 a16, F96_S5 a17, F96_S6 a18); - - [Fact] - public static void TestSwiftFunc96() - { - Console.Write("Running SwiftFunc96: "); - long result = SwiftFunc96(21321, new F96_S0(3140378485759721513, 3334385568992933, 2434271617187235, new F96_S0_S0(unchecked((nint)6455348790423327394))), new F96_S1(new F96_S1_S0(new F96_S1_S0_S0(2421227444572952))), new F96_S2(72, 1265762), 13171, 4895217822310904030, unchecked((nint)5923562627585381292), 1083710828, 12717, unchecked((nuint)8000948766038488291), new F96_S3(43225), -19602, unchecked((nint)248571613858478112), 17, 514773482, 1555810858, new F96_S4(unchecked((nint)5975988026010739585)), new F96_S5(new F96_S5_S0(231)), new F96_S6(4299230038366602170)); - Assert.Equal(-9154394486464436217, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F97_S0 - { - public float F0; - public float F1; - public nint F2; - public nint F3; - public nint F4; - - public F97_S0(float f0, float f1, nint f2, nint f3, nint f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc972a02a12a22a32a42a52a62a7Sis4Int8V_s5Int32Vs5UInt8Vs6UInt32VApA6F97_S0VALSitF")] - private static extern nint SwiftFunc97(sbyte a0, int a1, byte a2, uint a3, byte a4, F97_S0 a5, sbyte a6, nint a7); - - [Fact] - public static void TestSwiftFunc97() - { - Console.Write("Running SwiftFunc97: "); - long result = SwiftFunc97(-90, 2040542494, 255, 990214241, 129, new F97_S0(3372147, 5204115, unchecked((nint)4061871110726583367), unchecked((nint)5498225315328650601), unchecked((nint)4096658558391048200)), -91, unchecked((nint)8125330763927981736)); - Assert.Equal(-4028368897548286667, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F98_S0_S0_S0 - { - public float F0; - - public F98_S0_S0_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct F98_S0_S0 - { - public nuint F0; - public F98_S0_S0_S0 F1; - - public F98_S0_S0(nuint f0, F98_S0_S0_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct F98_S0 - { - public long F0; - public F98_S0_S0 F1; - public nuint F2; - - public F98_S0(long f0, F98_S0_S0 f1, nuint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc982a02a12a22a32a42a5SiAA6F98_S0V_s6UInt16VALs5Int16Vs4Int8Vs6UInt32VtF")] - private static extern nint SwiftFunc98(F98_S0 a0, ushort a1, ushort a2, short a3, sbyte a4, uint a5); - - [Fact] - public static void TestSwiftFunc98() - { - Console.Write("Running SwiftFunc98: "); - long result = SwiftFunc98(new F98_S0(3497167808648160462, new F98_S0_S0(unchecked((nuint)2747735625017321807), new F98_S0_S0_S0(4681050)), unchecked((nuint)3446511732552970390)), 61052, 18880, -20869, 35, 1056152744); - Assert.Equal(7350111494379160095, result); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct F99_S0 - { - public ulong F0; - public ushort F1; - public float F2; - public ulong F3; - - public F99_S0(ulong f0, ushort f1, float f2, ulong f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F99_S1_S0 - { - public uint F0; - - public F99_S1_S0(uint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct F99_S1 - { - public F99_S1_S0 F0; - - public F99_S1(F99_S1_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s14SwiftAbiStress11swiftFunc992a02a12a22a3SiAA6F99_S0V_s4Int8VAA0J3_S1Vs5Int64VtF")] - private static extern nint SwiftFunc99(F99_S0 a0, sbyte a1, F99_S1 a2, long a3); - - [Fact] - public static void TestSwiftFunc99() - { - Console.Write("Running SwiftFunc99: "); - long result = SwiftFunc99(new F99_S0(1210929052346596858, 3796, 3904675, 8849045203219202310), 97, new F99_S1(new F99_S1_S0(498956895)), 241968587946267390); - Assert.Equal(7941122870613797512, result); - Console.WriteLine("OK"); - } - -} diff --git a/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.csproj b/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.csproj deleted file mode 100644 index a57cd84cf8842c..00000000000000 --- a/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - true - true - - true - - - - - - - - - diff --git a/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.swift b/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.swift deleted file mode 100644 index 91e4fd03683afa..00000000000000 --- a/src/tests/Interop/Swift/SwiftAbiStress/SwiftAbiStress.swift +++ /dev/null @@ -1,4659 +0,0 @@ -import Foundation - -struct HasherFNV1a { - - private var hash: UInt = 14_695_981_039_346_656_037 - private let prime: UInt = 1_099_511_628_211 - - mutating func combine(_ val: T) { - for byte in withUnsafeBytes(of: val, Array.init) { - hash ^= UInt(byte) - hash = hash &* prime - } - } - - func finalize() -> Int { - Int(truncatingIfNeeded: hash) - } -} - -@frozen -public struct F0_S0 -{ - public let f0 : Double; - public let f1 : UInt32; - public let f2 : UInt16; -} - -@frozen -public struct F0_S1 -{ - public let f0 : UInt64; -} - -@frozen -public struct F0_S2 -{ - public let f0 : Float; -} - -public func swiftFunc0(a0: Int16, a1: Int32, a2: UInt64, a3: UInt16, a4: F0_S0, a5: F0_S1, a6: UInt8, a7: F0_S2) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7.f0); - return hasher.finalize() -} - -@frozen -public struct F1_S0 -{ - public let f0 : Int64; - public let f1 : Double; - public let f2 : Int8; - public let f3 : Int32; - public let f4 : UInt16; -} - -@frozen -public struct F1_S1 -{ - public let f0 : UInt8; -} - -@frozen -public struct F1_S2 -{ - public let f0 : Int16; -} - -public func swiftFunc1(a0: F1_S0, a1: UInt8, a2: F1_S1, a3: F1_S2) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a0.f4); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a3.f0); - return hasher.finalize() -} - -@frozen -public struct F2_S0 -{ - public let f0 : Int; - public let f1 : UInt; -} - -@frozen -public struct F2_S1 -{ - public let f0 : Int64; - public let f1 : Int32; - public let f2 : Int16; - public let f3 : Int64; - public let f4 : UInt16; -} - -@frozen -public struct F2_S2_S0_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F2_S2_S0 -{ - public let f0 : F2_S2_S0_S0; -} - -@frozen -public struct F2_S2 -{ - public let f0 : F2_S2_S0; -} - -@frozen -public struct F2_S3 -{ - public let f0 : UInt8; -} - -@frozen -public struct F2_S4 -{ - public let f0 : Int32; - public let f1 : UInt; -} - -@frozen -public struct F2_S5 -{ - public let f0 : Float; -} - -public func swiftFunc2(a0: Int64, a1: Int16, a2: Int32, a3: F2_S0, a4: UInt8, a5: Int32, a6: F2_S1, a7: F2_S2, a8: UInt16, a9: Float, a10: F2_S3, a11: F2_S4, a12: F2_S5, a13: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a6.f4); - hasher.combine(a7.f0.f0.f0); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a11.f0); - hasher.combine(a11.f1); - hasher.combine(a12.f0); - hasher.combine(a13); - return hasher.finalize() -} - -@frozen -public struct F3_S0_S0 -{ - public let f0 : Int; - public let f1 : UInt32; -} - -@frozen -public struct F3_S0 -{ - public let f0 : Int8; - public let f1 : F3_S0_S0; - public let f2 : UInt32; -} - -@frozen -public struct F3_S1 -{ - public let f0 : Int64; - public let f1 : Float; -} - -@frozen -public struct F3_S2 -{ - public let f0 : Float; -} - -@frozen -public struct F3_S3 -{ - public let f0 : UInt8; - public let f1 : Int; -} - -@frozen -public struct F3_S4 -{ - public let f0 : UInt; - public let f1 : Float; - public let f2 : UInt16; -} - -@frozen -public struct F3_S5 -{ - public let f0 : UInt32; - public let f1 : Int64; -} - -@frozen -public struct F3_S6_S0 -{ - public let f0 : Int16; - public let f1 : UInt8; -} - -@frozen -public struct F3_S6 -{ - public let f0 : F3_S6_S0; - public let f1 : Int8; - public let f2 : UInt8; -} - -@frozen -public struct F3_S7 -{ - public let f0 : UInt64; -} - -public func swiftFunc3(a0: Int, a1: F3_S0, a2: F3_S1, a3: Double, a4: Int, a5: F3_S2, a6: F3_S3, a7: F3_S4, a8: F3_S5, a9: UInt16, a10: Int32, a11: F3_S6, a12: Int, a13: F3_S7) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1.f0); - hasher.combine(a1.f1.f1); - hasher.combine(a1.f2); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a8.f0); - hasher.combine(a8.f1); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11.f0.f0); - hasher.combine(a11.f0.f1); - hasher.combine(a11.f1); - hasher.combine(a11.f2); - hasher.combine(a12); - hasher.combine(a13.f0); - return hasher.finalize() -} - -@frozen -public struct F4_S0 -{ - public let f0 : UInt16; - public let f1 : Int16; - public let f2 : Int16; -} - -@frozen -public struct F4_S1_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F4_S1 -{ - public let f0 : F4_S1_S0; - public let f1 : Float; -} - -@frozen -public struct F4_S2_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F4_S2 -{ - public let f0 : F4_S2_S0; - public let f1 : Int; -} - -@frozen -public struct F4_S3 -{ - public let f0 : UInt64; - public let f1 : UInt64; - public let f2 : Int64; -} - -public func swiftFunc4(a0: Int, a1: F4_S0, a2: UInt, a3: UInt64, a4: Int8, a5: Double, a6: F4_S1, a7: UInt8, a8: Int32, a9: UInt32, a10: UInt64, a11: F4_S2, a12: Int16, a13: Int, a14: F4_S3, a15: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0.f0); - hasher.combine(a6.f1); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11.f0.f0); - hasher.combine(a11.f1); - hasher.combine(a12); - hasher.combine(a13); - hasher.combine(a14.f0); - hasher.combine(a14.f1); - hasher.combine(a14.f2); - hasher.combine(a15); - return hasher.finalize() -} - -@frozen -public struct F5_S0 -{ - public let f0 : UInt; -} - -public func swiftFunc5(a0: UInt, a1: UInt64, a2: UInt8, a3: F5_S0) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - return hasher.finalize() -} - -@frozen -public struct F6_S0 -{ - public let f0 : Int32; - public let f1 : Int; - public let f2 : UInt8; -} - -@frozen -public struct F6_S1 -{ - public let f0 : Int; - public let f1 : Float; -} - -@frozen -public struct F6_S2_S0 -{ - public let f0 : Double; -} - -@frozen -public struct F6_S2 -{ - public let f0 : F6_S2_S0; - public let f1 : UInt16; -} - -@frozen -public struct F6_S3 -{ - public let f0 : Double; - public let f1 : Double; - public let f2 : UInt64; -} - -@frozen -public struct F6_S4 -{ - public let f0 : Int8; -} - -@frozen -public struct F6_S5 -{ - public let f0 : Int16; -} - -public func swiftFunc6(a0: Int64, a1: F6_S0, a2: F6_S1, a3: UInt, a4: UInt8, a5: Int32, a6: F6_S2, a7: Float, a8: Int16, a9: F6_S3, a10: UInt16, a11: Double, a12: UInt32, a13: F6_S4, a14: F6_S5) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0.f0); - hasher.combine(a6.f1); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9.f0); - hasher.combine(a9.f1); - hasher.combine(a9.f2); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12); - hasher.combine(a13.f0); - hasher.combine(a14.f0); - return hasher.finalize() -} - -@frozen -public struct F7_S0 -{ - public let f0 : Int16; - public let f1 : Int; -} - -@frozen -public struct F7_S1 -{ - public let f0 : UInt8; -} - -public func swiftFunc7(a0: Int64, a1: Int, a2: UInt8, a3: F7_S0, a4: F7_S1, a5: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4.f0); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F8_S0 -{ - public let f0 : Int32; -} - -public func swiftFunc8(a0: UInt16, a1: UInt, a2: UInt16, a3: UInt64, a4: F8_S0, a5: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F9_S0 -{ - public let f0 : Double; -} - -@frozen -public struct F9_S1 -{ - public let f0 : Int32; -} - -public func swiftFunc9(a0: Int64, a1: Float, a2: F9_S0, a3: UInt16, a4: F9_S1, a5: UInt16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F10_S0 -{ - public let f0 : Int64; - public let f1 : UInt32; -} - -@frozen -public struct F10_S1 -{ - public let f0 : Float; - public let f1 : UInt8; - public let f2 : UInt; -} - -@frozen -public struct F10_S2 -{ - public let f0 : UInt; - public let f1 : UInt64; -} - -@frozen -public struct F10_S3 -{ - public let f0 : Float; -} - -@frozen -public struct F10_S4 -{ - public let f0 : Int64; -} - -public func swiftFunc10(a0: UInt16, a1: UInt16, a2: F10_S0, a3: UInt64, a4: Float, a5: Int8, a6: Int64, a7: UInt64, a8: Int64, a9: Float, a10: Int32, a11: Int32, a12: Int64, a13: UInt64, a14: F10_S1, a15: Int64, a16: F10_S2, a17: F10_S3, a18: F10_S4) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12); - hasher.combine(a13); - hasher.combine(a14.f0); - hasher.combine(a14.f1); - hasher.combine(a14.f2); - hasher.combine(a15); - hasher.combine(a16.f0); - hasher.combine(a16.f1); - hasher.combine(a17.f0); - hasher.combine(a18.f0); - return hasher.finalize() -} - -@frozen -public struct F11_S0 -{ - public let f0 : Int16; - public let f1 : Int8; - public let f2 : UInt64; - public let f3 : Int16; -} - -@frozen -public struct F11_S1 -{ - public let f0 : UInt; -} - -@frozen -public struct F11_S2 -{ - public let f0 : Int16; -} - -@frozen -public struct F11_S3_S0 -{ - public let f0 : Float; -} - -@frozen -public struct F11_S3 -{ - public let f0 : F11_S3_S0; -} - -public func swiftFunc11(a0: Int, a1: UInt64, a2: UInt8, a3: Int16, a4: F11_S0, a5: F11_S1, a6: UInt16, a7: Double, a8: Int, a9: UInt32, a10: F11_S2, a11: F11_S3, a12: Int8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a4.f3); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a11.f0.f0); - hasher.combine(a12); - return hasher.finalize() -} - -@frozen -public struct F12_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F12_S1 -{ - public let f0 : UInt8; -} - -@frozen -public struct F12_S2 -{ - public let f0 : UInt; -} - -public func swiftFunc12(a0: UInt8, a1: Int32, a2: F12_S0, a3: Int8, a4: F12_S1, a5: F12_S2, a6: UInt32, a7: Int16, a8: Int8, a9: Int8, a10: UInt32, a11: UInt8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11); - return hasher.finalize() -} - -@frozen -public struct F13_S0_S0_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct F13_S0_S0 -{ - public let f0 : F13_S0_S0_S0; -} - -@frozen -public struct F13_S0 -{ - public let f0 : Int8; - public let f1 : F13_S0_S0; -} - -@frozen -public struct F13_S1_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct F13_S1 -{ - public let f0 : F13_S1_S0; -} - -public func swiftFunc13(a0: Int8, a1: Double, a2: F13_S0, a3: F13_S1, a4: Int8, a5: Double) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1.f0.f0); - hasher.combine(a3.f0.f0); - hasher.combine(a4); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F14_S0 -{ - public let f0 : Int; -} - -public func swiftFunc14(a0: Int8, a1: Int, a2: F14_S0, a3: Float, a4: UInt) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a3); - hasher.combine(a4); - return hasher.finalize() -} - -@frozen -public struct F15_S0 -{ - public let f0 : Float; - public let f1 : Int16; - public let f2 : UInt8; - public let f3 : Int64; - public let f4 : Double; -} - -@frozen -public struct F15_S1_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct F15_S1 -{ - public let f0 : UInt32; - public let f1 : F15_S1_S0; - public let f2 : UInt; - public let f3 : Int32; -} - -public func swiftFunc15(a0: F15_S0, a1: UInt64, a2: UInt32, a3: UInt, a4: UInt64, a5: Int16, a6: F15_S1, a7: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a0.f4); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1.f0); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a7); - return hasher.finalize() -} - -@frozen -public struct F16_S0_S0 -{ - public let f0 : Double; -} - -@frozen -public struct F16_S0 -{ - public let f0 : Int; - public let f1 : Int; - public let f2 : F16_S0_S0; -} - -@frozen -public struct F16_S1 -{ - public let f0 : Int16; - public let f1 : UInt64; - public let f2 : UInt32; -} - -@frozen -public struct F16_S2 -{ - public let f0 : UInt8; - public let f1 : UInt64; - public let f2 : Float; -} - -@frozen -public struct F16_S3 -{ - public let f0 : Int32; -} - -public func swiftFunc16(a0: UInt64, a1: F16_S0, a2: F16_S1, a3: UInt16, a4: Int16, a5: F16_S2, a6: F16_S3) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2.f0); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a5.f1); - hasher.combine(a5.f2); - hasher.combine(a6.f0); - return hasher.finalize() -} - -@frozen -public struct F17_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F17_S1 -{ - public let f0 : Int64; - public let f1 : UInt; - public let f2 : UInt64; -} - -@frozen -public struct F17_S2 -{ - public let f0 : Int8; -} - -@frozen -public struct F17_S3 -{ - public let f0 : Int8; - public let f1 : UInt32; -} - -@frozen -public struct F17_S4 -{ - public let f0 : UInt64; -} - -@frozen -public struct F17_S5 -{ - public let f0 : Int64; -} - -public func swiftFunc17(a0: F17_S0, a1: Int8, a2: F17_S1, a3: Int8, a4: UInt, a5: F17_S2, a6: Int64, a7: F17_S3, a8: F17_S4, a9: F17_S5) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a8.f0); - hasher.combine(a9.f0); - return hasher.finalize() -} - -@frozen -public struct F18_S0_S0 -{ - public let f0 : UInt16; - public let f1 : Int16; -} - -@frozen -public struct F18_S0 -{ - public let f0 : UInt32; - public let f1 : F18_S0_S0; - public let f2 : UInt16; -} - -@frozen -public struct F18_S1 -{ - public let f0 : Int; - public let f1 : Int; -} - -@frozen -public struct F18_S2_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct F18_S2 -{ - public let f0 : UInt64; - public let f1 : Int64; - public let f2 : UInt8; - public let f3 : F18_S2_S0; -} - -public func swiftFunc18(a0: UInt8, a1: Double, a2: F18_S0, a3: F18_S1, a4: UInt16, a5: Int64, a6: UInt64, a7: F18_S2, a8: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1.f0); - hasher.combine(a2.f1.f1); - hasher.combine(a2.f2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a7.f3.f0); - hasher.combine(a8); - return hasher.finalize() -} - -@frozen -public struct F19_S0 -{ - public let f0 : Int; - public let f1 : Double; - public let f2 : UInt16; -} - -public func swiftFunc19(a0: UInt, a1: F19_S0, a2: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2); - return hasher.finalize() -} - -@frozen -public struct F20_S0 -{ - public let f0 : UInt16; - public let f1 : Int8; - public let f2 : UInt64; - public let f3 : UInt32; - public let f4 : UInt64; -} - -@frozen -public struct F20_S1 -{ - public let f0 : Int64; -} - -public func swiftFunc20(a0: Int8, a1: F20_S0, a2: UInt64, a3: Int, a4: F20_S1, a5: UInt8, a6: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a1.f4); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5); - hasher.combine(a6); - return hasher.finalize() -} - -@frozen -public struct F21_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F21_S1 -{ - public let f0 : Int; - public let f1 : UInt32; - public let f2 : UInt8; - public let f3 : Int16; -} - -@frozen -public struct F21_S2 -{ - public let f0 : Int8; - public let f1 : UInt64; - public let f2 : Int64; - public let f3 : UInt8; -} - -@frozen -public struct F21_S3 -{ - public let f0 : Double; - public let f1 : Int; -} - -public func swiftFunc21(a0: UInt64, a1: Int8, a2: UInt, a3: Double, a4: Float, a5: Int, a6: F21_S0, a7: F21_S1, a8: UInt16, a9: F21_S2, a10: UInt8, a11: F21_S3, a12: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a7.f3); - hasher.combine(a8); - hasher.combine(a9.f0); - hasher.combine(a9.f1); - hasher.combine(a9.f2); - hasher.combine(a9.f3); - hasher.combine(a10); - hasher.combine(a11.f0); - hasher.combine(a11.f1); - hasher.combine(a12); - return hasher.finalize() -} - -@frozen -public struct F22_S0 -{ - public let f0 : UInt16; - public let f1 : UInt32; - public let f2 : Int16; - public let f3 : Float; -} - -@frozen -public struct F22_S1 -{ - public let f0 : UInt16; - public let f1 : Int8; - public let f2 : UInt8; - public let f3 : Int; - public let f4 : Int; -} - -@frozen -public struct F22_S2_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct F22_S2 -{ - public let f0 : Int32; - public let f1 : Int32; - public let f2 : UInt32; - public let f3 : UInt8; - public let f4 : F22_S2_S0; -} - -@frozen -public struct F22_S3 -{ - public let f0 : Int16; - public let f1 : Double; - public let f2 : Double; - public let f3 : Int32; -} - -public func swiftFunc22(a0: Int8, a1: Int32, a2: F22_S0, a3: F22_S1, a4: F22_S2, a5: UInt64, a6: F22_S3, a7: UInt) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a3.f3); - hasher.combine(a3.f4); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a4.f3); - hasher.combine(a4.f4.f0); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a7); - return hasher.finalize() -} - -@frozen -public struct F23_S0 -{ - public let f0 : UInt32; - public let f1 : Int16; -} - -@frozen -public struct F23_S1 -{ - public let f0 : UInt; - public let f1 : UInt32; -} - -@frozen -public struct F23_S2 -{ - public let f0 : Double; - public let f1 : UInt32; - public let f2 : Int32; - public let f3 : UInt8; -} - -public func swiftFunc23(a0: F23_S0, a1: F23_S1, a2: F23_S2, a3: Double, a4: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3); - hasher.combine(a3); - hasher.combine(a4); - return hasher.finalize() -} - -@frozen -public struct F24_S0 -{ - public let f0 : Int8; - public let f1 : Int32; -} - -@frozen -public struct F24_S1 -{ - public let f0 : Int8; -} - -@frozen -public struct F24_S2 -{ - public let f0 : UInt16; - public let f1 : Int16; - public let f2 : Double; - public let f3 : UInt; -} - -@frozen -public struct F24_S3 -{ - public let f0 : Int; -} - -public func swiftFunc24(a0: F24_S0, a1: F24_S1, a2: F24_S2, a3: F24_S3, a4: UInt, a5: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1.f0); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3); - hasher.combine(a3.f0); - hasher.combine(a4); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F25_S0_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct F25_S0 -{ - public let f0 : Float; - public let f1 : F25_S0_S0; - public let f2 : UInt32; -} - -@frozen -public struct F25_S1 -{ - public let f0 : Int16; - public let f1 : Int8; - public let f2 : Float; -} - -@frozen -public struct F25_S2 -{ - public let f0 : Int64; - public let f1 : UInt16; -} - -@frozen -public struct F25_S3 -{ - public let f0 : UInt64; -} - -@frozen -public struct F25_S4 -{ - public let f0 : UInt16; -} - -public func swiftFunc25(a0: Float, a1: F25_S0, a2: Int64, a3: UInt8, a4: F25_S1, a5: Int, a6: F25_S2, a7: Int32, a8: Int32, a9: UInt, a10: UInt64, a11: F25_S3, a12: F25_S4) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1.f0); - hasher.combine(a1.f2); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11.f0); - hasher.combine(a12.f0); - return hasher.finalize() -} - -@frozen -public struct F26_S0 -{ - public let f0 : Double; -} - -public func swiftFunc26(a0: UInt16, a1: Double, a2: Int64, a3: F26_S0, a4: UInt8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a4); - return hasher.finalize() -} - -@frozen -public struct F27_S0_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F27_S0 -{ - public let f0 : UInt16; - public let f1 : F27_S0_S0; - public let f2 : Double; -} - -@frozen -public struct F27_S1 -{ - public let f0 : Int; - public let f1 : Int8; - public let f2 : Int16; - public let f3 : UInt8; -} - -@frozen -public struct F27_S2 -{ - public let f0 : UInt16; -} - -@frozen -public struct F27_S3 -{ - public let f0 : UInt64; - public let f1 : UInt32; -} - -@frozen -public struct F27_S4 -{ - public let f0 : UInt8; -} - -public func swiftFunc27(a0: F27_S0, a1: Double, a2: Double, a3: Int8, a4: Int8, a5: F27_S1, a6: Int16, a7: F27_S2, a8: Int8, a9: UInt16, a10: F27_S3, a11: F27_S4, a12: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1.f0); - hasher.combine(a0.f2); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a5.f1); - hasher.combine(a5.f2); - hasher.combine(a5.f3); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a10.f1); - hasher.combine(a11.f0); - hasher.combine(a12); - return hasher.finalize() -} - -@frozen -public struct F28_S0 -{ - public let f0 : Double; - public let f1 : Int16; - public let f2 : Double; - public let f3 : UInt64; -} - -@frozen -public struct F28_S1 -{ - public let f0 : Int; - public let f1 : UInt32; - public let f2 : UInt64; - public let f3 : Float; -} - -@frozen -public struct F28_S2 -{ - public let f0 : Double; - public let f1 : UInt64; -} - -@frozen -public struct F28_S3 -{ - public let f0 : Int16; - public let f1 : UInt64; - public let f2 : Double; - public let f3 : Int32; -} - -@frozen -public struct F28_S4 -{ - public let f0 : Int; -} - -public func swiftFunc28(a0: UInt8, a1: UInt16, a2: F28_S0, a3: F28_S1, a4: F28_S2, a5: UInt64, a6: Int32, a7: Int64, a8: Double, a9: UInt16, a10: F28_S3, a11: F28_S4, a12: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a3.f3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a10.f1); - hasher.combine(a10.f2); - hasher.combine(a10.f3); - hasher.combine(a11.f0); - hasher.combine(a12); - return hasher.finalize() -} - -@frozen -public struct F29_S0 -{ - public let f0 : Int32; - public let f1 : Float; - public let f2 : Int16; -} - -@frozen -public struct F29_S1 -{ - public let f0 : Int16; - public let f1 : Int8; - public let f2 : UInt; -} - -@frozen -public struct F29_S2 -{ - public let f0 : UInt16; -} - -@frozen -public struct F29_S3 -{ - public let f0 : Int64; - public let f1 : Int64; -} - -public func swiftFunc29(a0: Int8, a1: F29_S0, a2: Int32, a3: UInt, a4: F29_S1, a5: UInt64, a6: F29_S2, a7: Int16, a8: Int64, a9: UInt32, a10: UInt64, a11: Int, a12: F29_S3, a13: UInt8, a14: Int8, a15: Double) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12.f0); - hasher.combine(a12.f1); - hasher.combine(a13); - hasher.combine(a14); - hasher.combine(a15); - return hasher.finalize() -} - -@frozen -public struct F30_S0 -{ - public let f0 : UInt; - public let f1 : Float; -} - -@frozen -public struct F30_S1 -{ - public let f0 : UInt64; - public let f1 : UInt8; - public let f2 : Double; - public let f3 : Int; -} - -@frozen -public struct F30_S2_S0 -{ - public let f0 : Int16; - public let f1 : Int16; -} - -@frozen -public struct F30_S2_S1 -{ - public let f0 : Int64; -} - -@frozen -public struct F30_S2 -{ - public let f0 : F30_S2_S0; - public let f1 : F30_S2_S1; -} - -@frozen -public struct F30_S3 -{ - public let f0 : Int8; - public let f1 : UInt8; - public let f2 : UInt64; - public let f3 : UInt32; -} - -@frozen -public struct F30_S4 -{ - public let f0 : UInt16; -} - -public func swiftFunc30(a0: UInt16, a1: Int16, a2: UInt16, a3: F30_S0, a4: F30_S1, a5: F30_S2, a6: UInt64, a7: Int32, a8: UInt, a9: F30_S3, a10: UInt16, a11: F30_S4, a12: Int8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a4.f3); - hasher.combine(a5.f0.f0); - hasher.combine(a5.f0.f1); - hasher.combine(a5.f1.f0); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9.f0); - hasher.combine(a9.f1); - hasher.combine(a9.f2); - hasher.combine(a9.f3); - hasher.combine(a10); - hasher.combine(a11.f0); - hasher.combine(a12); - return hasher.finalize() -} - -@frozen -public struct F31_S0 -{ - public let f0 : Int; - public let f1 : Float; - public let f2 : UInt32; - public let f3 : Int; -} - -public func swiftFunc31(a0: Int64, a1: F31_S0, a2: UInt32, a3: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a2); - hasher.combine(a3); - return hasher.finalize() -} - -@frozen -public struct F32_S0 -{ - public let f0 : Int16; - public let f1 : Float; - public let f2 : Int64; -} - -@frozen -public struct F32_S1_S0 -{ - public let f0 : UInt; -} - -@frozen -public struct F32_S1 -{ - public let f0 : UInt8; - public let f1 : F32_S1_S0; -} - -@frozen -public struct F32_S2 -{ - public let f0 : UInt32; - public let f1 : UInt8; - public let f2 : UInt; -} - -@frozen -public struct F32_S3_S0 -{ - public let f0 : UInt; -} - -@frozen -public struct F32_S3 -{ - public let f0 : UInt64; - public let f1 : F32_S3_S0; - public let f2 : UInt64; -} - -@frozen -public struct F32_S4 -{ - public let f0 : Double; - public let f1 : Int64; - public let f2 : Int64; - public let f3 : Float; -} - -public func swiftFunc32(a0: UInt64, a1: F32_S0, a2: Double, a3: F32_S1, a4: F32_S2, a5: UInt64, a6: Float, a7: F32_S3, a8: F32_S4, a9: UInt32, a10: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1.f0); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1.f0); - hasher.combine(a7.f2); - hasher.combine(a8.f0); - hasher.combine(a8.f1); - hasher.combine(a8.f2); - hasher.combine(a8.f3); - hasher.combine(a9); - hasher.combine(a10); - return hasher.finalize() -} - -@frozen -public struct F33_S0 -{ - public let f0 : Int8; - public let f1 : UInt8; -} - -@frozen -public struct F33_S1 -{ - public let f0 : UInt16; - public let f1 : UInt8; - public let f2 : Int64; -} - -@frozen -public struct F33_S2_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F33_S2 -{ - public let f0 : F33_S2_S0; - public let f1 : UInt; - public let f2 : Float; - public let f3 : Double; - public let f4 : UInt16; -} - -@frozen -public struct F33_S3 -{ - public let f0 : UInt; -} - -public func swiftFunc33(a0: Float, a1: F33_S0, a2: UInt64, a3: Int64, a4: F33_S1, a5: UInt16, a6: UInt, a7: UInt16, a8: F33_S2, a9: F33_S3, a10: Int) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8.f0.f0); - hasher.combine(a8.f1); - hasher.combine(a8.f2); - hasher.combine(a8.f3); - hasher.combine(a8.f4); - hasher.combine(a9.f0); - hasher.combine(a10); - return hasher.finalize() -} - -@frozen -public struct F34_S0 -{ - public let f0 : UInt8; -} - -public func swiftFunc34(a0: Int64, a1: F34_S0, a2: UInt, a3: UInt, a4: UInt8, a5: Double) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F35_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F35_S1_S0 -{ - public let f0 : UInt16; - public let f1 : Int8; -} - -@frozen -public struct F35_S1 -{ - public let f0 : Int64; - public let f1 : F35_S1_S0; - public let f2 : Float; -} - -@frozen -public struct F35_S2 -{ - public let f0 : UInt64; - public let f1 : Int8; - public let f2 : UInt32; - public let f3 : Int64; -} - -@frozen -public struct F35_S3_S0_S0 -{ - public let f0 : UInt32; - public let f1 : UInt8; -} - -@frozen -public struct F35_S3_S0 -{ - public let f0 : UInt16; - public let f1 : F35_S3_S0_S0; - public let f2 : Double; -} - -@frozen -public struct F35_S3 -{ - public let f0 : F35_S3_S0; - public let f1 : UInt32; -} - -@frozen -public struct F35_S4 -{ - public let f0 : Float; -} - -public func swiftFunc35(a0: UInt8, a1: F35_S0, a2: UInt8, a3: UInt8, a4: F35_S1, a5: Int32, a6: F35_S2, a7: Int, a8: UInt32, a9: F35_S3, a10: F35_S4) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1.f0); - hasher.combine(a4.f1.f1); - hasher.combine(a4.f2); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9.f0.f0); - hasher.combine(a9.f0.f1.f0); - hasher.combine(a9.f0.f1.f1); - hasher.combine(a9.f0.f2); - hasher.combine(a9.f1); - hasher.combine(a10.f0); - return hasher.finalize() -} - -@frozen -public struct F36_S0 -{ - public let f0 : UInt64; - public let f1 : Int8; -} - -@frozen -public struct F36_S1 -{ - public let f0 : Int64; - public let f1 : UInt; - public let f2 : Int; - public let f3 : Int32; -} - -@frozen -public struct F36_S2 -{ - public let f0 : Int; -} - -@frozen -public struct F36_S3_S0 -{ - public let f0 : Float; -} - -@frozen -public struct F36_S3 -{ - public let f0 : Int64; - public let f1 : Int8; - public let f2 : F36_S3_S0; -} - -@frozen -public struct F36_S4 -{ - public let f0 : UInt; - public let f1 : Int64; - public let f2 : Double; - public let f3 : Double; -} - -@frozen -public struct F36_S5 -{ - public let f0 : UInt8; - public let f1 : UInt8; -} - -@frozen -public struct F36_S6 -{ - public let f0 : UInt16; -} - -public func swiftFunc36(a0: F36_S0, a1: Double, a2: UInt64, a3: F36_S1, a4: F36_S2, a5: F36_S3, a6: F36_S4, a7: Float, a8: F36_S5, a9: UInt8, a10: Double, a11: F36_S6) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a3.f3); - hasher.combine(a4.f0); - hasher.combine(a5.f0); - hasher.combine(a5.f1); - hasher.combine(a5.f2.f0); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a7); - hasher.combine(a8.f0); - hasher.combine(a8.f1); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11.f0); - return hasher.finalize() -} - -@frozen -public struct F37_S0 -{ - public let f0 : Int32; -} - -@frozen -public struct F37_S1 -{ - public let f0 : UInt32; - public let f1 : UInt32; - public let f2 : Float; -} - -@frozen -public struct F37_S2 -{ - public let f0 : Int32; - public let f1 : UInt32; - public let f2 : Double; - public let f3 : UInt; -} - -@frozen -public struct F37_S3_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F37_S3 -{ - public let f0 : F37_S3_S0; -} - -public func swiftFunc37(a0: Int, a1: UInt64, a2: UInt32, a3: Int32, a4: Int8, a5: UInt8, a6: UInt64, a7: F37_S0, a8: F37_S1, a9: Int16, a10: F37_S2, a11: UInt, a12: F37_S3, a13: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a8.f0); - hasher.combine(a8.f1); - hasher.combine(a8.f2); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a10.f1); - hasher.combine(a10.f2); - hasher.combine(a10.f3); - hasher.combine(a11); - hasher.combine(a12.f0.f0); - hasher.combine(a13); - return hasher.finalize() -} - -@frozen -public struct F38_S0 -{ - public let f0 : UInt16; - public let f1 : Int16; - public let f2 : Int16; -} - -@frozen -public struct F38_S1 -{ - public let f0 : Int32; -} - -@frozen -public struct F38_S2 -{ - public let f0 : UInt; -} - -public func swiftFunc38(a0: UInt32, a1: Int32, a2: F38_S0, a3: F38_S1, a4: F38_S2) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3.f0); - hasher.combine(a4.f0); - return hasher.finalize() -} - -@frozen -public struct F39_S0_S0 -{ - public let f0 : UInt; -} - -@frozen -public struct F39_S0_S1 -{ - public let f0 : Int32; -} - -@frozen -public struct F39_S0 -{ - public let f0 : Int; - public let f1 : Int64; - public let f2 : UInt32; - public let f3 : F39_S0_S0; - public let f4 : F39_S0_S1; -} - -@frozen -public struct F39_S1 -{ - public let f0 : UInt; - public let f1 : Double; -} - -public func swiftFunc39(a0: UInt, a1: UInt, a2: F39_S0, a3: F39_S1, a4: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3.f0); - hasher.combine(a2.f4.f0); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4); - return hasher.finalize() -} - -public func swiftFunc40(a0: Int32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - return hasher.finalize() -} - -@frozen -public struct F41_S0 -{ - public let f0 : Int16; - public let f1 : Float; - public let f2 : UInt16; -} - -@frozen -public struct F41_S1 -{ - public let f0 : UInt16; - public let f1 : UInt64; - public let f2 : Int8; - public let f3 : Float; - public let f4 : UInt64; -} - -@frozen -public struct F41_S2_S0_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F41_S2_S0 -{ - public let f0 : F41_S2_S0_S0; -} - -@frozen -public struct F41_S2 -{ - public let f0 : Int32; - public let f1 : Int16; - public let f2 : UInt64; - public let f3 : Float; - public let f4 : F41_S2_S0; -} - -public func swiftFunc41(a0: Float, a1: F41_S0, a2: F41_S1, a3: F41_S2, a4: UInt32, a5: UInt, a6: UInt32, a7: Int, a8: Int8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3); - hasher.combine(a2.f4); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a3.f3); - hasher.combine(a3.f4.f0.f0); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - return hasher.finalize() -} - -@frozen -public struct F42_S0 -{ - public let f0 : UInt32; - public let f1 : UInt64; - public let f2 : UInt64; -} - -@frozen -public struct F42_S1 -{ - public let f0 : Double; - public let f1 : Double; -} - -@frozen -public struct F42_S2_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F42_S2 -{ - public let f0 : UInt8; - public let f1 : Int64; - public let f2 : F42_S2_S0; - public let f3 : Int; -} - -@frozen -public struct F42_S3_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F42_S3 -{ - public let f0 : Float; - public let f1 : F42_S3_S0; -} - -@frozen -public struct F42_S4 -{ - public let f0 : UInt32; -} - -@frozen -public struct F42_S5_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F42_S5 -{ - public let f0 : F42_S5_S0; -} - -@frozen -public struct F42_S6 -{ - public let f0 : UInt; -} - -public func swiftFunc42(a0: F42_S0, a1: F42_S1, a2: UInt16, a3: F42_S2, a4: F42_S3, a5: F42_S4, a6: F42_S5, a7: F42_S6, a8: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2.f0); - hasher.combine(a3.f3); - hasher.combine(a4.f0); - hasher.combine(a4.f1.f0); - hasher.combine(a5.f0); - hasher.combine(a6.f0.f0); - hasher.combine(a7.f0); - hasher.combine(a8); - return hasher.finalize() -} - -@frozen -public struct F43_S0_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F43_S0 -{ - public let f0 : F43_S0_S0; -} - -public func swiftFunc43(a0: Int64, a1: UInt8, a2: Int8, a3: Float, a4: Int64, a5: Int, a6: F43_S0) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0.f0); - return hasher.finalize() -} - -@frozen -public struct F44_S0 -{ - public let f0 : UInt64; -} - -public func swiftFunc44(a0: F44_S0) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - return hasher.finalize() -} - -@frozen -public struct F45_S0 -{ - public let f0 : Double; - public let f1 : Int; -} - -@frozen -public struct F45_S1_S0 -{ - public let f0 : Double; -} - -@frozen -public struct F45_S1_S1 -{ - public let f0 : Float; -} - -@frozen -public struct F45_S1 -{ - public let f0 : UInt16; - public let f1 : Int8; - public let f2 : F45_S1_S0; - public let f3 : F45_S1_S1; -} - -@frozen -public struct F45_S2 -{ - public let f0 : UInt64; - public let f1 : Float; - public let f2 : UInt16; -} - -public func swiftFunc45(a0: F45_S0, a1: F45_S1, a2: F45_S2, a3: Int) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2.f0); - hasher.combine(a1.f3.f0); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3); - return hasher.finalize() -} - -@frozen -public struct F46_S0 -{ - public let f0 : Int64; - public let f1 : UInt8; - public let f2 : UInt; - public let f3 : Int8; -} - -@frozen -public struct F46_S1 -{ - public let f0 : UInt8; -} - -@frozen -public struct F46_S2 -{ - public let f0 : Int; -} - -@frozen -public struct F46_S3 -{ - public let f0 : UInt64; - public let f1 : Int64; -} - -@frozen -public struct F46_S4 -{ - public let f0 : Int16; - public let f1 : Int32; - public let f2 : UInt32; -} - -@frozen -public struct F46_S5 -{ - public let f0 : UInt64; -} - -public func swiftFunc46(a0: F46_S0, a1: F46_S1, a2: Int8, a3: Float, a4: F46_S2, a5: Int16, a6: F46_S3, a7: Int16, a8: Float, a9: F46_S4, a10: UInt16, a11: Float, a12: Int8, a13: F46_S5) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1.f0); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9.f0); - hasher.combine(a9.f1); - hasher.combine(a9.f2); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12); - hasher.combine(a13.f0); - return hasher.finalize() -} - -@frozen -public struct F47_S0_S0 -{ - public let f0 : UInt16; - public let f1 : Int8; -} - -@frozen -public struct F47_S0 -{ - public let f0 : F47_S0_S0; - public let f1 : UInt16; - public let f2 : UInt; - public let f3 : Int64; -} - -@frozen -public struct F47_S1 -{ - public let f0 : Int64; - public let f1 : UInt8; -} - -public func swiftFunc47(a0: Int, a1: F47_S0, a2: F47_S1, a3: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0.f0); - hasher.combine(a1.f0.f1); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a3); - return hasher.finalize() -} - -public func swiftFunc48(a0: Int8, a1: UInt32, a2: Int16, a3: Float, a4: Int, a5: Float, a6: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - return hasher.finalize() -} - -@frozen -public struct F49_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct F49_S1_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F49_S1_S1 -{ - public let f0 : UInt16; -} - -@frozen -public struct F49_S1 -{ - public let f0 : F49_S1_S0; - public let f1 : Int32; - public let f2 : F49_S1_S1; - public let f3 : UInt; -} - -@frozen -public struct F49_S2 -{ - public let f0 : UInt16; - public let f1 : UInt8; - public let f2 : Float; - public let f3 : Int64; -} - -@frozen -public struct F49_S3 -{ - public let f0 : Int32; - public let f1 : Float; -} - -@frozen -public struct F49_S4 -{ - public let f0 : UInt32; - public let f1 : Int; - public let f2 : Int; -} - -public func swiftFunc49(a0: UInt64, a1: UInt8, a2: F49_S0, a3: F49_S1, a4: UInt, a5: UInt32, a6: Double, a7: F49_S2, a8: F49_S3, a9: Int8, a10: F49_S4, a11: Int32, a12: UInt64, a13: UInt8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a3.f0.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2.f0); - hasher.combine(a3.f3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a7.f3); - hasher.combine(a8.f0); - hasher.combine(a8.f1); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a10.f1); - hasher.combine(a10.f2); - hasher.combine(a11); - hasher.combine(a12); - hasher.combine(a13); - return hasher.finalize() -} - -@frozen -public struct F50_S0 -{ - public let f0 : Int8; - public let f1 : Int16; - public let f2 : Int32; - public let f3 : UInt32; -} - -@frozen -public struct F50_S1 -{ - public let f0 : Int32; -} - -public func swiftFunc50(a0: F50_S0, a1: UInt8, a2: F50_S1) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1); - hasher.combine(a2.f0); - return hasher.finalize() -} - -public func swiftFunc51(a0: UInt16, a1: Int8, a2: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - return hasher.finalize() -} - -public func swiftFunc52(a0: UInt8, a1: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - return hasher.finalize() -} - -@frozen -public struct F53_S0_S0 -{ - public let f0 : Int64; - public let f1 : UInt; -} - -@frozen -public struct F53_S0 -{ - public let f0 : UInt64; - public let f1 : F53_S0_S0; - public let f2 : Int16; - public let f3 : UInt8; -} - -@frozen -public struct F53_S1_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F53_S1 -{ - public let f0 : F53_S1_S0; -} - -@frozen -public struct F53_S2 -{ - public let f0 : UInt8; - public let f1 : UInt64; - public let f2 : Double; -} - -public func swiftFunc53(a0: F53_S0, a1: UInt, a2: UInt64, a3: Float, a4: UInt32, a5: F53_S1, a6: F53_S2, a7: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1.f0); - hasher.combine(a0.f1.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0.f0); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a7); - return hasher.finalize() -} - -@frozen -public struct F54_S0_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F54_S0 -{ - public let f0 : F54_S0_S0; -} - -@frozen -public struct F54_S1 -{ - public let f0 : UInt32; -} - -public func swiftFunc54(a0: Int8, a1: Int32, a2: UInt32, a3: F54_S0, a4: Float, a5: UInt8, a6: F54_S1) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0.f0); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0); - return hasher.finalize() -} - -@frozen -public struct F55_S0 -{ - public let f0 : Double; -} - -public func swiftFunc55(a0: F55_S0) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - return hasher.finalize() -} - -@frozen -public struct F56_S0_S0 -{ - public let f0 : UInt8; -} - -@frozen -public struct F56_S0 -{ - public let f0 : Float; - public let f1 : F56_S0_S0; -} - -@frozen -public struct F56_S1_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F56_S1 -{ - public let f0 : F56_S1_S0; - public let f1 : Double; - public let f2 : UInt; - public let f3 : UInt32; -} - -@frozen -public struct F56_S2 -{ - public let f0 : Int16; - public let f1 : Int16; -} - -@frozen -public struct F56_S3 -{ - public let f0 : UInt16; -} - -@frozen -public struct F56_S4 -{ - public let f0 : UInt; -} - -public func swiftFunc56(a0: F56_S0, a1: F56_S1, a2: F56_S2, a3: F56_S3, a4: F56_S4) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1.f0); - hasher.combine(a1.f0.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a3.f0); - hasher.combine(a4.f0); - return hasher.finalize() -} - -@frozen -public struct F57_S0 -{ - public let f0 : Int8; - public let f1 : UInt32; -} - -@frozen -public struct F57_S1_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F57_S1_S1 -{ - public let f0 : UInt; -} - -@frozen -public struct F57_S1 -{ - public let f0 : F57_S1_S0; - public let f1 : F57_S1_S1; - public let f2 : Int16; -} - -@frozen -public struct F57_S2 -{ - public let f0 : UInt; -} - -public func swiftFunc57(a0: UInt32, a1: F57_S0, a2: F57_S1, a3: UInt, a4: F57_S2, a5: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a2.f0.f0); - hasher.combine(a2.f1.f0); - hasher.combine(a2.f2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F58_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F58_S1 -{ - public let f0 : UInt; - public let f1 : Int; - public let f2 : UInt; - public let f3 : UInt16; -} - -public func swiftFunc58(a0: UInt8, a1: UInt8, a2: Int, a3: F58_S0, a4: Float, a5: UInt64, a6: Int8, a7: F58_S1, a8: UInt16, a9: Int64, a10: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a7.f3); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - return hasher.finalize() -} - -@frozen -public struct F59_S0 -{ - public let f0 : UInt; - public let f1 : UInt8; - public let f2 : Float; - public let f3 : Int; -} - -@frozen -public struct F59_S1 -{ - public let f0 : UInt8; - public let f1 : Int32; -} - -@frozen -public struct F59_S2 -{ - public let f0 : Int; - public let f1 : UInt32; - public let f2 : Int8; -} - -@frozen -public struct F59_S3 -{ - public let f0 : Int8; - public let f1 : Float; - public let f2 : Int32; -} - -@frozen -public struct F59_S4_S0 -{ - public let f0 : UInt8; -} - -@frozen -public struct F59_S4 -{ - public let f0 : F59_S4_S0; -} - -public func swiftFunc59(a0: F59_S0, a1: Float, a2: UInt32, a3: F59_S1, a4: F59_S2, a5: UInt16, a6: Float, a7: Int, a8: Int, a9: UInt, a10: UInt, a11: Int16, a12: F59_S3, a13: F59_S4) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12.f0); - hasher.combine(a12.f1); - hasher.combine(a12.f2); - hasher.combine(a13.f0.f0); - return hasher.finalize() -} - -@frozen -public struct F60_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F60_S1 -{ - public let f0 : UInt32; -} - -public func swiftFunc60(a0: Int32, a1: Int8, a2: Int32, a3: UInt16, a4: Float, a5: F60_S0, a6: F60_S1) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a6.f0); - return hasher.finalize() -} - -@frozen -public struct F61_S0 -{ - public let f0 : UInt16; - public let f1 : Int32; - public let f2 : Int8; -} - -@frozen -public struct F61_S1 -{ - public let f0 : Double; - public let f1 : Int; -} - -@frozen -public struct F61_S2 -{ - public let f0 : Int; - public let f1 : Int8; - public let f2 : Float; - public let f3 : UInt16; - public let f4 : Float; -} - -@frozen -public struct F61_S3 -{ - public let f0 : UInt32; - public let f1 : UInt64; - public let f2 : UInt; - public let f3 : UInt; -} - -@frozen -public struct F61_S4_S0 -{ - public let f0 : UInt8; - public let f1 : UInt64; -} - -@frozen -public struct F61_S4 -{ - public let f0 : F61_S4_S0; - public let f1 : Int64; -} - -public func swiftFunc61(a0: F61_S0, a1: UInt8, a2: Float, a3: F61_S1, a4: Int8, a5: Int64, a6: F61_S2, a7: F61_S3, a8: F61_S4, a9: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a6.f4); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a7.f3); - hasher.combine(a8.f0.f0); - hasher.combine(a8.f0.f1); - hasher.combine(a8.f1); - hasher.combine(a9); - return hasher.finalize() -} - -@frozen -public struct F62_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F62_S1 -{ - public let f0 : Float; -} - -public func swiftFunc62(a0: F62_S0, a1: Int16, a2: Int32, a3: F62_S1) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - return hasher.finalize() -} - -@frozen -public struct F63_S0 -{ - public let f0 : Int; -} - -public func swiftFunc63(a0: F63_S0) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - return hasher.finalize() -} - -@frozen -public struct F64_S0 -{ - public let f0 : Double; - public let f1 : UInt16; - public let f2 : Int32; - public let f3 : Int; - public let f4 : Double; -} - -@frozen -public struct F64_S1 -{ - public let f0 : Int32; - public let f1 : Float; - public let f2 : UInt32; -} - -public func swiftFunc64(a0: Double, a1: F64_S0, a2: UInt8, a3: F64_S1, a4: Int32, a5: UInt64, a6: Int8, a7: Int8, a8: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a1.f4); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - return hasher.finalize() -} - -public func swiftFunc65(a0: Float, a1: Float, a2: UInt, a3: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - return hasher.finalize() -} - -@frozen -public struct F66_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F66_S1_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F66_S1 -{ - public let f0 : F66_S1_S0; - public let f1 : Float; -} - -@frozen -public struct F66_S2 -{ - public let f0 : Double; - public let f1 : UInt8; -} - -@frozen -public struct F66_S3 -{ - public let f0 : UInt; -} - -public func swiftFunc66(a0: F66_S0, a1: F66_S1, a2: F66_S2, a3: F66_S3) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a1.f0.f0); - hasher.combine(a1.f1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a3.f0); - return hasher.finalize() -} - -@frozen -public struct F67_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F67_S1_S0_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F67_S1_S0 -{ - public let f0 : F67_S1_S0_S0; -} - -@frozen -public struct F67_S1 -{ - public let f0 : F67_S1_S0; - public let f1 : UInt32; - public let f2 : Int16; -} - -public func swiftFunc67(a0: UInt64, a1: UInt32, a2: UInt16, a3: Int8, a4: F67_S0, a5: UInt64, a6: F67_S1, a7: UInt, a8: UInt64, a9: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5); - hasher.combine(a6.f0.f0.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - return hasher.finalize() -} - -@frozen -public struct F68_S0_S0_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F68_S0_S0 -{ - public let f0 : F68_S0_S0_S0; -} - -@frozen -public struct F68_S0 -{ - public let f0 : F68_S0_S0; -} - -@frozen -public struct F68_S1 -{ - public let f0 : UInt64; - public let f1 : UInt16; -} - -@frozen -public struct F68_S2 -{ - public let f0 : UInt; - public let f1 : Int; - public let f2 : UInt64; - public let f3 : Double; -} - -@frozen -public struct F68_S3 -{ - public let f0 : Int; - public let f1 : UInt32; - public let f2 : UInt32; - public let f3 : UInt; -} - -@frozen -public struct F68_S4 -{ - public let f0 : Int32; -} - -public func swiftFunc68(a0: UInt16, a1: Int64, a2: Int16, a3: UInt64, a4: Int8, a5: Int32, a6: UInt8, a7: F68_S0, a8: UInt8, a9: F68_S1, a10: Int16, a11: F68_S2, a12: Int16, a13: Int16, a14: F68_S3, a15: F68_S4) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0.f0.f0); - hasher.combine(a8); - hasher.combine(a9.f0); - hasher.combine(a9.f1); - hasher.combine(a10); - hasher.combine(a11.f0); - hasher.combine(a11.f1); - hasher.combine(a11.f2); - hasher.combine(a11.f3); - hasher.combine(a12); - hasher.combine(a13); - hasher.combine(a14.f0); - hasher.combine(a14.f1); - hasher.combine(a14.f2); - hasher.combine(a14.f3); - hasher.combine(a15.f0); - return hasher.finalize() -} - -@frozen -public struct F69_S0 -{ - public let f0 : UInt32; - public let f1 : UInt; -} - -@frozen -public struct F69_S1_S0_S0 -{ - public let f0 : UInt8; -} - -@frozen -public struct F69_S1_S0 -{ - public let f0 : F69_S1_S0_S0; - public let f1 : Int8; -} - -@frozen -public struct F69_S1 -{ - public let f0 : F69_S1_S0; - public let f1 : UInt; - public let f2 : Int; -} - -@frozen -public struct F69_S2 -{ - public let f0 : Float; - public let f1 : UInt32; - public let f2 : UInt16; - public let f3 : Int8; -} - -@frozen -public struct F69_S3 -{ - public let f0 : UInt8; - public let f1 : Double; -} - -@frozen -public struct F69_S4 -{ - public let f0 : Double; -} - -@frozen -public struct F69_S5 -{ - public let f0 : UInt64; -} - -public func swiftFunc69(a0: F69_S0, a1: F69_S1, a2: Int, a3: Int, a4: UInt16, a5: Int16, a6: Double, a7: F69_S2, a8: F69_S3, a9: F69_S4, a10: Int, a11: Int32, a12: F69_S5, a13: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1.f0.f0.f0); - hasher.combine(a1.f0.f1); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a7.f3); - hasher.combine(a8.f0); - hasher.combine(a8.f1); - hasher.combine(a9.f0); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12.f0); - hasher.combine(a13); - return hasher.finalize() -} - -@frozen -public struct F70_S0 -{ - public let f0 : Float; - public let f1 : Int64; -} - -@frozen -public struct F70_S1 -{ - public let f0 : UInt16; - public let f1 : Int8; - public let f2 : Int16; -} - -@frozen -public struct F70_S2 -{ - public let f0 : UInt16; -} - -@frozen -public struct F70_S3 -{ - public let f0 : UInt16; -} - -public func swiftFunc70(a0: UInt64, a1: F70_S0, a2: UInt16, a3: Int8, a4: Float, a5: F70_S1, a6: Int, a7: F70_S2, a8: F70_S3, a9: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a5.f1); - hasher.combine(a5.f2); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a8.f0); - hasher.combine(a9); - return hasher.finalize() -} - -@frozen -public struct F71_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F71_S1 -{ - public let f0 : UInt64; -} - -public func swiftFunc71(a0: Int64, a1: F71_S0, a2: Int8, a3: F71_S1, a4: Float, a5: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a4); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F72_S0_S0 -{ - public let f0 : Int; - public let f1 : Double; -} - -@frozen -public struct F72_S0 -{ - public let f0 : F72_S0_S0; - public let f1 : UInt32; -} - -@frozen -public struct F72_S1 -{ - public let f0 : Int; -} - -@frozen -public struct F72_S2 -{ - public let f0 : Double; -} - -public func swiftFunc72(a0: F72_S0, a1: F72_S1, a2: F72_S2) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0.f0); - hasher.combine(a0.f0.f1); - hasher.combine(a0.f1); - hasher.combine(a1.f0); - hasher.combine(a2.f0); - return hasher.finalize() -} - -public func swiftFunc73(a0: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - return hasher.finalize() -} - -@frozen -public struct F74_S0 -{ - public let f0 : UInt8; - public let f1 : UInt8; - public let f2 : Double; - public let f3 : UInt8; -} - -@frozen -public struct F74_S1 -{ - public let f0 : Int16; - public let f1 : UInt16; - public let f2 : Int64; - public let f3 : UInt; -} - -@frozen -public struct F74_S2 -{ - public let f0 : Int16; - public let f1 : Double; - public let f2 : Float; -} - -@frozen -public struct F74_S3 -{ - public let f0 : Int16; -} - -public func swiftFunc74(a0: F74_S0, a1: F74_S1, a2: Int32, a3: F74_S2, a4: Int, a5: Int64, a6: Int16, a7: Int32, a8: F74_S3, a9: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8.f0); - hasher.combine(a9); - return hasher.finalize() -} - -@frozen -public struct F75_S0_S0_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct F75_S0_S0 -{ - public let f0 : F75_S0_S0_S0; -} - -@frozen -public struct F75_S0 -{ - public let f0 : F75_S0_S0; - public let f1 : Double; - public let f2 : Int32; -} - -@frozen -public struct F75_S1_S0_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F75_S1_S0 -{ - public let f0 : UInt; - public let f1 : F75_S1_S0_S0; - public let f2 : Int64; -} - -@frozen -public struct F75_S1 -{ - public let f0 : F75_S1_S0; - public let f1 : Int; -} - -@frozen -public struct F75_S2 -{ - public let f0 : UInt64; -} - -public func swiftFunc75(a0: F75_S0, a1: Double, a2: Int, a3: UInt, a4: Int, a5: F75_S1, a6: F75_S2) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0.f0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0.f0); - hasher.combine(a5.f0.f1.f0); - hasher.combine(a5.f0.f2); - hasher.combine(a5.f1); - hasher.combine(a6.f0); - return hasher.finalize() -} - -@frozen -public struct F76_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F76_S1 -{ - public let f0 : UInt64; - public let f1 : Int32; - public let f2 : Int16; -} - -@frozen -public struct F76_S2 -{ - public let f0 : UInt32; -} - -@frozen -public struct F76_S3 -{ - public let f0 : Int; -} - -public func swiftFunc76(a0: Double, a1: Int64, a2: UInt16, a3: Float, a4: Float, a5: F76_S0, a6: Int16, a7: F76_S1, a8: Int64, a9: UInt64, a10: UInt16, a11: UInt8, a12: Int8, a13: Int, a14: Int64, a15: Int8, a16: Int8, a17: Int16, a18: UInt16, a19: F76_S2, a20: F76_S3) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a7.f1); - hasher.combine(a7.f2); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12); - hasher.combine(a13); - hasher.combine(a14); - hasher.combine(a15); - hasher.combine(a16); - hasher.combine(a17); - hasher.combine(a18); - hasher.combine(a19.f0); - hasher.combine(a20.f0); - return hasher.finalize() -} - -@frozen -public struct F77_S0_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct F77_S0 -{ - public let f0 : UInt64; - public let f1 : F77_S0_S0; - public let f2 : Int8; -} - -@frozen -public struct F77_S1 -{ - public let f0 : UInt64; - public let f1 : Int; - public let f2 : Int32; -} - -@frozen -public struct F77_S2_S0_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F77_S2_S0 -{ - public let f0 : F77_S2_S0_S0; -} - -@frozen -public struct F77_S2 -{ - public let f0 : F77_S2_S0; - public let f1 : Int16; - public let f2 : Int8; - public let f3 : UInt8; -} - -@frozen -public struct F77_S3 -{ - public let f0 : Int; - public let f1 : Int; - public let f2 : Int; - public let f3 : Int16; -} - -@frozen -public struct F77_S4 -{ - public let f0 : Double; - public let f1 : Int8; - public let f2 : UInt32; - public let f3 : Int16; - public let f4 : UInt32; -} - -@frozen -public struct F77_S5 -{ - public let f0 : UInt; -} - -public func swiftFunc77(a0: F77_S0, a1: Int16, a2: F77_S1, a3: UInt32, a4: F77_S2, a5: F77_S3, a6: F77_S4, a7: UInt64, a8: F77_S5, a9: UInt16, a10: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1.f0); - hasher.combine(a0.f2); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3); - hasher.combine(a4.f0.f0.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a4.f3); - hasher.combine(a5.f0); - hasher.combine(a5.f1); - hasher.combine(a5.f2); - hasher.combine(a5.f3); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a6.f4); - hasher.combine(a7); - hasher.combine(a8.f0); - hasher.combine(a9); - hasher.combine(a10); - return hasher.finalize() -} - -@frozen -public struct F78_S0 -{ - public let f0 : UInt16; - public let f1 : UInt; -} - -public func swiftFunc78(a0: F78_S0, a1: UInt64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1); - return hasher.finalize() -} - -@frozen -public struct F79_S0 -{ - public let f0 : Double; -} - -public func swiftFunc79(a0: UInt32, a1: F79_S0, a2: Int16, a3: Double) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a2); - hasher.combine(a3); - return hasher.finalize() -} - -@frozen -public struct F80_S0 -{ - public let f0 : UInt64; - public let f1 : Double; -} - -@frozen -public struct F80_S1_S0 -{ - public let f0 : UInt8; -} - -@frozen -public struct F80_S1 -{ - public let f0 : Int32; - public let f1 : UInt16; - public let f2 : UInt32; - public let f3 : F80_S1_S0; -} - -@frozen -public struct F80_S2 -{ - public let f0 : UInt64; - public let f1 : Int64; - public let f2 : UInt32; - public let f3 : UInt16; -} - -@frozen -public struct F80_S3_S0_S0 -{ - public let f0 : Int; - public let f1 : Int64; - public let f2 : UInt64; -} - -@frozen -public struct F80_S3_S0 -{ - public let f0 : F80_S3_S0_S0; - public let f1 : UInt32; -} - -@frozen -public struct F80_S3 -{ - public let f0 : F80_S3_S0; - public let f1 : Int32; -} - -@frozen -public struct F80_S4_S0 -{ - public let f0 : Float; -} - -@frozen -public struct F80_S4 -{ - public let f0 : F80_S4_S0; -} - -public func swiftFunc80(a0: F80_S0, a1: F80_S1, a2: UInt16, a3: Int64, a4: F80_S2, a5: Double, a6: UInt64, a7: Int32, a8: F80_S3, a9: F80_S4, a10: UInt8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3.f0); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a4.f2); - hasher.combine(a4.f3); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8.f0.f0.f0); - hasher.combine(a8.f0.f0.f1); - hasher.combine(a8.f0.f0.f2); - hasher.combine(a8.f0.f1); - hasher.combine(a8.f1); - hasher.combine(a9.f0.f0); - hasher.combine(a10); - return hasher.finalize() -} - -@frozen -public struct F81_S0 -{ - public let f0 : Double; - public let f1 : UInt64; - public let f2 : UInt32; - public let f3 : UInt8; - public let f4 : UInt8; -} - -@frozen -public struct F81_S1 -{ - public let f0 : UInt32; -} - -public func swiftFunc81(a0: F81_S0, a1: Int32, a2: Float, a3: F81_S1) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a0.f4); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3.f0); - return hasher.finalize() -} - -@frozen -public struct F82_S0 -{ - public let f0 : Int32; - public let f1 : Int16; - public let f2 : UInt64; - public let f3 : Int8; -} - -@frozen -public struct F82_S1_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F82_S1 -{ - public let f0 : Int; - public let f1 : Int32; - public let f2 : F82_S1_S0; -} - -@frozen -public struct F82_S2 -{ - public let f0 : Int; - public let f1 : Int64; - public let f2 : UInt32; - public let f3 : UInt16; - public let f4 : Int64; -} - -@frozen -public struct F82_S3 -{ - public let f0 : UInt8; -} - -public func swiftFunc82(a0: F82_S0, a1: F82_S1, a2: F82_S2, a3: UInt32, a4: Int, a5: F82_S3) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2.f0); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a2.f3); - hasher.combine(a2.f4); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - return hasher.finalize() -} - -@frozen -public struct F83_S0_S0 -{ - public let f0 : UInt8; -} - -@frozen -public struct F83_S0 -{ - public let f0 : F83_S0_S0; - public let f1 : Int; - public let f2 : Float; -} - -@frozen -public struct F83_S1_S0 -{ - public let f0 : Double; -} - -@frozen -public struct F83_S1_S1_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F83_S1_S1 -{ - public let f0 : F83_S1_S1_S0; -} - -@frozen -public struct F83_S1 -{ - public let f0 : UInt32; - public let f1 : F83_S1_S0; - public let f2 : F83_S1_S1; -} - -@frozen -public struct F83_S2 -{ - public let f0 : Int; -} - -public func swiftFunc83(a0: Float, a1: F83_S0, a2: F83_S1, a3: Int16, a4: Int, a5: Float, a6: F83_S2, a7: UInt16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a2.f0); - hasher.combine(a2.f1.f0); - hasher.combine(a2.f2.f0.f0); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a7); - return hasher.finalize() -} - -@frozen -public struct F84_S0 -{ - public let f0 : Int16; - public let f1 : Int8; - public let f2 : UInt16; - public let f3 : Int64; - public let f4 : Int16; -} - -@frozen -public struct F84_S1 -{ - public let f0 : Int32; -} - -@frozen -public struct F84_S2_S0 -{ - public let f0 : UInt8; - public let f1 : UInt64; -} - -@frozen -public struct F84_S2 -{ - public let f0 : UInt; - public let f1 : F84_S2_S0; - public let f2 : Int8; - public let f3 : Double; -} - -@frozen -public struct F84_S3 -{ - public let f0 : UInt32; -} - -@frozen -public struct F84_S4 -{ - public let f0 : Float; -} - -public func swiftFunc84(a0: F84_S0, a1: F84_S1, a2: UInt64, a3: F84_S2, a4: UInt32, a5: F84_S3, a6: UInt, a7: F84_S4, a8: UInt64, a9: UInt64, a10: UInt16, a11: Int16, a12: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a0.f4); - hasher.combine(a1.f0); - hasher.combine(a2); - hasher.combine(a3.f0); - hasher.combine(a3.f1.f0); - hasher.combine(a3.f1.f1); - hasher.combine(a3.f2); - hasher.combine(a3.f3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11); - hasher.combine(a12); - return hasher.finalize() -} - -@frozen -public struct F85_S0_S0_S0 -{ - public let f0 : Float; -} - -@frozen -public struct F85_S0_S0 -{ - public let f0 : Int32; - public let f1 : F85_S0_S0_S0; -} - -@frozen -public struct F85_S0 -{ - public let f0 : Float; - public let f1 : F85_S0_S0; - public let f2 : Int; - public let f3 : Int64; -} - -@frozen -public struct F85_S1 -{ - public let f0 : UInt32; - public let f1 : Int32; -} - -@frozen -public struct F85_S2 -{ - public let f0 : UInt; -} - -public func swiftFunc85(a0: F85_S0, a1: F85_S1, a2: F85_S2, a3: Int8, a4: UInt32, a5: Int16) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1.f0); - hasher.combine(a0.f1.f1.f0); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a2.f0); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F86_S0 -{ - public let f0 : Int32; - public let f1 : Int64; - public let f2 : Int32; - public let f3 : UInt16; -} - -@frozen -public struct F86_S1_S0 -{ - public let f0 : UInt; -} - -@frozen -public struct F86_S1 -{ - public let f0 : F86_S1_S0; - public let f1 : UInt16; -} - -@frozen -public struct F86_S2 -{ - public let f0 : UInt32; -} - -@frozen -public struct F86_S3 -{ - public let f0 : Int16; -} - -@frozen -public struct F86_S4 -{ - public let f0 : Int; -} - -@frozen -public struct F86_S5 -{ - public let f0 : Int16; -} - -public func swiftFunc86(a0: F86_S0, a1: Int, a2: Int, a3: UInt, a4: F86_S1, a5: F86_S2, a6: UInt64, a7: F86_S3, a8: F86_S4, a9: F86_S5) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0.f0); - hasher.combine(a4.f1); - hasher.combine(a5.f0); - hasher.combine(a6); - hasher.combine(a7.f0); - hasher.combine(a8.f0); - hasher.combine(a9.f0); - return hasher.finalize() -} - -@frozen -public struct F87_S0_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F87_S0 -{ - public let f0 : F87_S0_S0; - public let f1 : Float; - public let f2 : Int64; - public let f3 : Double; -} - -@frozen -public struct F87_S1 -{ - public let f0 : Int32; -} - -@frozen -public struct F87_S2_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct F87_S2 -{ - public let f0 : F87_S2_S0; -} - -@frozen -public struct F87_S3 -{ - public let f0 : Int32; -} - -public func swiftFunc87(a0: Int64, a1: F87_S0, a2: UInt, a3: UInt8, a4: Double, a5: Int16, a6: UInt64, a7: Double, a8: Float, a9: F87_S1, a10: Int64, a11: F87_S2, a12: F87_S3, a13: Float) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9.f0); - hasher.combine(a10); - hasher.combine(a11.f0.f0); - hasher.combine(a12.f0); - hasher.combine(a13); - return hasher.finalize() -} - -@frozen -public struct F88_S0 -{ - public let f0 : UInt8; - public let f1 : Int64; - public let f2 : UInt64; - public let f3 : Int; -} - -@frozen -public struct F88_S1 -{ - public let f0 : Int64; - public let f1 : UInt8; - public let f2 : UInt16; -} - -@frozen -public struct F88_S2 -{ - public let f0 : UInt32; -} - -@frozen -public struct F88_S3_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F88_S3 -{ - public let f0 : Int32; - public let f1 : F88_S3_S0; - public let f2 : Int8; - public let f3 : UInt16; -} - -@frozen -public struct F88_S4_S0 -{ - public let f0 : Float; -} - -@frozen -public struct F88_S4 -{ - public let f0 : UInt16; - public let f1 : UInt; - public let f2 : Int8; - public let f3 : Int; - public let f4 : F88_S4_S0; -} - -@frozen -public struct F88_S5 -{ - public let f0 : Float; -} - -@frozen -public struct F88_S6 -{ - public let f0 : UInt32; -} - -@frozen -public struct F88_S7_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F88_S7 -{ - public let f0 : F88_S7_S0; -} - -public func swiftFunc88(a0: F88_S0, a1: Int8, a2: F88_S1, a3: UInt64, a4: F88_S2, a5: F88_S3, a6: F88_S4, a7: Int16, a8: F88_S5, a9: F88_S6, a10: F88_S7) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a5.f0); - hasher.combine(a5.f1.f0); - hasher.combine(a5.f2); - hasher.combine(a5.f3); - hasher.combine(a6.f0); - hasher.combine(a6.f1); - hasher.combine(a6.f2); - hasher.combine(a6.f3); - hasher.combine(a6.f4.f0); - hasher.combine(a7); - hasher.combine(a8.f0); - hasher.combine(a9.f0); - hasher.combine(a10.f0.f0); - return hasher.finalize() -} - -@frozen -public struct F89_S0 -{ - public let f0 : UInt8; - public let f1 : Int8; -} - -@frozen -public struct F89_S1 -{ - public let f0 : Int32; -} - -@frozen -public struct F89_S2 -{ - public let f0 : UInt16; -} - -@frozen -public struct F89_S3 -{ - public let f0 : Double; - public let f1 : Double; -} - -@frozen -public struct F89_S4 -{ - public let f0 : UInt32; -} - -public func swiftFunc89(a0: F89_S0, a1: F89_S1, a2: F89_S2, a3: UInt8, a4: F89_S3, a5: F89_S4, a6: Int32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1.f0); - hasher.combine(a2.f0); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a5.f0); - hasher.combine(a6); - return hasher.finalize() -} - -@frozen -public struct F90_S0 -{ - public let f0 : UInt16; - public let f1 : Int; -} - -@frozen -public struct F90_S1_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F90_S1 -{ - public let f0 : F90_S1_S0; - public let f1 : UInt; - public let f2 : Double; -} - -@frozen -public struct F90_S2 -{ - public let f0 : UInt64; - public let f1 : Int; - public let f2 : UInt16; -} - -@frozen -public struct F90_S3_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct F90_S3 -{ - public let f0 : F90_S3_S0; -} - -@frozen -public struct F90_S4 -{ - public let f0 : Int64; -} - -public func swiftFunc90(a0: F90_S0, a1: Int8, a2: F90_S1, a3: F90_S2, a4: F90_S3, a5: UInt32, a6: F90_S4, a7: UInt8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a1); - hasher.combine(a2.f0.f0); - hasher.combine(a2.f1); - hasher.combine(a2.f2); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a3.f2); - hasher.combine(a4.f0.f0); - hasher.combine(a5); - hasher.combine(a6.f0); - hasher.combine(a7); - return hasher.finalize() -} - -@frozen -public struct F91_S0_S0 -{ - public let f0 : Int32; -} - -@frozen -public struct F91_S0 -{ - public let f0 : F91_S0_S0; - public let f1 : UInt32; - public let f2 : Int; -} - -public func swiftFunc91(a0: F91_S0, a1: UInt8) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a1); - return hasher.finalize() -} - -@frozen -public struct F92_S0 -{ - public let f0 : UInt16; - public let f1 : UInt16; -} - -@frozen -public struct F92_S1 -{ - public let f0 : UInt64; -} - -@frozen -public struct F92_S2 -{ - public let f0 : UInt64; - public let f1 : UInt64; -} - -@frozen -public struct F92_S3 -{ - public let f0 : UInt; -} - -public func swiftFunc92(a0: Int16, a1: UInt64, a2: UInt, a3: Int64, a4: F92_S0, a5: Int64, a6: Double, a7: UInt8, a8: Int8, a9: UInt32, a10: Int8, a11: F92_S1, a12: UInt32, a13: Float, a14: UInt64, a15: UInt8, a16: Int32, a17: UInt32, a18: F92_S2, a19: F92_S3) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4.f0); - hasher.combine(a4.f1); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10); - hasher.combine(a11.f0); - hasher.combine(a12); - hasher.combine(a13); - hasher.combine(a14); - hasher.combine(a15); - hasher.combine(a16); - hasher.combine(a17); - hasher.combine(a18.f0); - hasher.combine(a18.f1); - hasher.combine(a19.f0); - return hasher.finalize() -} - -@frozen -public struct F93_S0 -{ - public let f0 : Int32; - public let f1 : UInt; - public let f2 : Double; -} - -@frozen -public struct F93_S1 -{ - public let f0 : UInt32; -} - -@frozen -public struct F93_S2 -{ - public let f0 : Double; -} - -public func swiftFunc93(a0: F93_S0, a1: Int, a2: F93_S1, a3: Double, a4: F93_S2) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a1); - hasher.combine(a2.f0); - hasher.combine(a3); - hasher.combine(a4.f0); - return hasher.finalize() -} - -public func swiftFunc94(a0: UInt64, a1: Int32, a2: UInt64, a3: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - return hasher.finalize() -} - -@frozen -public struct F95_S0 -{ - public let f0 : Int64; -} - -public func swiftFunc95(a0: F95_S0, a1: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a1); - return hasher.finalize() -} - -@frozen -public struct F96_S0_S0 -{ - public let f0 : Int; -} - -@frozen -public struct F96_S0 -{ - public let f0 : UInt64; - public let f1 : Double; - public let f2 : Double; - public let f3 : F96_S0_S0; -} - -@frozen -public struct F96_S1_S0_S0 -{ - public let f0 : Double; -} - -@frozen -public struct F96_S1_S0 -{ - public let f0 : F96_S1_S0_S0; -} - -@frozen -public struct F96_S1 -{ - public let f0 : F96_S1_S0; -} - -@frozen -public struct F96_S2 -{ - public let f0 : UInt8; - public let f1 : Float; -} - -@frozen -public struct F96_S3 -{ - public let f0 : UInt16; -} - -@frozen -public struct F96_S4 -{ - public let f0 : Int; -} - -@frozen -public struct F96_S5_S0 -{ - public let f0 : UInt8; -} - -@frozen -public struct F96_S5 -{ - public let f0 : F96_S5_S0; -} - -@frozen -public struct F96_S6 -{ - public let f0 : UInt64; -} - -public func swiftFunc96(a0: UInt16, a1: F96_S0, a2: F96_S1, a3: F96_S2, a4: UInt16, a5: UInt64, a6: Int, a7: Int32, a8: Int16, a9: UInt, a10: F96_S3, a11: Int16, a12: Int, a13: Int8, a14: Int32, a15: UInt32, a16: F96_S4, a17: F96_S5, a18: F96_S6) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1.f0); - hasher.combine(a1.f1); - hasher.combine(a1.f2); - hasher.combine(a1.f3.f0); - hasher.combine(a2.f0.f0.f0); - hasher.combine(a3.f0); - hasher.combine(a3.f1); - hasher.combine(a4); - hasher.combine(a5); - hasher.combine(a6); - hasher.combine(a7); - hasher.combine(a8); - hasher.combine(a9); - hasher.combine(a10.f0); - hasher.combine(a11); - hasher.combine(a12); - hasher.combine(a13); - hasher.combine(a14); - hasher.combine(a15); - hasher.combine(a16.f0); - hasher.combine(a17.f0.f0); - hasher.combine(a18.f0); - return hasher.finalize() -} - -@frozen -public struct F97_S0 -{ - public let f0 : Float; - public let f1 : Float; - public let f2 : Int; - public let f3 : Int; - public let f4 : Int; -} - -public func swiftFunc97(a0: Int8, a1: Int32, a2: UInt8, a3: UInt32, a4: UInt8, a5: F97_S0, a6: Int8, a7: Int) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5.f0); - hasher.combine(a5.f1); - hasher.combine(a5.f2); - hasher.combine(a5.f3); - hasher.combine(a5.f4); - hasher.combine(a6); - hasher.combine(a7); - return hasher.finalize() -} - -@frozen -public struct F98_S0_S0_S0 -{ - public let f0 : Float; -} - -@frozen -public struct F98_S0_S0 -{ - public let f0 : UInt; - public let f1 : F98_S0_S0_S0; -} - -@frozen -public struct F98_S0 -{ - public let f0 : Int64; - public let f1 : F98_S0_S0; - public let f2 : UInt; -} - -public func swiftFunc98(a0: F98_S0, a1: UInt16, a2: UInt16, a3: Int16, a4: Int8, a5: UInt32) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1.f0); - hasher.combine(a0.f1.f1.f0); - hasher.combine(a0.f2); - hasher.combine(a1); - hasher.combine(a2); - hasher.combine(a3); - hasher.combine(a4); - hasher.combine(a5); - return hasher.finalize() -} - -@frozen -public struct F99_S0 -{ - public let f0 : UInt64; - public let f1 : UInt16; - public let f2 : Float; - public let f3 : UInt64; -} - -@frozen -public struct F99_S1_S0 -{ - public let f0 : UInt32; -} - -@frozen -public struct F99_S1 -{ - public let f0 : F99_S1_S0; -} - -public func swiftFunc99(a0: F99_S0, a1: Int8, a2: F99_S1, a3: Int64) -> Int { - var hasher = HasherFNV1a() - hasher.combine(a0.f0); - hasher.combine(a0.f1); - hasher.combine(a0.f2); - hasher.combine(a0.f3); - hasher.combine(a1); - hasher.combine(a2.f0.f0); - hasher.combine(a3); - return hasher.finalize() -} - diff --git a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs index d4b81bafcd4c8a..67a398d357e113 100644 --- a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs +++ b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs @@ -26,9 +26,6 @@ public class ErrorHandlingTests [DllImport(SwiftLib, EntryPoint = "$s18SwiftErrorHandling05getMyB7Message4from13messageLengthSPys6UInt16VGSgs0B0_p_s5Int32VztF")] public unsafe static extern void* GetErrorMessage(void* handle, out int length); - [DllImport(SwiftLib, EntryPoint = "$s18SwiftErrorHandling16freeStringBuffer6bufferySpys6UInt16VG_tF")] - public unsafe static extern void FreeErrorMessageBuffer(void* stringPtr); - [Fact] public unsafe static void TestSwiftErrorThrown() { @@ -102,7 +99,7 @@ private unsafe static string GetErrorMessageFromSwift(SwiftError error) { void* pointer = GetErrorMessage(error.Value, out int messageLength); string errorMessage = Marshal.PtrToStringUni((IntPtr)pointer, messageLength); - FreeErrorMessageBuffer(pointer); + NativeMemory.Free((void*)pointer); return errorMessage; } } diff --git a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj index 89eda99352fd20..a57cd84cf8842c 100644 --- a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj +++ b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj @@ -3,7 +3,7 @@ true true - + true diff --git a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift index 5058014a42ce3a..20022c0dba3e28 100644 --- a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift +++ b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift @@ -33,7 +33,3 @@ public func getMyErrorMessage(from error: Error, messageLength: inout Int32) -> } return nil } - -public func freeStringBuffer(buffer: UnsafeMutablePointer) { - buffer.deallocate() -} diff --git a/src/tests/Interop/Swift/SwiftInvalidCallConv/SwiftInvalidCallConv.csproj b/src/tests/Interop/Swift/SwiftInvalidCallConv/SwiftInvalidCallConv.csproj index 49be10b9393911..a57cd84cf8842c 100644 --- a/src/tests/Interop/Swift/SwiftInvalidCallConv/SwiftInvalidCallConv.csproj +++ b/src/tests/Interop/Swift/SwiftInvalidCallConv/SwiftInvalidCallConv.csproj @@ -3,10 +3,8 @@ true true - - true - true + true diff --git a/src/tests/Interop/Swift/SwiftRetAbiStress/CMakeLists.txt b/src/tests/Interop/Swift/SwiftRetAbiStress/CMakeLists.txt deleted file mode 100644 index 9f0e2a3423b2a2..00000000000000 --- a/src/tests/Interop/Swift/SwiftRetAbiStress/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -project(SwiftRetAbiStress) -include ("${CLR_INTEROP_TEST_ROOT}/Interop.cmake") - -set(SOURCE SwiftRetAbiStress) - -if (NOT SWIFT_COMPILER_TARGET AND CLR_CMAKE_TARGET_OSX) - set(SWIFT_PLATFORM "macosx") - set(SWIFT_PLATFORM_SUFFIX "") - set(SWIFT_DEPLOYMENT_TARGET ${CMAKE_OSX_DEPLOYMENT_TARGET}) - set(SWIFT_COMPILER_TARGET "${CMAKE_OSX_ARCHITECTURES}-apple-${SWIFT_PLATFORM}${SWIFT_DEPLOYMENT_TARGET}${SWIFT_PLATFORM_SUFFIX}") -endif() - -add_custom_target(${SOURCE} ALL - COMMAND xcrun swiftc -target ${SWIFT_COMPILER_TARGET} -emit-library ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}.swift -o ${CMAKE_CURRENT_BINARY_DIR}/lib${SOURCE}.dylib - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${SOURCE}.swift - COMMENT "Generating ${SOURCE} library" -) - -install(FILES ${CMAKE_CURRENT_BINARY_DIR}/lib${SOURCE}.dylib - DESTINATION bin -) diff --git a/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.cs b/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.cs deleted file mode 100644 index bbe4b8858d39cb..00000000000000 --- a/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.cs +++ /dev/null @@ -1,3916 +0,0 @@ -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.InteropServices.Swift; -using Xunit; - -public class SwiftRetAbiStress -{ - private const string SwiftLib = "libSwiftRetAbiStress.dylib"; - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S0 - { - public short F0; - public int F1; - public ulong F2; - - public S0(short f0, int f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func0AA2S0VyF")] - private static extern S0 SwiftRetFunc0(); - - [Fact] - public static void TestSwiftRetFunc0() - { - Console.Write("Running SwiftRetFunc0: "); - S0 val = SwiftRetFunc0(); - Assert.Equal((short)-17813, val.F0); - Assert.Equal((int)318006528, val.F1); - Assert.Equal((ulong)1195162122024233590, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S1 - { - public short F0; - public float F1; - public long F2; - public uint F3; - - public S1(short f0, float f1, long f2, uint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func1AA2S1VyF")] - private static extern S1 SwiftRetFunc1(); - - [Fact] - public static void TestSwiftRetFunc1() - { - Console.Write("Running SwiftRetFunc1: "); - S1 val = SwiftRetFunc1(); - Assert.Equal((short)-29793, val.F0); - Assert.Equal((float)7351779, val.F1); - Assert.Equal((long)133491708229548754, val.F2); - Assert.Equal((uint)665726990, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S2_S0 - { - public ulong F0; - - public S2_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S2 - { - public S2_S0 F0; - public byte F1; - public ushort F2; - public float F3; - public int F4; - - public S2(S2_S0 f0, byte f1, ushort f2, float f3, int f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func2AA2S2VyF")] - private static extern S2 SwiftRetFunc2(); - - [Fact] - public static void TestSwiftRetFunc2() - { - Console.Write("Running SwiftRetFunc2: "); - S2 val = SwiftRetFunc2(); - Assert.Equal((ulong)2153637757371267722, val.F0.F0); - Assert.Equal((byte)150, val.F1); - Assert.Equal((ushort)48920, val.F2); - Assert.Equal((float)3564327, val.F3); - Assert.Equal((int)1310569731, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S3 - { - public long F0; - public double F1; - public sbyte F2; - public int F3; - public ushort F4; - public byte F5; - public double F6; - - public S3(long f0, double f1, sbyte f2, int f3, ushort f4, byte f5, double f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func3AA2S3VyF")] - private static extern S3 SwiftRetFunc3(); - - [Fact] - public static void TestSwiftRetFunc3() - { - Console.Write("Running SwiftRetFunc3: "); - S3 val = SwiftRetFunc3(); - Assert.Equal((long)5610153900386943274, val.F0); - Assert.Equal((double)2431035148834736, val.F1); - Assert.Equal((sbyte)111, val.F2); - Assert.Equal((int)772269424, val.F3); - Assert.Equal((ushort)19240, val.F4); - Assert.Equal((byte)146, val.F5); - Assert.Equal((double)821805530740405, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S4 - { - public sbyte F0; - public uint F1; - public ulong F2; - public long F3; - - public S4(sbyte f0, uint f1, ulong f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func4AA2S4VyF")] - private static extern S4 SwiftRetFunc4(); - - [Fact] - public static void TestSwiftRetFunc4() - { - Console.Write("Running SwiftRetFunc4: "); - S4 val = SwiftRetFunc4(); - Assert.Equal((sbyte)125, val.F0); - Assert.Equal((uint)377073381, val.F1); - Assert.Equal((ulong)964784376430620335, val.F2); - Assert.Equal((long)5588038704850976624, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S5_S0 - { - public uint F0; - public double F1; - - public S5_S0(uint f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 49)] - struct S5 - { - public ulong F0; - public sbyte F1; - public nuint F2; - public S5_S0 F3; - public nint F4; - public byte F5; - - public S5(ulong f0, sbyte f1, nuint f2, S5_S0 f3, nint f4, byte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func5AA2S5VyF")] - private static extern S5 SwiftRetFunc5(); - - [Fact] - public static void TestSwiftRetFunc5() - { - Console.Write("Running SwiftRetFunc5: "); - S5 val = SwiftRetFunc5(); - Assert.Equal((ulong)5315019731968023493, val.F0); - Assert.Equal((sbyte)114, val.F1); - Assert.Equal((nuint)unchecked((nuint)1154655179105889397), val.F2); - Assert.Equal((uint)1468030771, val.F3.F0); - Assert.Equal((double)3066473182924818, val.F3.F1); - Assert.Equal((nint)unchecked((nint)6252650621827449809), val.F4); - Assert.Equal((byte)129, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct S6 - { - public int F0; - public short F1; - public long F2; - public ushort F3; - - public S6(int f0, short f1, long f2, ushort f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func6AA2S6VyF")] - private static extern S6 SwiftRetFunc6(); - - [Fact] - public static void TestSwiftRetFunc6() - { - Console.Write("Running SwiftRetFunc6: "); - S6 val = SwiftRetFunc6(); - Assert.Equal((int)743741783, val.F0); - Assert.Equal((short)-6821, val.F1); - Assert.Equal((long)5908745692727636656, val.F2); - Assert.Equal((ushort)64295, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S7_S0 - { - public nint F0; - - public S7_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S7 - { - public S7_S0 F0; - - public S7(S7_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func7AA2S7VyF")] - private static extern S7 SwiftRetFunc7(); - - [Fact] - public static void TestSwiftRetFunc7() - { - Console.Write("Running SwiftRetFunc7: "); - S7 val = SwiftRetFunc7(); - Assert.Equal((nint)unchecked((nint)7625368278886567558), val.F0.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S8 - { - public nint F0; - - public S8(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func8AA2S8VyF")] - private static extern S8 SwiftRetFunc8(); - - [Fact] - public static void TestSwiftRetFunc8() - { - Console.Write("Running SwiftRetFunc8: "); - S8 val = SwiftRetFunc8(); - Assert.Equal((nint)unchecked((nint)775279004683334365), val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S9_S0 - { - public short F0; - public int F1; - - public S9_S0(short f0, int f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct S9 - { - public uint F0; - public nint F1; - public S9_S0 F2; - public ushort F3; - - public S9(uint f0, nint f1, S9_S0 f2, ushort f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB5Func9AA2S9VyF")] - private static extern S9 SwiftRetFunc9(); - - [Fact] - public static void TestSwiftRetFunc9() - { - Console.Write("Running SwiftRetFunc9: "); - S9 val = SwiftRetFunc9(); - Assert.Equal((uint)1223030410, val.F0); - Assert.Equal((nint)unchecked((nint)4720638462358523954), val.F1); - Assert.Equal((short)30631, val.F2.F0); - Assert.Equal((int)1033774469, val.F2.F1); - Assert.Equal((ushort)64474, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S10 - { - public float F0; - public float F1; - - public S10(float f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func10AA3S10VyF")] - private static extern S10 SwiftRetFunc10(); - - [Fact] - public static void TestSwiftRetFunc10() - { - Console.Write("Running SwiftRetFunc10: "); - S10 val = SwiftRetFunc10(); - Assert.Equal((float)3276917, val.F0); - Assert.Equal((float)6694615, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 21)] - struct S11 - { - public double F0; - public nint F1; - public uint F2; - public sbyte F3; - - public S11(double f0, nint f1, uint f2, sbyte f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func11AA3S11VyF")] - private static extern S11 SwiftRetFunc11(); - - [Fact] - public static void TestSwiftRetFunc11() - { - Console.Write("Running SwiftRetFunc11: "); - S11 val = SwiftRetFunc11(); - Assert.Equal((double)938206348036312, val.F0); - Assert.Equal((nint)unchecked((nint)6559514243876905696), val.F1); - Assert.Equal((uint)1357772248, val.F2); - Assert.Equal((sbyte)59, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S12 - { - public double F0; - - public S12(double f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func12AA3S12VyF")] - private static extern S12 SwiftRetFunc12(); - - [Fact] - public static void TestSwiftRetFunc12() - { - Console.Write("Running SwiftRetFunc12: "); - S12 val = SwiftRetFunc12(); - Assert.Equal((double)1580503485222363, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S13 - { - public uint F0; - - public S13(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func13AA3S13VyF")] - private static extern S13 SwiftRetFunc13(); - - [Fact] - public static void TestSwiftRetFunc13() - { - Console.Write("Running SwiftRetFunc13: "); - S13 val = SwiftRetFunc13(); - Assert.Equal((uint)1381551558, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct S14_S0_S0 - { - public sbyte F0; - - public S14_S0_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct S14_S0 - { - public S14_S0_S0 F0; - - public S14_S0(S14_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct S14 - { - public int F0; - public ushort F1; - public sbyte F2; - public float F3; - public ulong F4; - public S14_S0 F5; - public sbyte F6; - - public S14(int f0, ushort f1, sbyte f2, float f3, ulong f4, S14_S0 f5, sbyte f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func14AA3S14VyF")] - private static extern S14 SwiftRetFunc14(); - - [Fact] - public static void TestSwiftRetFunc14() - { - Console.Write("Running SwiftRetFunc14: "); - S14 val = SwiftRetFunc14(); - Assert.Equal((int)1765691191, val.F0); - Assert.Equal((ushort)56629, val.F1); - Assert.Equal((sbyte)25, val.F2); - Assert.Equal((float)2944946, val.F3); - Assert.Equal((ulong)951929105049584033, val.F4); - Assert.Equal((sbyte)-30, val.F5.F0.F0); - Assert.Equal((sbyte)66, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct S15_S0 - { - public nuint F0; - public float F1; - - public S15_S0(nuint f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct S15 - { - public nint F0; - public S15_S0 F1; - public ushort F2; - public int F3; - - public S15(nint f0, S15_S0 f1, ushort f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func15AA3S15VyF")] - private static extern S15 SwiftRetFunc15(); - - [Fact] - public static void TestSwiftRetFunc15() - { - Console.Write("Running SwiftRetFunc15: "); - S15 val = SwiftRetFunc15(); - Assert.Equal((nint)unchecked((nint)2090703541638269172), val.F0); - Assert.Equal((nuint)unchecked((nuint)6408314016925514463), val.F1.F0); - Assert.Equal((float)6534515, val.F1.F1); - Assert.Equal((ushort)30438, val.F2); - Assert.Equal((int)1745811802, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 33)] - struct S16 - { - public uint F0; - public ulong F1; - public byte F2; - public int F3; - public nuint F4; - public sbyte F5; - - public S16(uint f0, ulong f1, byte f2, int f3, nuint f4, sbyte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func16AA3S16VyF")] - private static extern S16 SwiftRetFunc16(); - - [Fact] - public static void TestSwiftRetFunc16() - { - Console.Write("Running SwiftRetFunc16: "); - S16 val = SwiftRetFunc16(); - Assert.Equal((uint)585220635, val.F0); - Assert.Equal((ulong)4034210936973794153, val.F1); - Assert.Equal((byte)48, val.F2); - Assert.Equal((int)1155081155, val.F3); - Assert.Equal((nuint)unchecked((nuint)806384837403045657), val.F4); - Assert.Equal((sbyte)54, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 3)] - struct S17 - { - public byte F0; - public sbyte F1; - public byte F2; - - public S17(byte f0, sbyte f1, byte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func17AA3S17VyF")] - private static extern S17 SwiftRetFunc17(); - - [Fact] - public static void TestSwiftRetFunc17() - { - Console.Write("Running SwiftRetFunc17: "); - S17 val = SwiftRetFunc17(); - Assert.Equal((byte)23, val.F0); - Assert.Equal((sbyte)112, val.F1); - Assert.Equal((byte)15, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S18_S0 - { - public uint F0; - public float F1; - - public S18_S0(uint f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S18 - { - public S18_S0 F0; - public nint F1; - public int F2; - public ushort F3; - public short F4; - - public S18(S18_S0 f0, nint f1, int f2, ushort f3, short f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func18AA3S18VyF")] - private static extern S18 SwiftRetFunc18(); - - [Fact] - public static void TestSwiftRetFunc18() - { - Console.Write("Running SwiftRetFunc18: "); - S18 val = SwiftRetFunc18(); - Assert.Equal((uint)1964425016, val.F0.F0); - Assert.Equal((float)2767295, val.F0.F1); - Assert.Equal((nint)unchecked((nint)6016563774923595868), val.F1); - Assert.Equal((int)1648562735, val.F2); - Assert.Equal((ushort)378, val.F3); - Assert.Equal((short)-20536, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S19 - { - public byte F0; - public ushort F1; - public float F2; - public ulong F3; - public int F4; - - public S19(byte f0, ushort f1, float f2, ulong f3, int f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func19AA3S19VyF")] - private static extern S19 SwiftRetFunc19(); - - [Fact] - public static void TestSwiftRetFunc19() - { - Console.Write("Running SwiftRetFunc19: "); - S19 val = SwiftRetFunc19(); - Assert.Equal((byte)188, val.F0); - Assert.Equal((ushort)47167, val.F1); - Assert.Equal((float)6781297, val.F2); - Assert.Equal((ulong)8140268502944465472, val.F3); - Assert.Equal((int)708690468, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S20_S0 - { - public uint F0; - public float F1; - - public S20_S0(uint f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 9)] - struct S20 - { - public S20_S0 F0; - public byte F1; - - public S20(S20_S0 f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func20AA3S20VyF")] - private static extern S20 SwiftRetFunc20(); - - [Fact] - public static void TestSwiftRetFunc20() - { - Console.Write("Running SwiftRetFunc20: "); - S20 val = SwiftRetFunc20(); - Assert.Equal((uint)2019361333, val.F0.F0); - Assert.Equal((float)938975, val.F0.F1); - Assert.Equal((byte)192, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct S21_S0_S0 - { - public ushort F0; - - public S21_S0_S0(ushort f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct S21_S0 - { - public S21_S0_S0 F0; - - public S21_S0(S21_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 42)] - struct S21 - { - public double F0; - public double F1; - public nuint F2; - public nint F3; - public ulong F4; - public S21_S0 F5; - - public S21(double f0, double f1, nuint f2, nint f3, ulong f4, S21_S0 f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func21AA3S21VyF")] - private static extern S21 SwiftRetFunc21(); - - [Fact] - public static void TestSwiftRetFunc21() - { - Console.Write("Running SwiftRetFunc21: "); - S21 val = SwiftRetFunc21(); - Assert.Equal((double)1693878073402490, val.F0); - Assert.Equal((double)3392111340517811, val.F1); - Assert.Equal((nuint)unchecked((nuint)3584917502172813732), val.F2); - Assert.Equal((nint)unchecked((nint)665495086154608745), val.F3); - Assert.Equal((ulong)2918107814961929578, val.F4); - Assert.Equal((ushort)4634, val.F5.F0.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S22 - { - public uint F0; - - public S22(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func22AA3S22VyF")] - private static extern S22 SwiftRetFunc22(); - - [Fact] - public static void TestSwiftRetFunc22() - { - Console.Write("Running SwiftRetFunc22: "); - S22 val = SwiftRetFunc22(); - Assert.Equal((uint)640156952, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 41)] - struct S23 - { - public byte F0; - public short F1; - public ulong F2; - public nuint F3; - public nuint F4; - public ulong F5; - public byte F6; - - public S23(byte f0, short f1, ulong f2, nuint f3, nuint f4, ulong f5, byte f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func23AA3S23VyF")] - private static extern S23 SwiftRetFunc23(); - - [Fact] - public static void TestSwiftRetFunc23() - { - Console.Write("Running SwiftRetFunc23: "); - S23 val = SwiftRetFunc23(); - Assert.Equal((byte)122, val.F0); - Assert.Equal((short)28995, val.F1); - Assert.Equal((ulong)25673626033589541, val.F2); - Assert.Equal((nuint)unchecked((nuint)828363978755325884), val.F3); - Assert.Equal((nuint)unchecked((nuint)3065573182429720699), val.F4); - Assert.Equal((ulong)1484484917001276079, val.F5); - Assert.Equal((byte)209, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S24 - { - public ulong F0; - public ulong F1; - - public S24(ulong f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func24AA3S24VyF")] - private static extern S24 SwiftRetFunc24(); - - [Fact] - public static void TestSwiftRetFunc24() - { - Console.Write("Running SwiftRetFunc24: "); - S24 val = SwiftRetFunc24(); - Assert.Equal((ulong)2621245238416080387, val.F0); - Assert.Equal((ulong)6541787564638363256, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S25_S0 - { - public nint F0; - - public S25_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S25 - { - public sbyte F0; - public sbyte F1; - public byte F2; - public S25_S0 F3; - public uint F4; - - public S25(sbyte f0, sbyte f1, byte f2, S25_S0 f3, uint f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func25AA3S25VyF")] - private static extern S25 SwiftRetFunc25(); - - [Fact] - public static void TestSwiftRetFunc25() - { - Console.Write("Running SwiftRetFunc25: "); - S25 val = SwiftRetFunc25(); - Assert.Equal((sbyte)30, val.F0); - Assert.Equal((sbyte)-8, val.F1); - Assert.Equal((byte)168, val.F2); - Assert.Equal((nint)unchecked((nint)7601538494489501573), val.F3.F0); - Assert.Equal((uint)814523741, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S26 - { - public float F0; - - public S26(float f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func26AA3S26VyF")] - private static extern S26 SwiftRetFunc26(); - - [Fact] - public static void TestSwiftRetFunc26() - { - Console.Write("Running SwiftRetFunc26: "); - S26 val = SwiftRetFunc26(); - Assert.Equal((float)3681545, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 48)] - struct S27 - { - public long F0; - public double F1; - public sbyte F2; - public nint F3; - public short F4; - public long F5; - - public S27(long f0, double f1, sbyte f2, nint f3, short f4, long f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func27AA3S27VyF")] - private static extern S27 SwiftRetFunc27(); - - [Fact] - public static void TestSwiftRetFunc27() - { - Console.Write("Running SwiftRetFunc27: "); - S27 val = SwiftRetFunc27(); - Assert.Equal((long)4847421047018330189, val.F0); - Assert.Equal((double)3655171692392280, val.F1); - Assert.Equal((sbyte)46, val.F2); - Assert.Equal((nint)unchecked((nint)4476120319602257660), val.F3); - Assert.Equal((short)-6106, val.F4); - Assert.Equal((long)5756567968111212829, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S28_S0 - { - public double F0; - - public S28_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S28 - { - public float F0; - public short F1; - public S28_S0 F2; - public double F3; - public ulong F4; - - public S28(float f0, short f1, S28_S0 f2, double f3, ulong f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func28AA3S28VyF")] - private static extern S28 SwiftRetFunc28(); - - [Fact] - public static void TestSwiftRetFunc28() - { - Console.Write("Running SwiftRetFunc28: "); - S28 val = SwiftRetFunc28(); - Assert.Equal((float)3491512, val.F0); - Assert.Equal((short)5249, val.F1); - Assert.Equal((double)1107064327388314, val.F2.F0); - Assert.Equal((double)2170381648425673, val.F3); - Assert.Equal((ulong)5138313315157580943, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 34)] - struct S29 - { - public ushort F0; - public uint F1; - public short F2; - public int F3; - public int F4; - public ulong F5; - public short F6; - - public S29(ushort f0, uint f1, short f2, int f3, int f4, ulong f5, short f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func29AA3S29VyF")] - private static extern S29 SwiftRetFunc29(); - - [Fact] - public static void TestSwiftRetFunc29() - { - Console.Write("Running SwiftRetFunc29: "); - S29 val = SwiftRetFunc29(); - Assert.Equal((ushort)39000, val.F0); - Assert.Equal((uint)408611655, val.F1); - Assert.Equal((short)18090, val.F2); - Assert.Equal((int)351857085, val.F3); - Assert.Equal((int)1103441843, val.F4); - Assert.Equal((ulong)5162040247631126074, val.F5); - Assert.Equal((short)-27930, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S30_S0 - { - public sbyte F0; - public sbyte F1; - public int F2; - - public S30_S0(sbyte f0, sbyte f1, int f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S30_S1 - { - public float F0; - - public S30_S1(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S30 - { - public float F0; - public S30_S0 F1; - public S30_S1 F2; - public long F3; - - public S30(float f0, S30_S0 f1, S30_S1 f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func30AA3S30VyF")] - private static extern S30 SwiftRetFunc30(); - - [Fact] - public static void TestSwiftRetFunc30() - { - Console.Write("Running SwiftRetFunc30: "); - S30 val = SwiftRetFunc30(); - Assert.Equal((float)6492602, val.F0); - Assert.Equal((sbyte)76, val.F1.F0); - Assert.Equal((sbyte)-26, val.F1.F1); - Assert.Equal((int)1777644423, val.F1.F2); - Assert.Equal((float)6558571, val.F2.F0); - Assert.Equal((long)5879147675377398012, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 21)] - struct S31 - { - public long F0; - public ulong F1; - public ushort F2; - public ushort F3; - public sbyte F4; - - public S31(long f0, ulong f1, ushort f2, ushort f3, sbyte f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func31AA3S31VyF")] - private static extern S31 SwiftRetFunc31(); - - [Fact] - public static void TestSwiftRetFunc31() - { - Console.Write("Running SwiftRetFunc31: "); - S31 val = SwiftRetFunc31(); - Assert.Equal((long)4699402628739628277, val.F0); - Assert.Equal((ulong)7062790893852687562, val.F1); - Assert.Equal((ushort)28087, val.F2); - Assert.Equal((ushort)11088, val.F3); - Assert.Equal((sbyte)69, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S32 - { - public int F0; - public ulong F1; - public ulong F2; - public uint F3; - public short F4; - public ushort F5; - - public S32(int f0, ulong f1, ulong f2, uint f3, short f4, ushort f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func32AA3S32VyF")] - private static extern S32 SwiftRetFunc32(); - - [Fact] - public static void TestSwiftRetFunc32() - { - Console.Write("Running SwiftRetFunc32: "); - S32 val = SwiftRetFunc32(); - Assert.Equal((int)688805466, val.F0); - Assert.Equal((ulong)8860655326984381661, val.F1); - Assert.Equal((ulong)6943423675662271404, val.F2); - Assert.Equal((uint)196368476, val.F3); - Assert.Equal((short)14229, val.F4); - Assert.Equal((ushort)34635, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S33 - { - public ushort F0; - public uint F1; - public int F2; - public ushort F3; - public float F4; - public ulong F5; - public nint F6; - - public S33(ushort f0, uint f1, int f2, ushort f3, float f4, ulong f5, nint f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func33AA3S33VyF")] - private static extern S33 SwiftRetFunc33(); - - [Fact] - public static void TestSwiftRetFunc33() - { - Console.Write("Running SwiftRetFunc33: "); - S33 val = SwiftRetFunc33(); - Assert.Equal((ushort)9297, val.F0); - Assert.Equal((uint)7963252, val.F1); - Assert.Equal((int)556244690, val.F2); - Assert.Equal((ushort)19447, val.F3); - Assert.Equal((float)6930550, val.F4); - Assert.Equal((ulong)126294981263481729, val.F5); - Assert.Equal((nint)unchecked((nint)2540579257616511618), val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S34 - { - public long F0; - public uint F1; - public ulong F2; - - public S34(long f0, uint f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func34AA3S34VyF")] - private static extern S34 SwiftRetFunc34(); - - [Fact] - public static void TestSwiftRetFunc34() - { - Console.Write("Running SwiftRetFunc34: "); - S34 val = SwiftRetFunc34(); - Assert.Equal((long)5845561428743737556, val.F0); - Assert.Equal((uint)1358941228, val.F1); - Assert.Equal((ulong)3701080255861218446, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 34)] - struct S35 - { - public float F0; - public float F1; - public long F2; - public byte F3; - public double F4; - public ushort F5; - - public S35(float f0, float f1, long f2, byte f3, double f4, ushort f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func35AA3S35VyF")] - private static extern S35 SwiftRetFunc35(); - - [Fact] - public static void TestSwiftRetFunc35() - { - Console.Write("Running SwiftRetFunc35: "); - S35 val = SwiftRetFunc35(); - Assert.Equal((float)5982956, val.F0); - Assert.Equal((float)3675164, val.F1); - Assert.Equal((long)229451138397478297, val.F2); - Assert.Equal((byte)163, val.F3); - Assert.Equal((double)2925293762193390, val.F4); - Assert.Equal((ushort)5018, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S36 - { - public int F0; - public long F1; - public ulong F2; - - public S36(int f0, long f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func36AA3S36VyF")] - private static extern S36 SwiftRetFunc36(); - - [Fact] - public static void TestSwiftRetFunc36() - { - Console.Write("Running SwiftRetFunc36: "); - S36 val = SwiftRetFunc36(); - Assert.Equal((int)1915776502, val.F0); - Assert.Equal((long)2197655909333830531, val.F1); - Assert.Equal((ulong)6072941592567177049, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S37 - { - public byte F0; - public double F1; - - public S37(byte f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func37AA3S37VyF")] - private static extern S37 SwiftRetFunc37(); - - [Fact] - public static void TestSwiftRetFunc37() - { - Console.Write("Running SwiftRetFunc37: "); - S37 val = SwiftRetFunc37(); - Assert.Equal((byte)18, val.F0); - Assert.Equal((double)4063164371882658, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S38 - { - public nuint F0; - public long F1; - public byte F2; - public nuint F3; - - public S38(nuint f0, long f1, byte f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func38AA3S38VyF")] - private static extern S38 SwiftRetFunc38(); - - [Fact] - public static void TestSwiftRetFunc38() - { - Console.Write("Running SwiftRetFunc38: "); - S38 val = SwiftRetFunc38(); - Assert.Equal((nuint)unchecked((nuint)7389960750529773276), val.F0); - Assert.Equal((long)2725802169582362061, val.F1); - Assert.Equal((byte)2, val.F2); - Assert.Equal((nuint)unchecked((nuint)3659261019360356514), val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S39 - { - public int F0; - public int F1; - public nint F2; - public short F3; - public ushort F4; - - public S39(int f0, int f1, nint f2, short f3, ushort f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func39AA3S39VyF")] - private static extern S39 SwiftRetFunc39(); - - [Fact] - public static void TestSwiftRetFunc39() - { - Console.Write("Running SwiftRetFunc39: "); - S39 val = SwiftRetFunc39(); - Assert.Equal((int)50995691, val.F0); - Assert.Equal((int)1623216479, val.F1); - Assert.Equal((nint)unchecked((nint)2906650346451599789), val.F2); - Assert.Equal((short)28648, val.F3); - Assert.Equal((ushort)8278, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S40_S0 - { - public float F0; - public byte F1; - public sbyte F2; - public nuint F3; - public double F4; - - public S40_S0(float f0, byte f1, sbyte f2, nuint f3, double f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct S40 - { - public S40_S0 F0; - public short F1; - public short F2; - - public S40(S40_S0 f0, short f1, short f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func40AA3S40VyF")] - private static extern S40 SwiftRetFunc40(); - - [Fact] - public static void TestSwiftRetFunc40() - { - Console.Write("Running SwiftRetFunc40: "); - S40 val = SwiftRetFunc40(); - Assert.Equal((float)7087264, val.F0.F0); - Assert.Equal((byte)37, val.F0.F1); - Assert.Equal((sbyte)-5, val.F0.F2); - Assert.Equal((nuint)unchecked((nuint)479915249821490487), val.F0.F3); - Assert.Equal((double)144033730096589, val.F0.F4); - Assert.Equal((short)28654, val.F1); - Assert.Equal((short)16398, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S41 - { - public nuint F0; - public nuint F1; - - public S41(nuint f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func41AA3S41VyF")] - private static extern S41 SwiftRetFunc41(); - - [Fact] - public static void TestSwiftRetFunc41() - { - Console.Write("Running SwiftRetFunc41: "); - S41 val = SwiftRetFunc41(); - Assert.Equal((nuint)unchecked((nuint)7923718819069382599), val.F0); - Assert.Equal((nuint)unchecked((nuint)1539666179674725957), val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S42_S0 - { - public int F0; - - public S42_S0(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S42 - { - public uint F0; - public long F1; - public S42_S0 F2; - public nuint F3; - - public S42(uint f0, long f1, S42_S0 f2, nuint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func42AA3S42VyF")] - private static extern S42 SwiftRetFunc42(); - - [Fact] - public static void TestSwiftRetFunc42() - { - Console.Write("Running SwiftRetFunc42: "); - S42 val = SwiftRetFunc42(); - Assert.Equal((uint)1046060439, val.F0); - Assert.Equal((long)8249831314190867613, val.F1); - Assert.Equal((int)1097582349, val.F2.F0); - Assert.Equal((nuint)unchecked((nuint)2864677262092469436), val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S43_S0_S0 - { - public float F0; - - public S43_S0_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S43_S0 - { - public S43_S0_S0 F0; - - public S43_S0(S43_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 5)] - struct S43 - { - public S43_S0 F0; - public sbyte F1; - - public S43(S43_S0 f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func43AA3S43VyF")] - private static extern S43 SwiftRetFunc43(); - - [Fact] - public static void TestSwiftRetFunc43() - { - Console.Write("Running SwiftRetFunc43: "); - S43 val = SwiftRetFunc43(); - Assert.Equal((float)1586338, val.F0.F0.F0); - Assert.Equal((sbyte)104, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S44 - { - public byte F0; - public int F1; - public nint F2; - public uint F3; - - public S44(byte f0, int f1, nint f2, uint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func44AA3S44VyF")] - private static extern S44 SwiftRetFunc44(); - - [Fact] - public static void TestSwiftRetFunc44() - { - Console.Write("Running SwiftRetFunc44: "); - S44 val = SwiftRetFunc44(); - Assert.Equal((byte)94, val.F0); - Assert.Equal((int)1109076022, val.F1); - Assert.Equal((nint)unchecked((nint)3135595850598607828), val.F2); - Assert.Equal((uint)760084013, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S45_S0 - { - public long F0; - - public S45_S0(long f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S45 - { - public short F0; - public ulong F1; - public nint F2; - public S45_S0 F3; - - public S45(short f0, ulong f1, nint f2, S45_S0 f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func45AA3S45VyF")] - private static extern S45 SwiftRetFunc45(); - - [Fact] - public static void TestSwiftRetFunc45() - { - Console.Write("Running SwiftRetFunc45: "); - S45 val = SwiftRetFunc45(); - Assert.Equal((short)3071, val.F0); - Assert.Equal((ulong)5908138438609341766, val.F1); - Assert.Equal((nint)unchecked((nint)5870206722419946629), val.F2); - Assert.Equal((long)8128455876189744801, val.F3.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S46 - { - public short F0; - public sbyte F1; - public sbyte F2; - public uint F3; - public byte F4; - public int F5; - - public S46(short f0, sbyte f1, sbyte f2, uint f3, byte f4, int f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func46AA3S46VyF")] - private static extern S46 SwiftRetFunc46(); - - [Fact] - public static void TestSwiftRetFunc46() - { - Console.Write("Running SwiftRetFunc46: "); - S46 val = SwiftRetFunc46(); - Assert.Equal((short)14794, val.F0); - Assert.Equal((sbyte)60, val.F1); - Assert.Equal((sbyte)-77, val.F2); - Assert.Equal((uint)653898879, val.F3); - Assert.Equal((byte)224, val.F4); - Assert.Equal((int)266602433, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct S47_S0 - { - public sbyte F0; - - public S47_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 9)] - struct S47 - { - public double F0; - public S47_S0 F1; - - public S47(double f0, S47_S0 f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func47AA3S47VyF")] - private static extern S47 SwiftRetFunc47(); - - [Fact] - public static void TestSwiftRetFunc47() - { - Console.Write("Running SwiftRetFunc47: "); - S47 val = SwiftRetFunc47(); - Assert.Equal((double)3195976594911793, val.F0); - Assert.Equal((sbyte)-91, val.F1.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S48 - { - public nint F0; - - public S48(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func48AA3S48VyF")] - private static extern S48 SwiftRetFunc48(); - - [Fact] - public static void TestSwiftRetFunc48() - { - Console.Write("Running SwiftRetFunc48: "); - S48 val = SwiftRetFunc48(); - Assert.Equal((nint)unchecked((nint)778504172538154682), val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S49_S0_S0 - { - public ulong F0; - - public S49_S0_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S49_S0 - { - public S49_S0_S0 F0; - - public S49_S0(S49_S0_S0 f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S49 - { - public ulong F0; - public S49_S0 F1; - public sbyte F2; - public double F3; - public uint F4; - public uint F5; - - public S49(ulong f0, S49_S0 f1, sbyte f2, double f3, uint f4, uint f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func49AA3S49VyF")] - private static extern S49 SwiftRetFunc49(); - - [Fact] - public static void TestSwiftRetFunc49() - { - Console.Write("Running SwiftRetFunc49: "); - S49 val = SwiftRetFunc49(); - Assert.Equal((ulong)4235011519458710874, val.F0); - Assert.Equal((ulong)3120420438742285733, val.F1.F0.F0); - Assert.Equal((sbyte)-8, val.F2); - Assert.Equal((double)1077419570643725, val.F3); - Assert.Equal((uint)1985303212, val.F4); - Assert.Equal((uint)264580506, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S50 - { - public int F0; - - public S50(int f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func50AA3S50VyF")] - private static extern S50 SwiftRetFunc50(); - - [Fact] - public static void TestSwiftRetFunc50() - { - Console.Write("Running SwiftRetFunc50: "); - S50 val = SwiftRetFunc50(); - Assert.Equal((int)1043912405, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S51_S0_S0_S0 - { - public float F0; - - public S51_S0_S0_S0(float f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 6)] - struct S51_S0_S0 - { - public S51_S0_S0_S0 F0; - public short F1; - - public S51_S0_S0(S51_S0_S0_S0 f0, short f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S51_S0 - { - public double F0; - public S51_S0_S0 F1; - public byte F2; - public long F3; - - public S51_S0(double f0, S51_S0_S0 f1, byte f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S51 - { - public S51_S0 F0; - public double F1; - - public S51(S51_S0 f0, double f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func51AA3S51VyF")] - private static extern S51 SwiftRetFunc51(); - - [Fact] - public static void TestSwiftRetFunc51() - { - Console.Write("Running SwiftRetFunc51: "); - S51 val = SwiftRetFunc51(); - Assert.Equal((double)3266680719186600, val.F0.F0); - Assert.Equal((float)428247, val.F0.F1.F0.F0); - Assert.Equal((short)-24968, val.F0.F1.F1); - Assert.Equal((byte)76, val.F0.F2); - Assert.Equal((long)183022772513065490, val.F0.F3); - Assert.Equal((double)2661928101793033, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 41)] - struct S52 - { - public uint F0; - public long F1; - public uint F2; - public ulong F3; - public nint F4; - public sbyte F5; - - public S52(uint f0, long f1, uint f2, ulong f3, nint f4, sbyte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func52AA3S52VyF")] - private static extern S52 SwiftRetFunc52(); - - [Fact] - public static void TestSwiftRetFunc52() - { - Console.Write("Running SwiftRetFunc52: "); - S52 val = SwiftRetFunc52(); - Assert.Equal((uint)1812191671, val.F0); - Assert.Equal((long)6594574760089190928, val.F1); - Assert.Equal((uint)831147243, val.F2); - Assert.Equal((ulong)3301835731003365248, val.F3); - Assert.Equal((nint)unchecked((nint)5382332538247340743), val.F4); - Assert.Equal((sbyte)-77, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S53_S0 - { - public sbyte F0; - public nuint F1; - - public S53_S0(sbyte f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 37)] - struct S53 - { - public S53_S0 F0; - public int F1; - public long F2; - public float F3; - public sbyte F4; - - public S53(S53_S0 f0, int f1, long f2, float f3, sbyte f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func53AA3S53VyF")] - private static extern S53 SwiftRetFunc53(); - - [Fact] - public static void TestSwiftRetFunc53() - { - Console.Write("Running SwiftRetFunc53: "); - S53 val = SwiftRetFunc53(); - Assert.Equal((sbyte)-123, val.F0.F0); - Assert.Equal((nuint)unchecked((nuint)3494916243607193741), val.F0.F1); - Assert.Equal((int)1406699798, val.F1); - Assert.Equal((long)4018943158751734338, val.F2); - Assert.Equal((float)1084415, val.F3); - Assert.Equal((sbyte)-8, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S54_S0 - { - public double F0; - - public S54_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S54 - { - public nint F0; - public nint F1; - public S54_S0 F2; - public long F3; - - public S54(nint f0, nint f1, S54_S0 f2, long f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func54AA3S54VyF")] - private static extern S54 SwiftRetFunc54(); - - [Fact] - public static void TestSwiftRetFunc54() - { - Console.Write("Running SwiftRetFunc54: "); - S54 val = SwiftRetFunc54(); - Assert.Equal((nint)unchecked((nint)8623517456704997133), val.F0); - Assert.Equal((nint)unchecked((nint)1521939500434086364), val.F1); - Assert.Equal((double)3472783299414218, val.F2.F0); - Assert.Equal((long)4761507229870258916, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 22)] - struct S55 - { - public short F0; - public uint F1; - public long F2; - public uint F3; - public sbyte F4; - public byte F5; - - public S55(short f0, uint f1, long f2, uint f3, sbyte f4, byte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func55AA3S55VyF")] - private static extern S55 SwiftRetFunc55(); - - [Fact] - public static void TestSwiftRetFunc55() - { - Console.Write("Running SwiftRetFunc55: "); - S55 val = SwiftRetFunc55(); - Assert.Equal((short)-28051, val.F0); - Assert.Equal((uint)1759912152, val.F1); - Assert.Equal((long)2038322238348454200, val.F2); - Assert.Equal((uint)601094102, val.F3); - Assert.Equal((sbyte)5, val.F4); - Assert.Equal((byte)75, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S56 - { - public ulong F0; - public float F1; - public sbyte F2; - public int F3; - - public S56(ulong f0, float f1, sbyte f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func56AA3S56VyF")] - private static extern S56 SwiftRetFunc56(); - - [Fact] - public static void TestSwiftRetFunc56() - { - Console.Write("Running SwiftRetFunc56: "); - S56 val = SwiftRetFunc56(); - Assert.Equal((ulong)6313168909786453069, val.F0); - Assert.Equal((float)6254558, val.F1); - Assert.Equal((sbyte)115, val.F2); - Assert.Equal((int)847834891, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S57 - { - public nuint F0; - public short F1; - public sbyte F2; - public int F3; - - public S57(nuint f0, short f1, sbyte f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func57AA3S57VyF")] - private static extern S57 SwiftRetFunc57(); - - [Fact] - public static void TestSwiftRetFunc57() - { - Console.Write("Running SwiftRetFunc57: "); - S57 val = SwiftRetFunc57(); - Assert.Equal((nuint)unchecked((nuint)546304219852233452), val.F0); - Assert.Equal((short)-27416, val.F1); - Assert.Equal((sbyte)47, val.F2); - Assert.Equal((int)1094575684, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S58 - { - public ulong F0; - public ulong F1; - - public S58(ulong f0, ulong f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func58AA3S58VyF")] - private static extern S58 SwiftRetFunc58(); - - [Fact] - public static void TestSwiftRetFunc58() - { - Console.Write("Running SwiftRetFunc58: "); - S58 val = SwiftRetFunc58(); - Assert.Equal((ulong)4612004722568513699, val.F0); - Assert.Equal((ulong)2222525519606580195, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 41)] - struct S59 - { - public sbyte F0; - public nuint F1; - public nint F2; - public sbyte F3; - public long F4; - public byte F5; - - public S59(sbyte f0, nuint f1, nint f2, sbyte f3, long f4, byte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func59AA3S59VyF")] - private static extern S59 SwiftRetFunc59(); - - [Fact] - public static void TestSwiftRetFunc59() - { - Console.Write("Running SwiftRetFunc59: "); - S59 val = SwiftRetFunc59(); - Assert.Equal((sbyte)-92, val.F0); - Assert.Equal((nuint)unchecked((nuint)7281011081566942937), val.F1); - Assert.Equal((nint)unchecked((nint)8203439771560005792), val.F2); - Assert.Equal((sbyte)103, val.F3); - Assert.Equal((long)1003386607251132236, val.F4); - Assert.Equal((byte)6, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S60 - { - public ulong F0; - public nint F1; - - public S60(ulong f0, nint f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func60AA3S60VyF")] - private static extern S60 SwiftRetFunc60(); - - [Fact] - public static void TestSwiftRetFunc60() - { - Console.Write("Running SwiftRetFunc60: "); - S60 val = SwiftRetFunc60(); - Assert.Equal((ulong)6922353269487057763, val.F0); - Assert.Equal((nint)unchecked((nint)103032455997325768), val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 20)] - struct S61_S0 - { - public long F0; - public long F1; - public float F2; - - public S61_S0(long f0, long f1, float f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 36)] - struct S61 - { - public ulong F0; - public S61_S0 F1; - public short F2; - public int F3; - - public S61(ulong f0, S61_S0 f1, short f2, int f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func61AA3S61VyF")] - private static extern S61 SwiftRetFunc61(); - - [Fact] - public static void TestSwiftRetFunc61() - { - Console.Write("Running SwiftRetFunc61: "); - S61 val = SwiftRetFunc61(); - Assert.Equal((ulong)3465845922566501572, val.F0); - Assert.Equal((long)8266662359091888314, val.F1.F0); - Assert.Equal((long)7511705648638703076, val.F1.F1); - Assert.Equal((float)535470, val.F1.F2); - Assert.Equal((short)-5945, val.F2); - Assert.Equal((int)523043523, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S62_S0_S0 - { - public nint F0; - - public S62_S0_S0(nint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S62_S0 - { - public ushort F0; - public short F1; - public ushort F2; - public S62_S0_S0 F3; - - public S62_S0(ushort f0, short f1, ushort f2, S62_S0_S0 f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct S62 - { - public S62_S0 F0; - public nint F1; - public ushort F2; - - public S62(S62_S0 f0, nint f1, ushort f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func62AA3S62VyF")] - private static extern S62 SwiftRetFunc62(); - - [Fact] - public static void TestSwiftRetFunc62() - { - Console.Write("Running SwiftRetFunc62: "); - S62 val = SwiftRetFunc62(); - Assert.Equal((ushort)50789, val.F0.F0); - Assert.Equal((short)30245, val.F0.F1); - Assert.Equal((ushort)35063, val.F0.F2); - Assert.Equal((nint)unchecked((nint)3102684963408623932), val.F0.F3.F0); - Assert.Equal((nint)unchecked((nint)792877586576090769), val.F1); - Assert.Equal((ushort)24697, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S63 - { - public double F0; - public nint F1; - public double F2; - public sbyte F3; - public float F4; - - public S63(double f0, nint f1, double f2, sbyte f3, float f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func63AA3S63VyF")] - private static extern S63 SwiftRetFunc63(); - - [Fact] - public static void TestSwiftRetFunc63() - { - Console.Write("Running SwiftRetFunc63: "); - S63 val = SwiftRetFunc63(); - Assert.Equal((double)4097323000009314, val.F0); - Assert.Equal((nint)unchecked((nint)4162427097168837193), val.F1); - Assert.Equal((double)140736061437152, val.F2); - Assert.Equal((sbyte)-59, val.F3); - Assert.Equal((float)7331757, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S64_S0 - { - public ulong F0; - - public S64_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S64 - { - public S64_S0 F0; - public ulong F1; - public long F2; - public nint F3; - - public S64(S64_S0 f0, ulong f1, long f2, nint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func64AA3S64VyF")] - private static extern S64 SwiftRetFunc64(); - - [Fact] - public static void TestSwiftRetFunc64() - { - Console.Write("Running SwiftRetFunc64: "); - S64 val = SwiftRetFunc64(); - Assert.Equal((ulong)2624461610177878495, val.F0.F0); - Assert.Equal((ulong)5222178027019975511, val.F1); - Assert.Equal((long)9006949357929457355, val.F2); - Assert.Equal((nint)unchecked((nint)7966680593035770540), val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S65 - { - public nint F0; - public double F1; - public ushort F2; - public short F3; - public byte F4; - public int F5; - public ulong F6; - - public S65(nint f0, double f1, ushort f2, short f3, byte f4, int f5, ulong f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func65AA3S65VyF")] - private static extern S65 SwiftRetFunc65(); - - [Fact] - public static void TestSwiftRetFunc65() - { - Console.Write("Running SwiftRetFunc65: "); - S65 val = SwiftRetFunc65(); - Assert.Equal((nint)unchecked((nint)6080968957098434687), val.F0); - Assert.Equal((double)3067343828504927, val.F1); - Assert.Equal((ushort)56887, val.F2); - Assert.Equal((short)804, val.F3); - Assert.Equal((byte)235, val.F4); - Assert.Equal((int)121742660, val.F5); - Assert.Equal((ulong)9218677163034827308, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S66 - { - public sbyte F0; - public ulong F1; - public uint F2; - public ulong F3; - public ulong F4; - - public S66(sbyte f0, ulong f1, uint f2, ulong f3, ulong f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func66AA3S66VyF")] - private static extern S66 SwiftRetFunc66(); - - [Fact] - public static void TestSwiftRetFunc66() - { - Console.Write("Running SwiftRetFunc66: "); - S66 val = SwiftRetFunc66(); - Assert.Equal((sbyte)-16, val.F0); - Assert.Equal((ulong)7967447403042597794, val.F1); - Assert.Equal((uint)2029697750, val.F2); - Assert.Equal((ulong)4180031087394830849, val.F3); - Assert.Equal((ulong)5847795120921557969, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S67_S0 - { - public ulong F0; - - public S67_S0(ulong f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 33)] - struct S67 - { - public S67_S0 F0; - public byte F1; - public ushort F2; - public ulong F3; - public ulong F4; - public sbyte F5; - - public S67(S67_S0 f0, byte f1, ushort f2, ulong f3, ulong f4, sbyte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func67AA3S67VyF")] - private static extern S67 SwiftRetFunc67(); - - [Fact] - public static void TestSwiftRetFunc67() - { - Console.Write("Running SwiftRetFunc67: "); - S67 val = SwiftRetFunc67(); - Assert.Equal((ulong)4844204675254434929, val.F0.F0); - Assert.Equal((byte)135, val.F1); - Assert.Equal((ushort)13969, val.F2); - Assert.Equal((ulong)4897129719050177731, val.F3); - Assert.Equal((ulong)7233638107485862921, val.F4); - Assert.Equal((sbyte)-11, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S68_S0 - { - public double F0; - - public S68_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 37)] - struct S68 - { - public int F0; - public ulong F1; - public uint F2; - public S68_S0 F3; - public int F4; - public sbyte F5; - - public S68(int f0, ulong f1, uint f2, S68_S0 f3, int f4, sbyte f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func68AA3S68VyF")] - private static extern S68 SwiftRetFunc68(); - - [Fact] - public static void TestSwiftRetFunc68() - { - Console.Write("Running SwiftRetFunc68: "); - S68 val = SwiftRetFunc68(); - Assert.Equal((int)1708606840, val.F0); - Assert.Equal((ulong)1768121573985581212, val.F1); - Assert.Equal((uint)1033319213, val.F2); - Assert.Equal((double)2741322436867931, val.F3.F0); - Assert.Equal((int)955320338, val.F4); - Assert.Equal((sbyte)12, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S69 - { - public uint F0; - - public S69(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func69AA3S69VyF")] - private static extern S69 SwiftRetFunc69(); - - [Fact] - public static void TestSwiftRetFunc69() - { - Console.Write("Running SwiftRetFunc69: "); - S69 val = SwiftRetFunc69(); - Assert.Equal((uint)2092746473, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S70 - { - public byte F0; - public float F1; - - public S70(byte f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func70AA3S70VyF")] - private static extern S70 SwiftRetFunc70(); - - [Fact] - public static void TestSwiftRetFunc70() - { - Console.Write("Running SwiftRetFunc70: "); - S70 val = SwiftRetFunc70(); - Assert.Equal((byte)76, val.F0); - Assert.Equal((float)4138467, val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S71_S0 - { - public sbyte F0; - public ulong F1; - public long F2; - - public S71_S0(sbyte f0, ulong f1, long f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 26)] - struct S71 - { - public S71_S0 F0; - public byte F1; - public byte F2; - - public S71(S71_S0 f0, byte f1, byte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func71AA3S71VyF")] - private static extern S71 SwiftRetFunc71(); - - [Fact] - public static void TestSwiftRetFunc71() - { - Console.Write("Running SwiftRetFunc71: "); - S71 val = SwiftRetFunc71(); - Assert.Equal((sbyte)-98, val.F0.F0); - Assert.Equal((ulong)8603744544763953916, val.F0.F1); - Assert.Equal((long)8460721064583106347, val.F0.F2); - Assert.Equal((byte)10, val.F1); - Assert.Equal((byte)88, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S72 - { - public uint F0; - - public S72(uint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func72AA3S72VyF")] - private static extern S72 SwiftRetFunc72(); - - [Fact] - public static void TestSwiftRetFunc72() - { - Console.Write("Running SwiftRetFunc72: "); - S72 val = SwiftRetFunc72(); - Assert.Equal((uint)2021509367, val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 48)] - struct S73 - { - public nint F0; - public short F1; - public ulong F2; - public float F3; - public int F4; - public nuint F5; - public nuint F6; - - public S73(nint f0, short f1, ulong f2, float f3, int f4, nuint f5, nuint f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func73AA3S73VyF")] - private static extern S73 SwiftRetFunc73(); - - [Fact] - public static void TestSwiftRetFunc73() - { - Console.Write("Running SwiftRetFunc73: "); - S73 val = SwiftRetFunc73(); - Assert.Equal((nint)unchecked((nint)6222563427944465437), val.F0); - Assert.Equal((short)28721, val.F1); - Assert.Equal((ulong)1313300783845289148, val.F2); - Assert.Equal((float)6761, val.F3); - Assert.Equal((int)2074171265, val.F4); - Assert.Equal((nuint)unchecked((nuint)6232209228889209160), val.F5); - Assert.Equal((nuint)unchecked((nuint)1423931135184844265), val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 19)] - struct S74 - { - public short F0; - public float F1; - public double F2; - public ushort F3; - public sbyte F4; - - public S74(short f0, float f1, double f2, ushort f3, sbyte f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func74AA3S74VyF")] - private static extern S74 SwiftRetFunc74(); - - [Fact] - public static void TestSwiftRetFunc74() - { - Console.Write("Running SwiftRetFunc74: "); - S74 val = SwiftRetFunc74(); - Assert.Equal((short)27115, val.F0); - Assert.Equal((float)1416098, val.F1); - Assert.Equal((double)4468576755457331, val.F2); - Assert.Equal((ushort)58864, val.F3); - Assert.Equal((sbyte)81, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 1)] - struct S75_S0_S0 - { - public sbyte F0; - - public S75_S0_S0(sbyte f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct S75_S0 - { - public S75_S0_S0 F0; - public byte F1; - - public S75_S0(S75_S0_S0 f0, byte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 11)] - struct S75 - { - public ulong F0; - public S75_S0 F1; - public byte F2; - - public S75(ulong f0, S75_S0 f1, byte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func75AA3S75VyF")] - private static extern S75 SwiftRetFunc75(); - - [Fact] - public static void TestSwiftRetFunc75() - { - Console.Write("Running SwiftRetFunc75: "); - S75 val = SwiftRetFunc75(); - Assert.Equal((ulong)8532911974860912350, val.F0); - Assert.Equal((sbyte)-60, val.F1.F0.F0); - Assert.Equal((byte)66, val.F1.F1); - Assert.Equal((byte)200, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct S76_S0_S0 - { - public short F0; - - public S76_S0_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S76_S0 - { - public sbyte F0; - public ulong F1; - public S76_S0_S0 F2; - public double F3; - - public S76_S0(sbyte f0, ulong f1, S76_S0_S0 f2, double f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 48)] - struct S76 - { - public byte F0; - public S76_S0 F1; - public double F2; - - public S76(byte f0, S76_S0 f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func76AA3S76VyF")] - private static extern S76 SwiftRetFunc76(); - - [Fact] - public static void TestSwiftRetFunc76() - { - Console.Write("Running SwiftRetFunc76: "); - S76 val = SwiftRetFunc76(); - Assert.Equal((byte)69, val.F0); - Assert.Equal((sbyte)-29, val.F1.F0); - Assert.Equal((ulong)4872234474620951743, val.F1.F1); - Assert.Equal((short)11036, val.F1.F2.F0); - Assert.Equal((double)585486652063917, val.F1.F3); - Assert.Equal((double)2265391710186639, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 18)] - struct S77 - { - public int F0; - public int F1; - public int F2; - public uint F3; - public short F4; - - public S77(int f0, int f1, int f2, uint f3, short f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func77AA3S77VyF")] - private static extern S77 SwiftRetFunc77(); - - [Fact] - public static void TestSwiftRetFunc77() - { - Console.Write("Running SwiftRetFunc77: "); - S77 val = SwiftRetFunc77(); - Assert.Equal((int)4495211, val.F0); - Assert.Equal((int)1364377405, val.F1); - Assert.Equal((int)773989694, val.F2); - Assert.Equal((uint)1121696315, val.F3); - Assert.Equal((short)7589, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S78 - { - public uint F0; - public nuint F1; - - public S78(uint f0, nuint f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func78AA3S78VyF")] - private static extern S78 SwiftRetFunc78(); - - [Fact] - public static void TestSwiftRetFunc78() - { - Console.Write("Running SwiftRetFunc78: "); - S78 val = SwiftRetFunc78(); - Assert.Equal((uint)1767839225, val.F0); - Assert.Equal((nuint)unchecked((nuint)7917317019379224114), val.F1); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S79_S0 - { - public double F0; - public uint F1; - public int F2; - - public S79_S0(double f0, uint f1, int f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S79 - { - public S79_S0 F0; - public byte F1; - public double F2; - - public S79(S79_S0 f0, byte f1, double f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func79AA3S79VyF")] - private static extern S79 SwiftRetFunc79(); - - [Fact] - public static void TestSwiftRetFunc79() - { - Console.Write("Running SwiftRetFunc79: "); - S79 val = SwiftRetFunc79(); - Assert.Equal((double)495074072703635, val.F0.F0); - Assert.Equal((uint)417605286, val.F0.F1); - Assert.Equal((int)171326442, val.F0.F2); - Assert.Equal((byte)203, val.F1); - Assert.Equal((double)2976663235490421, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 7)] - struct S80 - { - public int F0; - public short F1; - public sbyte F2; - - public S80(int f0, short f1, sbyte f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func80AA3S80VyF")] - private static extern S80 SwiftRetFunc80(); - - [Fact] - public static void TestSwiftRetFunc80() - { - Console.Write("Running SwiftRetFunc80: "); - S80 val = SwiftRetFunc80(); - Assert.Equal((int)999559959, val.F0); - Assert.Equal((short)19977, val.F1); - Assert.Equal((sbyte)-4, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S81_S0 - { - public nuint F0; - - public S81_S0(nuint f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S81 - { - public int F0; - public S81_S0 F1; - public float F2; - public long F3; - public uint F4; - public byte F5; - public short F6; - - public S81(int f0, S81_S0 f1, float f2, long f3, uint f4, byte f5, short f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func81AA3S81VyF")] - private static extern S81 SwiftRetFunc81(); - - [Fact] - public static void TestSwiftRetFunc81() - { - Console.Write("Running SwiftRetFunc81: "); - S81 val = SwiftRetFunc81(); - Assert.Equal((int)452603110, val.F0); - Assert.Equal((nuint)unchecked((nuint)6240652733420985265), val.F1.F0); - Assert.Equal((float)6469988, val.F2); - Assert.Equal((long)5775316279348621124, val.F3); - Assert.Equal((uint)1398033592, val.F4); - Assert.Equal((byte)105, val.F5); - Assert.Equal((short)21937, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S82 - { - public nint F0; - - public S82(nint f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func82AA3S82VyF")] - private static extern S82 SwiftRetFunc82(); - - [Fact] - public static void TestSwiftRetFunc82() - { - Console.Write("Running SwiftRetFunc82: "); - S82 val = SwiftRetFunc82(); - Assert.Equal((nint)unchecked((nint)6454754584537364459), val.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S83 - { - public ulong F0; - public uint F1; - public float F2; - public byte F3; - public float F4; - - public S83(ulong f0, uint f1, float f2, byte f3, float f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func83AA3S83VyF")] - private static extern S83 SwiftRetFunc83(); - - [Fact] - public static void TestSwiftRetFunc83() - { - Console.Write("Running SwiftRetFunc83: "); - S83 val = SwiftRetFunc83(); - Assert.Equal((ulong)2998238441521688907, val.F0); - Assert.Equal((uint)9623946, val.F1); - Assert.Equal((float)2577885, val.F2); - Assert.Equal((byte)156, val.F3); - Assert.Equal((float)6678807, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct S84_S0 - { - public short F0; - - public S84_S0(short f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 2)] - struct S84 - { - public S84_S0 F0; - - public S84(S84_S0 f0) - { - F0 = f0; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func84AA3S84VyF")] - private static extern S84 SwiftRetFunc84(); - - [Fact] - public static void TestSwiftRetFunc84() - { - Console.Write("Running SwiftRetFunc84: "); - S84 val = SwiftRetFunc84(); - Assert.Equal((short)16213, val.F0.F0); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 3)] - struct S85_S0 - { - public short F0; - public sbyte F1; - - public S85_S0(short f0, sbyte f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S85 - { - public long F0; - public byte F1; - public S85_S0 F2; - public float F3; - public nint F4; - - public S85(long f0, byte f1, S85_S0 f2, float f3, nint f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func85AA3S85VyF")] - private static extern S85 SwiftRetFunc85(); - - [Fact] - public static void TestSwiftRetFunc85() - { - Console.Write("Running SwiftRetFunc85: "); - S85 val = SwiftRetFunc85(); - Assert.Equal((long)8858924985061791416, val.F0); - Assert.Equal((byte)200, val.F1); - Assert.Equal((short)4504, val.F2.F0); - Assert.Equal((sbyte)60, val.F2.F1); - Assert.Equal((float)5572917, val.F3); - Assert.Equal((nint)unchecked((nint)6546369836182556538), val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct S86 - { - public ushort F0; - public float F1; - public uint F2; - - public S86(ushort f0, float f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func86AA3S86VyF")] - private static extern S86 SwiftRetFunc86(); - - [Fact] - public static void TestSwiftRetFunc86() - { - Console.Write("Running SwiftRetFunc86: "); - S86 val = SwiftRetFunc86(); - Assert.Equal((ushort)22762, val.F0); - Assert.Equal((float)4672435, val.F1); - Assert.Equal((uint)719927700, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S87 - { - public int F0; - public nuint F1; - public ulong F2; - - public S87(int f0, nuint f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func87AA3S87VyF")] - private static extern S87 SwiftRetFunc87(); - - [Fact] - public static void TestSwiftRetFunc87() - { - Console.Write("Running SwiftRetFunc87: "); - S87 val = SwiftRetFunc87(); - Assert.Equal((int)361750184, val.F0); - Assert.Equal((nuint)unchecked((nuint)4206825694012787823), val.F1); - Assert.Equal((ulong)2885153391732919282, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 12)] - struct S88 - { - public uint F0; - public short F1; - public uint F2; - - public S88(uint f0, short f1, uint f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func88AA3S88VyF")] - private static extern S88 SwiftRetFunc88(); - - [Fact] - public static void TestSwiftRetFunc88() - { - Console.Write("Running SwiftRetFunc88: "); - S88 val = SwiftRetFunc88(); - Assert.Equal((uint)2125094198, val.F0); - Assert.Equal((short)-10705, val.F1); - Assert.Equal((uint)182007583, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S89 - { - public byte F0; - public uint F1; - public int F2; - public sbyte F3; - public long F4; - - public S89(byte f0, uint f1, int f2, sbyte f3, long f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func89AA3S89VyF")] - private static extern S89 SwiftRetFunc89(); - - [Fact] - public static void TestSwiftRetFunc89() - { - Console.Write("Running SwiftRetFunc89: "); - S89 val = SwiftRetFunc89(); - Assert.Equal((byte)175, val.F0); - Assert.Equal((uint)1062985476, val.F1); - Assert.Equal((int)1019006263, val.F2); - Assert.Equal((sbyte)-22, val.F3); - Assert.Equal((long)6888877252788498422, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S90 - { - public byte F0; - public int F1; - public short F2; - public nint F3; - public uint F4; - public uint F5; - public long F6; - - public S90(byte f0, int f1, short f2, nint f3, uint f4, uint f5, long f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func90AA3S90VyF")] - private static extern S90 SwiftRetFunc90(); - - [Fact] - public static void TestSwiftRetFunc90() - { - Console.Write("Running SwiftRetFunc90: "); - S90 val = SwiftRetFunc90(); - Assert.Equal((byte)221, val.F0); - Assert.Equal((int)225825436, val.F1); - Assert.Equal((short)-26231, val.F2); - Assert.Equal((nint)unchecked((nint)5122880520199505508), val.F3); - Assert.Equal((uint)907657092, val.F4); - Assert.Equal((uint)707089277, val.F5); - Assert.Equal((long)6091814344013414920, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 28)] - struct S91 - { - public double F0; - public sbyte F1; - public sbyte F2; - public uint F3; - public nint F4; - public sbyte F5; - public short F6; - - public S91(double f0, sbyte f1, sbyte f2, uint f3, nint f4, sbyte f5, short f6) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - F6 = f6; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func91AA3S91VyF")] - private static extern S91 SwiftRetFunc91(); - - [Fact] - public static void TestSwiftRetFunc91() - { - Console.Write("Running SwiftRetFunc91: "); - S91 val = SwiftRetFunc91(); - Assert.Equal((double)3265110225161261, val.F0); - Assert.Equal((sbyte)62, val.F1); - Assert.Equal((sbyte)-38, val.F2); - Assert.Equal((uint)946023589, val.F3); - Assert.Equal((nint)unchecked((nint)4109819715069879890), val.F4); - Assert.Equal((sbyte)-73, val.F5); - Assert.Equal((short)20363, val.F6); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S92_S0 - { - public float F0; - public long F1; - - public S92_S0(float f0, long f1) - { - F0 = f0; - F1 = f1; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 44)] - struct S92 - { - public long F0; - public nuint F1; - public S92_S0 F2; - public int F3; - public float F4; - public float F5; - - public S92(long f0, nuint f1, S92_S0 f2, int f3, float f4, float f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func92AA3S92VyF")] - private static extern S92 SwiftRetFunc92(); - - [Fact] - public static void TestSwiftRetFunc92() - { - Console.Write("Running SwiftRetFunc92: "); - S92 val = SwiftRetFunc92(); - Assert.Equal((long)3230438394207610137, val.F0); - Assert.Equal((nuint)unchecked((nuint)3003396252681176136), val.F1); - Assert.Equal((float)6494422, val.F2.F0); - Assert.Equal((long)2971773224350614312, val.F2.F1); - Assert.Equal((int)2063694141, val.F3); - Assert.Equal((float)3117041, val.F4); - Assert.Equal((float)1003760, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 32)] - struct S93 - { - public nint F0; - public byte F1; - public uint F2; - public uint F3; - public ulong F4; - - public S93(nint f0, byte f1, uint f2, uint f3, ulong f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func93AA3S93VyF")] - private static extern S93 SwiftRetFunc93(); - - [Fact] - public static void TestSwiftRetFunc93() - { - Console.Write("Running SwiftRetFunc93: "); - S93 val = SwiftRetFunc93(); - Assert.Equal((nint)unchecked((nint)5170226481546239050), val.F0); - Assert.Equal((byte)11, val.F1); - Assert.Equal((uint)1120259582, val.F2); - Assert.Equal((uint)1947849905, val.F3); - Assert.Equal((ulong)3690113387392112192, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 40)] - struct S94 - { - public ushort F0; - public double F1; - public short F2; - public double F3; - public ulong F4; - - public S94(ushort f0, double f1, short f2, double f3, ulong f4) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func94AA3S94VyF")] - private static extern S94 SwiftRetFunc94(); - - [Fact] - public static void TestSwiftRetFunc94() - { - Console.Write("Running SwiftRetFunc94: "); - S94 val = SwiftRetFunc94(); - Assert.Equal((ushort)57111, val.F0); - Assert.Equal((double)1718940123307098, val.F1); - Assert.Equal((short)-16145, val.F2); - Assert.Equal((double)1099321301986326, val.F3); - Assert.Equal((ulong)2972912419231960385, val.F4); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S95_S0 - { - public double F0; - - public S95_S0(double f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 24)] - struct S95 - { - public short F0; - public S95_S0 F1; - public ulong F2; - - public S95(short f0, S95_S0 f1, ulong f2) - { - F0 = f0; - F1 = f1; - F2 = f2; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func95AA3S95VyF")] - private static extern S95 SwiftRetFunc95(); - - [Fact] - public static void TestSwiftRetFunc95() - { - Console.Write("Running SwiftRetFunc95: "); - S95 val = SwiftRetFunc95(); - Assert.Equal((short)12620, val.F0); - Assert.Equal((double)3232445258308074, val.F1.F0); - Assert.Equal((ulong)97365157264460373, val.F2); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 48)] - struct S96 - { - public sbyte F0; - public double F1; - public ulong F2; - public ulong F3; - public int F4; - public long F5; - - public S96(sbyte f0, double f1, ulong f2, ulong f3, int f4, long f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func96AA3S96VyF")] - private static extern S96 SwiftRetFunc96(); - - [Fact] - public static void TestSwiftRetFunc96() - { - Console.Write("Running SwiftRetFunc96: "); - S96 val = SwiftRetFunc96(); - Assert.Equal((sbyte)3, val.F0); - Assert.Equal((double)242355060906873, val.F1); - Assert.Equal((ulong)3087879465791321798, val.F2); - Assert.Equal((ulong)7363229136420263380, val.F3); - Assert.Equal((int)46853328, val.F4); - Assert.Equal((long)4148307028758236491, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 16)] - struct S97 - { - public ushort F0; - public int F1; - public ushort F2; - public uint F3; - - public S97(ushort f0, int f1, ushort f2, uint f3) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func97AA3S97VyF")] - private static extern S97 SwiftRetFunc97(); - - [Fact] - public static void TestSwiftRetFunc97() - { - Console.Write("Running SwiftRetFunc97: "); - S97 val = SwiftRetFunc97(); - Assert.Equal((ushort)10651, val.F0); - Assert.Equal((int)2068379463, val.F1); - Assert.Equal((ushort)57307, val.F2); - Assert.Equal((uint)329271020, val.F3); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 48)] - struct S98 - { - public double F0; - public int F1; - public long F2; - public nint F3; - public float F4; - public double F5; - - public S98(double f0, int f1, long f2, nint f3, float f4, double f5) - { - F0 = f0; - F1 = f1; - F2 = f2; - F3 = f3; - F4 = f4; - F5 = f5; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func98AA3S98VyF")] - private static extern S98 SwiftRetFunc98(); - - [Fact] - public static void TestSwiftRetFunc98() - { - Console.Write("Running SwiftRetFunc98: "); - S98 val = SwiftRetFunc98(); - Assert.Equal((double)2250389231883613, val.F0); - Assert.Equal((int)1755058358, val.F1); - Assert.Equal((long)6686142382639170849, val.F2); - Assert.Equal((nint)unchecked((nint)6456632014163315773), val.F3); - Assert.Equal((float)2818253, val.F4); - Assert.Equal((double)1085859434505817, val.F5); - Console.WriteLine("OK"); - } - - [StructLayout(LayoutKind.Sequential, Size = 4)] - struct S99_S0 - { - public int F0; - - public S99_S0(int f0) - { - F0 = f0; - } - } - - [StructLayout(LayoutKind.Sequential, Size = 8)] - struct S99 - { - public S99_S0 F0; - public float F1; - - public S99(S99_S0 f0, float f1) - { - F0 = f0; - F1 = f1; - } - } - - [UnmanagedCallConv(CallConvs = new Type[] { typeof(CallConvSwift) })] - [DllImport(SwiftLib, EntryPoint = "$s17SwiftRetAbiStress05swiftB6Func99AA3S99VyF")] - private static extern S99 SwiftRetFunc99(); - - [Fact] - public static void TestSwiftRetFunc99() - { - Console.Write("Running SwiftRetFunc99: "); - S99 val = SwiftRetFunc99(); - Assert.Equal((int)1117297545, val.F0.F0); - Assert.Equal((float)1539294, val.F1); - Console.WriteLine("OK"); - } - -} diff --git a/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.csproj b/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.csproj deleted file mode 100644 index a57cd84cf8842c..00000000000000 --- a/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.csproj +++ /dev/null @@ -1,16 +0,0 @@ - - - - true - true - - true - - - - - - - - - diff --git a/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.swift b/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.swift deleted file mode 100644 index 2373b510687567..00000000000000 --- a/src/tests/Interop/Swift/SwiftRetAbiStress/SwiftRetAbiStress.swift +++ /dev/null @@ -1,1611 +0,0 @@ -import Foundation - -@frozen -public struct S0 -{ - public let f0 : Int16; - public let f1 : Int32; - public let f2 : UInt64; -} - -public func swiftRetFunc0() -> S0 { - return S0(f0: -17813, f1: 318006528, f2: 1195162122024233590) -} - -@frozen -public struct S1 -{ - public let f0 : Int16; - public let f1 : Float; - public let f2 : Int64; - public let f3 : UInt32; -} - -public func swiftRetFunc1() -> S1 { - return S1(f0: -29793, f1: 7351779, f2: 133491708229548754, f3: 665726990) -} - -@frozen -public struct S2_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct S2 -{ - public let f0 : S2_S0; - public let f1 : UInt8; - public let f2 : UInt16; - public let f3 : Float; - public let f4 : Int32; -} - -public func swiftRetFunc2() -> S2 { - return S2(f0: S2_S0(f0: 2153637757371267722), f1: 150, f2: 48920, f3: 3564327, f4: 1310569731) -} - -@frozen -public struct S3 -{ - public let f0 : Int64; - public let f1 : Double; - public let f2 : Int8; - public let f3 : Int32; - public let f4 : UInt16; - public let f5 : UInt8; - public let f6 : Double; -} - -public func swiftRetFunc3() -> S3 { - return S3(f0: 5610153900386943274, f1: 2431035148834736, f2: 111, f3: 772269424, f4: 19240, f5: 146, f6: 821805530740405) -} - -@frozen -public struct S4 -{ - public let f0 : Int8; - public let f1 : UInt32; - public let f2 : UInt64; - public let f3 : Int64; -} - -public func swiftRetFunc4() -> S4 { - return S4(f0: 125, f1: 377073381, f2: 964784376430620335, f3: 5588038704850976624) -} - -@frozen -public struct S5_S0 -{ - public let f0 : UInt32; - public let f1 : Double; -} - -@frozen -public struct S5 -{ - public let f0 : UInt64; - public let f1 : Int8; - public let f2 : UInt; - public let f3 : S5_S0; - public let f4 : Int; - public let f5 : UInt8; -} - -public func swiftRetFunc5() -> S5 { - return S5(f0: 5315019731968023493, f1: 114, f2: 1154655179105889397, f3: S5_S0(f0: 1468030771, f1: 3066473182924818), f4: 6252650621827449809, f5: 129) -} - -@frozen -public struct S6 -{ - public let f0 : Int32; - public let f1 : Int16; - public let f2 : Int64; - public let f3 : UInt16; -} - -public func swiftRetFunc6() -> S6 { - return S6(f0: 743741783, f1: -6821, f2: 5908745692727636656, f3: 64295) -} - -@frozen -public struct S7_S0 -{ - public let f0 : Int; -} - -@frozen -public struct S7 -{ - public let f0 : S7_S0; -} - -public func swiftRetFunc7() -> S7 { - return S7(f0: S7_S0(f0: 7625368278886567558)) -} - -@frozen -public struct S8 -{ - public let f0 : Int; -} - -public func swiftRetFunc8() -> S8 { - return S8(f0: 775279004683334365) -} - -@frozen -public struct S9_S0 -{ - public let f0 : Int16; - public let f1 : Int32; -} - -@frozen -public struct S9 -{ - public let f0 : UInt32; - public let f1 : Int; - public let f2 : S9_S0; - public let f3 : UInt16; -} - -public func swiftRetFunc9() -> S9 { - return S9(f0: 1223030410, f1: 4720638462358523954, f2: S9_S0(f0: 30631, f1: 1033774469), f3: 64474) -} - -@frozen -public struct S10 -{ - public let f0 : Float; - public let f1 : Float; -} - -public func swiftRetFunc10() -> S10 { - return S10(f0: 3276917, f1: 6694615) -} - -@frozen -public struct S11 -{ - public let f0 : Double; - public let f1 : Int; - public let f2 : UInt32; - public let f3 : Int8; -} - -public func swiftRetFunc11() -> S11 { - return S11(f0: 938206348036312, f1: 6559514243876905696, f2: 1357772248, f3: 59) -} - -@frozen -public struct S12 -{ - public let f0 : Double; -} - -public func swiftRetFunc12() -> S12 { - return S12(f0: 1580503485222363) -} - -@frozen -public struct S13 -{ - public let f0 : UInt32; -} - -public func swiftRetFunc13() -> S13 { - return S13(f0: 1381551558) -} - -@frozen -public struct S14_S0_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct S14_S0 -{ - public let f0 : S14_S0_S0; -} - -@frozen -public struct S14 -{ - public let f0 : Int32; - public let f1 : UInt16; - public let f2 : Int8; - public let f3 : Float; - public let f4 : UInt64; - public let f5 : S14_S0; - public let f6 : Int8; -} - -public func swiftRetFunc14() -> S14 { - return S14(f0: 1765691191, f1: 56629, f2: 25, f3: 2944946, f4: 951929105049584033, f5: S14_S0(f0: S14_S0_S0(f0: -30)), f6: 66) -} - -@frozen -public struct S15_S0 -{ - public let f0 : UInt; - public let f1 : Float; -} - -@frozen -public struct S15 -{ - public let f0 : Int; - public let f1 : S15_S0; - public let f2 : UInt16; - public let f3 : Int32; -} - -public func swiftRetFunc15() -> S15 { - return S15(f0: 2090703541638269172, f1: S15_S0(f0: 6408314016925514463, f1: 6534515), f2: 30438, f3: 1745811802) -} - -@frozen -public struct S16 -{ - public let f0 : UInt32; - public let f1 : UInt64; - public let f2 : UInt8; - public let f3 : Int32; - public let f4 : UInt; - public let f5 : Int8; -} - -public func swiftRetFunc16() -> S16 { - return S16(f0: 585220635, f1: 4034210936973794153, f2: 48, f3: 1155081155, f4: 806384837403045657, f5: 54) -} - -@frozen -public struct S17 -{ - public let f0 : UInt8; - public let f1 : Int8; - public let f2 : UInt8; -} - -public func swiftRetFunc17() -> S17 { - return S17(f0: 23, f1: 112, f2: 15) -} - -@frozen -public struct S18_S0 -{ - public let f0 : UInt32; - public let f1 : Float; -} - -@frozen -public struct S18 -{ - public let f0 : S18_S0; - public let f1 : Int; - public let f2 : Int32; - public let f3 : UInt16; - public let f4 : Int16; -} - -public func swiftRetFunc18() -> S18 { - return S18(f0: S18_S0(f0: 1964425016, f1: 2767295), f1: 6016563774923595868, f2: 1648562735, f3: 378, f4: -20536) -} - -@frozen -public struct S19 -{ - public let f0 : UInt8; - public let f1 : UInt16; - public let f2 : Float; - public let f3 : UInt64; - public let f4 : Int32; -} - -public func swiftRetFunc19() -> S19 { - return S19(f0: 188, f1: 47167, f2: 6781297, f3: 8140268502944465472, f4: 708690468) -} - -@frozen -public struct S20_S0 -{ - public let f0 : UInt32; - public let f1 : Float; -} - -@frozen -public struct S20 -{ - public let f0 : S20_S0; - public let f1 : UInt8; -} - -public func swiftRetFunc20() -> S20 { - return S20(f0: S20_S0(f0: 2019361333, f1: 938975), f1: 192) -} - -@frozen -public struct S21_S0_S0 -{ - public let f0 : UInt16; -} - -@frozen -public struct S21_S0 -{ - public let f0 : S21_S0_S0; -} - -@frozen -public struct S21 -{ - public let f0 : Double; - public let f1 : Double; - public let f2 : UInt; - public let f3 : Int; - public let f4 : UInt64; - public let f5 : S21_S0; -} - -public func swiftRetFunc21() -> S21 { - return S21(f0: 1693878073402490, f1: 3392111340517811, f2: 3584917502172813732, f3: 665495086154608745, f4: 2918107814961929578, f5: S21_S0(f0: S21_S0_S0(f0: 4634))) -} - -@frozen -public struct S22 -{ - public let f0 : UInt32; -} - -public func swiftRetFunc22() -> S22 { - return S22(f0: 640156952) -} - -@frozen -public struct S23 -{ - public let f0 : UInt8; - public let f1 : Int16; - public let f2 : UInt64; - public let f3 : UInt; - public let f4 : UInt; - public let f5 : UInt64; - public let f6 : UInt8; -} - -public func swiftRetFunc23() -> S23 { - return S23(f0: 122, f1: 28995, f2: 25673626033589541, f3: 828363978755325884, f4: 3065573182429720699, f5: 1484484917001276079, f6: 209) -} - -@frozen -public struct S24 -{ - public let f0 : UInt64; - public let f1 : UInt64; -} - -public func swiftRetFunc24() -> S24 { - return S24(f0: 2621245238416080387, f1: 6541787564638363256) -} - -@frozen -public struct S25_S0 -{ - public let f0 : Int; -} - -@frozen -public struct S25 -{ - public let f0 : Int8; - public let f1 : Int8; - public let f2 : UInt8; - public let f3 : S25_S0; - public let f4 : UInt32; -} - -public func swiftRetFunc25() -> S25 { - return S25(f0: 30, f1: -8, f2: 168, f3: S25_S0(f0: 7601538494489501573), f4: 814523741) -} - -@frozen -public struct S26 -{ - public let f0 : Float; -} - -public func swiftRetFunc26() -> S26 { - return S26(f0: 3681545) -} - -@frozen -public struct S27 -{ - public let f0 : Int64; - public let f1 : Double; - public let f2 : Int8; - public let f3 : Int; - public let f4 : Int16; - public let f5 : Int64; -} - -public func swiftRetFunc27() -> S27 { - return S27(f0: 4847421047018330189, f1: 3655171692392280, f2: 46, f3: 4476120319602257660, f4: -6106, f5: 5756567968111212829) -} - -@frozen -public struct S28_S0 -{ - public let f0 : Double; -} - -@frozen -public struct S28 -{ - public let f0 : Float; - public let f1 : Int16; - public let f2 : S28_S0; - public let f3 : Double; - public let f4 : UInt64; -} - -public func swiftRetFunc28() -> S28 { - return S28(f0: 3491512, f1: 5249, f2: S28_S0(f0: 1107064327388314), f3: 2170381648425673, f4: 5138313315157580943) -} - -@frozen -public struct S29 -{ - public let f0 : UInt16; - public let f1 : UInt32; - public let f2 : Int16; - public let f3 : Int32; - public let f4 : Int32; - public let f5 : UInt64; - public let f6 : Int16; -} - -public func swiftRetFunc29() -> S29 { - return S29(f0: 39000, f1: 408611655, f2: 18090, f3: 351857085, f4: 1103441843, f5: 5162040247631126074, f6: -27930) -} - -@frozen -public struct S30_S0 -{ - public let f0 : Int8; - public let f1 : Int8; - public let f2 : Int32; -} - -@frozen -public struct S30_S1 -{ - public let f0 : Float; -} - -@frozen -public struct S30 -{ - public let f0 : Float; - public let f1 : S30_S0; - public let f2 : S30_S1; - public let f3 : Int64; -} - -public func swiftRetFunc30() -> S30 { - return S30(f0: 6492602, f1: S30_S0(f0: 76, f1: -26, f2: 1777644423), f2: S30_S1(f0: 6558571), f3: 5879147675377398012) -} - -@frozen -public struct S31 -{ - public let f0 : Int64; - public let f1 : UInt64; - public let f2 : UInt16; - public let f3 : UInt16; - public let f4 : Int8; -} - -public func swiftRetFunc31() -> S31 { - return S31(f0: 4699402628739628277, f1: 7062790893852687562, f2: 28087, f3: 11088, f4: 69) -} - -@frozen -public struct S32 -{ - public let f0 : Int32; - public let f1 : UInt64; - public let f2 : UInt64; - public let f3 : UInt32; - public let f4 : Int16; - public let f5 : UInt16; -} - -public func swiftRetFunc32() -> S32 { - return S32(f0: 688805466, f1: 8860655326984381661, f2: 6943423675662271404, f3: 196368476, f4: 14229, f5: 34635) -} - -@frozen -public struct S33 -{ - public let f0 : UInt16; - public let f1 : UInt32; - public let f2 : Int32; - public let f3 : UInt16; - public let f4 : Float; - public let f5 : UInt64; - public let f6 : Int; -} - -public func swiftRetFunc33() -> S33 { - return S33(f0: 9297, f1: 7963252, f2: 556244690, f3: 19447, f4: 6930550, f5: 126294981263481729, f6: 2540579257616511618) -} - -@frozen -public struct S34 -{ - public let f0 : Int64; - public let f1 : UInt32; - public let f2 : UInt64; -} - -public func swiftRetFunc34() -> S34 { - return S34(f0: 5845561428743737556, f1: 1358941228, f2: 3701080255861218446) -} - -@frozen -public struct S35 -{ - public let f0 : Float; - public let f1 : Float; - public let f2 : Int64; - public let f3 : UInt8; - public let f4 : Double; - public let f5 : UInt16; -} - -public func swiftRetFunc35() -> S35 { - return S35(f0: 5982956, f1: 3675164, f2: 229451138397478297, f3: 163, f4: 2925293762193390, f5: 5018) -} - -@frozen -public struct S36 -{ - public let f0 : Int32; - public let f1 : Int64; - public let f2 : UInt64; -} - -public func swiftRetFunc36() -> S36 { - return S36(f0: 1915776502, f1: 2197655909333830531, f2: 6072941592567177049) -} - -@frozen -public struct S37 -{ - public let f0 : UInt8; - public let f1 : Double; -} - -public func swiftRetFunc37() -> S37 { - return S37(f0: 18, f1: 4063164371882658) -} - -@frozen -public struct S38 -{ - public let f0 : UInt; - public let f1 : Int64; - public let f2 : UInt8; - public let f3 : UInt; -} - -public func swiftRetFunc38() -> S38 { - return S38(f0: 7389960750529773276, f1: 2725802169582362061, f2: 2, f3: 3659261019360356514) -} - -@frozen -public struct S39 -{ - public let f0 : Int32; - public let f1 : Int32; - public let f2 : Int; - public let f3 : Int16; - public let f4 : UInt16; -} - -public func swiftRetFunc39() -> S39 { - return S39(f0: 50995691, f1: 1623216479, f2: 2906650346451599789, f3: 28648, f4: 8278) -} - -@frozen -public struct S40_S0 -{ - public let f0 : Float; - public let f1 : UInt8; - public let f2 : Int8; - public let f3 : UInt; - public let f4 : Double; -} - -@frozen -public struct S40 -{ - public let f0 : S40_S0; - public let f1 : Int16; - public let f2 : Int16; -} - -public func swiftRetFunc40() -> S40 { - return S40(f0: S40_S0(f0: 7087264, f1: 37, f2: -5, f3: 479915249821490487, f4: 144033730096589), f1: 28654, f2: 16398) -} - -@frozen -public struct S41 -{ - public let f0 : UInt; - public let f1 : UInt; -} - -public func swiftRetFunc41() -> S41 { - return S41(f0: 7923718819069382599, f1: 1539666179674725957) -} - -@frozen -public struct S42_S0 -{ - public let f0 : Int32; -} - -@frozen -public struct S42 -{ - public let f0 : UInt32; - public let f1 : Int64; - public let f2 : S42_S0; - public let f3 : UInt; -} - -public func swiftRetFunc42() -> S42 { - return S42(f0: 1046060439, f1: 8249831314190867613, f2: S42_S0(f0: 1097582349), f3: 2864677262092469436) -} - -@frozen -public struct S43_S0_S0 -{ - public let f0 : Float; -} - -@frozen -public struct S43_S0 -{ - public let f0 : S43_S0_S0; -} - -@frozen -public struct S43 -{ - public let f0 : S43_S0; - public let f1 : Int8; -} - -public func swiftRetFunc43() -> S43 { - return S43(f0: S43_S0(f0: S43_S0_S0(f0: 1586338)), f1: 104) -} - -@frozen -public struct S44 -{ - public let f0 : UInt8; - public let f1 : Int32; - public let f2 : Int; - public let f3 : UInt32; -} - -public func swiftRetFunc44() -> S44 { - return S44(f0: 94, f1: 1109076022, f2: 3135595850598607828, f3: 760084013) -} - -@frozen -public struct S45_S0 -{ - public let f0 : Int64; -} - -@frozen -public struct S45 -{ - public let f0 : Int16; - public let f1 : UInt64; - public let f2 : Int; - public let f3 : S45_S0; -} - -public func swiftRetFunc45() -> S45 { - return S45(f0: 3071, f1: 5908138438609341766, f2: 5870206722419946629, f3: S45_S0(f0: 8128455876189744801)) -} - -@frozen -public struct S46 -{ - public let f0 : Int16; - public let f1 : Int8; - public let f2 : Int8; - public let f3 : UInt32; - public let f4 : UInt8; - public let f5 : Int32; -} - -public func swiftRetFunc46() -> S46 { - return S46(f0: 14794, f1: 60, f2: -77, f3: 653898879, f4: 224, f5: 266602433) -} - -@frozen -public struct S47_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct S47 -{ - public let f0 : Double; - public let f1 : S47_S0; -} - -public func swiftRetFunc47() -> S47 { - return S47(f0: 3195976594911793, f1: S47_S0(f0: -91)) -} - -@frozen -public struct S48 -{ - public let f0 : Int; -} - -public func swiftRetFunc48() -> S48 { - return S48(f0: 778504172538154682) -} - -@frozen -public struct S49_S0_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct S49_S0 -{ - public let f0 : S49_S0_S0; -} - -@frozen -public struct S49 -{ - public let f0 : UInt64; - public let f1 : S49_S0; - public let f2 : Int8; - public let f3 : Double; - public let f4 : UInt32; - public let f5 : UInt32; -} - -public func swiftRetFunc49() -> S49 { - return S49(f0: 4235011519458710874, f1: S49_S0(f0: S49_S0_S0(f0: 3120420438742285733)), f2: -8, f3: 1077419570643725, f4: 1985303212, f5: 264580506) -} - -@frozen -public struct S50 -{ - public let f0 : Int32; -} - -public func swiftRetFunc50() -> S50 { - return S50(f0: 1043912405) -} - -@frozen -public struct S51_S0_S0_S0 -{ - public let f0 : Float; -} - -@frozen -public struct S51_S0_S0 -{ - public let f0 : S51_S0_S0_S0; - public let f1 : Int16; -} - -@frozen -public struct S51_S0 -{ - public let f0 : Double; - public let f1 : S51_S0_S0; - public let f2 : UInt8; - public let f3 : Int64; -} - -@frozen -public struct S51 -{ - public let f0 : S51_S0; - public let f1 : Double; -} - -public func swiftRetFunc51() -> S51 { - return S51(f0: S51_S0(f0: 3266680719186600, f1: S51_S0_S0(f0: S51_S0_S0_S0(f0: 428247), f1: -24968), f2: 76, f3: 183022772513065490), f1: 2661928101793033) -} - -@frozen -public struct S52 -{ - public let f0 : UInt32; - public let f1 : Int64; - public let f2 : UInt32; - public let f3 : UInt64; - public let f4 : Int; - public let f5 : Int8; -} - -public func swiftRetFunc52() -> S52 { - return S52(f0: 1812191671, f1: 6594574760089190928, f2: 831147243, f3: 3301835731003365248, f4: 5382332538247340743, f5: -77) -} - -@frozen -public struct S53_S0 -{ - public let f0 : Int8; - public let f1 : UInt; -} - -@frozen -public struct S53 -{ - public let f0 : S53_S0; - public let f1 : Int32; - public let f2 : Int64; - public let f3 : Float; - public let f4 : Int8; -} - -public func swiftRetFunc53() -> S53 { - return S53(f0: S53_S0(f0: -123, f1: 3494916243607193741), f1: 1406699798, f2: 4018943158751734338, f3: 1084415, f4: -8) -} - -@frozen -public struct S54_S0 -{ - public let f0 : Double; -} - -@frozen -public struct S54 -{ - public let f0 : Int; - public let f1 : Int; - public let f2 : S54_S0; - public let f3 : Int64; -} - -public func swiftRetFunc54() -> S54 { - return S54(f0: 8623517456704997133, f1: 1521939500434086364, f2: S54_S0(f0: 3472783299414218), f3: 4761507229870258916) -} - -@frozen -public struct S55 -{ - public let f0 : Int16; - public let f1 : UInt32; - public let f2 : Int64; - public let f3 : UInt32; - public let f4 : Int8; - public let f5 : UInt8; -} - -public func swiftRetFunc55() -> S55 { - return S55(f0: -28051, f1: 1759912152, f2: 2038322238348454200, f3: 601094102, f4: 5, f5: 75) -} - -@frozen -public struct S56 -{ - public let f0 : UInt64; - public let f1 : Float; - public let f2 : Int8; - public let f3 : Int32; -} - -public func swiftRetFunc56() -> S56 { - return S56(f0: 6313168909786453069, f1: 6254558, f2: 115, f3: 847834891) -} - -@frozen -public struct S57 -{ - public let f0 : UInt; - public let f1 : Int16; - public let f2 : Int8; - public let f3 : Int32; -} - -public func swiftRetFunc57() -> S57 { - return S57(f0: 546304219852233452, f1: -27416, f2: 47, f3: 1094575684) -} - -@frozen -public struct S58 -{ - public let f0 : UInt64; - public let f1 : UInt64; -} - -public func swiftRetFunc58() -> S58 { - return S58(f0: 4612004722568513699, f1: 2222525519606580195) -} - -@frozen -public struct S59 -{ - public let f0 : Int8; - public let f1 : UInt; - public let f2 : Int; - public let f3 : Int8; - public let f4 : Int64; - public let f5 : UInt8; -} - -public func swiftRetFunc59() -> S59 { - return S59(f0: -92, f1: 7281011081566942937, f2: 8203439771560005792, f3: 103, f4: 1003386607251132236, f5: 6) -} - -@frozen -public struct S60 -{ - public let f0 : UInt64; - public let f1 : Int; -} - -public func swiftRetFunc60() -> S60 { - return S60(f0: 6922353269487057763, f1: 103032455997325768) -} - -@frozen -public struct S61_S0 -{ - public let f0 : Int64; - public let f1 : Int64; - public let f2 : Float; -} - -@frozen -public struct S61 -{ - public let f0 : UInt64; - public let f1 : S61_S0; - public let f2 : Int16; - public let f3 : Int32; -} - -public func swiftRetFunc61() -> S61 { - return S61(f0: 3465845922566501572, f1: S61_S0(f0: 8266662359091888314, f1: 7511705648638703076, f2: 535470), f2: -5945, f3: 523043523) -} - -@frozen -public struct S62_S0_S0 -{ - public let f0 : Int; -} - -@frozen -public struct S62_S0 -{ - public let f0 : UInt16; - public let f1 : Int16; - public let f2 : UInt16; - public let f3 : S62_S0_S0; -} - -@frozen -public struct S62 -{ - public let f0 : S62_S0; - public let f1 : Int; - public let f2 : UInt16; -} - -public func swiftRetFunc62() -> S62 { - return S62(f0: S62_S0(f0: 50789, f1: 30245, f2: 35063, f3: S62_S0_S0(f0: 3102684963408623932)), f1: 792877586576090769, f2: 24697) -} - -@frozen -public struct S63 -{ - public let f0 : Double; - public let f1 : Int; - public let f2 : Double; - public let f3 : Int8; - public let f4 : Float; -} - -public func swiftRetFunc63() -> S63 { - return S63(f0: 4097323000009314, f1: 4162427097168837193, f2: 140736061437152, f3: -59, f4: 7331757) -} - -@frozen -public struct S64_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct S64 -{ - public let f0 : S64_S0; - public let f1 : UInt64; - public let f2 : Int64; - public let f3 : Int; -} - -public func swiftRetFunc64() -> S64 { - return S64(f0: S64_S0(f0: 2624461610177878495), f1: 5222178027019975511, f2: 9006949357929457355, f3: 7966680593035770540) -} - -@frozen -public struct S65 -{ - public let f0 : Int; - public let f1 : Double; - public let f2 : UInt16; - public let f3 : Int16; - public let f4 : UInt8; - public let f5 : Int32; - public let f6 : UInt64; -} - -public func swiftRetFunc65() -> S65 { - return S65(f0: 6080968957098434687, f1: 3067343828504927, f2: 56887, f3: 804, f4: 235, f5: 121742660, f6: 9218677163034827308) -} - -@frozen -public struct S66 -{ - public let f0 : Int8; - public let f1 : UInt64; - public let f2 : UInt32; - public let f3 : UInt64; - public let f4 : UInt64; -} - -public func swiftRetFunc66() -> S66 { - return S66(f0: -16, f1: 7967447403042597794, f2: 2029697750, f3: 4180031087394830849, f4: 5847795120921557969) -} - -@frozen -public struct S67_S0 -{ - public let f0 : UInt64; -} - -@frozen -public struct S67 -{ - public let f0 : S67_S0; - public let f1 : UInt8; - public let f2 : UInt16; - public let f3 : UInt64; - public let f4 : UInt64; - public let f5 : Int8; -} - -public func swiftRetFunc67() -> S67 { - return S67(f0: S67_S0(f0: 4844204675254434929), f1: 135, f2: 13969, f3: 4897129719050177731, f4: 7233638107485862921, f5: -11) -} - -@frozen -public struct S68_S0 -{ - public let f0 : Double; -} - -@frozen -public struct S68 -{ - public let f0 : Int32; - public let f1 : UInt64; - public let f2 : UInt32; - public let f3 : S68_S0; - public let f4 : Int32; - public let f5 : Int8; -} - -public func swiftRetFunc68() -> S68 { - return S68(f0: 1708606840, f1: 1768121573985581212, f2: 1033319213, f3: S68_S0(f0: 2741322436867931), f4: 955320338, f5: 12) -} - -@frozen -public struct S69 -{ - public let f0 : UInt32; -} - -public func swiftRetFunc69() -> S69 { - return S69(f0: 2092746473) -} - -@frozen -public struct S70 -{ - public let f0 : UInt8; - public let f1 : Float; -} - -public func swiftRetFunc70() -> S70 { - return S70(f0: 76, f1: 4138467) -} - -@frozen -public struct S71_S0 -{ - public let f0 : Int8; - public let f1 : UInt64; - public let f2 : Int64; -} - -@frozen -public struct S71 -{ - public let f0 : S71_S0; - public let f1 : UInt8; - public let f2 : UInt8; -} - -public func swiftRetFunc71() -> S71 { - return S71(f0: S71_S0(f0: -98, f1: 8603744544763953916, f2: 8460721064583106347), f1: 10, f2: 88) -} - -@frozen -public struct S72 -{ - public let f0 : UInt32; -} - -public func swiftRetFunc72() -> S72 { - return S72(f0: 2021509367) -} - -@frozen -public struct S73 -{ - public let f0 : Int; - public let f1 : Int16; - public let f2 : UInt64; - public let f3 : Float; - public let f4 : Int32; - public let f5 : UInt; - public let f6 : UInt; -} - -public func swiftRetFunc73() -> S73 { - return S73(f0: 6222563427944465437, f1: 28721, f2: 1313300783845289148, f3: 6761, f4: 2074171265, f5: 6232209228889209160, f6: 1423931135184844265) -} - -@frozen -public struct S74 -{ - public let f0 : Int16; - public let f1 : Float; - public let f2 : Double; - public let f3 : UInt16; - public let f4 : Int8; -} - -public func swiftRetFunc74() -> S74 { - return S74(f0: 27115, f1: 1416098, f2: 4468576755457331, f3: 58864, f4: 81) -} - -@frozen -public struct S75_S0_S0 -{ - public let f0 : Int8; -} - -@frozen -public struct S75_S0 -{ - public let f0 : S75_S0_S0; - public let f1 : UInt8; -} - -@frozen -public struct S75 -{ - public let f0 : UInt64; - public let f1 : S75_S0; - public let f2 : UInt8; -} - -public func swiftRetFunc75() -> S75 { - return S75(f0: 8532911974860912350, f1: S75_S0(f0: S75_S0_S0(f0: -60), f1: 66), f2: 200) -} - -@frozen -public struct S76_S0_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct S76_S0 -{ - public let f0 : Int8; - public let f1 : UInt64; - public let f2 : S76_S0_S0; - public let f3 : Double; -} - -@frozen -public struct S76 -{ - public let f0 : UInt8; - public let f1 : S76_S0; - public let f2 : Double; -} - -public func swiftRetFunc76() -> S76 { - return S76(f0: 69, f1: S76_S0(f0: -29, f1: 4872234474620951743, f2: S76_S0_S0(f0: 11036), f3: 585486652063917), f2: 2265391710186639) -} - -@frozen -public struct S77 -{ - public let f0 : Int32; - public let f1 : Int32; - public let f2 : Int32; - public let f3 : UInt32; - public let f4 : Int16; -} - -public func swiftRetFunc77() -> S77 { - return S77(f0: 4495211, f1: 1364377405, f2: 773989694, f3: 1121696315, f4: 7589) -} - -@frozen -public struct S78 -{ - public let f0 : UInt32; - public let f1 : UInt; -} - -public func swiftRetFunc78() -> S78 { - return S78(f0: 1767839225, f1: 7917317019379224114) -} - -@frozen -public struct S79_S0 -{ - public let f0 : Double; - public let f1 : UInt32; - public let f2 : Int32; -} - -@frozen -public struct S79 -{ - public let f0 : S79_S0; - public let f1 : UInt8; - public let f2 : Double; -} - -public func swiftRetFunc79() -> S79 { - return S79(f0: S79_S0(f0: 495074072703635, f1: 417605286, f2: 171326442), f1: 203, f2: 2976663235490421) -} - -@frozen -public struct S80 -{ - public let f0 : Int32; - public let f1 : Int16; - public let f2 : Int8; -} - -public func swiftRetFunc80() -> S80 { - return S80(f0: 999559959, f1: 19977, f2: -4) -} - -@frozen -public struct S81_S0 -{ - public let f0 : UInt; -} - -@frozen -public struct S81 -{ - public let f0 : Int32; - public let f1 : S81_S0; - public let f2 : Float; - public let f3 : Int64; - public let f4 : UInt32; - public let f5 : UInt8; - public let f6 : Int16; -} - -public func swiftRetFunc81() -> S81 { - return S81(f0: 452603110, f1: S81_S0(f0: 6240652733420985265), f2: 6469988, f3: 5775316279348621124, f4: 1398033592, f5: 105, f6: 21937) -} - -@frozen -public struct S82 -{ - public let f0 : Int; -} - -public func swiftRetFunc82() -> S82 { - return S82(f0: 6454754584537364459) -} - -@frozen -public struct S83 -{ - public let f0 : UInt64; - public let f1 : UInt32; - public let f2 : Float; - public let f3 : UInt8; - public let f4 : Float; -} - -public func swiftRetFunc83() -> S83 { - return S83(f0: 2998238441521688907, f1: 9623946, f2: 2577885, f3: 156, f4: 6678807) -} - -@frozen -public struct S84_S0 -{ - public let f0 : Int16; -} - -@frozen -public struct S84 -{ - public let f0 : S84_S0; -} - -public func swiftRetFunc84() -> S84 { - return S84(f0: S84_S0(f0: 16213)) -} - -@frozen -public struct S85_S0 -{ - public let f0 : Int16; - public let f1 : Int8; -} - -@frozen -public struct S85 -{ - public let f0 : Int64; - public let f1 : UInt8; - public let f2 : S85_S0; - public let f3 : Float; - public let f4 : Int; -} - -public func swiftRetFunc85() -> S85 { - return S85(f0: 8858924985061791416, f1: 200, f2: S85_S0(f0: 4504, f1: 60), f3: 5572917, f4: 6546369836182556538) -} - -@frozen -public struct S86 -{ - public let f0 : UInt16; - public let f1 : Float; - public let f2 : UInt32; -} - -public func swiftRetFunc86() -> S86 { - return S86(f0: 22762, f1: 4672435, f2: 719927700) -} - -@frozen -public struct S87 -{ - public let f0 : Int32; - public let f1 : UInt; - public let f2 : UInt64; -} - -public func swiftRetFunc87() -> S87 { - return S87(f0: 361750184, f1: 4206825694012787823, f2: 2885153391732919282) -} - -@frozen -public struct S88 -{ - public let f0 : UInt32; - public let f1 : Int16; - public let f2 : UInt32; -} - -public func swiftRetFunc88() -> S88 { - return S88(f0: 2125094198, f1: -10705, f2: 182007583) -} - -@frozen -public struct S89 -{ - public let f0 : UInt8; - public let f1 : UInt32; - public let f2 : Int32; - public let f3 : Int8; - public let f4 : Int64; -} - -public func swiftRetFunc89() -> S89 { - return S89(f0: 175, f1: 1062985476, f2: 1019006263, f3: -22, f4: 6888877252788498422) -} - -@frozen -public struct S90 -{ - public let f0 : UInt8; - public let f1 : Int32; - public let f2 : Int16; - public let f3 : Int; - public let f4 : UInt32; - public let f5 : UInt32; - public let f6 : Int64; -} - -public func swiftRetFunc90() -> S90 { - return S90(f0: 221, f1: 225825436, f2: -26231, f3: 5122880520199505508, f4: 907657092, f5: 707089277, f6: 6091814344013414920) -} - -@frozen -public struct S91 -{ - public let f0 : Double; - public let f1 : Int8; - public let f2 : Int8; - public let f3 : UInt32; - public let f4 : Int; - public let f5 : Int8; - public let f6 : Int16; -} - -public func swiftRetFunc91() -> S91 { - return S91(f0: 3265110225161261, f1: 62, f2: -38, f3: 946023589, f4: 4109819715069879890, f5: -73, f6: 20363) -} - -@frozen -public struct S92_S0 -{ - public let f0 : Float; - public let f1 : Int64; -} - -@frozen -public struct S92 -{ - public let f0 : Int64; - public let f1 : UInt; - public let f2 : S92_S0; - public let f3 : Int32; - public let f4 : Float; - public let f5 : Float; -} - -public func swiftRetFunc92() -> S92 { - return S92(f0: 3230438394207610137, f1: 3003396252681176136, f2: S92_S0(f0: 6494422, f1: 2971773224350614312), f3: 2063694141, f4: 3117041, f5: 1003760) -} - -@frozen -public struct S93 -{ - public let f0 : Int; - public let f1 : UInt8; - public let f2 : UInt32; - public let f3 : UInt32; - public let f4 : UInt64; -} - -public func swiftRetFunc93() -> S93 { - return S93(f0: 5170226481546239050, f1: 11, f2: 1120259582, f3: 1947849905, f4: 3690113387392112192) -} - -@frozen -public struct S94 -{ - public let f0 : UInt16; - public let f1 : Double; - public let f2 : Int16; - public let f3 : Double; - public let f4 : UInt64; -} - -public func swiftRetFunc94() -> S94 { - return S94(f0: 57111, f1: 1718940123307098, f2: -16145, f3: 1099321301986326, f4: 2972912419231960385) -} - -@frozen -public struct S95_S0 -{ - public let f0 : Double; -} - -@frozen -public struct S95 -{ - public let f0 : Int16; - public let f1 : S95_S0; - public let f2 : UInt64; -} - -public func swiftRetFunc95() -> S95 { - return S95(f0: 12620, f1: S95_S0(f0: 3232445258308074), f2: 97365157264460373) -} - -@frozen -public struct S96 -{ - public let f0 : Int8; - public let f1 : Double; - public let f2 : UInt64; - public let f3 : UInt64; - public let f4 : Int32; - public let f5 : Int64; -} - -public func swiftRetFunc96() -> S96 { - return S96(f0: 3, f1: 242355060906873, f2: 3087879465791321798, f3: 7363229136420263380, f4: 46853328, f5: 4148307028758236491) -} - -@frozen -public struct S97 -{ - public let f0 : UInt16; - public let f1 : Int32; - public let f2 : UInt16; - public let f3 : UInt32; -} - -public func swiftRetFunc97() -> S97 { - return S97(f0: 10651, f1: 2068379463, f2: 57307, f3: 329271020) -} - -@frozen -public struct S98 -{ - public let f0 : Double; - public let f1 : Int32; - public let f2 : Int64; - public let f3 : Int; - public let f4 : Float; - public let f5 : Double; -} - -public func swiftRetFunc98() -> S98 { - return S98(f0: 2250389231883613, f1: 1755058358, f2: 6686142382639170849, f3: 6456632014163315773, f4: 2818253, f5: 1085859434505817) -} - -@frozen -public struct S99_S0 -{ - public let f0 : Int32; -} - -@frozen -public struct S99 -{ - public let f0 : S99_S0; - public let f1 : Float; -} - -public func swiftRetFunc99() -> S99 { - return S99(f0: S99_S0(f0: 1117297545), f1: 1539294) -} - diff --git a/src/tests/Interop/Swift/SwiftSelfContext/SwiftSelfContext.csproj b/src/tests/Interop/Swift/SwiftSelfContext/SwiftSelfContext.csproj index 89eda99352fd20..a57cd84cf8842c 100644 --- a/src/tests/Interop/Swift/SwiftSelfContext/SwiftSelfContext.csproj +++ b/src/tests/Interop/Swift/SwiftSelfContext/SwiftSelfContext.csproj @@ -3,7 +3,7 @@ true true - + true diff --git a/src/tests/JIT/Directed/debugging/poisoning/poison.cs b/src/tests/JIT/Directed/debugging/poisoning/poison.cs index 07543fcc53446c..c5bcccded9270e 100644 --- a/src/tests/JIT/Directed/debugging/poisoning/poison.cs +++ b/src/tests/JIT/Directed/debugging/poisoning/poison.cs @@ -8,7 +8,6 @@ public class Program [Fact] public static unsafe int TestEntryPoint() { -#pragma warning disable CS8500 // takes address of managed type bool result = true; int poisoned; @@ -17,7 +16,7 @@ public static unsafe int TestEntryPoint() GCRef zeroed; Unsafe.SkipInit(out zeroed); - result &= VerifyZero(&zeroed, sizeof(GCRef)); + result &= VerifyZero(Unsafe.AsPointer(ref zeroed), Unsafe.SizeOf()); WithoutGCRef poisoned2; Unsafe.SkipInit(out poisoned2); @@ -37,10 +36,9 @@ public static unsafe int TestEntryPoint() GCRef zeroed2; Unsafe.SkipInit(out zeroed2); - result &= VerifyZero(&zeroed2, sizeof(GCRef)); + result &= VerifyZero(Unsafe.AsPointer(ref zeroed2), Unsafe.SizeOf()); return result ? 100 : 101; -#pragma warning restore CS8500 } [MethodImpl(MethodImplOptions.NoInlining)] diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template deleted file mode 100644 index 09aaf2f442e136..00000000000000 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveLoadMaskedUnOpTest.template +++ /dev/null @@ -1,203 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -/****************************************************************************** - * This file is auto-generated from a template file by the GenerateTests.csx * - * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * - * changes, please update the corresponding template and run according to the * - * directions listed in the file. * - ******************************************************************************/ - -using System; -using System.Numerics; -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using Xunit; - -namespace JIT.HardwareIntrinsics.Arm -{ - public static partial class Program - { - [Fact] - public static void {TestName}() - { - var test = new LoadUnaryOpTest__{TestName}(); - - if (test.IsSupported) - { - // Validates basic functionality works - test.RunBasicScenario_Load(); - - // Validates calling via reflection works - // TODO-SVE: Enable once register allocation exists for predicates. - // test.RunReflectionScenario_Load(); - } - else - { - // Validates we throw on unsupported hardware - test.RunUnsupportedScenario(); - } - - if (!test.Succeeded) - { - throw new Exception("One or more scenarios did not complete as expected."); - } - } - } - - public sealed unsafe class LoadUnaryOpTest__{TestName} - { - private struct DataTable - { - private byte[] inArray1; - private byte[] outArray; - - private GCHandle inHandle1; - private GCHandle outHandle; - - private ulong alignment; - - public DataTable({Op2BaseType}[] inArray1, {RetBaseType}[] outArray, int alignment) - { - int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op2BaseType}>(); - int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); - if ((alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfoutArray) - { - throw new ArgumentException("Invalid value of alignment"); - } - - this.inArray1 = new byte[alignment * 2]; - this.outArray = new byte[alignment * 2]; - - this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); - this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); - - this.alignment = (ulong)alignment; - - Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); - - } - - public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); - public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); - - public void Dispose() - { - inHandle1.Free(); - outHandle.Free(); - } - - private static unsafe void* Align(byte* buffer, ulong expectedAlignment) - { - return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); - } - } - - private static readonly int LargestVectorSize = {LargestVectorSize}; - - private static readonly int Op2ElementCount = Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>() / sizeof({Op2BaseType}); - private static readonly int RetElementCount = Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>() / sizeof({RetBaseType}); - - private static {Op2BaseType}[] _data = new {Op2BaseType}[Op2ElementCount]; - - private DataTable _dataTable; - - public LoadUnaryOpTest__{TestName}() - { - Succeeded = true; - - for (var i = 0; i < Op2ElementCount; i++) { _data[i] = {NextValueOp2}; } - _dataTable = new DataTable(_data, new {RetBaseType}[RetElementCount], LargestVectorSize); - } - - public bool IsSupported => {Isa}.IsSupported; - - public bool Succeeded { get; set; } - - public void RunBasicScenario_Load() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); - - //TODO-SVE: Once register allocation exists for predicates, move loadMask into DataTable - {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); - - var result = {Isa}.{Method}( - loadMask, - ({Op2BaseType}*)(_dataTable.inArray1Ptr) - ); - - Unsafe.Write(_dataTable.outArrayPtr, result); - ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); - } - - public void RunReflectionScenario_Load() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_Load)); - - {Op1VectorType}<{Op1BaseType}> loadMask = Sve.CreateTrueMask{RetBaseType}(SveMaskPattern.All); - - var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof(Vector<{Op2BaseType}>), typeof({Op2BaseType}*) }) - .Invoke(null, new object[] { - loadMask, - Pointer.Box(_dataTable.inArray1Ptr, typeof({Op2BaseType}*)) - }); - - Unsafe.Write(_dataTable.outArrayPtr, ({RetVectorType}<{RetBaseType}>)(result)); - ValidateResult(_dataTable.inArray1Ptr, _dataTable.outArrayPtr); - } - - public void RunUnsupportedScenario() - { - TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); - - Succeeded = false; - - try - { - RunBasicScenario_Load(); - } - catch (PlatformNotSupportedException) - { - Succeeded = true; - } - } - - private void ValidateResult(void* firstOp, void* result, [CallerMemberName] string method = "") - { - {Op2BaseType}[] inArray = new {Op2BaseType}[Op2ElementCount]; - {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; - - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray[0]), ref Unsafe.AsRef(firstOp), (uint)Unsafe.SizeOf<{RetVectorType}<{Op2BaseType}>>()); - Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>()); - - ValidateResult(inArray, outArray, method); - } - - private void ValidateResult({Op2BaseType}[] firstOp, {RetBaseType}[] result, [CallerMemberName] string method = "") - { - bool succeeded = true; - - for (var i = 0; i < RetElementCount; i++) - { - if ({ValidateIterResult}) - { - succeeded = false; - break; - } - } - - if (!succeeded) - { - TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1VectorType}<{Op1BaseType}>, {Op2BaseType}): {method} failed:"); - TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})"); - TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); - TestLibrary.TestFramework.LogInformation(string.Empty); - - Succeeded = false; - } - } - } -} diff --git a/src/tests/JIT/Regression/Dev11/External/Dev11_243742/app.cs b/src/tests/JIT/Regression/Dev11/External/Dev11_243742/app.cs index 1842f13cc9e099..393691bd4336d7 100644 --- a/src/tests/JIT/Regression/Dev11/External/Dev11_243742/app.cs +++ b/src/tests/JIT/Regression/Dev11/External/Dev11_243742/app.cs @@ -3,6 +3,10 @@ /* * Regression test for Dev11 243742 [Triton] +* precommands: +* set DOTNET_ZAPREQUIRE=2 +* set CORECLR_PREJITType=MDIL +* del /q nitype.signal * * Execute: * %CORE_ROOT%\fxprun.exe App.exe diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_61510/Runtime_61510.cs b/src/tests/JIT/Regression/JitBlue/Runtime_61510/Runtime_61510.cs index 122e8d0c604f28..fe76c604ef1b0e 100644 --- a/src/tests/JIT/Regression/JitBlue/Runtime_61510/Runtime_61510.cs +++ b/src/tests/JIT/Regression/JitBlue/Runtime_61510/Runtime_61510.cs @@ -12,7 +12,6 @@ public unsafe class Runtime_61510 [Fact] public static int TestEntryPoint() { - // Unsafe.AsPointer is safe since static field is marked with [FixedAddressValueType] ref byte result = ref AddZeroByrefToNativeInt((nint)Unsafe.AsPointer(ref s_field)); return Unsafe.AreSame(ref s_field, ref result) ? 100 : 101; diff --git a/src/tests/JIT/Regression/JitBlue/Runtime_92349/Runtime_92349.cs b/src/tests/JIT/Regression/JitBlue/Runtime_92349/Runtime_92349.cs index 5ddf458793104f..5de0a28895b268 100644 --- a/src/tests/JIT/Regression/JitBlue/Runtime_92349/Runtime_92349.cs +++ b/src/tests/JIT/Regression/JitBlue/Runtime_92349/Runtime_92349.cs @@ -22,7 +22,7 @@ public unsafe static void EntryPoint() if (Sse2.IsSupported) { ulong value = 0; - Test((byte*)&value); + Test((byte*)Unsafe.AsPointer(ref value)); Assert.True(value == 246); } } diff --git a/src/tests/JIT/jit64/mcc/common/common.il b/src/tests/JIT/jit64/mcc/common/common.il index 83eac3d270fc36..3b45f9c46e48af 100644 --- a/src/tests/JIT/jit64/mcc/common/common.il +++ b/src/tests/JIT/jit64/mcc/common/common.il @@ -16207,6 +16207,6919 @@ } // end of class MCCTest.VType8 +.class public sequential ansi sealed beforefieldinit MCCTest.VType9 + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public float32 f1 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: conv.r4 + IL_0004: stfld float32 MCCTest.VType9::f1 + IL_0009: ret + } // end of method VType9::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VType9::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VType9::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VType9::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VType9::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VType9 val) cil managed + { + // Code size 22 (0x16) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: dup + IL_0003: ldfld float32 MCCTest.VType9::f1 + IL_0008: ldarga.s val + IL_000a: ldfld float32 MCCTest.VType9::f1 + IL_000f: add + IL_0010: stfld float32 MCCTest.VType9::f1 + IL_0015: ret + } // end of method VType9::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VType9 expected) cil managed + { + // Code size 68 (0x44) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2, + bool V_3) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldfld float32 MCCTest.VType9::f1 + IL_0007: ldarga.s expected + IL_0009: ldfld float32 MCCTest.VType9::f1 + IL_000e: ceq + IL_0010: stloc.3 + IL_0011: ldloc.3 + IL_0012: brtrue.s IL_002f + + IL_0014: nop + IL_0015: ldstr "f1" + IL_001a: ldarg.0 + IL_001b: ldfld float32 MCCTest.VType9::f1 + IL_0020: conv.r8 + IL_0021: ldarga.s expected + IL_0023: ldfld float32 MCCTest.VType9::f1 + IL_0028: conv.r8 + IL_0029: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_002e: throw + + IL_002f: ldnull + IL_0030: stloc.0 + IL_0031: ldnull + IL_0032: stloc.1 + .try + { + IL_0033: nop + IL_0034: nop + IL_0035: leave.s IL_0042 + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_0037: stloc.2 + IL_0038: nop + IL_0039: ldloc.0 + IL_003a: ldloc.1 + IL_003b: ldloc.2 + IL_003c: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_0041: throw + + } // end handler + IL_0042: nop + IL_0043: ret + } // end of method VType9::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 54 (0x36) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "f1 = " + IL_0015: ldarg.0 + IL_0016: ldfld float32 MCCTest.VType9::f1 + IL_001b: box [mscorlib]System.Single + IL_0020: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0025: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_002a: nop + IL_002b: ldloc.1 + IL_002c: callvirt instance string [mscorlib]System.Object::ToString() + IL_0031: stloc.2 + IL_0032: br.s IL_0034 + + IL_0034: ldloc.2 + IL_0035: ret + } // end of method VType9::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VType9::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VType9::Dump + +} // end of class MCCTest.VType9 + +.class public sequential ansi sealed beforefieldinit MCCTest.VTypeA + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public class MCCTest.RType4 f1 + .field public class MCCTest.RType4 f2 + .field public class MCCTest.RType4 f3 + .field public class MCCTest.RType4 f4 + .field public class MCCTest.RType4 f5 + .field public class MCCTest.RType4 f6 + .field public class MCCTest.RType4 f7 + .field public class MCCTest.RType4 f8 + .field public class MCCTest.RType4 f9 + .field public class MCCTest.RType4 f10 + .field public class MCCTest.RType4 f11 + .field public class MCCTest.RType4 f12 + .field public class MCCTest.RType4 f13 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 314 (0x13a) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: newobj instance void MCCTest.RType4::.ctor() + IL_0007: stfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_000c: ldarg.0 + IL_000d: newobj instance void MCCTest.RType4::.ctor() + IL_0012: stfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_0017: ldarg.0 + IL_0018: newobj instance void MCCTest.RType4::.ctor() + IL_001d: stfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_0022: ldarg.0 + IL_0023: newobj instance void MCCTest.RType4::.ctor() + IL_0028: stfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_002d: ldarg.0 + IL_002e: newobj instance void MCCTest.RType4::.ctor() + IL_0033: stfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_0038: ldarg.0 + IL_0039: newobj instance void MCCTest.RType4::.ctor() + IL_003e: stfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_0043: ldarg.0 + IL_0044: newobj instance void MCCTest.RType4::.ctor() + IL_0049: stfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_004e: ldarg.0 + IL_004f: newobj instance void MCCTest.RType4::.ctor() + IL_0054: stfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_0059: ldarg.0 + IL_005a: newobj instance void MCCTest.RType4::.ctor() + IL_005f: stfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_0064: ldarg.0 + IL_0065: newobj instance void MCCTest.RType4::.ctor() + IL_006a: stfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_006f: ldarg.0 + IL_0070: newobj instance void MCCTest.RType4::.ctor() + IL_0075: stfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_007a: ldarg.0 + IL_007b: newobj instance void MCCTest.RType4::.ctor() + IL_0080: stfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_0085: ldarg.0 + IL_0086: newobj instance void MCCTest.RType4::.ctor() + IL_008b: stfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_0090: ldarg.0 + IL_0091: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_0096: ldarg.1 + IL_0097: callvirt instance void MCCTest.RType4::Init(int32) + IL_009c: nop + IL_009d: ldarg.0 + IL_009e: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_00a3: ldarg.1 + IL_00a4: callvirt instance void MCCTest.RType4::Init(int32) + IL_00a9: nop + IL_00aa: ldarg.0 + IL_00ab: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_00b0: ldarg.1 + IL_00b1: callvirt instance void MCCTest.RType4::Init(int32) + IL_00b6: nop + IL_00b7: ldarg.0 + IL_00b8: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_00bd: ldarg.1 + IL_00be: callvirt instance void MCCTest.RType4::Init(int32) + IL_00c3: nop + IL_00c4: ldarg.0 + IL_00c5: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_00ca: ldarg.1 + IL_00cb: callvirt instance void MCCTest.RType4::Init(int32) + IL_00d0: nop + IL_00d1: ldarg.0 + IL_00d2: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_00d7: ldarg.1 + IL_00d8: callvirt instance void MCCTest.RType4::Init(int32) + IL_00dd: nop + IL_00de: ldarg.0 + IL_00df: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_00e4: ldarg.1 + IL_00e5: callvirt instance void MCCTest.RType4::Init(int32) + IL_00ea: nop + IL_00eb: ldarg.0 + IL_00ec: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_00f1: ldarg.1 + IL_00f2: callvirt instance void MCCTest.RType4::Init(int32) + IL_00f7: nop + IL_00f8: ldarg.0 + IL_00f9: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_00fe: ldarg.1 + IL_00ff: callvirt instance void MCCTest.RType4::Init(int32) + IL_0104: nop + IL_0105: ldarg.0 + IL_0106: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_010b: ldarg.1 + IL_010c: callvirt instance void MCCTest.RType4::Init(int32) + IL_0111: nop + IL_0112: ldarg.0 + IL_0113: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_0118: ldarg.1 + IL_0119: callvirt instance void MCCTest.RType4::Init(int32) + IL_011e: nop + IL_011f: ldarg.0 + IL_0120: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_0125: ldarg.1 + IL_0126: callvirt instance void MCCTest.RType4::Init(int32) + IL_012b: nop + IL_012c: ldarg.0 + IL_012d: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_0132: ldarg.1 + IL_0133: callvirt instance void MCCTest.RType4::Init(int32) + IL_0138: nop + IL_0139: ret + } // end of method VTypeA::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VTypeA::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeA::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VTypeA::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeA::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VTypeA val) cil managed + { + // Code size 249 (0xf9) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_0007: ldarga.s val + IL_0009: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_000e: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0013: nop + IL_0014: ldarg.0 + IL_0015: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_001a: ldarga.s val + IL_001c: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_0021: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0026: nop + IL_0027: ldarg.0 + IL_0028: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_002d: ldarga.s val + IL_002f: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_0034: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0039: nop + IL_003a: ldarg.0 + IL_003b: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_0040: ldarga.s val + IL_0042: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_0047: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_004c: nop + IL_004d: ldarg.0 + IL_004e: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_0053: ldarga.s val + IL_0055: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_005a: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_005f: nop + IL_0060: ldarg.0 + IL_0061: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_0066: ldarga.s val + IL_0068: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_006d: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0072: nop + IL_0073: ldarg.0 + IL_0074: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_0079: ldarga.s val + IL_007b: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_0080: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0085: nop + IL_0086: ldarg.0 + IL_0087: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_008c: ldarga.s val + IL_008e: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_0093: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0098: nop + IL_0099: ldarg.0 + IL_009a: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_009f: ldarga.s val + IL_00a1: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_00a6: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00ab: nop + IL_00ac: ldarg.0 + IL_00ad: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_00b2: ldarga.s val + IL_00b4: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_00b9: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00be: nop + IL_00bf: ldarg.0 + IL_00c0: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_00c5: ldarga.s val + IL_00c7: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_00cc: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00d1: nop + IL_00d2: ldarg.0 + IL_00d3: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_00d8: ldarga.s val + IL_00da: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_00df: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00e4: nop + IL_00e5: ldarg.0 + IL_00e6: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_00eb: ldarga.s val + IL_00ed: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_00f2: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00f7: nop + IL_00f8: ret + } // end of method VTypeA::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VTypeA expected) cil managed + { + // Code size 503 (0x1f7) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2) + IL_0000: nop + IL_0001: ldnull + IL_0002: stloc.0 + IL_0003: ldnull + IL_0004: stloc.1 + .try + { + IL_0005: nop + IL_0006: ldstr "f1" + IL_000b: stloc.0 + IL_000c: ldarg.0 + IL_000d: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_0012: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0017: stloc.1 + IL_0018: ldarg.0 + IL_0019: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_001e: ldarga.s expected + IL_0020: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_0025: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_002a: nop + IL_002b: ldstr "f2" + IL_0030: stloc.0 + IL_0031: ldarg.0 + IL_0032: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_0037: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_003c: stloc.1 + IL_003d: ldarg.0 + IL_003e: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_0043: ldarga.s expected + IL_0045: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_004a: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_004f: nop + IL_0050: ldstr "f3" + IL_0055: stloc.0 + IL_0056: ldarg.0 + IL_0057: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_005c: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0061: stloc.1 + IL_0062: ldarg.0 + IL_0063: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_0068: ldarga.s expected + IL_006a: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_006f: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0074: nop + IL_0075: ldstr "f4" + IL_007a: stloc.0 + IL_007b: ldarg.0 + IL_007c: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_0081: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0086: stloc.1 + IL_0087: ldarg.0 + IL_0088: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_008d: ldarga.s expected + IL_008f: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_0094: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0099: nop + IL_009a: ldstr "f5" + IL_009f: stloc.0 + IL_00a0: ldarg.0 + IL_00a1: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_00a6: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00ab: stloc.1 + IL_00ac: ldarg.0 + IL_00ad: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_00b2: ldarga.s expected + IL_00b4: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_00b9: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_00be: nop + IL_00bf: ldstr "f6" + IL_00c4: stloc.0 + IL_00c5: ldarg.0 + IL_00c6: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_00cb: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00d0: stloc.1 + IL_00d1: ldarg.0 + IL_00d2: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_00d7: ldarga.s expected + IL_00d9: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_00de: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_00e3: nop + IL_00e4: ldstr "f7" + IL_00e9: stloc.0 + IL_00ea: ldarg.0 + IL_00eb: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_00f0: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00f5: stloc.1 + IL_00f6: ldarg.0 + IL_00f7: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_00fc: ldarga.s expected + IL_00fe: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_0103: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0108: nop + IL_0109: ldstr "f8" + IL_010e: stloc.0 + IL_010f: ldarg.0 + IL_0110: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_0115: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_011a: stloc.1 + IL_011b: ldarg.0 + IL_011c: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_0121: ldarga.s expected + IL_0123: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_0128: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_012d: nop + IL_012e: ldstr "f9" + IL_0133: stloc.0 + IL_0134: ldarg.0 + IL_0135: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_013a: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_013f: stloc.1 + IL_0140: ldarg.0 + IL_0141: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_0146: ldarga.s expected + IL_0148: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_014d: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0152: nop + IL_0153: ldstr "f10" + IL_0158: stloc.0 + IL_0159: ldarg.0 + IL_015a: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_015f: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0164: stloc.1 + IL_0165: ldarg.0 + IL_0166: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_016b: ldarga.s expected + IL_016d: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_0172: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0177: nop + IL_0178: ldstr "f11" + IL_017d: stloc.0 + IL_017e: ldarg.0 + IL_017f: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_0184: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0189: stloc.1 + IL_018a: ldarg.0 + IL_018b: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_0190: ldarga.s expected + IL_0192: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_0197: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_019c: nop + IL_019d: ldstr "f12" + IL_01a2: stloc.0 + IL_01a3: ldarg.0 + IL_01a4: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_01a9: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ae: stloc.1 + IL_01af: ldarg.0 + IL_01b0: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_01b5: ldarga.s expected + IL_01b7: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_01bc: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_01c1: nop + IL_01c2: ldstr "f13" + IL_01c7: stloc.0 + IL_01c8: ldarg.0 + IL_01c9: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_01ce: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01d3: stloc.1 + IL_01d4: ldarg.0 + IL_01d5: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_01da: ldarga.s expected + IL_01dc: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_01e1: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_01e6: nop + IL_01e7: nop + IL_01e8: leave.s IL_01f5 + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_01ea: stloc.2 + IL_01eb: nop + IL_01ec: ldloc.0 + IL_01ed: ldloc.1 + IL_01ee: ldloc.2 + IL_01ef: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_01f4: throw + + } // end handler + IL_01f5: nop + IL_01f6: ret + } // end of method VTypeA::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 740 (0x2e4) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "[Field f1] [Type '{0}']" + IL_0015: call string [mscorlib]System.String::Concat(string, + string) + IL_001a: ldarg.0 + IL_001b: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_0020: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0025: callvirt instance string [mscorlib]System.Object::ToString() + IL_002a: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_002f: nop + IL_0030: ldloc.1 + IL_0031: ldarg.0 + IL_0032: ldfld class MCCTest.RType4 MCCTest.VTypeA::f1 + IL_0037: ldarg.1 + IL_0038: ldc.i4.1 + IL_0039: add + IL_003a: callvirt instance string MCCTest.RType4::Dump(int32) + IL_003f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0044: nop + IL_0045: ldloc.1 + IL_0046: ldloc.0 + IL_0047: ldstr "[Field f2] [Type '{0}']" + IL_004c: call string [mscorlib]System.String::Concat(string, + string) + IL_0051: ldarg.0 + IL_0052: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_0057: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_005c: callvirt instance string [mscorlib]System.Object::ToString() + IL_0061: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0066: nop + IL_0067: ldloc.1 + IL_0068: ldarg.0 + IL_0069: ldfld class MCCTest.RType4 MCCTest.VTypeA::f2 + IL_006e: ldarg.1 + IL_006f: ldc.i4.1 + IL_0070: add + IL_0071: callvirt instance string MCCTest.RType4::Dump(int32) + IL_0076: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_007b: nop + IL_007c: ldloc.1 + IL_007d: ldloc.0 + IL_007e: ldstr "[Field f3] [Type '{0}']" + IL_0083: call string [mscorlib]System.String::Concat(string, + string) + IL_0088: ldarg.0 + IL_0089: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_008e: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0093: callvirt instance string [mscorlib]System.Object::ToString() + IL_0098: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_009d: nop + IL_009e: ldloc.1 + IL_009f: ldarg.0 + IL_00a0: ldfld class MCCTest.RType4 MCCTest.VTypeA::f3 + IL_00a5: ldarg.1 + IL_00a6: ldc.i4.1 + IL_00a7: add + IL_00a8: callvirt instance string MCCTest.RType4::Dump(int32) + IL_00ad: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00b2: nop + IL_00b3: ldloc.1 + IL_00b4: ldloc.0 + IL_00b5: ldstr "[Field f4] [Type '{0}']" + IL_00ba: call string [mscorlib]System.String::Concat(string, + string) + IL_00bf: ldarg.0 + IL_00c0: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_00c5: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00ca: callvirt instance string [mscorlib]System.Object::ToString() + IL_00cf: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00d4: nop + IL_00d5: ldloc.1 + IL_00d6: ldarg.0 + IL_00d7: ldfld class MCCTest.RType4 MCCTest.VTypeA::f4 + IL_00dc: ldarg.1 + IL_00dd: ldc.i4.1 + IL_00de: add + IL_00df: callvirt instance string MCCTest.RType4::Dump(int32) + IL_00e4: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00e9: nop + IL_00ea: ldloc.1 + IL_00eb: ldloc.0 + IL_00ec: ldstr "[Field f5] [Type '{0}']" + IL_00f1: call string [mscorlib]System.String::Concat(string, + string) + IL_00f6: ldarg.0 + IL_00f7: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_00fc: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0101: callvirt instance string [mscorlib]System.Object::ToString() + IL_0106: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_010b: nop + IL_010c: ldloc.1 + IL_010d: ldarg.0 + IL_010e: ldfld class MCCTest.RType4 MCCTest.VTypeA::f5 + IL_0113: ldarg.1 + IL_0114: ldc.i4.1 + IL_0115: add + IL_0116: callvirt instance string MCCTest.RType4::Dump(int32) + IL_011b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0120: nop + IL_0121: ldloc.1 + IL_0122: ldloc.0 + IL_0123: ldstr "[Field f6] [Type '{0}']" + IL_0128: call string [mscorlib]System.String::Concat(string, + string) + IL_012d: ldarg.0 + IL_012e: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_0133: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0138: callvirt instance string [mscorlib]System.Object::ToString() + IL_013d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0142: nop + IL_0143: ldloc.1 + IL_0144: ldarg.0 + IL_0145: ldfld class MCCTest.RType4 MCCTest.VTypeA::f6 + IL_014a: ldarg.1 + IL_014b: ldc.i4.1 + IL_014c: add + IL_014d: callvirt instance string MCCTest.RType4::Dump(int32) + IL_0152: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0157: nop + IL_0158: ldloc.1 + IL_0159: ldloc.0 + IL_015a: ldstr "[Field f7] [Type '{0}']" + IL_015f: call string [mscorlib]System.String::Concat(string, + string) + IL_0164: ldarg.0 + IL_0165: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_016a: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_016f: callvirt instance string [mscorlib]System.Object::ToString() + IL_0174: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0179: nop + IL_017a: ldloc.1 + IL_017b: ldarg.0 + IL_017c: ldfld class MCCTest.RType4 MCCTest.VTypeA::f7 + IL_0181: ldarg.1 + IL_0182: ldc.i4.1 + IL_0183: add + IL_0184: callvirt instance string MCCTest.RType4::Dump(int32) + IL_0189: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_018e: nop + IL_018f: ldloc.1 + IL_0190: ldloc.0 + IL_0191: ldstr "[Field f8] [Type '{0}']" + IL_0196: call string [mscorlib]System.String::Concat(string, + string) + IL_019b: ldarg.0 + IL_019c: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_01a1: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01a6: callvirt instance string [mscorlib]System.Object::ToString() + IL_01ab: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01b0: nop + IL_01b1: ldloc.1 + IL_01b2: ldarg.0 + IL_01b3: ldfld class MCCTest.RType4 MCCTest.VTypeA::f8 + IL_01b8: ldarg.1 + IL_01b9: ldc.i4.1 + IL_01ba: add + IL_01bb: callvirt instance string MCCTest.RType4::Dump(int32) + IL_01c0: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01c5: nop + IL_01c6: ldloc.1 + IL_01c7: ldloc.0 + IL_01c8: ldstr "[Field f9] [Type '{0}']" + IL_01cd: call string [mscorlib]System.String::Concat(string, + string) + IL_01d2: ldarg.0 + IL_01d3: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_01d8: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01dd: callvirt instance string [mscorlib]System.Object::ToString() + IL_01e2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01e7: nop + IL_01e8: ldloc.1 + IL_01e9: ldarg.0 + IL_01ea: ldfld class MCCTest.RType4 MCCTest.VTypeA::f9 + IL_01ef: ldarg.1 + IL_01f0: ldc.i4.1 + IL_01f1: add + IL_01f2: callvirt instance string MCCTest.RType4::Dump(int32) + IL_01f7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01fc: nop + IL_01fd: ldloc.1 + IL_01fe: ldloc.0 + IL_01ff: ldstr "[Field f10] [Type '{0}']" + IL_0204: call string [mscorlib]System.String::Concat(string, + string) + IL_0209: ldarg.0 + IL_020a: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_020f: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0214: callvirt instance string [mscorlib]System.Object::ToString() + IL_0219: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_021e: nop + IL_021f: ldloc.1 + IL_0220: ldarg.0 + IL_0221: ldfld class MCCTest.RType4 MCCTest.VTypeA::f10 + IL_0226: ldarg.1 + IL_0227: ldc.i4.1 + IL_0228: add + IL_0229: callvirt instance string MCCTest.RType4::Dump(int32) + IL_022e: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0233: nop + IL_0234: ldloc.1 + IL_0235: ldloc.0 + IL_0236: ldstr "[Field f11] [Type '{0}']" + IL_023b: call string [mscorlib]System.String::Concat(string, + string) + IL_0240: ldarg.0 + IL_0241: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_0246: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_024b: callvirt instance string [mscorlib]System.Object::ToString() + IL_0250: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0255: nop + IL_0256: ldloc.1 + IL_0257: ldarg.0 + IL_0258: ldfld class MCCTest.RType4 MCCTest.VTypeA::f11 + IL_025d: ldarg.1 + IL_025e: ldc.i4.1 + IL_025f: add + IL_0260: callvirt instance string MCCTest.RType4::Dump(int32) + IL_0265: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_026a: nop + IL_026b: ldloc.1 + IL_026c: ldloc.0 + IL_026d: ldstr "[Field f12] [Type '{0}']" + IL_0272: call string [mscorlib]System.String::Concat(string, + string) + IL_0277: ldarg.0 + IL_0278: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_027d: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0282: callvirt instance string [mscorlib]System.Object::ToString() + IL_0287: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_028c: nop + IL_028d: ldloc.1 + IL_028e: ldarg.0 + IL_028f: ldfld class MCCTest.RType4 MCCTest.VTypeA::f12 + IL_0294: ldarg.1 + IL_0295: ldc.i4.1 + IL_0296: add + IL_0297: callvirt instance string MCCTest.RType4::Dump(int32) + IL_029c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02a1: nop + IL_02a2: ldloc.1 + IL_02a3: ldloc.0 + IL_02a4: ldstr "[Field f13] [Type '{0}']" + IL_02a9: call string [mscorlib]System.String::Concat(string, + string) + IL_02ae: ldarg.0 + IL_02af: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_02b4: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02b9: callvirt instance string [mscorlib]System.Object::ToString() + IL_02be: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02c3: nop + IL_02c4: ldloc.1 + IL_02c5: ldarg.0 + IL_02c6: ldfld class MCCTest.RType4 MCCTest.VTypeA::f13 + IL_02cb: ldarg.1 + IL_02cc: ldc.i4.1 + IL_02cd: add + IL_02ce: callvirt instance string MCCTest.RType4::Dump(int32) + IL_02d3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02d8: nop + IL_02d9: ldloc.1 + IL_02da: callvirt instance string [mscorlib]System.Object::ToString() + IL_02df: stloc.2 + IL_02e0: br.s IL_02e2 + + IL_02e2: ldloc.2 + IL_02e3: ret + } // end of method VTypeA::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VTypeA::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VTypeA::Dump + +} // end of class MCCTest.VTypeA + +.class public sequential ansi sealed beforefieldinit MCCTest.VTypeB + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public valuetype MCCTest.VType5 f1 + .field public valuetype MCCTest.VType5 f2 + .field public valuetype MCCTest.VType5 f3 + .field public valuetype MCCTest.VType5 f4 + .field public valuetype MCCTest.VType5 f5 + .field public valuetype MCCTest.VType5 f6 + .field public valuetype MCCTest.VType5 f7 + .field public valuetype MCCTest.VType5 f8 + .field public valuetype MCCTest.VType5 f9 + .field public valuetype MCCTest.VType5 f10 + .field public valuetype MCCTest.VType5 f11 + .field public valuetype MCCTest.VType5 f12 + .field public valuetype MCCTest.VType5 f13 + .field public valuetype MCCTest.VType5 f14 + .field public valuetype MCCTest.VType5 f15 + .field public valuetype MCCTest.VType5 f16 + .field public valuetype MCCTest.VType5 f17 + .field public valuetype MCCTest.VType5 f18 + .field public valuetype MCCTest.VType5 f19 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 249 (0xf9) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_0007: ldarg.1 + IL_0008: call instance void MCCTest.VType5::Init(int32) + IL_000d: nop + IL_000e: ldarg.0 + IL_000f: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_0014: ldarg.1 + IL_0015: call instance void MCCTest.VType5::Init(int32) + IL_001a: nop + IL_001b: ldarg.0 + IL_001c: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_0021: ldarg.1 + IL_0022: call instance void MCCTest.VType5::Init(int32) + IL_0027: nop + IL_0028: ldarg.0 + IL_0029: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_002e: ldarg.1 + IL_002f: call instance void MCCTest.VType5::Init(int32) + IL_0034: nop + IL_0035: ldarg.0 + IL_0036: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_003b: ldarg.1 + IL_003c: call instance void MCCTest.VType5::Init(int32) + IL_0041: nop + IL_0042: ldarg.0 + IL_0043: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_0048: ldarg.1 + IL_0049: call instance void MCCTest.VType5::Init(int32) + IL_004e: nop + IL_004f: ldarg.0 + IL_0050: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_0055: ldarg.1 + IL_0056: call instance void MCCTest.VType5::Init(int32) + IL_005b: nop + IL_005c: ldarg.0 + IL_005d: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_0062: ldarg.1 + IL_0063: call instance void MCCTest.VType5::Init(int32) + IL_0068: nop + IL_0069: ldarg.0 + IL_006a: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_006f: ldarg.1 + IL_0070: call instance void MCCTest.VType5::Init(int32) + IL_0075: nop + IL_0076: ldarg.0 + IL_0077: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_007c: ldarg.1 + IL_007d: call instance void MCCTest.VType5::Init(int32) + IL_0082: nop + IL_0083: ldarg.0 + IL_0084: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_0089: ldarg.1 + IL_008a: call instance void MCCTest.VType5::Init(int32) + IL_008f: nop + IL_0090: ldarg.0 + IL_0091: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_0096: ldarg.1 + IL_0097: call instance void MCCTest.VType5::Init(int32) + IL_009c: nop + IL_009d: ldarg.0 + IL_009e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_00a3: ldarg.1 + IL_00a4: call instance void MCCTest.VType5::Init(int32) + IL_00a9: nop + IL_00aa: ldarg.0 + IL_00ab: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_00b0: ldarg.1 + IL_00b1: call instance void MCCTest.VType5::Init(int32) + IL_00b6: nop + IL_00b7: ldarg.0 + IL_00b8: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_00bd: ldarg.1 + IL_00be: call instance void MCCTest.VType5::Init(int32) + IL_00c3: nop + IL_00c4: ldarg.0 + IL_00c5: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_00ca: ldarg.1 + IL_00cb: call instance void MCCTest.VType5::Init(int32) + IL_00d0: nop + IL_00d1: ldarg.0 + IL_00d2: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_00d7: ldarg.1 + IL_00d8: call instance void MCCTest.VType5::Init(int32) + IL_00dd: nop + IL_00de: ldarg.0 + IL_00df: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_00e4: ldarg.1 + IL_00e5: call instance void MCCTest.VType5::Init(int32) + IL_00ea: nop + IL_00eb: ldarg.0 + IL_00ec: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_00f1: ldarg.1 + IL_00f2: call instance void MCCTest.VType5::Init(int32) + IL_00f7: nop + IL_00f8: ret + } // end of method VTypeB::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VTypeB::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeB::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VTypeB::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeB::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VTypeB val) cil managed + { + // Code size 363 (0x16b) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_0007: ldarga.s val + IL_0009: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_000e: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0013: nop + IL_0014: ldarg.0 + IL_0015: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_001a: ldarga.s val + IL_001c: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_0021: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0026: nop + IL_0027: ldarg.0 + IL_0028: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_002d: ldarga.s val + IL_002f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_0034: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0039: nop + IL_003a: ldarg.0 + IL_003b: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_0040: ldarga.s val + IL_0042: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_0047: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_004c: nop + IL_004d: ldarg.0 + IL_004e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_0053: ldarga.s val + IL_0055: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_005a: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_005f: nop + IL_0060: ldarg.0 + IL_0061: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_0066: ldarga.s val + IL_0068: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_006d: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0072: nop + IL_0073: ldarg.0 + IL_0074: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_0079: ldarga.s val + IL_007b: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_0080: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0085: nop + IL_0086: ldarg.0 + IL_0087: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_008c: ldarga.s val + IL_008e: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_0093: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0098: nop + IL_0099: ldarg.0 + IL_009a: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_009f: ldarga.s val + IL_00a1: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_00a6: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00ab: nop + IL_00ac: ldarg.0 + IL_00ad: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_00b2: ldarga.s val + IL_00b4: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_00b9: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00be: nop + IL_00bf: ldarg.0 + IL_00c0: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_00c5: ldarga.s val + IL_00c7: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_00cc: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00d1: nop + IL_00d2: ldarg.0 + IL_00d3: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_00d8: ldarga.s val + IL_00da: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_00df: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00e4: nop + IL_00e5: ldarg.0 + IL_00e6: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_00eb: ldarga.s val + IL_00ed: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_00f2: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00f7: nop + IL_00f8: ldarg.0 + IL_00f9: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_00fe: ldarga.s val + IL_0100: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_0105: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_010a: nop + IL_010b: ldarg.0 + IL_010c: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_0111: ldarga.s val + IL_0113: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_0118: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_011d: nop + IL_011e: ldarg.0 + IL_011f: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_0124: ldarga.s val + IL_0126: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_012b: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0130: nop + IL_0131: ldarg.0 + IL_0132: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_0137: ldarga.s val + IL_0139: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_013e: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0143: nop + IL_0144: ldarg.0 + IL_0145: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_014a: ldarga.s val + IL_014c: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_0151: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0156: nop + IL_0157: ldarg.0 + IL_0158: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_015d: ldarga.s val + IL_015f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_0164: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0169: nop + IL_016a: ret + } // end of method VTypeB::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VTypeB expected) cil managed + { + // Code size 820 (0x334) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2) + IL_0000: nop + IL_0001: ldnull + IL_0002: stloc.0 + IL_0003: ldnull + IL_0004: stloc.1 + .try + { + IL_0005: nop + IL_0006: ldstr "f1" + IL_000b: stloc.0 + IL_000c: ldarg.0 + IL_000d: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_0012: box MCCTest.VType5 + IL_0017: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_001c: stloc.1 + IL_001d: ldarg.0 + IL_001e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_0023: ldarga.s expected + IL_0025: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_002a: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_002f: nop + IL_0030: ldstr "f2" + IL_0035: stloc.0 + IL_0036: ldarg.0 + IL_0037: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_003c: box MCCTest.VType5 + IL_0041: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0046: stloc.1 + IL_0047: ldarg.0 + IL_0048: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_004d: ldarga.s expected + IL_004f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_0054: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0059: nop + IL_005a: ldstr "f3" + IL_005f: stloc.0 + IL_0060: ldarg.0 + IL_0061: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_0066: box MCCTest.VType5 + IL_006b: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0070: stloc.1 + IL_0071: ldarg.0 + IL_0072: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_0077: ldarga.s expected + IL_0079: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_007e: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0083: nop + IL_0084: ldstr "f4" + IL_0089: stloc.0 + IL_008a: ldarg.0 + IL_008b: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_0090: box MCCTest.VType5 + IL_0095: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_009a: stloc.1 + IL_009b: ldarg.0 + IL_009c: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_00a1: ldarga.s expected + IL_00a3: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_00a8: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_00ad: nop + IL_00ae: ldstr "f5" + IL_00b3: stloc.0 + IL_00b4: ldarg.0 + IL_00b5: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_00ba: box MCCTest.VType5 + IL_00bf: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00c4: stloc.1 + IL_00c5: ldarg.0 + IL_00c6: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_00cb: ldarga.s expected + IL_00cd: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_00d2: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_00d7: nop + IL_00d8: ldstr "f6" + IL_00dd: stloc.0 + IL_00de: ldarg.0 + IL_00df: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_00e4: box MCCTest.VType5 + IL_00e9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00ee: stloc.1 + IL_00ef: ldarg.0 + IL_00f0: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_00f5: ldarga.s expected + IL_00f7: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_00fc: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0101: nop + IL_0102: ldstr "f7" + IL_0107: stloc.0 + IL_0108: ldarg.0 + IL_0109: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_010e: box MCCTest.VType5 + IL_0113: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0118: stloc.1 + IL_0119: ldarg.0 + IL_011a: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_011f: ldarga.s expected + IL_0121: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_0126: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_012b: nop + IL_012c: ldstr "f8" + IL_0131: stloc.0 + IL_0132: ldarg.0 + IL_0133: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_0138: box MCCTest.VType5 + IL_013d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0142: stloc.1 + IL_0143: ldarg.0 + IL_0144: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_0149: ldarga.s expected + IL_014b: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_0150: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0155: nop + IL_0156: ldstr "f9" + IL_015b: stloc.0 + IL_015c: ldarg.0 + IL_015d: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_0162: box MCCTest.VType5 + IL_0167: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_016c: stloc.1 + IL_016d: ldarg.0 + IL_016e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_0173: ldarga.s expected + IL_0175: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_017a: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_017f: nop + IL_0180: ldstr "f10" + IL_0185: stloc.0 + IL_0186: ldarg.0 + IL_0187: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_018c: box MCCTest.VType5 + IL_0191: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0196: stloc.1 + IL_0197: ldarg.0 + IL_0198: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_019d: ldarga.s expected + IL_019f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_01a4: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_01a9: nop + IL_01aa: ldstr "f11" + IL_01af: stloc.0 + IL_01b0: ldarg.0 + IL_01b1: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_01b6: box MCCTest.VType5 + IL_01bb: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01c0: stloc.1 + IL_01c1: ldarg.0 + IL_01c2: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_01c7: ldarga.s expected + IL_01c9: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_01ce: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_01d3: nop + IL_01d4: ldstr "f12" + IL_01d9: stloc.0 + IL_01da: ldarg.0 + IL_01db: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_01e0: box MCCTest.VType5 + IL_01e5: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ea: stloc.1 + IL_01eb: ldarg.0 + IL_01ec: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_01f1: ldarga.s expected + IL_01f3: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_01f8: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_01fd: nop + IL_01fe: ldstr "f13" + IL_0203: stloc.0 + IL_0204: ldarg.0 + IL_0205: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_020a: box MCCTest.VType5 + IL_020f: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0214: stloc.1 + IL_0215: ldarg.0 + IL_0216: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_021b: ldarga.s expected + IL_021d: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_0222: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0227: nop + IL_0228: ldstr "f14" + IL_022d: stloc.0 + IL_022e: ldarg.0 + IL_022f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_0234: box MCCTest.VType5 + IL_0239: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_023e: stloc.1 + IL_023f: ldarg.0 + IL_0240: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_0245: ldarga.s expected + IL_0247: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_024c: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0251: nop + IL_0252: ldstr "f15" + IL_0257: stloc.0 + IL_0258: ldarg.0 + IL_0259: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_025e: box MCCTest.VType5 + IL_0263: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0268: stloc.1 + IL_0269: ldarg.0 + IL_026a: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_026f: ldarga.s expected + IL_0271: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_0276: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_027b: nop + IL_027c: ldstr "f16" + IL_0281: stloc.0 + IL_0282: ldarg.0 + IL_0283: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_0288: box MCCTest.VType5 + IL_028d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0292: stloc.1 + IL_0293: ldarg.0 + IL_0294: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_0299: ldarga.s expected + IL_029b: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_02a0: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_02a5: nop + IL_02a6: ldstr "f17" + IL_02ab: stloc.0 + IL_02ac: ldarg.0 + IL_02ad: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_02b2: box MCCTest.VType5 + IL_02b7: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02bc: stloc.1 + IL_02bd: ldarg.0 + IL_02be: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_02c3: ldarga.s expected + IL_02c5: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_02ca: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_02cf: nop + IL_02d0: ldstr "f18" + IL_02d5: stloc.0 + IL_02d6: ldarg.0 + IL_02d7: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_02dc: box MCCTest.VType5 + IL_02e1: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02e6: stloc.1 + IL_02e7: ldarg.0 + IL_02e8: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_02ed: ldarga.s expected + IL_02ef: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_02f4: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_02f9: nop + IL_02fa: ldstr "f19" + IL_02ff: stloc.0 + IL_0300: ldarg.0 + IL_0301: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_0306: box MCCTest.VType5 + IL_030b: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0310: stloc.1 + IL_0311: ldarg.0 + IL_0312: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_0317: ldarga.s expected + IL_0319: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_031e: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0323: nop + IL_0324: nop + IL_0325: leave.s IL_0332 + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_0327: stloc.2 + IL_0328: nop + IL_0329: ldloc.0 + IL_032a: ldloc.1 + IL_032b: ldloc.2 + IL_032c: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_0331: throw + + } // end handler + IL_0332: nop + IL_0333: ret + } // end of method VTypeB::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 1165 (0x48d) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "[Field f1] [Type '{0}']" + IL_0015: call string [mscorlib]System.String::Concat(string, + string) + IL_001a: ldarg.0 + IL_001b: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_0020: box MCCTest.VType5 + IL_0025: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_002a: callvirt instance string [mscorlib]System.Object::ToString() + IL_002f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0034: nop + IL_0035: ldloc.1 + IL_0036: ldarg.0 + IL_0037: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f1 + IL_003c: ldarg.1 + IL_003d: ldc.i4.1 + IL_003e: add + IL_003f: call instance string MCCTest.VType5::Dump(int32) + IL_0044: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0049: nop + IL_004a: ldloc.1 + IL_004b: ldloc.0 + IL_004c: ldstr "[Field f2] [Type '{0}']" + IL_0051: call string [mscorlib]System.String::Concat(string, + string) + IL_0056: ldarg.0 + IL_0057: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_005c: box MCCTest.VType5 + IL_0061: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0066: callvirt instance string [mscorlib]System.Object::ToString() + IL_006b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0070: nop + IL_0071: ldloc.1 + IL_0072: ldarg.0 + IL_0073: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f2 + IL_0078: ldarg.1 + IL_0079: ldc.i4.1 + IL_007a: add + IL_007b: call instance string MCCTest.VType5::Dump(int32) + IL_0080: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0085: nop + IL_0086: ldloc.1 + IL_0087: ldloc.0 + IL_0088: ldstr "[Field f3] [Type '{0}']" + IL_008d: call string [mscorlib]System.String::Concat(string, + string) + IL_0092: ldarg.0 + IL_0093: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_0098: box MCCTest.VType5 + IL_009d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00a2: callvirt instance string [mscorlib]System.Object::ToString() + IL_00a7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00ac: nop + IL_00ad: ldloc.1 + IL_00ae: ldarg.0 + IL_00af: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f3 + IL_00b4: ldarg.1 + IL_00b5: ldc.i4.1 + IL_00b6: add + IL_00b7: call instance string MCCTest.VType5::Dump(int32) + IL_00bc: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00c1: nop + IL_00c2: ldloc.1 + IL_00c3: ldloc.0 + IL_00c4: ldstr "[Field f4] [Type '{0}']" + IL_00c9: call string [mscorlib]System.String::Concat(string, + string) + IL_00ce: ldarg.0 + IL_00cf: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_00d4: box MCCTest.VType5 + IL_00d9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00de: callvirt instance string [mscorlib]System.Object::ToString() + IL_00e3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00e8: nop + IL_00e9: ldloc.1 + IL_00ea: ldarg.0 + IL_00eb: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f4 + IL_00f0: ldarg.1 + IL_00f1: ldc.i4.1 + IL_00f2: add + IL_00f3: call instance string MCCTest.VType5::Dump(int32) + IL_00f8: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00fd: nop + IL_00fe: ldloc.1 + IL_00ff: ldloc.0 + IL_0100: ldstr "[Field f5] [Type '{0}']" + IL_0105: call string [mscorlib]System.String::Concat(string, + string) + IL_010a: ldarg.0 + IL_010b: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_0110: box MCCTest.VType5 + IL_0115: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_011a: callvirt instance string [mscorlib]System.Object::ToString() + IL_011f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0124: nop + IL_0125: ldloc.1 + IL_0126: ldarg.0 + IL_0127: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f5 + IL_012c: ldarg.1 + IL_012d: ldc.i4.1 + IL_012e: add + IL_012f: call instance string MCCTest.VType5::Dump(int32) + IL_0134: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0139: nop + IL_013a: ldloc.1 + IL_013b: ldloc.0 + IL_013c: ldstr "[Field f6] [Type '{0}']" + IL_0141: call string [mscorlib]System.String::Concat(string, + string) + IL_0146: ldarg.0 + IL_0147: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_014c: box MCCTest.VType5 + IL_0151: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0156: callvirt instance string [mscorlib]System.Object::ToString() + IL_015b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0160: nop + IL_0161: ldloc.1 + IL_0162: ldarg.0 + IL_0163: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f6 + IL_0168: ldarg.1 + IL_0169: ldc.i4.1 + IL_016a: add + IL_016b: call instance string MCCTest.VType5::Dump(int32) + IL_0170: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0175: nop + IL_0176: ldloc.1 + IL_0177: ldloc.0 + IL_0178: ldstr "[Field f7] [Type '{0}']" + IL_017d: call string [mscorlib]System.String::Concat(string, + string) + IL_0182: ldarg.0 + IL_0183: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_0188: box MCCTest.VType5 + IL_018d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0192: callvirt instance string [mscorlib]System.Object::ToString() + IL_0197: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_019c: nop + IL_019d: ldloc.1 + IL_019e: ldarg.0 + IL_019f: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f7 + IL_01a4: ldarg.1 + IL_01a5: ldc.i4.1 + IL_01a6: add + IL_01a7: call instance string MCCTest.VType5::Dump(int32) + IL_01ac: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01b1: nop + IL_01b2: ldloc.1 + IL_01b3: ldloc.0 + IL_01b4: ldstr "[Field f8] [Type '{0}']" + IL_01b9: call string [mscorlib]System.String::Concat(string, + string) + IL_01be: ldarg.0 + IL_01bf: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_01c4: box MCCTest.VType5 + IL_01c9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ce: callvirt instance string [mscorlib]System.Object::ToString() + IL_01d3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01d8: nop + IL_01d9: ldloc.1 + IL_01da: ldarg.0 + IL_01db: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f8 + IL_01e0: ldarg.1 + IL_01e1: ldc.i4.1 + IL_01e2: add + IL_01e3: call instance string MCCTest.VType5::Dump(int32) + IL_01e8: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01ed: nop + IL_01ee: ldloc.1 + IL_01ef: ldloc.0 + IL_01f0: ldstr "[Field f9] [Type '{0}']" + IL_01f5: call string [mscorlib]System.String::Concat(string, + string) + IL_01fa: ldarg.0 + IL_01fb: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_0200: box MCCTest.VType5 + IL_0205: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_020a: callvirt instance string [mscorlib]System.Object::ToString() + IL_020f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0214: nop + IL_0215: ldloc.1 + IL_0216: ldarg.0 + IL_0217: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f9 + IL_021c: ldarg.1 + IL_021d: ldc.i4.1 + IL_021e: add + IL_021f: call instance string MCCTest.VType5::Dump(int32) + IL_0224: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0229: nop + IL_022a: ldloc.1 + IL_022b: ldloc.0 + IL_022c: ldstr "[Field f10] [Type '{0}']" + IL_0231: call string [mscorlib]System.String::Concat(string, + string) + IL_0236: ldarg.0 + IL_0237: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_023c: box MCCTest.VType5 + IL_0241: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0246: callvirt instance string [mscorlib]System.Object::ToString() + IL_024b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0250: nop + IL_0251: ldloc.1 + IL_0252: ldarg.0 + IL_0253: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f10 + IL_0258: ldarg.1 + IL_0259: ldc.i4.1 + IL_025a: add + IL_025b: call instance string MCCTest.VType5::Dump(int32) + IL_0260: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0265: nop + IL_0266: ldloc.1 + IL_0267: ldloc.0 + IL_0268: ldstr "[Field f11] [Type '{0}']" + IL_026d: call string [mscorlib]System.String::Concat(string, + string) + IL_0272: ldarg.0 + IL_0273: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_0278: box MCCTest.VType5 + IL_027d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0282: callvirt instance string [mscorlib]System.Object::ToString() + IL_0287: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_028c: nop + IL_028d: ldloc.1 + IL_028e: ldarg.0 + IL_028f: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f11 + IL_0294: ldarg.1 + IL_0295: ldc.i4.1 + IL_0296: add + IL_0297: call instance string MCCTest.VType5::Dump(int32) + IL_029c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02a1: nop + IL_02a2: ldloc.1 + IL_02a3: ldloc.0 + IL_02a4: ldstr "[Field f12] [Type '{0}']" + IL_02a9: call string [mscorlib]System.String::Concat(string, + string) + IL_02ae: ldarg.0 + IL_02af: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_02b4: box MCCTest.VType5 + IL_02b9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02be: callvirt instance string [mscorlib]System.Object::ToString() + IL_02c3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02c8: nop + IL_02c9: ldloc.1 + IL_02ca: ldarg.0 + IL_02cb: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f12 + IL_02d0: ldarg.1 + IL_02d1: ldc.i4.1 + IL_02d2: add + IL_02d3: call instance string MCCTest.VType5::Dump(int32) + IL_02d8: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02dd: nop + IL_02de: ldloc.1 + IL_02df: ldloc.0 + IL_02e0: ldstr "[Field f13] [Type '{0}']" + IL_02e5: call string [mscorlib]System.String::Concat(string, + string) + IL_02ea: ldarg.0 + IL_02eb: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_02f0: box MCCTest.VType5 + IL_02f5: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02fa: callvirt instance string [mscorlib]System.Object::ToString() + IL_02ff: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0304: nop + IL_0305: ldloc.1 + IL_0306: ldarg.0 + IL_0307: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f13 + IL_030c: ldarg.1 + IL_030d: ldc.i4.1 + IL_030e: add + IL_030f: call instance string MCCTest.VType5::Dump(int32) + IL_0314: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0319: nop + IL_031a: ldloc.1 + IL_031b: ldloc.0 + IL_031c: ldstr "[Field f14] [Type '{0}']" + IL_0321: call string [mscorlib]System.String::Concat(string, + string) + IL_0326: ldarg.0 + IL_0327: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_032c: box MCCTest.VType5 + IL_0331: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0336: callvirt instance string [mscorlib]System.Object::ToString() + IL_033b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0340: nop + IL_0341: ldloc.1 + IL_0342: ldarg.0 + IL_0343: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f14 + IL_0348: ldarg.1 + IL_0349: ldc.i4.1 + IL_034a: add + IL_034b: call instance string MCCTest.VType5::Dump(int32) + IL_0350: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0355: nop + IL_0356: ldloc.1 + IL_0357: ldloc.0 + IL_0358: ldstr "[Field f15] [Type '{0}']" + IL_035d: call string [mscorlib]System.String::Concat(string, + string) + IL_0362: ldarg.0 + IL_0363: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_0368: box MCCTest.VType5 + IL_036d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0372: callvirt instance string [mscorlib]System.Object::ToString() + IL_0377: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_037c: nop + IL_037d: ldloc.1 + IL_037e: ldarg.0 + IL_037f: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f15 + IL_0384: ldarg.1 + IL_0385: ldc.i4.1 + IL_0386: add + IL_0387: call instance string MCCTest.VType5::Dump(int32) + IL_038c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0391: nop + IL_0392: ldloc.1 + IL_0393: ldloc.0 + IL_0394: ldstr "[Field f16] [Type '{0}']" + IL_0399: call string [mscorlib]System.String::Concat(string, + string) + IL_039e: ldarg.0 + IL_039f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_03a4: box MCCTest.VType5 + IL_03a9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03ae: callvirt instance string [mscorlib]System.Object::ToString() + IL_03b3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_03b8: nop + IL_03b9: ldloc.1 + IL_03ba: ldarg.0 + IL_03bb: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f16 + IL_03c0: ldarg.1 + IL_03c1: ldc.i4.1 + IL_03c2: add + IL_03c3: call instance string MCCTest.VType5::Dump(int32) + IL_03c8: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_03cd: nop + IL_03ce: ldloc.1 + IL_03cf: ldloc.0 + IL_03d0: ldstr "[Field f17] [Type '{0}']" + IL_03d5: call string [mscorlib]System.String::Concat(string, + string) + IL_03da: ldarg.0 + IL_03db: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_03e0: box MCCTest.VType5 + IL_03e5: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03ea: callvirt instance string [mscorlib]System.Object::ToString() + IL_03ef: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_03f4: nop + IL_03f5: ldloc.1 + IL_03f6: ldarg.0 + IL_03f7: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f17 + IL_03fc: ldarg.1 + IL_03fd: ldc.i4.1 + IL_03fe: add + IL_03ff: call instance string MCCTest.VType5::Dump(int32) + IL_0404: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0409: nop + IL_040a: ldloc.1 + IL_040b: ldloc.0 + IL_040c: ldstr "[Field f18] [Type '{0}']" + IL_0411: call string [mscorlib]System.String::Concat(string, + string) + IL_0416: ldarg.0 + IL_0417: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_041c: box MCCTest.VType5 + IL_0421: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0426: callvirt instance string [mscorlib]System.Object::ToString() + IL_042b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0430: nop + IL_0431: ldloc.1 + IL_0432: ldarg.0 + IL_0433: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f18 + IL_0438: ldarg.1 + IL_0439: ldc.i4.1 + IL_043a: add + IL_043b: call instance string MCCTest.VType5::Dump(int32) + IL_0440: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0445: nop + IL_0446: ldloc.1 + IL_0447: ldloc.0 + IL_0448: ldstr "[Field f19] [Type '{0}']" + IL_044d: call string [mscorlib]System.String::Concat(string, + string) + IL_0452: ldarg.0 + IL_0453: ldfld valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_0458: box MCCTest.VType5 + IL_045d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0462: callvirt instance string [mscorlib]System.Object::ToString() + IL_0467: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_046c: nop + IL_046d: ldloc.1 + IL_046e: ldarg.0 + IL_046f: ldflda valuetype MCCTest.VType5 MCCTest.VTypeB::f19 + IL_0474: ldarg.1 + IL_0475: ldc.i4.1 + IL_0476: add + IL_0477: call instance string MCCTest.VType5::Dump(int32) + IL_047c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0481: nop + IL_0482: ldloc.1 + IL_0483: callvirt instance string [mscorlib]System.Object::ToString() + IL_0488: stloc.2 + IL_0489: br.s IL_048b + + IL_048b: ldloc.2 + IL_048c: ret + } // end of method VTypeB::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VTypeB::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VTypeB::Dump + +} // end of class MCCTest.VTypeB + +.class public sequential ansi sealed beforefieldinit MCCTest.VTypeC + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public class MCCTest.RType4 f1 + .field public valuetype MCCTest.VType5 f2 + .field public valuetype MCCTest.VType6 f3 + .field public class MCCTest.RType4 f4 + .field public valuetype MCCTest.VType5 f5 + .field public valuetype MCCTest.VType6 f6 + .field public class MCCTest.RType4 f7 + .field public valuetype MCCTest.VType5 f8 + .field public valuetype MCCTest.VType6 f9 + .field public class MCCTest.RType4 f10 + .field public valuetype MCCTest.VType5 f11 + .field public valuetype MCCTest.VType6 f12 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 202 (0xca) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: newobj instance void MCCTest.RType4::.ctor() + IL_0007: stfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_000c: ldarg.0 + IL_000d: newobj instance void MCCTest.RType4::.ctor() + IL_0012: stfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_0017: ldarg.0 + IL_0018: newobj instance void MCCTest.RType4::.ctor() + IL_001d: stfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0022: ldarg.0 + IL_0023: newobj instance void MCCTest.RType4::.ctor() + IL_0028: stfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_002d: ldarg.0 + IL_002e: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_0033: ldarg.1 + IL_0034: callvirt instance void MCCTest.RType4::Init(int32) + IL_0039: nop + IL_003a: ldarg.0 + IL_003b: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_0040: ldarg.1 + IL_0041: call instance void MCCTest.VType5::Init(int32) + IL_0046: nop + IL_0047: ldarg.0 + IL_0048: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_004d: ldarg.1 + IL_004e: call instance void MCCTest.VType6::Init(int32) + IL_0053: nop + IL_0054: ldarg.0 + IL_0055: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_005a: ldarg.1 + IL_005b: callvirt instance void MCCTest.RType4::Init(int32) + IL_0060: nop + IL_0061: ldarg.0 + IL_0062: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_0067: ldarg.1 + IL_0068: call instance void MCCTest.VType5::Init(int32) + IL_006d: nop + IL_006e: ldarg.0 + IL_006f: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_0074: ldarg.1 + IL_0075: call instance void MCCTest.VType6::Init(int32) + IL_007a: nop + IL_007b: ldarg.0 + IL_007c: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0081: ldarg.1 + IL_0082: callvirt instance void MCCTest.RType4::Init(int32) + IL_0087: nop + IL_0088: ldarg.0 + IL_0089: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_008e: ldarg.1 + IL_008f: call instance void MCCTest.VType5::Init(int32) + IL_0094: nop + IL_0095: ldarg.0 + IL_0096: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_009b: ldarg.1 + IL_009c: call instance void MCCTest.VType6::Init(int32) + IL_00a1: nop + IL_00a2: ldarg.0 + IL_00a3: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_00a8: ldarg.1 + IL_00a9: callvirt instance void MCCTest.RType4::Init(int32) + IL_00ae: nop + IL_00af: ldarg.0 + IL_00b0: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_00b5: ldarg.1 + IL_00b6: call instance void MCCTest.VType5::Init(int32) + IL_00bb: nop + IL_00bc: ldarg.0 + IL_00bd: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_00c2: ldarg.1 + IL_00c3: call instance void MCCTest.VType6::Init(int32) + IL_00c8: nop + IL_00c9: ret + } // end of method VTypeC::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VTypeC::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeC::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VTypeC::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeC::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VTypeC val) cil managed + { + // Code size 230 (0xe6) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_0007: ldarga.s val + IL_0009: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_000e: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0013: nop + IL_0014: ldarg.0 + IL_0015: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_001a: ldarga.s val + IL_001c: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_0021: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0026: nop + IL_0027: ldarg.0 + IL_0028: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_002d: ldarga.s val + IL_002f: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_0034: call instance void MCCTest.VType6::Add(valuetype MCCTest.VType6) + IL_0039: nop + IL_003a: ldarg.0 + IL_003b: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_0040: ldarga.s val + IL_0042: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_0047: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_004c: nop + IL_004d: ldarg.0 + IL_004e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_0053: ldarga.s val + IL_0055: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_005a: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_005f: nop + IL_0060: ldarg.0 + IL_0061: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_0066: ldarga.s val + IL_0068: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_006d: call instance void MCCTest.VType6::Add(valuetype MCCTest.VType6) + IL_0072: nop + IL_0073: ldarg.0 + IL_0074: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0079: ldarga.s val + IL_007b: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0080: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0085: nop + IL_0086: ldarg.0 + IL_0087: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_008c: ldarga.s val + IL_008e: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_0093: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0098: nop + IL_0099: ldarg.0 + IL_009a: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_009f: ldarga.s val + IL_00a1: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_00a6: call instance void MCCTest.VType6::Add(valuetype MCCTest.VType6) + IL_00ab: nop + IL_00ac: ldarg.0 + IL_00ad: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_00b2: ldarga.s val + IL_00b4: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_00b9: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00be: nop + IL_00bf: ldarg.0 + IL_00c0: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_00c5: ldarga.s val + IL_00c7: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_00cc: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00d1: nop + IL_00d2: ldarg.0 + IL_00d3: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_00d8: ldarga.s val + IL_00da: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_00df: call instance void MCCTest.VType6::Add(valuetype MCCTest.VType6) + IL_00e4: nop + IL_00e5: ret + } // end of method VTypeC::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VTypeC expected) cil managed + { + // Code size 506 (0x1fa) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2) + IL_0000: nop + IL_0001: ldnull + IL_0002: stloc.0 + IL_0003: ldnull + IL_0004: stloc.1 + .try + { + IL_0005: nop + IL_0006: ldstr "f1" + IL_000b: stloc.0 + IL_000c: ldarg.0 + IL_000d: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_0012: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0017: stloc.1 + IL_0018: ldarg.0 + IL_0019: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_001e: ldarga.s expected + IL_0020: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_0025: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_002a: nop + IL_002b: ldstr "f2" + IL_0030: stloc.0 + IL_0031: ldarg.0 + IL_0032: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_0037: box MCCTest.VType5 + IL_003c: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0041: stloc.1 + IL_0042: ldarg.0 + IL_0043: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_0048: ldarga.s expected + IL_004a: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_004f: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0054: nop + IL_0055: ldstr "f3" + IL_005a: stloc.0 + IL_005b: ldarg.0 + IL_005c: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_0061: box MCCTest.VType6 + IL_0066: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_006b: stloc.1 + IL_006c: ldarg.0 + IL_006d: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_0072: ldarga.s expected + IL_0074: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_0079: call instance void MCCTest.VType6::Check(valuetype MCCTest.VType6) + IL_007e: nop + IL_007f: ldstr "f4" + IL_0084: stloc.0 + IL_0085: ldarg.0 + IL_0086: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_008b: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0090: stloc.1 + IL_0091: ldarg.0 + IL_0092: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_0097: ldarga.s expected + IL_0099: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_009e: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_00a3: nop + IL_00a4: ldstr "f5" + IL_00a9: stloc.0 + IL_00aa: ldarg.0 + IL_00ab: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_00b0: box MCCTest.VType5 + IL_00b5: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00ba: stloc.1 + IL_00bb: ldarg.0 + IL_00bc: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_00c1: ldarga.s expected + IL_00c3: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_00c8: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_00cd: nop + IL_00ce: ldstr "f6" + IL_00d3: stloc.0 + IL_00d4: ldarg.0 + IL_00d5: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_00da: box MCCTest.VType6 + IL_00df: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00e4: stloc.1 + IL_00e5: ldarg.0 + IL_00e6: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_00eb: ldarga.s expected + IL_00ed: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_00f2: call instance void MCCTest.VType6::Check(valuetype MCCTest.VType6) + IL_00f7: nop + IL_00f8: ldstr "f7" + IL_00fd: stloc.0 + IL_00fe: ldarg.0 + IL_00ff: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0104: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0109: stloc.1 + IL_010a: ldarg.0 + IL_010b: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0110: ldarga.s expected + IL_0112: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0117: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_011c: nop + IL_011d: ldstr "f8" + IL_0122: stloc.0 + IL_0123: ldarg.0 + IL_0124: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_0129: box MCCTest.VType5 + IL_012e: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0133: stloc.1 + IL_0134: ldarg.0 + IL_0135: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_013a: ldarga.s expected + IL_013c: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_0141: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_0146: nop + IL_0147: ldstr "f9" + IL_014c: stloc.0 + IL_014d: ldarg.0 + IL_014e: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_0153: box MCCTest.VType6 + IL_0158: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_015d: stloc.1 + IL_015e: ldarg.0 + IL_015f: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_0164: ldarga.s expected + IL_0166: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_016b: call instance void MCCTest.VType6::Check(valuetype MCCTest.VType6) + IL_0170: nop + IL_0171: ldstr "f10" + IL_0176: stloc.0 + IL_0177: ldarg.0 + IL_0178: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_017d: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0182: stloc.1 + IL_0183: ldarg.0 + IL_0184: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_0189: ldarga.s expected + IL_018b: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_0190: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0195: nop + IL_0196: ldstr "f11" + IL_019b: stloc.0 + IL_019c: ldarg.0 + IL_019d: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_01a2: box MCCTest.VType5 + IL_01a7: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ac: stloc.1 + IL_01ad: ldarg.0 + IL_01ae: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_01b3: ldarga.s expected + IL_01b5: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_01ba: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_01bf: nop + IL_01c0: ldstr "f12" + IL_01c5: stloc.0 + IL_01c6: ldarg.0 + IL_01c7: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_01cc: box MCCTest.VType6 + IL_01d1: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01d6: stloc.1 + IL_01d7: ldarg.0 + IL_01d8: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_01dd: ldarga.s expected + IL_01df: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_01e4: call instance void MCCTest.VType6::Check(valuetype MCCTest.VType6) + IL_01e9: nop + IL_01ea: nop + IL_01eb: leave.s IL_01f8 + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_01ed: stloc.2 + IL_01ee: nop + IL_01ef: ldloc.0 + IL_01f0: ldloc.1 + IL_01f1: ldloc.2 + IL_01f2: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_01f7: throw + + } // end handler + IL_01f8: nop + IL_01f9: ret + } // end of method VTypeC::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 725 (0x2d5) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "[Field f1] [Type '{0}']" + IL_0015: call string [mscorlib]System.String::Concat(string, + string) + IL_001a: ldarg.0 + IL_001b: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_0020: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0025: callvirt instance string [mscorlib]System.Object::ToString() + IL_002a: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_002f: nop + IL_0030: ldloc.1 + IL_0031: ldarg.0 + IL_0032: ldfld class MCCTest.RType4 MCCTest.VTypeC::f1 + IL_0037: ldarg.1 + IL_0038: ldc.i4.1 + IL_0039: add + IL_003a: callvirt instance string MCCTest.RType4::Dump(int32) + IL_003f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0044: nop + IL_0045: ldloc.1 + IL_0046: ldloc.0 + IL_0047: ldstr "[Field f2] [Type '{0}']" + IL_004c: call string [mscorlib]System.String::Concat(string, + string) + IL_0051: ldarg.0 + IL_0052: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_0057: box MCCTest.VType5 + IL_005c: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0061: callvirt instance string [mscorlib]System.Object::ToString() + IL_0066: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_006b: nop + IL_006c: ldloc.1 + IL_006d: ldarg.0 + IL_006e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f2 + IL_0073: ldarg.1 + IL_0074: ldc.i4.1 + IL_0075: add + IL_0076: call instance string MCCTest.VType5::Dump(int32) + IL_007b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0080: nop + IL_0081: ldloc.1 + IL_0082: ldloc.0 + IL_0083: ldstr "[Field f3] [Type '{0}']" + IL_0088: call string [mscorlib]System.String::Concat(string, + string) + IL_008d: ldarg.0 + IL_008e: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_0093: box MCCTest.VType6 + IL_0098: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_009d: callvirt instance string [mscorlib]System.Object::ToString() + IL_00a2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00a7: nop + IL_00a8: ldloc.1 + IL_00a9: ldarg.0 + IL_00aa: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f3 + IL_00af: ldarg.1 + IL_00b0: ldc.i4.1 + IL_00b1: add + IL_00b2: call instance string MCCTest.VType6::Dump(int32) + IL_00b7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00bc: nop + IL_00bd: ldloc.1 + IL_00be: ldloc.0 + IL_00bf: ldstr "[Field f4] [Type '{0}']" + IL_00c4: call string [mscorlib]System.String::Concat(string, + string) + IL_00c9: ldarg.0 + IL_00ca: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_00cf: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00d4: callvirt instance string [mscorlib]System.Object::ToString() + IL_00d9: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00de: nop + IL_00df: ldloc.1 + IL_00e0: ldarg.0 + IL_00e1: ldfld class MCCTest.RType4 MCCTest.VTypeC::f4 + IL_00e6: ldarg.1 + IL_00e7: ldc.i4.1 + IL_00e8: add + IL_00e9: callvirt instance string MCCTest.RType4::Dump(int32) + IL_00ee: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00f3: nop + IL_00f4: ldloc.1 + IL_00f5: ldloc.0 + IL_00f6: ldstr "[Field f5] [Type '{0}']" + IL_00fb: call string [mscorlib]System.String::Concat(string, + string) + IL_0100: ldarg.0 + IL_0101: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_0106: box MCCTest.VType5 + IL_010b: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0110: callvirt instance string [mscorlib]System.Object::ToString() + IL_0115: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_011a: nop + IL_011b: ldloc.1 + IL_011c: ldarg.0 + IL_011d: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f5 + IL_0122: ldarg.1 + IL_0123: ldc.i4.1 + IL_0124: add + IL_0125: call instance string MCCTest.VType5::Dump(int32) + IL_012a: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_012f: nop + IL_0130: ldloc.1 + IL_0131: ldloc.0 + IL_0132: ldstr "[Field f6] [Type '{0}']" + IL_0137: call string [mscorlib]System.String::Concat(string, + string) + IL_013c: ldarg.0 + IL_013d: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_0142: box MCCTest.VType6 + IL_0147: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_014c: callvirt instance string [mscorlib]System.Object::ToString() + IL_0151: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0156: nop + IL_0157: ldloc.1 + IL_0158: ldarg.0 + IL_0159: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f6 + IL_015e: ldarg.1 + IL_015f: ldc.i4.1 + IL_0160: add + IL_0161: call instance string MCCTest.VType6::Dump(int32) + IL_0166: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_016b: nop + IL_016c: ldloc.1 + IL_016d: ldloc.0 + IL_016e: ldstr "[Field f7] [Type '{0}']" + IL_0173: call string [mscorlib]System.String::Concat(string, + string) + IL_0178: ldarg.0 + IL_0179: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_017e: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0183: callvirt instance string [mscorlib]System.Object::ToString() + IL_0188: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_018d: nop + IL_018e: ldloc.1 + IL_018f: ldarg.0 + IL_0190: ldfld class MCCTest.RType4 MCCTest.VTypeC::f7 + IL_0195: ldarg.1 + IL_0196: ldc.i4.1 + IL_0197: add + IL_0198: callvirt instance string MCCTest.RType4::Dump(int32) + IL_019d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01a2: nop + IL_01a3: ldloc.1 + IL_01a4: ldloc.0 + IL_01a5: ldstr "[Field f8] [Type '{0}']" + IL_01aa: call string [mscorlib]System.String::Concat(string, + string) + IL_01af: ldarg.0 + IL_01b0: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_01b5: box MCCTest.VType5 + IL_01ba: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01bf: callvirt instance string [mscorlib]System.Object::ToString() + IL_01c4: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01c9: nop + IL_01ca: ldloc.1 + IL_01cb: ldarg.0 + IL_01cc: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f8 + IL_01d1: ldarg.1 + IL_01d2: ldc.i4.1 + IL_01d3: add + IL_01d4: call instance string MCCTest.VType5::Dump(int32) + IL_01d9: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01de: nop + IL_01df: ldloc.1 + IL_01e0: ldloc.0 + IL_01e1: ldstr "[Field f9] [Type '{0}']" + IL_01e6: call string [mscorlib]System.String::Concat(string, + string) + IL_01eb: ldarg.0 + IL_01ec: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_01f1: box MCCTest.VType6 + IL_01f6: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01fb: callvirt instance string [mscorlib]System.Object::ToString() + IL_0200: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0205: nop + IL_0206: ldloc.1 + IL_0207: ldarg.0 + IL_0208: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f9 + IL_020d: ldarg.1 + IL_020e: ldc.i4.1 + IL_020f: add + IL_0210: call instance string MCCTest.VType6::Dump(int32) + IL_0215: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_021a: nop + IL_021b: ldloc.1 + IL_021c: ldloc.0 + IL_021d: ldstr "[Field f10] [Type '{0}']" + IL_0222: call string [mscorlib]System.String::Concat(string, + string) + IL_0227: ldarg.0 + IL_0228: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_022d: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0232: callvirt instance string [mscorlib]System.Object::ToString() + IL_0237: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_023c: nop + IL_023d: ldloc.1 + IL_023e: ldarg.0 + IL_023f: ldfld class MCCTest.RType4 MCCTest.VTypeC::f10 + IL_0244: ldarg.1 + IL_0245: ldc.i4.1 + IL_0246: add + IL_0247: callvirt instance string MCCTest.RType4::Dump(int32) + IL_024c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0251: nop + IL_0252: ldloc.1 + IL_0253: ldloc.0 + IL_0254: ldstr "[Field f11] [Type '{0}']" + IL_0259: call string [mscorlib]System.String::Concat(string, + string) + IL_025e: ldarg.0 + IL_025f: ldfld valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_0264: box MCCTest.VType5 + IL_0269: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_026e: callvirt instance string [mscorlib]System.Object::ToString() + IL_0273: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0278: nop + IL_0279: ldloc.1 + IL_027a: ldarg.0 + IL_027b: ldflda valuetype MCCTest.VType5 MCCTest.VTypeC::f11 + IL_0280: ldarg.1 + IL_0281: ldc.i4.1 + IL_0282: add + IL_0283: call instance string MCCTest.VType5::Dump(int32) + IL_0288: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_028d: nop + IL_028e: ldloc.1 + IL_028f: ldloc.0 + IL_0290: ldstr "[Field f12] [Type '{0}']" + IL_0295: call string [mscorlib]System.String::Concat(string, + string) + IL_029a: ldarg.0 + IL_029b: ldfld valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_02a0: box MCCTest.VType6 + IL_02a5: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02aa: callvirt instance string [mscorlib]System.Object::ToString() + IL_02af: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02b4: nop + IL_02b5: ldloc.1 + IL_02b6: ldarg.0 + IL_02b7: ldflda valuetype MCCTest.VType6 MCCTest.VTypeC::f12 + IL_02bc: ldarg.1 + IL_02bd: ldc.i4.1 + IL_02be: add + IL_02bf: call instance string MCCTest.VType6::Dump(int32) + IL_02c4: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02c9: nop + IL_02ca: ldloc.1 + IL_02cb: callvirt instance string [mscorlib]System.Object::ToString() + IL_02d0: stloc.2 + IL_02d1: br.s IL_02d3 + + IL_02d3: ldloc.2 + IL_02d4: ret + } // end of method VTypeC::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VTypeC::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VTypeC::Dump + +} // end of class MCCTest.VTypeC + +.class public sequential ansi sealed beforefieldinit MCCTest.VTypeD + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public int32 f1 + .field public valuetype MCCTest.VType3 f2 + .field public float64 f3 + .field public class MCCTest.RType4 f4 + .field public valuetype MCCTest.VType7 f5 + .field public uint64 f6 + .field public float32 f7 + .field public class MCCTest.RType4 f8 + .field public valuetype MCCTest.VType6 f9 + .field public float64 f10 + .field public int16 f11 + .field public class MCCTest.RType4 f12 + .field public valuetype MCCTest.VType5 f13 + .field public valuetype MCCTest.VType3 f14 + .field public class MCCTest.RType4 f15 + .field public valuetype MCCTest.VType7 f16 + .field public uint32 f17 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 230 (0xe6) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: newobj instance void MCCTest.RType4::.ctor() + IL_0007: stfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_000c: ldarg.0 + IL_000d: newobj instance void MCCTest.RType4::.ctor() + IL_0012: stfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_0017: ldarg.0 + IL_0018: newobj instance void MCCTest.RType4::.ctor() + IL_001d: stfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_0022: ldarg.0 + IL_0023: newobj instance void MCCTest.RType4::.ctor() + IL_0028: stfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_002d: ldarg.0 + IL_002e: ldarg.1 + IL_002f: stfld int32 MCCTest.VTypeD::f1 + IL_0034: ldarg.0 + IL_0035: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_003a: ldarg.1 + IL_003b: call instance void MCCTest.VType3::Init(int32) + IL_0040: nop + IL_0041: ldarg.0 + IL_0042: ldarg.1 + IL_0043: conv.r8 + IL_0044: stfld float64 MCCTest.VTypeD::f3 + IL_0049: ldarg.0 + IL_004a: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_004f: ldarg.1 + IL_0050: callvirt instance void MCCTest.RType4::Init(int32) + IL_0055: nop + IL_0056: ldarg.0 + IL_0057: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_005c: ldarg.1 + IL_005d: call instance void MCCTest.VType7::Init(int32) + IL_0062: nop + IL_0063: ldarg.0 + IL_0064: ldarg.1 + IL_0065: conv.i8 + IL_0066: stfld uint64 MCCTest.VTypeD::f6 + IL_006b: ldarg.0 + IL_006c: ldarg.1 + IL_006d: conv.r4 + IL_006e: stfld float32 MCCTest.VTypeD::f7 + IL_0073: ldarg.0 + IL_0074: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_0079: ldarg.1 + IL_007a: callvirt instance void MCCTest.RType4::Init(int32) + IL_007f: nop + IL_0080: ldarg.0 + IL_0081: ldflda valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_0086: ldarg.1 + IL_0087: call instance void MCCTest.VType6::Init(int32) + IL_008c: nop + IL_008d: ldarg.0 + IL_008e: ldarg.1 + IL_008f: conv.r8 + IL_0090: stfld float64 MCCTest.VTypeD::f10 + IL_0095: ldarg.0 + IL_0096: ldarg.1 + IL_0097: conv.i2 + IL_0098: stfld int16 MCCTest.VTypeD::f11 + IL_009d: ldarg.0 + IL_009e: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_00a3: ldarg.1 + IL_00a4: callvirt instance void MCCTest.RType4::Init(int32) + IL_00a9: nop + IL_00aa: ldarg.0 + IL_00ab: ldflda valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_00b0: ldarg.1 + IL_00b1: call instance void MCCTest.VType5::Init(int32) + IL_00b6: nop + IL_00b7: ldarg.0 + IL_00b8: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_00bd: ldarg.1 + IL_00be: call instance void MCCTest.VType3::Init(int32) + IL_00c3: nop + IL_00c4: ldarg.0 + IL_00c5: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_00ca: ldarg.1 + IL_00cb: callvirt instance void MCCTest.RType4::Init(int32) + IL_00d0: nop + IL_00d1: ldarg.0 + IL_00d2: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_00d7: ldarg.1 + IL_00d8: call instance void MCCTest.VType7::Init(int32) + IL_00dd: nop + IL_00de: ldarg.0 + IL_00df: ldarg.1 + IL_00e0: stfld uint32 MCCTest.VTypeD::f17 + IL_00e5: ret + } // end of method VTypeD::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VTypeD::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeD::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VTypeD::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeD::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VTypeD val) cil managed + { + // Code size 333 (0x14d) + .maxstack 3 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: dup + IL_0003: ldfld int32 MCCTest.VTypeD::f1 + IL_0008: ldarga.s val + IL_000a: ldfld int32 MCCTest.VTypeD::f1 + IL_000f: add + IL_0010: stfld int32 MCCTest.VTypeD::f1 + IL_0015: ldarg.0 + IL_0016: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_001b: ldarga.s val + IL_001d: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_0022: call instance void MCCTest.VType3::Add(valuetype MCCTest.VType3) + IL_0027: nop + IL_0028: ldarg.0 + IL_0029: dup + IL_002a: ldfld float64 MCCTest.VTypeD::f3 + IL_002f: ldarga.s val + IL_0031: ldfld float64 MCCTest.VTypeD::f3 + IL_0036: add + IL_0037: stfld float64 MCCTest.VTypeD::f3 + IL_003c: ldarg.0 + IL_003d: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_0042: ldarga.s val + IL_0044: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_0049: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_004e: nop + IL_004f: ldarg.0 + IL_0050: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_0055: ldarga.s val + IL_0057: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_005c: call instance void MCCTest.VType7::Add(valuetype MCCTest.VType7) + IL_0061: nop + IL_0062: ldarg.0 + IL_0063: dup + IL_0064: ldfld uint64 MCCTest.VTypeD::f6 + IL_0069: ldarga.s val + IL_006b: ldfld uint64 MCCTest.VTypeD::f6 + IL_0070: add + IL_0071: stfld uint64 MCCTest.VTypeD::f6 + IL_0076: ldarg.0 + IL_0077: dup + IL_0078: ldfld float32 MCCTest.VTypeD::f7 + IL_007d: ldarga.s val + IL_007f: ldfld float32 MCCTest.VTypeD::f7 + IL_0084: add + IL_0085: stfld float32 MCCTest.VTypeD::f7 + IL_008a: ldarg.0 + IL_008b: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_0090: ldarga.s val + IL_0092: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_0097: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_009c: nop + IL_009d: ldarg.0 + IL_009e: ldflda valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_00a3: ldarga.s val + IL_00a5: ldfld valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_00aa: call instance void MCCTest.VType6::Add(valuetype MCCTest.VType6) + IL_00af: nop + IL_00b0: ldarg.0 + IL_00b1: dup + IL_00b2: ldfld float64 MCCTest.VTypeD::f10 + IL_00b7: ldarga.s val + IL_00b9: ldfld float64 MCCTest.VTypeD::f10 + IL_00be: add + IL_00bf: stfld float64 MCCTest.VTypeD::f10 + IL_00c4: ldarg.0 + IL_00c5: dup + IL_00c6: ldfld int16 MCCTest.VTypeD::f11 + IL_00cb: ldarga.s val + IL_00cd: ldfld int16 MCCTest.VTypeD::f11 + IL_00d2: add + IL_00d3: conv.i2 + IL_00d4: stfld int16 MCCTest.VTypeD::f11 + IL_00d9: ldarg.0 + IL_00da: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_00df: ldarga.s val + IL_00e1: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_00e6: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_00eb: nop + IL_00ec: ldarg.0 + IL_00ed: ldflda valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_00f2: ldarga.s val + IL_00f4: ldfld valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_00f9: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_00fe: nop + IL_00ff: ldarg.0 + IL_0100: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_0105: ldarga.s val + IL_0107: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_010c: call instance void MCCTest.VType3::Add(valuetype MCCTest.VType3) + IL_0111: nop + IL_0112: ldarg.0 + IL_0113: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_0118: ldarga.s val + IL_011a: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_011f: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0124: nop + IL_0125: ldarg.0 + IL_0126: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_012b: ldarga.s val + IL_012d: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_0132: call instance void MCCTest.VType7::Add(valuetype MCCTest.VType7) + IL_0137: nop + IL_0138: ldarg.0 + IL_0139: dup + IL_013a: ldfld uint32 MCCTest.VTypeD::f17 + IL_013f: ldarga.s val + IL_0141: ldfld uint32 MCCTest.VTypeD::f17 + IL_0146: add + IL_0147: stfld uint32 MCCTest.VTypeD::f17 + IL_014c: ret + } // end of method VTypeD::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VTypeD expected) cil managed + { + // Code size 742 (0x2e6) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2, + bool V_3) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldfld int32 MCCTest.VTypeD::f1 + IL_0007: ldarga.s expected + IL_0009: ldfld int32 MCCTest.VTypeD::f1 + IL_000e: ceq + IL_0010: stloc.3 + IL_0011: ldloc.3 + IL_0012: brtrue.s IL_002f + + IL_0014: nop + IL_0015: ldstr "f1" + IL_001a: ldarg.0 + IL_001b: ldfld int32 MCCTest.VTypeD::f1 + IL_0020: conv.i8 + IL_0021: ldarga.s expected + IL_0023: ldfld int32 MCCTest.VTypeD::f1 + IL_0028: conv.i8 + IL_0029: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_002e: throw + + IL_002f: ldarg.0 + IL_0030: ldfld float64 MCCTest.VTypeD::f3 + IL_0035: ldarga.s expected + IL_0037: ldfld float64 MCCTest.VTypeD::f3 + IL_003c: ceq + IL_003e: stloc.3 + IL_003f: ldloc.3 + IL_0040: brtrue.s IL_005b + + IL_0042: nop + IL_0043: ldstr "f3" + IL_0048: ldarg.0 + IL_0049: ldfld float64 MCCTest.VTypeD::f3 + IL_004e: ldarga.s expected + IL_0050: ldfld float64 MCCTest.VTypeD::f3 + IL_0055: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_005a: throw + + IL_005b: ldarg.0 + IL_005c: ldfld uint64 MCCTest.VTypeD::f6 + IL_0061: ldarga.s expected + IL_0063: ldfld uint64 MCCTest.VTypeD::f6 + IL_0068: ceq + IL_006a: stloc.3 + IL_006b: ldloc.3 + IL_006c: brtrue.s IL_008b + + IL_006e: nop + IL_006f: ldstr "f6" + IL_0074: ldarg.0 + IL_0075: ldfld uint64 MCCTest.VTypeD::f6 + IL_007a: conv.r.un + IL_007b: conv.r8 + IL_007c: ldarga.s expected + IL_007e: ldfld uint64 MCCTest.VTypeD::f6 + IL_0083: conv.r.un + IL_0084: conv.r8 + IL_0085: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_008a: throw + + IL_008b: ldarg.0 + IL_008c: ldfld float32 MCCTest.VTypeD::f7 + IL_0091: ldarga.s expected + IL_0093: ldfld float32 MCCTest.VTypeD::f7 + IL_0098: ceq + IL_009a: stloc.3 + IL_009b: ldloc.3 + IL_009c: brtrue.s IL_00b9 + + IL_009e: nop + IL_009f: ldstr "f7" + IL_00a4: ldarg.0 + IL_00a5: ldfld float32 MCCTest.VTypeD::f7 + IL_00aa: conv.r8 + IL_00ab: ldarga.s expected + IL_00ad: ldfld float32 MCCTest.VTypeD::f7 + IL_00b2: conv.r8 + IL_00b3: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_00b8: throw + + IL_00b9: ldarg.0 + IL_00ba: ldfld float64 MCCTest.VTypeD::f10 + IL_00bf: ldarga.s expected + IL_00c1: ldfld float64 MCCTest.VTypeD::f10 + IL_00c6: ceq + IL_00c8: stloc.3 + IL_00c9: ldloc.3 + IL_00ca: brtrue.s IL_00e5 + + IL_00cc: nop + IL_00cd: ldstr "f10" + IL_00d2: ldarg.0 + IL_00d3: ldfld float64 MCCTest.VTypeD::f10 + IL_00d8: ldarga.s expected + IL_00da: ldfld float64 MCCTest.VTypeD::f10 + IL_00df: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_00e4: throw + + IL_00e5: ldarg.0 + IL_00e6: ldfld int16 MCCTest.VTypeD::f11 + IL_00eb: ldarga.s expected + IL_00ed: ldfld int16 MCCTest.VTypeD::f11 + IL_00f2: ceq + IL_00f4: stloc.3 + IL_00f5: ldloc.3 + IL_00f6: brtrue.s IL_0113 + + IL_00f8: nop + IL_00f9: ldstr "f11" + IL_00fe: ldarg.0 + IL_00ff: ldfld int16 MCCTest.VTypeD::f11 + IL_0104: conv.i8 + IL_0105: ldarga.s expected + IL_0107: ldfld int16 MCCTest.VTypeD::f11 + IL_010c: conv.i8 + IL_010d: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_0112: throw + + IL_0113: ldarg.0 + IL_0114: ldfld uint32 MCCTest.VTypeD::f17 + IL_0119: ldarga.s expected + IL_011b: ldfld uint32 MCCTest.VTypeD::f17 + IL_0120: ceq + IL_0122: stloc.3 + IL_0123: ldloc.3 + IL_0124: brtrue.s IL_0141 + + IL_0126: nop + IL_0127: ldstr "f17" + IL_012c: ldarg.0 + IL_012d: ldfld uint32 MCCTest.VTypeD::f17 + IL_0132: conv.u8 + IL_0133: ldarga.s expected + IL_0135: ldfld uint32 MCCTest.VTypeD::f17 + IL_013a: conv.u8 + IL_013b: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_0140: throw + + IL_0141: ldnull + IL_0142: stloc.0 + IL_0143: ldnull + IL_0144: stloc.1 + .try + { + IL_0145: nop + IL_0146: ldstr "f2" + IL_014b: stloc.0 + IL_014c: ldarg.0 + IL_014d: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_0152: box MCCTest.VType3 + IL_0157: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_015c: stloc.1 + IL_015d: ldarg.0 + IL_015e: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_0163: ldarga.s expected + IL_0165: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_016a: call instance void MCCTest.VType3::Check(valuetype MCCTest.VType3) + IL_016f: nop + IL_0170: ldstr "f4" + IL_0175: stloc.0 + IL_0176: ldarg.0 + IL_0177: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_017c: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0181: stloc.1 + IL_0182: ldarg.0 + IL_0183: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_0188: ldarga.s expected + IL_018a: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_018f: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0194: nop + IL_0195: ldstr "f5" + IL_019a: stloc.0 + IL_019b: ldarg.0 + IL_019c: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_01a1: box MCCTest.VType7 + IL_01a6: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ab: stloc.1 + IL_01ac: ldarg.0 + IL_01ad: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_01b2: ldarga.s expected + IL_01b4: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_01b9: call instance void MCCTest.VType7::Check(valuetype MCCTest.VType7) + IL_01be: nop + IL_01bf: ldstr "f8" + IL_01c4: stloc.0 + IL_01c5: ldarg.0 + IL_01c6: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_01cb: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01d0: stloc.1 + IL_01d1: ldarg.0 + IL_01d2: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_01d7: ldarga.s expected + IL_01d9: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_01de: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_01e3: nop + IL_01e4: ldstr "f9" + IL_01e9: stloc.0 + IL_01ea: ldarg.0 + IL_01eb: ldfld valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_01f0: box MCCTest.VType6 + IL_01f5: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01fa: stloc.1 + IL_01fb: ldarg.0 + IL_01fc: ldflda valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_0201: ldarga.s expected + IL_0203: ldfld valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_0208: call instance void MCCTest.VType6::Check(valuetype MCCTest.VType6) + IL_020d: nop + IL_020e: ldstr "f12" + IL_0213: stloc.0 + IL_0214: ldarg.0 + IL_0215: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_021a: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_021f: stloc.1 + IL_0220: ldarg.0 + IL_0221: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_0226: ldarga.s expected + IL_0228: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_022d: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0232: nop + IL_0233: ldstr "f13" + IL_0238: stloc.0 + IL_0239: ldarg.0 + IL_023a: ldfld valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_023f: box MCCTest.VType5 + IL_0244: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0249: stloc.1 + IL_024a: ldarg.0 + IL_024b: ldflda valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_0250: ldarga.s expected + IL_0252: ldfld valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_0257: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_025c: nop + IL_025d: ldstr "f14" + IL_0262: stloc.0 + IL_0263: ldarg.0 + IL_0264: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_0269: box MCCTest.VType3 + IL_026e: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0273: stloc.1 + IL_0274: ldarg.0 + IL_0275: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_027a: ldarga.s expected + IL_027c: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_0281: call instance void MCCTest.VType3::Check(valuetype MCCTest.VType3) + IL_0286: nop + IL_0287: ldstr "f15" + IL_028c: stloc.0 + IL_028d: ldarg.0 + IL_028e: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_0293: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0298: stloc.1 + IL_0299: ldarg.0 + IL_029a: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_029f: ldarga.s expected + IL_02a1: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_02a6: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_02ab: nop + IL_02ac: ldstr "f16" + IL_02b1: stloc.0 + IL_02b2: ldarg.0 + IL_02b3: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_02b8: box MCCTest.VType7 + IL_02bd: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02c2: stloc.1 + IL_02c3: ldarg.0 + IL_02c4: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_02c9: ldarga.s expected + IL_02cb: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_02d0: call instance void MCCTest.VType7::Check(valuetype MCCTest.VType7) + IL_02d5: nop + IL_02d6: nop + IL_02d7: leave.s IL_02e4 + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_02d9: stloc.2 + IL_02da: nop + IL_02db: ldloc.0 + IL_02dc: ldloc.1 + IL_02dd: ldloc.2 + IL_02de: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_02e3: throw + + } // end handler + IL_02e4: nop + IL_02e5: ret + } // end of method VTypeD::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 808 (0x328) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "f1 = " + IL_0015: ldarg.0 + IL_0016: ldfld int32 MCCTest.VTypeD::f1 + IL_001b: box [mscorlib]System.Int32 + IL_0020: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0025: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_002a: nop + IL_002b: ldloc.1 + IL_002c: ldloc.0 + IL_002d: ldstr "[Field f2] [Type '{0}']" + IL_0032: call string [mscorlib]System.String::Concat(string, + string) + IL_0037: ldarg.0 + IL_0038: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_003d: box MCCTest.VType3 + IL_0042: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0047: callvirt instance string [mscorlib]System.Object::ToString() + IL_004c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0051: nop + IL_0052: ldloc.1 + IL_0053: ldarg.0 + IL_0054: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f2 + IL_0059: ldarg.1 + IL_005a: ldc.i4.1 + IL_005b: add + IL_005c: call instance string MCCTest.VType3::Dump(int32) + IL_0061: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0066: nop + IL_0067: ldloc.1 + IL_0068: ldloc.0 + IL_0069: ldstr "f3 = " + IL_006e: ldarg.0 + IL_006f: ldfld float64 MCCTest.VTypeD::f3 + IL_0074: box [mscorlib]System.Double + IL_0079: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_007e: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0083: nop + IL_0084: ldloc.1 + IL_0085: ldloc.0 + IL_0086: ldstr "[Field f4] [Type '{0}']" + IL_008b: call string [mscorlib]System.String::Concat(string, + string) + IL_0090: ldarg.0 + IL_0091: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_0096: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_009b: callvirt instance string [mscorlib]System.Object::ToString() + IL_00a0: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00a5: nop + IL_00a6: ldloc.1 + IL_00a7: ldarg.0 + IL_00a8: ldfld class MCCTest.RType4 MCCTest.VTypeD::f4 + IL_00ad: ldarg.1 + IL_00ae: ldc.i4.1 + IL_00af: add + IL_00b0: callvirt instance string MCCTest.RType4::Dump(int32) + IL_00b5: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00ba: nop + IL_00bb: ldloc.1 + IL_00bc: ldloc.0 + IL_00bd: ldstr "[Field f5] [Type '{0}']" + IL_00c2: call string [mscorlib]System.String::Concat(string, + string) + IL_00c7: ldarg.0 + IL_00c8: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_00cd: box MCCTest.VType7 + IL_00d2: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00d7: callvirt instance string [mscorlib]System.Object::ToString() + IL_00dc: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00e1: nop + IL_00e2: ldloc.1 + IL_00e3: ldarg.0 + IL_00e4: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f5 + IL_00e9: ldarg.1 + IL_00ea: ldc.i4.1 + IL_00eb: add + IL_00ec: call instance string MCCTest.VType7::Dump(int32) + IL_00f1: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00f6: nop + IL_00f7: ldloc.1 + IL_00f8: ldloc.0 + IL_00f9: ldstr "f6 = " + IL_00fe: ldarg.0 + IL_00ff: ldfld uint64 MCCTest.VTypeD::f6 + IL_0104: box [mscorlib]System.UInt64 + IL_0109: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_010e: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0113: nop + IL_0114: ldloc.1 + IL_0115: ldloc.0 + IL_0116: ldstr "f7 = " + IL_011b: ldarg.0 + IL_011c: ldfld float32 MCCTest.VTypeD::f7 + IL_0121: box [mscorlib]System.Single + IL_0126: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_012b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0130: nop + IL_0131: ldloc.1 + IL_0132: ldloc.0 + IL_0133: ldstr "[Field f8] [Type '{0}']" + IL_0138: call string [mscorlib]System.String::Concat(string, + string) + IL_013d: ldarg.0 + IL_013e: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_0143: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0148: callvirt instance string [mscorlib]System.Object::ToString() + IL_014d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0152: nop + IL_0153: ldloc.1 + IL_0154: ldarg.0 + IL_0155: ldfld class MCCTest.RType4 MCCTest.VTypeD::f8 + IL_015a: ldarg.1 + IL_015b: ldc.i4.1 + IL_015c: add + IL_015d: callvirt instance string MCCTest.RType4::Dump(int32) + IL_0162: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0167: nop + IL_0168: ldloc.1 + IL_0169: ldloc.0 + IL_016a: ldstr "[Field f9] [Type '{0}']" + IL_016f: call string [mscorlib]System.String::Concat(string, + string) + IL_0174: ldarg.0 + IL_0175: ldfld valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_017a: box MCCTest.VType6 + IL_017f: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0184: callvirt instance string [mscorlib]System.Object::ToString() + IL_0189: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_018e: nop + IL_018f: ldloc.1 + IL_0190: ldarg.0 + IL_0191: ldflda valuetype MCCTest.VType6 MCCTest.VTypeD::f9 + IL_0196: ldarg.1 + IL_0197: ldc.i4.1 + IL_0198: add + IL_0199: call instance string MCCTest.VType6::Dump(int32) + IL_019e: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01a3: nop + IL_01a4: ldloc.1 + IL_01a5: ldloc.0 + IL_01a6: ldstr "f10 = " + IL_01ab: ldarg.0 + IL_01ac: ldfld float64 MCCTest.VTypeD::f10 + IL_01b1: box [mscorlib]System.Double + IL_01b6: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_01bb: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01c0: nop + IL_01c1: ldloc.1 + IL_01c2: ldloc.0 + IL_01c3: ldstr "f11 = " + IL_01c8: ldarg.0 + IL_01c9: ldfld int16 MCCTest.VTypeD::f11 + IL_01ce: box [mscorlib]System.Int16 + IL_01d3: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_01d8: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01dd: nop + IL_01de: ldloc.1 + IL_01df: ldloc.0 + IL_01e0: ldstr "[Field f12] [Type '{0}']" + IL_01e5: call string [mscorlib]System.String::Concat(string, + string) + IL_01ea: ldarg.0 + IL_01eb: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_01f0: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01f5: callvirt instance string [mscorlib]System.Object::ToString() + IL_01fa: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01ff: nop + IL_0200: ldloc.1 + IL_0201: ldarg.0 + IL_0202: ldfld class MCCTest.RType4 MCCTest.VTypeD::f12 + IL_0207: ldarg.1 + IL_0208: ldc.i4.1 + IL_0209: add + IL_020a: callvirt instance string MCCTest.RType4::Dump(int32) + IL_020f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0214: nop + IL_0215: ldloc.1 + IL_0216: ldloc.0 + IL_0217: ldstr "[Field f13] [Type '{0}']" + IL_021c: call string [mscorlib]System.String::Concat(string, + string) + IL_0221: ldarg.0 + IL_0222: ldfld valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_0227: box MCCTest.VType5 + IL_022c: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0231: callvirt instance string [mscorlib]System.Object::ToString() + IL_0236: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_023b: nop + IL_023c: ldloc.1 + IL_023d: ldarg.0 + IL_023e: ldflda valuetype MCCTest.VType5 MCCTest.VTypeD::f13 + IL_0243: ldarg.1 + IL_0244: ldc.i4.1 + IL_0245: add + IL_0246: call instance string MCCTest.VType5::Dump(int32) + IL_024b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0250: nop + IL_0251: ldloc.1 + IL_0252: ldloc.0 + IL_0253: ldstr "[Field f14] [Type '{0}']" + IL_0258: call string [mscorlib]System.String::Concat(string, + string) + IL_025d: ldarg.0 + IL_025e: ldfld valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_0263: box MCCTest.VType3 + IL_0268: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_026d: callvirt instance string [mscorlib]System.Object::ToString() + IL_0272: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0277: nop + IL_0278: ldloc.1 + IL_0279: ldarg.0 + IL_027a: ldflda valuetype MCCTest.VType3 MCCTest.VTypeD::f14 + IL_027f: ldarg.1 + IL_0280: ldc.i4.1 + IL_0281: add + IL_0282: call instance string MCCTest.VType3::Dump(int32) + IL_0287: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_028c: nop + IL_028d: ldloc.1 + IL_028e: ldloc.0 + IL_028f: ldstr "[Field f15] [Type '{0}']" + IL_0294: call string [mscorlib]System.String::Concat(string, + string) + IL_0299: ldarg.0 + IL_029a: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_029f: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02a4: callvirt instance string [mscorlib]System.Object::ToString() + IL_02a9: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02ae: nop + IL_02af: ldloc.1 + IL_02b0: ldarg.0 + IL_02b1: ldfld class MCCTest.RType4 MCCTest.VTypeD::f15 + IL_02b6: ldarg.1 + IL_02b7: ldc.i4.1 + IL_02b8: add + IL_02b9: callvirt instance string MCCTest.RType4::Dump(int32) + IL_02be: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02c3: nop + IL_02c4: ldloc.1 + IL_02c5: ldloc.0 + IL_02c6: ldstr "[Field f16] [Type '{0}']" + IL_02cb: call string [mscorlib]System.String::Concat(string, + string) + IL_02d0: ldarg.0 + IL_02d1: ldfld valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_02d6: box MCCTest.VType7 + IL_02db: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02e0: callvirt instance string [mscorlib]System.Object::ToString() + IL_02e5: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02ea: nop + IL_02eb: ldloc.1 + IL_02ec: ldarg.0 + IL_02ed: ldflda valuetype MCCTest.VType7 MCCTest.VTypeD::f16 + IL_02f2: ldarg.1 + IL_02f3: ldc.i4.1 + IL_02f4: add + IL_02f5: call instance string MCCTest.VType7::Dump(int32) + IL_02fa: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02ff: nop + IL_0300: ldloc.1 + IL_0301: ldloc.0 + IL_0302: ldstr "f17 = " + IL_0307: ldarg.0 + IL_0308: ldfld uint32 MCCTest.VTypeD::f17 + IL_030d: box [mscorlib]System.UInt32 + IL_0312: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0317: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_031c: nop + IL_031d: ldloc.1 + IL_031e: callvirt instance string [mscorlib]System.Object::ToString() + IL_0323: stloc.2 + IL_0324: br.s IL_0326 + + IL_0326: ldloc.2 + IL_0327: ret + } // end of method VTypeD::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VTypeD::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VTypeD::Dump + +} // end of class MCCTest.VTypeD + +.class public sequential ansi sealed beforefieldinit MCCTest.VTypeE + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public valuetype MCCTest.VType9 f1 + .field public class MCCTest.RType4 f2 + .field public valuetype MCCTest.VType8 f3 + .field public int16 f4 + .field public valuetype MCCTest.VType8 f5 + .field public valuetype MCCTest.VType8 f6 + .field public float64 f7 + .field public valuetype MCCTest.VTypeA f8 + .field public int16 f9 + .field public valuetype MCCTest.VTypeD f10 + .field public valuetype MCCTest.VType8 f11 + .field public valuetype MCCTest.VTypeC f12 + .field public valuetype MCCTest.VTypeA f13 + .field public float32 f14 + .field public int32 f15 + .field public valuetype MCCTest.VTypeC f16 + .field public valuetype MCCTest.VType7 f17 + .field public float64 f18 + .field public uint64 f19 + .field public valuetype MCCTest.VType8 f20 + .field public valuetype MCCTest.VType9 f21 + .field public valuetype MCCTest.VTypeC f22 + .field public valuetype MCCTest.VType8 f23 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 276 (0x114) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: newobj instance void MCCTest.RType4::.ctor() + IL_0007: stfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_000c: ldarg.0 + IL_000d: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_0012: ldarg.1 + IL_0013: call instance void MCCTest.VType9::Init(int32) + IL_0018: nop + IL_0019: ldarg.0 + IL_001a: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_001f: ldarg.1 + IL_0020: callvirt instance void MCCTest.RType4::Init(int32) + IL_0025: nop + IL_0026: ldarg.0 + IL_0027: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_002c: ldarg.1 + IL_002d: call instance void MCCTest.VType8::Init(int32) + IL_0032: nop + IL_0033: ldarg.0 + IL_0034: ldarg.1 + IL_0035: conv.i2 + IL_0036: stfld int16 MCCTest.VTypeE::f4 + IL_003b: ldarg.0 + IL_003c: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_0041: ldarg.1 + IL_0042: call instance void MCCTest.VType8::Init(int32) + IL_0047: nop + IL_0048: ldarg.0 + IL_0049: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_004e: ldarg.1 + IL_004f: call instance void MCCTest.VType8::Init(int32) + IL_0054: nop + IL_0055: ldarg.0 + IL_0056: ldarg.1 + IL_0057: conv.r8 + IL_0058: stfld float64 MCCTest.VTypeE::f7 + IL_005d: ldarg.0 + IL_005e: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_0063: ldarg.1 + IL_0064: call instance void MCCTest.VTypeA::Init(int32) + IL_0069: nop + IL_006a: ldarg.0 + IL_006b: ldarg.1 + IL_006c: conv.i2 + IL_006d: stfld int16 MCCTest.VTypeE::f9 + IL_0072: ldarg.0 + IL_0073: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_0078: ldarg.1 + IL_0079: call instance void MCCTest.VTypeD::Init(int32) + IL_007e: nop + IL_007f: ldarg.0 + IL_0080: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_0085: ldarg.1 + IL_0086: call instance void MCCTest.VType8::Init(int32) + IL_008b: nop + IL_008c: ldarg.0 + IL_008d: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_0092: ldarg.1 + IL_0093: call instance void MCCTest.VTypeC::Init(int32) + IL_0098: nop + IL_0099: ldarg.0 + IL_009a: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_009f: ldarg.1 + IL_00a0: call instance void MCCTest.VTypeA::Init(int32) + IL_00a5: nop + IL_00a6: ldarg.0 + IL_00a7: ldarg.1 + IL_00a8: conv.r4 + IL_00a9: stfld float32 MCCTest.VTypeE::f14 + IL_00ae: ldarg.0 + IL_00af: ldarg.1 + IL_00b0: stfld int32 MCCTest.VTypeE::f15 + IL_00b5: ldarg.0 + IL_00b6: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_00bb: ldarg.1 + IL_00bc: call instance void MCCTest.VTypeC::Init(int32) + IL_00c1: nop + IL_00c2: ldarg.0 + IL_00c3: ldflda valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_00c8: ldarg.1 + IL_00c9: call instance void MCCTest.VType7::Init(int32) + IL_00ce: nop + IL_00cf: ldarg.0 + IL_00d0: ldarg.1 + IL_00d1: conv.r8 + IL_00d2: stfld float64 MCCTest.VTypeE::f18 + IL_00d7: ldarg.0 + IL_00d8: ldarg.1 + IL_00d9: conv.i8 + IL_00da: stfld uint64 MCCTest.VTypeE::f19 + IL_00df: ldarg.0 + IL_00e0: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_00e5: ldarg.1 + IL_00e6: call instance void MCCTest.VType8::Init(int32) + IL_00eb: nop + IL_00ec: ldarg.0 + IL_00ed: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_00f2: ldarg.1 + IL_00f3: call instance void MCCTest.VType9::Init(int32) + IL_00f8: nop + IL_00f9: ldarg.0 + IL_00fa: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_00ff: ldarg.1 + IL_0100: call instance void MCCTest.VTypeC::Init(int32) + IL_0105: nop + IL_0106: ldarg.0 + IL_0107: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_010c: ldarg.1 + IL_010d: call instance void MCCTest.VType8::Init(int32) + IL_0112: nop + IL_0113: ret + } // end of method VTypeE::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VTypeE::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeE::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VTypeE::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeE::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VTypeE val) cil managed + { + // Code size 448 (0x1c0) + .maxstack 3 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_0007: ldarga.s val + IL_0009: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_000e: call instance void MCCTest.VType9::Add(valuetype MCCTest.VType9) + IL_0013: nop + IL_0014: ldarg.0 + IL_0015: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_001a: ldarga.s val + IL_001c: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_0021: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0026: nop + IL_0027: ldarg.0 + IL_0028: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_002d: ldarga.s val + IL_002f: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_0034: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_0039: nop + IL_003a: ldarg.0 + IL_003b: dup + IL_003c: ldfld int16 MCCTest.VTypeE::f4 + IL_0041: ldarga.s val + IL_0043: ldfld int16 MCCTest.VTypeE::f4 + IL_0048: add + IL_0049: conv.i2 + IL_004a: stfld int16 MCCTest.VTypeE::f4 + IL_004f: ldarg.0 + IL_0050: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_0055: ldarga.s val + IL_0057: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_005c: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_0061: nop + IL_0062: ldarg.0 + IL_0063: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_0068: ldarga.s val + IL_006a: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_006f: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_0074: nop + IL_0075: ldarg.0 + IL_0076: dup + IL_0077: ldfld float64 MCCTest.VTypeE::f7 + IL_007c: ldarga.s val + IL_007e: ldfld float64 MCCTest.VTypeE::f7 + IL_0083: add + IL_0084: stfld float64 MCCTest.VTypeE::f7 + IL_0089: ldarg.0 + IL_008a: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_008f: ldarga.s val + IL_0091: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_0096: call instance void MCCTest.VTypeA::Add(valuetype MCCTest.VTypeA) + IL_009b: nop + IL_009c: ldarg.0 + IL_009d: dup + IL_009e: ldfld int16 MCCTest.VTypeE::f9 + IL_00a3: ldarga.s val + IL_00a5: ldfld int16 MCCTest.VTypeE::f9 + IL_00aa: add + IL_00ab: conv.i2 + IL_00ac: stfld int16 MCCTest.VTypeE::f9 + IL_00b1: ldarg.0 + IL_00b2: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_00b7: ldarga.s val + IL_00b9: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_00be: call instance void MCCTest.VTypeD::Add(valuetype MCCTest.VTypeD) + IL_00c3: nop + IL_00c4: ldarg.0 + IL_00c5: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_00ca: ldarga.s val + IL_00cc: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_00d1: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_00d6: nop + IL_00d7: ldarg.0 + IL_00d8: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_00dd: ldarga.s val + IL_00df: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_00e4: call instance void MCCTest.VTypeC::Add(valuetype MCCTest.VTypeC) + IL_00e9: nop + IL_00ea: ldarg.0 + IL_00eb: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_00f0: ldarga.s val + IL_00f2: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_00f7: call instance void MCCTest.VTypeA::Add(valuetype MCCTest.VTypeA) + IL_00fc: nop + IL_00fd: ldarg.0 + IL_00fe: dup + IL_00ff: ldfld float32 MCCTest.VTypeE::f14 + IL_0104: ldarga.s val + IL_0106: ldfld float32 MCCTest.VTypeE::f14 + IL_010b: add + IL_010c: stfld float32 MCCTest.VTypeE::f14 + IL_0111: ldarg.0 + IL_0112: dup + IL_0113: ldfld int32 MCCTest.VTypeE::f15 + IL_0118: ldarga.s val + IL_011a: ldfld int32 MCCTest.VTypeE::f15 + IL_011f: add + IL_0120: stfld int32 MCCTest.VTypeE::f15 + IL_0125: ldarg.0 + IL_0126: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_012b: ldarga.s val + IL_012d: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_0132: call instance void MCCTest.VTypeC::Add(valuetype MCCTest.VTypeC) + IL_0137: nop + IL_0138: ldarg.0 + IL_0139: ldflda valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_013e: ldarga.s val + IL_0140: ldfld valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_0145: call instance void MCCTest.VType7::Add(valuetype MCCTest.VType7) + IL_014a: nop + IL_014b: ldarg.0 + IL_014c: dup + IL_014d: ldfld float64 MCCTest.VTypeE::f18 + IL_0152: ldarga.s val + IL_0154: ldfld float64 MCCTest.VTypeE::f18 + IL_0159: add + IL_015a: stfld float64 MCCTest.VTypeE::f18 + IL_015f: ldarg.0 + IL_0160: dup + IL_0161: ldfld uint64 MCCTest.VTypeE::f19 + IL_0166: ldarga.s val + IL_0168: ldfld uint64 MCCTest.VTypeE::f19 + IL_016d: add + IL_016e: stfld uint64 MCCTest.VTypeE::f19 + IL_0173: ldarg.0 + IL_0174: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_0179: ldarga.s val + IL_017b: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_0180: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_0185: nop + IL_0186: ldarg.0 + IL_0187: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_018c: ldarga.s val + IL_018e: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_0193: call instance void MCCTest.VType9::Add(valuetype MCCTest.VType9) + IL_0198: nop + IL_0199: ldarg.0 + IL_019a: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_019f: ldarga.s val + IL_01a1: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_01a6: call instance void MCCTest.VTypeC::Add(valuetype MCCTest.VTypeC) + IL_01ab: nop + IL_01ac: ldarg.0 + IL_01ad: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_01b2: ldarga.s val + IL_01b4: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_01b9: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_01be: nop + IL_01bf: ret + } // end of method VTypeE::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VTypeE expected) cil managed + { + // Code size 1009 (0x3f1) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2, + bool V_3) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldfld int16 MCCTest.VTypeE::f4 + IL_0007: ldarga.s expected + IL_0009: ldfld int16 MCCTest.VTypeE::f4 + IL_000e: ceq + IL_0010: stloc.3 + IL_0011: ldloc.3 + IL_0012: brtrue.s IL_002f + + IL_0014: nop + IL_0015: ldstr "f4" + IL_001a: ldarg.0 + IL_001b: ldfld int16 MCCTest.VTypeE::f4 + IL_0020: conv.i8 + IL_0021: ldarga.s expected + IL_0023: ldfld int16 MCCTest.VTypeE::f4 + IL_0028: conv.i8 + IL_0029: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_002e: throw + + IL_002f: ldarg.0 + IL_0030: ldfld float64 MCCTest.VTypeE::f7 + IL_0035: ldarga.s expected + IL_0037: ldfld float64 MCCTest.VTypeE::f7 + IL_003c: ceq + IL_003e: stloc.3 + IL_003f: ldloc.3 + IL_0040: brtrue.s IL_005b + + IL_0042: nop + IL_0043: ldstr "f7" + IL_0048: ldarg.0 + IL_0049: ldfld float64 MCCTest.VTypeE::f7 + IL_004e: ldarga.s expected + IL_0050: ldfld float64 MCCTest.VTypeE::f7 + IL_0055: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_005a: throw + + IL_005b: ldarg.0 + IL_005c: ldfld int16 MCCTest.VTypeE::f9 + IL_0061: ldarga.s expected + IL_0063: ldfld int16 MCCTest.VTypeE::f9 + IL_0068: ceq + IL_006a: stloc.3 + IL_006b: ldloc.3 + IL_006c: brtrue.s IL_0089 + + IL_006e: nop + IL_006f: ldstr "f9" + IL_0074: ldarg.0 + IL_0075: ldfld int16 MCCTest.VTypeE::f9 + IL_007a: conv.i8 + IL_007b: ldarga.s expected + IL_007d: ldfld int16 MCCTest.VTypeE::f9 + IL_0082: conv.i8 + IL_0083: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_0088: throw + + IL_0089: ldarg.0 + IL_008a: ldfld float32 MCCTest.VTypeE::f14 + IL_008f: ldarga.s expected + IL_0091: ldfld float32 MCCTest.VTypeE::f14 + IL_0096: ceq + IL_0098: stloc.3 + IL_0099: ldloc.3 + IL_009a: brtrue.s IL_00b7 + + IL_009c: nop + IL_009d: ldstr "f14" + IL_00a2: ldarg.0 + IL_00a3: ldfld float32 MCCTest.VTypeE::f14 + IL_00a8: conv.r8 + IL_00a9: ldarga.s expected + IL_00ab: ldfld float32 MCCTest.VTypeE::f14 + IL_00b0: conv.r8 + IL_00b1: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_00b6: throw + + IL_00b7: ldarg.0 + IL_00b8: ldfld int32 MCCTest.VTypeE::f15 + IL_00bd: ldarga.s expected + IL_00bf: ldfld int32 MCCTest.VTypeE::f15 + IL_00c4: ceq + IL_00c6: stloc.3 + IL_00c7: ldloc.3 + IL_00c8: brtrue.s IL_00e5 + + IL_00ca: nop + IL_00cb: ldstr "f15" + IL_00d0: ldarg.0 + IL_00d1: ldfld int32 MCCTest.VTypeE::f15 + IL_00d6: conv.i8 + IL_00d7: ldarga.s expected + IL_00d9: ldfld int32 MCCTest.VTypeE::f15 + IL_00de: conv.i8 + IL_00df: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_00e4: throw + + IL_00e5: ldarg.0 + IL_00e6: ldfld float64 MCCTest.VTypeE::f18 + IL_00eb: ldarga.s expected + IL_00ed: ldfld float64 MCCTest.VTypeE::f18 + IL_00f2: ceq + IL_00f4: stloc.3 + IL_00f5: ldloc.3 + IL_00f6: brtrue.s IL_0111 + + IL_00f8: nop + IL_00f9: ldstr "f18" + IL_00fe: ldarg.0 + IL_00ff: ldfld float64 MCCTest.VTypeE::f18 + IL_0104: ldarga.s expected + IL_0106: ldfld float64 MCCTest.VTypeE::f18 + IL_010b: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_0110: throw + + IL_0111: ldarg.0 + IL_0112: ldfld uint64 MCCTest.VTypeE::f19 + IL_0117: ldarga.s expected + IL_0119: ldfld uint64 MCCTest.VTypeE::f19 + IL_011e: ceq + IL_0120: stloc.3 + IL_0121: ldloc.3 + IL_0122: brtrue.s IL_0141 + + IL_0124: nop + IL_0125: ldstr "f19" + IL_012a: ldarg.0 + IL_012b: ldfld uint64 MCCTest.VTypeE::f19 + IL_0130: conv.r.un + IL_0131: conv.r8 + IL_0132: ldarga.s expected + IL_0134: ldfld uint64 MCCTest.VTypeE::f19 + IL_0139: conv.r.un + IL_013a: conv.r8 + IL_013b: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_0140: throw + + IL_0141: ldnull + IL_0142: stloc.0 + IL_0143: ldnull + IL_0144: stloc.1 + .try + { + IL_0145: nop + IL_0146: ldstr "f1" + IL_014b: stloc.0 + IL_014c: ldarg.0 + IL_014d: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_0152: box MCCTest.VType9 + IL_0157: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_015c: stloc.1 + IL_015d: ldarg.0 + IL_015e: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_0163: ldarga.s expected + IL_0165: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_016a: call instance void MCCTest.VType9::Check(valuetype MCCTest.VType9) + IL_016f: nop + IL_0170: ldstr "f2" + IL_0175: stloc.0 + IL_0176: ldarg.0 + IL_0177: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_017c: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0181: stloc.1 + IL_0182: ldarg.0 + IL_0183: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_0188: ldarga.s expected + IL_018a: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_018f: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0194: nop + IL_0195: ldstr "f3" + IL_019a: stloc.0 + IL_019b: ldarg.0 + IL_019c: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_01a1: box MCCTest.VType8 + IL_01a6: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ab: stloc.1 + IL_01ac: ldarg.0 + IL_01ad: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_01b2: ldarga.s expected + IL_01b4: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_01b9: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_01be: nop + IL_01bf: ldstr "f5" + IL_01c4: stloc.0 + IL_01c5: ldarg.0 + IL_01c6: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_01cb: box MCCTest.VType8 + IL_01d0: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01d5: stloc.1 + IL_01d6: ldarg.0 + IL_01d7: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_01dc: ldarga.s expected + IL_01de: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_01e3: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_01e8: nop + IL_01e9: ldstr "f6" + IL_01ee: stloc.0 + IL_01ef: ldarg.0 + IL_01f0: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_01f5: box MCCTest.VType8 + IL_01fa: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01ff: stloc.1 + IL_0200: ldarg.0 + IL_0201: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_0206: ldarga.s expected + IL_0208: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_020d: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_0212: nop + IL_0213: ldstr "f8" + IL_0218: stloc.0 + IL_0219: ldarg.0 + IL_021a: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_021f: box MCCTest.VTypeA + IL_0224: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0229: stloc.1 + IL_022a: ldarg.0 + IL_022b: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_0230: ldarga.s expected + IL_0232: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_0237: call instance void MCCTest.VTypeA::Check(valuetype MCCTest.VTypeA) + IL_023c: nop + IL_023d: ldstr "f10" + IL_0242: stloc.0 + IL_0243: ldarg.0 + IL_0244: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_0249: box MCCTest.VTypeD + IL_024e: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0253: stloc.1 + IL_0254: ldarg.0 + IL_0255: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_025a: ldarga.s expected + IL_025c: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_0261: call instance void MCCTest.VTypeD::Check(valuetype MCCTest.VTypeD) + IL_0266: nop + IL_0267: ldstr "f11" + IL_026c: stloc.0 + IL_026d: ldarg.0 + IL_026e: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_0273: box MCCTest.VType8 + IL_0278: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_027d: stloc.1 + IL_027e: ldarg.0 + IL_027f: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_0284: ldarga.s expected + IL_0286: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_028b: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_0290: nop + IL_0291: ldstr "f12" + IL_0296: stloc.0 + IL_0297: ldarg.0 + IL_0298: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_029d: box MCCTest.VTypeC + IL_02a2: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02a7: stloc.1 + IL_02a8: ldarg.0 + IL_02a9: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_02ae: ldarga.s expected + IL_02b0: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_02b5: call instance void MCCTest.VTypeC::Check(valuetype MCCTest.VTypeC) + IL_02ba: nop + IL_02bb: ldstr "f13" + IL_02c0: stloc.0 + IL_02c1: ldarg.0 + IL_02c2: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_02c7: box MCCTest.VTypeA + IL_02cc: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02d1: stloc.1 + IL_02d2: ldarg.0 + IL_02d3: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_02d8: ldarga.s expected + IL_02da: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_02df: call instance void MCCTest.VTypeA::Check(valuetype MCCTest.VTypeA) + IL_02e4: nop + IL_02e5: ldstr "f16" + IL_02ea: stloc.0 + IL_02eb: ldarg.0 + IL_02ec: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_02f1: box MCCTest.VTypeC + IL_02f6: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02fb: stloc.1 + IL_02fc: ldarg.0 + IL_02fd: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_0302: ldarga.s expected + IL_0304: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_0309: call instance void MCCTest.VTypeC::Check(valuetype MCCTest.VTypeC) + IL_030e: nop + IL_030f: ldstr "f17" + IL_0314: stloc.0 + IL_0315: ldarg.0 + IL_0316: ldfld valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_031b: box MCCTest.VType7 + IL_0320: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0325: stloc.1 + IL_0326: ldarg.0 + IL_0327: ldflda valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_032c: ldarga.s expected + IL_032e: ldfld valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_0333: call instance void MCCTest.VType7::Check(valuetype MCCTest.VType7) + IL_0338: nop + IL_0339: ldstr "f20" + IL_033e: stloc.0 + IL_033f: ldarg.0 + IL_0340: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_0345: box MCCTest.VType8 + IL_034a: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_034f: stloc.1 + IL_0350: ldarg.0 + IL_0351: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_0356: ldarga.s expected + IL_0358: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_035d: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_0362: nop + IL_0363: ldstr "f21" + IL_0368: stloc.0 + IL_0369: ldarg.0 + IL_036a: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_036f: box MCCTest.VType9 + IL_0374: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0379: stloc.1 + IL_037a: ldarg.0 + IL_037b: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_0380: ldarga.s expected + IL_0382: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_0387: call instance void MCCTest.VType9::Check(valuetype MCCTest.VType9) + IL_038c: nop + IL_038d: ldstr "f22" + IL_0392: stloc.0 + IL_0393: ldarg.0 + IL_0394: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_0399: box MCCTest.VTypeC + IL_039e: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03a3: stloc.1 + IL_03a4: ldarg.0 + IL_03a5: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_03aa: ldarga.s expected + IL_03ac: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_03b1: call instance void MCCTest.VTypeC::Check(valuetype MCCTest.VTypeC) + IL_03b6: nop + IL_03b7: ldstr "f23" + IL_03bc: stloc.0 + IL_03bd: ldarg.0 + IL_03be: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_03c3: box MCCTest.VType8 + IL_03c8: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03cd: stloc.1 + IL_03ce: ldarg.0 + IL_03cf: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_03d4: ldarga.s expected + IL_03d6: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_03db: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_03e0: nop + IL_03e1: nop + IL_03e2: leave.s IL_03ef + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_03e4: stloc.2 + IL_03e5: nop + IL_03e6: ldloc.0 + IL_03e7: ldloc.1 + IL_03e8: ldloc.2 + IL_03e9: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_03ee: throw + + } // end handler + IL_03ef: nop + IL_03f0: ret + } // end of method VTypeE::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 1183 (0x49f) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "[Field f1] [Type '{0}']" + IL_0015: call string [mscorlib]System.String::Concat(string, + string) + IL_001a: ldarg.0 + IL_001b: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_0020: box MCCTest.VType9 + IL_0025: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_002a: callvirt instance string [mscorlib]System.Object::ToString() + IL_002f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0034: nop + IL_0035: ldloc.1 + IL_0036: ldarg.0 + IL_0037: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f1 + IL_003c: ldarg.1 + IL_003d: ldc.i4.1 + IL_003e: add + IL_003f: call instance string MCCTest.VType9::Dump(int32) + IL_0044: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0049: nop + IL_004a: ldloc.1 + IL_004b: ldloc.0 + IL_004c: ldstr "[Field f2] [Type '{0}']" + IL_0051: call string [mscorlib]System.String::Concat(string, + string) + IL_0056: ldarg.0 + IL_0057: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_005c: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0061: callvirt instance string [mscorlib]System.Object::ToString() + IL_0066: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_006b: nop + IL_006c: ldloc.1 + IL_006d: ldarg.0 + IL_006e: ldfld class MCCTest.RType4 MCCTest.VTypeE::f2 + IL_0073: ldarg.1 + IL_0074: ldc.i4.1 + IL_0075: add + IL_0076: callvirt instance string MCCTest.RType4::Dump(int32) + IL_007b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0080: nop + IL_0081: ldloc.1 + IL_0082: ldloc.0 + IL_0083: ldstr "[Field f3] [Type '{0}']" + IL_0088: call string [mscorlib]System.String::Concat(string, + string) + IL_008d: ldarg.0 + IL_008e: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_0093: box MCCTest.VType8 + IL_0098: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_009d: callvirt instance string [mscorlib]System.Object::ToString() + IL_00a2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00a7: nop + IL_00a8: ldloc.1 + IL_00a9: ldarg.0 + IL_00aa: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f3 + IL_00af: ldarg.1 + IL_00b0: ldc.i4.1 + IL_00b1: add + IL_00b2: call instance string MCCTest.VType8::Dump(int32) + IL_00b7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00bc: nop + IL_00bd: ldloc.1 + IL_00be: ldloc.0 + IL_00bf: ldstr "f4 = " + IL_00c4: ldarg.0 + IL_00c5: ldfld int16 MCCTest.VTypeE::f4 + IL_00ca: box [mscorlib]System.Int16 + IL_00cf: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_00d4: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00d9: nop + IL_00da: ldloc.1 + IL_00db: ldloc.0 + IL_00dc: ldstr "[Field f5] [Type '{0}']" + IL_00e1: call string [mscorlib]System.String::Concat(string, + string) + IL_00e6: ldarg.0 + IL_00e7: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_00ec: box MCCTest.VType8 + IL_00f1: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00f6: callvirt instance string [mscorlib]System.Object::ToString() + IL_00fb: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0100: nop + IL_0101: ldloc.1 + IL_0102: ldarg.0 + IL_0103: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f5 + IL_0108: ldarg.1 + IL_0109: ldc.i4.1 + IL_010a: add + IL_010b: call instance string MCCTest.VType8::Dump(int32) + IL_0110: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0115: nop + IL_0116: ldloc.1 + IL_0117: ldloc.0 + IL_0118: ldstr "[Field f6] [Type '{0}']" + IL_011d: call string [mscorlib]System.String::Concat(string, + string) + IL_0122: ldarg.0 + IL_0123: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_0128: box MCCTest.VType8 + IL_012d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0132: callvirt instance string [mscorlib]System.Object::ToString() + IL_0137: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_013c: nop + IL_013d: ldloc.1 + IL_013e: ldarg.0 + IL_013f: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f6 + IL_0144: ldarg.1 + IL_0145: ldc.i4.1 + IL_0146: add + IL_0147: call instance string MCCTest.VType8::Dump(int32) + IL_014c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0151: nop + IL_0152: ldloc.1 + IL_0153: ldloc.0 + IL_0154: ldstr "f7 = " + IL_0159: ldarg.0 + IL_015a: ldfld float64 MCCTest.VTypeE::f7 + IL_015f: box [mscorlib]System.Double + IL_0164: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0169: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_016e: nop + IL_016f: ldloc.1 + IL_0170: ldloc.0 + IL_0171: ldstr "[Field f8] [Type '{0}']" + IL_0176: call string [mscorlib]System.String::Concat(string, + string) + IL_017b: ldarg.0 + IL_017c: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_0181: box MCCTest.VTypeA + IL_0186: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_018b: callvirt instance string [mscorlib]System.Object::ToString() + IL_0190: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0195: nop + IL_0196: ldloc.1 + IL_0197: ldarg.0 + IL_0198: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f8 + IL_019d: ldarg.1 + IL_019e: ldc.i4.1 + IL_019f: add + IL_01a0: call instance string MCCTest.VTypeA::Dump(int32) + IL_01a5: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01aa: nop + IL_01ab: ldloc.1 + IL_01ac: ldloc.0 + IL_01ad: ldstr "f9 = " + IL_01b2: ldarg.0 + IL_01b3: ldfld int16 MCCTest.VTypeE::f9 + IL_01b8: box [mscorlib]System.Int16 + IL_01bd: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_01c2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01c7: nop + IL_01c8: ldloc.1 + IL_01c9: ldloc.0 + IL_01ca: ldstr "[Field f10] [Type '{0}']" + IL_01cf: call string [mscorlib]System.String::Concat(string, + string) + IL_01d4: ldarg.0 + IL_01d5: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_01da: box MCCTest.VTypeD + IL_01df: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01e4: callvirt instance string [mscorlib]System.Object::ToString() + IL_01e9: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01ee: nop + IL_01ef: ldloc.1 + IL_01f0: ldarg.0 + IL_01f1: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeE::f10 + IL_01f6: ldarg.1 + IL_01f7: ldc.i4.1 + IL_01f8: add + IL_01f9: call instance string MCCTest.VTypeD::Dump(int32) + IL_01fe: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0203: nop + IL_0204: ldloc.1 + IL_0205: ldloc.0 + IL_0206: ldstr "[Field f11] [Type '{0}']" + IL_020b: call string [mscorlib]System.String::Concat(string, + string) + IL_0210: ldarg.0 + IL_0211: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_0216: box MCCTest.VType8 + IL_021b: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0220: callvirt instance string [mscorlib]System.Object::ToString() + IL_0225: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_022a: nop + IL_022b: ldloc.1 + IL_022c: ldarg.0 + IL_022d: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f11 + IL_0232: ldarg.1 + IL_0233: ldc.i4.1 + IL_0234: add + IL_0235: call instance string MCCTest.VType8::Dump(int32) + IL_023a: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_023f: nop + IL_0240: ldloc.1 + IL_0241: ldloc.0 + IL_0242: ldstr "[Field f12] [Type '{0}']" + IL_0247: call string [mscorlib]System.String::Concat(string, + string) + IL_024c: ldarg.0 + IL_024d: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_0252: box MCCTest.VTypeC + IL_0257: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_025c: callvirt instance string [mscorlib]System.Object::ToString() + IL_0261: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0266: nop + IL_0267: ldloc.1 + IL_0268: ldarg.0 + IL_0269: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f12 + IL_026e: ldarg.1 + IL_026f: ldc.i4.1 + IL_0270: add + IL_0271: call instance string MCCTest.VTypeC::Dump(int32) + IL_0276: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_027b: nop + IL_027c: ldloc.1 + IL_027d: ldloc.0 + IL_027e: ldstr "[Field f13] [Type '{0}']" + IL_0283: call string [mscorlib]System.String::Concat(string, + string) + IL_0288: ldarg.0 + IL_0289: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_028e: box MCCTest.VTypeA + IL_0293: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0298: callvirt instance string [mscorlib]System.Object::ToString() + IL_029d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02a2: nop + IL_02a3: ldloc.1 + IL_02a4: ldarg.0 + IL_02a5: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeE::f13 + IL_02aa: ldarg.1 + IL_02ab: ldc.i4.1 + IL_02ac: add + IL_02ad: call instance string MCCTest.VTypeA::Dump(int32) + IL_02b2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02b7: nop + IL_02b8: ldloc.1 + IL_02b9: ldloc.0 + IL_02ba: ldstr "f14 = " + IL_02bf: ldarg.0 + IL_02c0: ldfld float32 MCCTest.VTypeE::f14 + IL_02c5: box [mscorlib]System.Single + IL_02ca: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_02cf: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02d4: nop + IL_02d5: ldloc.1 + IL_02d6: ldloc.0 + IL_02d7: ldstr "f15 = " + IL_02dc: ldarg.0 + IL_02dd: ldfld int32 MCCTest.VTypeE::f15 + IL_02e2: box [mscorlib]System.Int32 + IL_02e7: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_02ec: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02f1: nop + IL_02f2: ldloc.1 + IL_02f3: ldloc.0 + IL_02f4: ldstr "[Field f16] [Type '{0}']" + IL_02f9: call string [mscorlib]System.String::Concat(string, + string) + IL_02fe: ldarg.0 + IL_02ff: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_0304: box MCCTest.VTypeC + IL_0309: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_030e: callvirt instance string [mscorlib]System.Object::ToString() + IL_0313: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0318: nop + IL_0319: ldloc.1 + IL_031a: ldarg.0 + IL_031b: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f16 + IL_0320: ldarg.1 + IL_0321: ldc.i4.1 + IL_0322: add + IL_0323: call instance string MCCTest.VTypeC::Dump(int32) + IL_0328: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_032d: nop + IL_032e: ldloc.1 + IL_032f: ldloc.0 + IL_0330: ldstr "[Field f17] [Type '{0}']" + IL_0335: call string [mscorlib]System.String::Concat(string, + string) + IL_033a: ldarg.0 + IL_033b: ldfld valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_0340: box MCCTest.VType7 + IL_0345: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_034a: callvirt instance string [mscorlib]System.Object::ToString() + IL_034f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0354: nop + IL_0355: ldloc.1 + IL_0356: ldarg.0 + IL_0357: ldflda valuetype MCCTest.VType7 MCCTest.VTypeE::f17 + IL_035c: ldarg.1 + IL_035d: ldc.i4.1 + IL_035e: add + IL_035f: call instance string MCCTest.VType7::Dump(int32) + IL_0364: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0369: nop + IL_036a: ldloc.1 + IL_036b: ldloc.0 + IL_036c: ldstr "f18 = " + IL_0371: ldarg.0 + IL_0372: ldfld float64 MCCTest.VTypeE::f18 + IL_0377: box [mscorlib]System.Double + IL_037c: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0381: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0386: nop + IL_0387: ldloc.1 + IL_0388: ldloc.0 + IL_0389: ldstr "f19 = " + IL_038e: ldarg.0 + IL_038f: ldfld uint64 MCCTest.VTypeE::f19 + IL_0394: box [mscorlib]System.UInt64 + IL_0399: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_039e: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_03a3: nop + IL_03a4: ldloc.1 + IL_03a5: ldloc.0 + IL_03a6: ldstr "[Field f20] [Type '{0}']" + IL_03ab: call string [mscorlib]System.String::Concat(string, + string) + IL_03b0: ldarg.0 + IL_03b1: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_03b6: box MCCTest.VType8 + IL_03bb: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03c0: callvirt instance string [mscorlib]System.Object::ToString() + IL_03c5: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_03ca: nop + IL_03cb: ldloc.1 + IL_03cc: ldarg.0 + IL_03cd: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f20 + IL_03d2: ldarg.1 + IL_03d3: ldc.i4.1 + IL_03d4: add + IL_03d5: call instance string MCCTest.VType8::Dump(int32) + IL_03da: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_03df: nop + IL_03e0: ldloc.1 + IL_03e1: ldloc.0 + IL_03e2: ldstr "[Field f21] [Type '{0}']" + IL_03e7: call string [mscorlib]System.String::Concat(string, + string) + IL_03ec: ldarg.0 + IL_03ed: ldfld valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_03f2: box MCCTest.VType9 + IL_03f7: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03fc: callvirt instance string [mscorlib]System.Object::ToString() + IL_0401: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0406: nop + IL_0407: ldloc.1 + IL_0408: ldarg.0 + IL_0409: ldflda valuetype MCCTest.VType9 MCCTest.VTypeE::f21 + IL_040e: ldarg.1 + IL_040f: ldc.i4.1 + IL_0410: add + IL_0411: call instance string MCCTest.VType9::Dump(int32) + IL_0416: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_041b: nop + IL_041c: ldloc.1 + IL_041d: ldloc.0 + IL_041e: ldstr "[Field f22] [Type '{0}']" + IL_0423: call string [mscorlib]System.String::Concat(string, + string) + IL_0428: ldarg.0 + IL_0429: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_042e: box MCCTest.VTypeC + IL_0433: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0438: callvirt instance string [mscorlib]System.Object::ToString() + IL_043d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0442: nop + IL_0443: ldloc.1 + IL_0444: ldarg.0 + IL_0445: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeE::f22 + IL_044a: ldarg.1 + IL_044b: ldc.i4.1 + IL_044c: add + IL_044d: call instance string MCCTest.VTypeC::Dump(int32) + IL_0452: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0457: nop + IL_0458: ldloc.1 + IL_0459: ldloc.0 + IL_045a: ldstr "[Field f23] [Type '{0}']" + IL_045f: call string [mscorlib]System.String::Concat(string, + string) + IL_0464: ldarg.0 + IL_0465: ldfld valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_046a: box MCCTest.VType8 + IL_046f: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0474: callvirt instance string [mscorlib]System.Object::ToString() + IL_0479: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_047e: nop + IL_047f: ldloc.1 + IL_0480: ldarg.0 + IL_0481: ldflda valuetype MCCTest.VType8 MCCTest.VTypeE::f23 + IL_0486: ldarg.1 + IL_0487: ldc.i4.1 + IL_0488: add + IL_0489: call instance string MCCTest.VType8::Dump(int32) + IL_048e: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0493: nop + IL_0494: ldloc.1 + IL_0495: callvirt instance string [mscorlib]System.Object::ToString() + IL_049a: stloc.2 + IL_049b: br.s IL_049d + + IL_049d: ldloc.2 + IL_049e: ret + } // end of method VTypeE::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VTypeE::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VTypeE::Dump + +} // end of class MCCTest.VTypeE + +.class public sequential ansi sealed beforefieldinit MCCTest.VTypeF + extends [mscorlib]System.ValueType + implements class MCCTest.CType`1 +{ + .field public char f1 + .field public float64 f2 + .field public float32 f3 + .field public valuetype MCCTest.VType9 f4 + .field public class MCCTest.RType4 f5 + .field public char f6 + .field public float32 f7 + .field public valuetype MCCTest.VTypeE f8 + .field public valuetype MCCTest.VType8 f9 + .field public valuetype MCCTest.VTypeE f10 + .field public valuetype MCCTest.VType3 f11 + .field public int64 f12 + .field public uint16 f13 + .field public valuetype MCCTest.VType5 f14 + .field public valuetype MCCTest.VType9 f15 + .field public class MCCTest.RType4 f16 + .field public class MCCTest.RType4 f17 + .field public int32 f18 + .field public valuetype MCCTest.VType8 f19 + .field public class MCCTest.RType4 f20 + .field public valuetype MCCTest.VTypeA f21 + .field public valuetype MCCTest.VTypeD f22 + .field public valuetype MCCTest.VType9 f23 + .field public valuetype MCCTest.VTypeE f24 + .field public class MCCTest.RType4 f25 + .field public float32 f26 + .field public uint8 f27 + .field public float64 f28 + .field public valuetype MCCTest.VTypeC f29 + .field public valuetype MCCTest.VType6 f30 + .field public float64 f31 + .field public valuetype MCCTest.VType8 f32 + .field public int8 f33 + .field public valuetype MCCTest.VTypeD f34 + .field public valuetype MCCTest.VTypeE f35 + .field public valuetype MCCTest.VTypeE f36 + .field public valuetype MCCTest.VType7 f37 + .field public valuetype MCCTest.VType9 f38 + .field public valuetype MCCTest.VType8 f39 + .field public float32 f40 + .field public int32 f41 + .field public char f42 + .field public valuetype MCCTest.VTypeB f43 + .field public valuetype MCCTest.VType8 f44 + .method public hidebysig newslot virtual final + instance void Init(int32 count) cil managed + { + // Code size 547 (0x223) + .maxstack 2 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: newobj instance void MCCTest.RType4::.ctor() + IL_0007: stfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_000c: ldarg.0 + IL_000d: newobj instance void MCCTest.RType4::.ctor() + IL_0012: stfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_0017: ldarg.0 + IL_0018: newobj instance void MCCTest.RType4::.ctor() + IL_001d: stfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_0022: ldarg.0 + IL_0023: newobj instance void MCCTest.RType4::.ctor() + IL_0028: stfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_002d: ldarg.0 + IL_002e: newobj instance void MCCTest.RType4::.ctor() + IL_0033: stfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_0038: ldarg.0 + IL_0039: ldarg.1 + IL_003a: conv.u2 + IL_003b: stfld char MCCTest.VTypeF::f1 + IL_0040: ldarg.0 + IL_0041: ldarg.1 + IL_0042: conv.r8 + IL_0043: stfld float64 MCCTest.VTypeF::f2 + IL_0048: ldarg.0 + IL_0049: ldarg.1 + IL_004a: conv.r4 + IL_004b: stfld float32 MCCTest.VTypeF::f3 + IL_0050: ldarg.0 + IL_0051: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_0056: ldarg.1 + IL_0057: call instance void MCCTest.VType9::Init(int32) + IL_005c: nop + IL_005d: ldarg.0 + IL_005e: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_0063: ldarg.1 + IL_0064: callvirt instance void MCCTest.RType4::Init(int32) + IL_0069: nop + IL_006a: ldarg.0 + IL_006b: ldarg.1 + IL_006c: conv.u2 + IL_006d: stfld char MCCTest.VTypeF::f6 + IL_0072: ldarg.0 + IL_0073: ldarg.1 + IL_0074: conv.r4 + IL_0075: stfld float32 MCCTest.VTypeF::f7 + IL_007a: ldarg.0 + IL_007b: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_0080: ldarg.1 + IL_0081: call instance void MCCTest.VTypeE::Init(int32) + IL_0086: nop + IL_0087: ldarg.0 + IL_0088: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_008d: ldarg.1 + IL_008e: call instance void MCCTest.VType8::Init(int32) + IL_0093: nop + IL_0094: ldarg.0 + IL_0095: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_009a: ldarg.1 + IL_009b: call instance void MCCTest.VTypeE::Init(int32) + IL_00a0: nop + IL_00a1: ldarg.0 + IL_00a2: ldflda valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_00a7: ldarg.1 + IL_00a8: call instance void MCCTest.VType3::Init(int32) + IL_00ad: nop + IL_00ae: ldarg.0 + IL_00af: ldarg.1 + IL_00b0: conv.i8 + IL_00b1: stfld int64 MCCTest.VTypeF::f12 + IL_00b6: ldarg.0 + IL_00b7: ldarg.1 + IL_00b8: conv.u2 + IL_00b9: stfld uint16 MCCTest.VTypeF::f13 + IL_00be: ldarg.0 + IL_00bf: ldflda valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_00c4: ldarg.1 + IL_00c5: call instance void MCCTest.VType5::Init(int32) + IL_00ca: nop + IL_00cb: ldarg.0 + IL_00cc: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_00d1: ldarg.1 + IL_00d2: call instance void MCCTest.VType9::Init(int32) + IL_00d7: nop + IL_00d8: ldarg.0 + IL_00d9: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_00de: ldarg.1 + IL_00df: callvirt instance void MCCTest.RType4::Init(int32) + IL_00e4: nop + IL_00e5: ldarg.0 + IL_00e6: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_00eb: ldarg.1 + IL_00ec: callvirt instance void MCCTest.RType4::Init(int32) + IL_00f1: nop + IL_00f2: ldarg.0 + IL_00f3: ldarg.1 + IL_00f4: stfld int32 MCCTest.VTypeF::f18 + IL_00f9: ldarg.0 + IL_00fa: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_00ff: ldarg.1 + IL_0100: call instance void MCCTest.VType8::Init(int32) + IL_0105: nop + IL_0106: ldarg.0 + IL_0107: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_010c: ldarg.1 + IL_010d: callvirt instance void MCCTest.RType4::Init(int32) + IL_0112: nop + IL_0113: ldarg.0 + IL_0114: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_0119: ldarg.1 + IL_011a: call instance void MCCTest.VTypeA::Init(int32) + IL_011f: nop + IL_0120: ldarg.0 + IL_0121: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_0126: ldarg.1 + IL_0127: call instance void MCCTest.VTypeD::Init(int32) + IL_012c: nop + IL_012d: ldarg.0 + IL_012e: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_0133: ldarg.1 + IL_0134: call instance void MCCTest.VType9::Init(int32) + IL_0139: nop + IL_013a: ldarg.0 + IL_013b: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_0140: ldarg.1 + IL_0141: call instance void MCCTest.VTypeE::Init(int32) + IL_0146: nop + IL_0147: ldarg.0 + IL_0148: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_014d: ldarg.1 + IL_014e: callvirt instance void MCCTest.RType4::Init(int32) + IL_0153: nop + IL_0154: ldarg.0 + IL_0155: ldarg.1 + IL_0156: conv.r4 + IL_0157: stfld float32 MCCTest.VTypeF::f26 + IL_015c: ldarg.0 + IL_015d: ldarg.1 + IL_015e: conv.u1 + IL_015f: stfld uint8 MCCTest.VTypeF::f27 + IL_0164: ldarg.0 + IL_0165: ldarg.1 + IL_0166: conv.r8 + IL_0167: stfld float64 MCCTest.VTypeF::f28 + IL_016c: ldarg.0 + IL_016d: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_0172: ldarg.1 + IL_0173: call instance void MCCTest.VTypeC::Init(int32) + IL_0178: nop + IL_0179: ldarg.0 + IL_017a: ldflda valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_017f: ldarg.1 + IL_0180: call instance void MCCTest.VType6::Init(int32) + IL_0185: nop + IL_0186: ldarg.0 + IL_0187: ldarg.1 + IL_0188: conv.r8 + IL_0189: stfld float64 MCCTest.VTypeF::f31 + IL_018e: ldarg.0 + IL_018f: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_0194: ldarg.1 + IL_0195: call instance void MCCTest.VType8::Init(int32) + IL_019a: nop + IL_019b: ldarg.0 + IL_019c: ldarg.1 + IL_019d: conv.i1 + IL_019e: stfld int8 MCCTest.VTypeF::f33 + IL_01a3: ldarg.0 + IL_01a4: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_01a9: ldarg.1 + IL_01aa: call instance void MCCTest.VTypeD::Init(int32) + IL_01af: nop + IL_01b0: ldarg.0 + IL_01b1: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_01b6: ldarg.1 + IL_01b7: call instance void MCCTest.VTypeE::Init(int32) + IL_01bc: nop + IL_01bd: ldarg.0 + IL_01be: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_01c3: ldarg.1 + IL_01c4: call instance void MCCTest.VTypeE::Init(int32) + IL_01c9: nop + IL_01ca: ldarg.0 + IL_01cb: ldflda valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_01d0: ldarg.1 + IL_01d1: call instance void MCCTest.VType7::Init(int32) + IL_01d6: nop + IL_01d7: ldarg.0 + IL_01d8: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_01dd: ldarg.1 + IL_01de: call instance void MCCTest.VType9::Init(int32) + IL_01e3: nop + IL_01e4: ldarg.0 + IL_01e5: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_01ea: ldarg.1 + IL_01eb: call instance void MCCTest.VType8::Init(int32) + IL_01f0: nop + IL_01f1: ldarg.0 + IL_01f2: ldarg.1 + IL_01f3: conv.r4 + IL_01f4: stfld float32 MCCTest.VTypeF::f40 + IL_01f9: ldarg.0 + IL_01fa: ldarg.1 + IL_01fb: stfld int32 MCCTest.VTypeF::f41 + IL_0200: ldarg.0 + IL_0201: ldarg.1 + IL_0202: conv.u2 + IL_0203: stfld char MCCTest.VTypeF::f42 + IL_0208: ldarg.0 + IL_0209: ldflda valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_020e: ldarg.1 + IL_020f: call instance void MCCTest.VTypeB::Init(int32) + IL_0214: nop + IL_0215: ldarg.0 + IL_0216: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_021b: ldarg.1 + IL_021c: call instance void MCCTest.VType8::Init(int32) + IL_0221: nop + IL_0222: ret + } // end of method VTypeF::Init + + .method public hidebysig newslot virtual final + instance void Init() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.1 + IL_0003: call instance void MCCTest.VTypeF::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeF::Init + + .method public hidebysig newslot virtual final + instance void Zero() cil managed + { + // Code size 10 (0xa) + .maxstack 8 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance void MCCTest.VTypeF::Init(int32) + IL_0008: nop + IL_0009: ret + } // end of method VTypeF::Zero + + .method public hidebysig instance void + Add(valuetype MCCTest.VTypeF val) cil managed + { + // Code size 860 (0x35c) + .maxstack 3 + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: dup + IL_0003: ldfld char MCCTest.VTypeF::f1 + IL_0008: ldarga.s val + IL_000a: ldfld char MCCTest.VTypeF::f1 + IL_000f: add + IL_0010: conv.u2 + IL_0011: stfld char MCCTest.VTypeF::f1 + IL_0016: ldarg.0 + IL_0017: dup + IL_0018: ldfld float64 MCCTest.VTypeF::f2 + IL_001d: ldarga.s val + IL_001f: ldfld float64 MCCTest.VTypeF::f2 + IL_0024: add + IL_0025: stfld float64 MCCTest.VTypeF::f2 + IL_002a: ldarg.0 + IL_002b: dup + IL_002c: ldfld float32 MCCTest.VTypeF::f3 + IL_0031: ldarga.s val + IL_0033: ldfld float32 MCCTest.VTypeF::f3 + IL_0038: add + IL_0039: stfld float32 MCCTest.VTypeF::f3 + IL_003e: ldarg.0 + IL_003f: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_0044: ldarga.s val + IL_0046: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_004b: call instance void MCCTest.VType9::Add(valuetype MCCTest.VType9) + IL_0050: nop + IL_0051: ldarg.0 + IL_0052: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_0057: ldarga.s val + IL_0059: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_005e: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0063: nop + IL_0064: ldarg.0 + IL_0065: dup + IL_0066: ldfld char MCCTest.VTypeF::f6 + IL_006b: ldarga.s val + IL_006d: ldfld char MCCTest.VTypeF::f6 + IL_0072: add + IL_0073: conv.u2 + IL_0074: stfld char MCCTest.VTypeF::f6 + IL_0079: ldarg.0 + IL_007a: dup + IL_007b: ldfld float32 MCCTest.VTypeF::f7 + IL_0080: ldarga.s val + IL_0082: ldfld float32 MCCTest.VTypeF::f7 + IL_0087: add + IL_0088: stfld float32 MCCTest.VTypeF::f7 + IL_008d: ldarg.0 + IL_008e: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_0093: ldarga.s val + IL_0095: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_009a: call instance void MCCTest.VTypeE::Add(valuetype MCCTest.VTypeE) + IL_009f: nop + IL_00a0: ldarg.0 + IL_00a1: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_00a6: ldarga.s val + IL_00a8: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_00ad: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_00b2: nop + IL_00b3: ldarg.0 + IL_00b4: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_00b9: ldarga.s val + IL_00bb: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_00c0: call instance void MCCTest.VTypeE::Add(valuetype MCCTest.VTypeE) + IL_00c5: nop + IL_00c6: ldarg.0 + IL_00c7: ldflda valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_00cc: ldarga.s val + IL_00ce: ldfld valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_00d3: call instance void MCCTest.VType3::Add(valuetype MCCTest.VType3) + IL_00d8: nop + IL_00d9: ldarg.0 + IL_00da: dup + IL_00db: ldfld int64 MCCTest.VTypeF::f12 + IL_00e0: ldarga.s val + IL_00e2: ldfld int64 MCCTest.VTypeF::f12 + IL_00e7: add + IL_00e8: stfld int64 MCCTest.VTypeF::f12 + IL_00ed: ldarg.0 + IL_00ee: dup + IL_00ef: ldfld uint16 MCCTest.VTypeF::f13 + IL_00f4: ldarga.s val + IL_00f6: ldfld uint16 MCCTest.VTypeF::f13 + IL_00fb: add + IL_00fc: conv.u2 + IL_00fd: stfld uint16 MCCTest.VTypeF::f13 + IL_0102: ldarg.0 + IL_0103: ldflda valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_0108: ldarga.s val + IL_010a: ldfld valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_010f: call instance void MCCTest.VType5::Add(valuetype MCCTest.VType5) + IL_0114: nop + IL_0115: ldarg.0 + IL_0116: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_011b: ldarga.s val + IL_011d: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_0122: call instance void MCCTest.VType9::Add(valuetype MCCTest.VType9) + IL_0127: nop + IL_0128: ldarg.0 + IL_0129: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_012e: ldarga.s val + IL_0130: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_0135: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_013a: nop + IL_013b: ldarg.0 + IL_013c: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_0141: ldarga.s val + IL_0143: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_0148: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_014d: nop + IL_014e: ldarg.0 + IL_014f: dup + IL_0150: ldfld int32 MCCTest.VTypeF::f18 + IL_0155: ldarga.s val + IL_0157: ldfld int32 MCCTest.VTypeF::f18 + IL_015c: add + IL_015d: stfld int32 MCCTest.VTypeF::f18 + IL_0162: ldarg.0 + IL_0163: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_0168: ldarga.s val + IL_016a: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_016f: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_0174: nop + IL_0175: ldarg.0 + IL_0176: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_017b: ldarga.s val + IL_017d: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_0182: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_0187: nop + IL_0188: ldarg.0 + IL_0189: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_018e: ldarga.s val + IL_0190: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_0195: call instance void MCCTest.VTypeA::Add(valuetype MCCTest.VTypeA) + IL_019a: nop + IL_019b: ldarg.0 + IL_019c: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_01a1: ldarga.s val + IL_01a3: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_01a8: call instance void MCCTest.VTypeD::Add(valuetype MCCTest.VTypeD) + IL_01ad: nop + IL_01ae: ldarg.0 + IL_01af: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_01b4: ldarga.s val + IL_01b6: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_01bb: call instance void MCCTest.VType9::Add(valuetype MCCTest.VType9) + IL_01c0: nop + IL_01c1: ldarg.0 + IL_01c2: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_01c7: ldarga.s val + IL_01c9: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_01ce: call instance void MCCTest.VTypeE::Add(valuetype MCCTest.VTypeE) + IL_01d3: nop + IL_01d4: ldarg.0 + IL_01d5: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_01da: ldarga.s val + IL_01dc: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_01e1: callvirt instance void MCCTest.RType4::Add(class MCCTest.RType4) + IL_01e6: nop + IL_01e7: ldarg.0 + IL_01e8: dup + IL_01e9: ldfld float32 MCCTest.VTypeF::f26 + IL_01ee: ldarga.s val + IL_01f0: ldfld float32 MCCTest.VTypeF::f26 + IL_01f5: add + IL_01f6: stfld float32 MCCTest.VTypeF::f26 + IL_01fb: ldarg.0 + IL_01fc: dup + IL_01fd: ldfld uint8 MCCTest.VTypeF::f27 + IL_0202: ldarga.s val + IL_0204: ldfld uint8 MCCTest.VTypeF::f27 + IL_0209: add + IL_020a: conv.u1 + IL_020b: stfld uint8 MCCTest.VTypeF::f27 + IL_0210: ldarg.0 + IL_0211: dup + IL_0212: ldfld float64 MCCTest.VTypeF::f28 + IL_0217: ldarga.s val + IL_0219: ldfld float64 MCCTest.VTypeF::f28 + IL_021e: add + IL_021f: stfld float64 MCCTest.VTypeF::f28 + IL_0224: ldarg.0 + IL_0225: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_022a: ldarga.s val + IL_022c: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_0231: call instance void MCCTest.VTypeC::Add(valuetype MCCTest.VTypeC) + IL_0236: nop + IL_0237: ldarg.0 + IL_0238: ldflda valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_023d: ldarga.s val + IL_023f: ldfld valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_0244: call instance void MCCTest.VType6::Add(valuetype MCCTest.VType6) + IL_0249: nop + IL_024a: ldarg.0 + IL_024b: dup + IL_024c: ldfld float64 MCCTest.VTypeF::f31 + IL_0251: ldarga.s val + IL_0253: ldfld float64 MCCTest.VTypeF::f31 + IL_0258: add + IL_0259: stfld float64 MCCTest.VTypeF::f31 + IL_025e: ldarg.0 + IL_025f: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_0264: ldarga.s val + IL_0266: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_026b: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_0270: nop + IL_0271: ldarg.0 + IL_0272: dup + IL_0273: ldfld int8 MCCTest.VTypeF::f33 + IL_0278: ldarga.s val + IL_027a: ldfld int8 MCCTest.VTypeF::f33 + IL_027f: add + IL_0280: conv.i1 + IL_0281: stfld int8 MCCTest.VTypeF::f33 + IL_0286: ldarg.0 + IL_0287: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_028c: ldarga.s val + IL_028e: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_0293: call instance void MCCTest.VTypeD::Add(valuetype MCCTest.VTypeD) + IL_0298: nop + IL_0299: ldarg.0 + IL_029a: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_029f: ldarga.s val + IL_02a1: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_02a6: call instance void MCCTest.VTypeE::Add(valuetype MCCTest.VTypeE) + IL_02ab: nop + IL_02ac: ldarg.0 + IL_02ad: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_02b2: ldarga.s val + IL_02b4: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_02b9: call instance void MCCTest.VTypeE::Add(valuetype MCCTest.VTypeE) + IL_02be: nop + IL_02bf: ldarg.0 + IL_02c0: ldflda valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_02c5: ldarga.s val + IL_02c7: ldfld valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_02cc: call instance void MCCTest.VType7::Add(valuetype MCCTest.VType7) + IL_02d1: nop + IL_02d2: ldarg.0 + IL_02d3: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_02d8: ldarga.s val + IL_02da: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_02df: call instance void MCCTest.VType9::Add(valuetype MCCTest.VType9) + IL_02e4: nop + IL_02e5: ldarg.0 + IL_02e6: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_02eb: ldarga.s val + IL_02ed: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_02f2: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_02f7: nop + IL_02f8: ldarg.0 + IL_02f9: dup + IL_02fa: ldfld float32 MCCTest.VTypeF::f40 + IL_02ff: ldarga.s val + IL_0301: ldfld float32 MCCTest.VTypeF::f40 + IL_0306: add + IL_0307: stfld float32 MCCTest.VTypeF::f40 + IL_030c: ldarg.0 + IL_030d: dup + IL_030e: ldfld int32 MCCTest.VTypeF::f41 + IL_0313: ldarga.s val + IL_0315: ldfld int32 MCCTest.VTypeF::f41 + IL_031a: add + IL_031b: stfld int32 MCCTest.VTypeF::f41 + IL_0320: ldarg.0 + IL_0321: dup + IL_0322: ldfld char MCCTest.VTypeF::f42 + IL_0327: ldarga.s val + IL_0329: ldfld char MCCTest.VTypeF::f42 + IL_032e: add + IL_032f: conv.u2 + IL_0330: stfld char MCCTest.VTypeF::f42 + IL_0335: ldarg.0 + IL_0336: ldflda valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_033b: ldarga.s val + IL_033d: ldfld valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_0342: call instance void MCCTest.VTypeB::Add(valuetype MCCTest.VTypeB) + IL_0347: nop + IL_0348: ldarg.0 + IL_0349: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_034e: ldarga.s val + IL_0350: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_0355: call instance void MCCTest.VType8::Add(valuetype MCCTest.VType8) + IL_035a: nop + IL_035b: ret + } // end of method VTypeF::Add + + .method public hidebysig newslot virtual final + instance void Check(valuetype MCCTest.VTypeF expected) cil managed + { + // Code size 1901 (0x76d) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.Type V_1, + class MCCTest.ResultVerificationException V_2, + bool V_3) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldfld char MCCTest.VTypeF::f1 + IL_0007: ldarga.s expected + IL_0009: ldfld char MCCTest.VTypeF::f1 + IL_000e: ceq + IL_0010: stloc.3 + IL_0011: ldloc.3 + IL_0012: brtrue.s IL_002f + + IL_0014: nop + IL_0015: ldstr "f1" + IL_001a: ldarg.0 + IL_001b: ldfld char MCCTest.VTypeF::f1 + IL_0020: conv.u8 + IL_0021: ldarga.s expected + IL_0023: ldfld char MCCTest.VTypeF::f1 + IL_0028: conv.u8 + IL_0029: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_002e: throw + + IL_002f: ldarg.0 + IL_0030: ldfld float64 MCCTest.VTypeF::f2 + IL_0035: ldarga.s expected + IL_0037: ldfld float64 MCCTest.VTypeF::f2 + IL_003c: ceq + IL_003e: stloc.3 + IL_003f: ldloc.3 + IL_0040: brtrue.s IL_005b + + IL_0042: nop + IL_0043: ldstr "f2" + IL_0048: ldarg.0 + IL_0049: ldfld float64 MCCTest.VTypeF::f2 + IL_004e: ldarga.s expected + IL_0050: ldfld float64 MCCTest.VTypeF::f2 + IL_0055: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_005a: throw + + IL_005b: ldarg.0 + IL_005c: ldfld float32 MCCTest.VTypeF::f3 + IL_0061: ldarga.s expected + IL_0063: ldfld float32 MCCTest.VTypeF::f3 + IL_0068: ceq + IL_006a: stloc.3 + IL_006b: ldloc.3 + IL_006c: brtrue.s IL_0089 + + IL_006e: nop + IL_006f: ldstr "f3" + IL_0074: ldarg.0 + IL_0075: ldfld float32 MCCTest.VTypeF::f3 + IL_007a: conv.r8 + IL_007b: ldarga.s expected + IL_007d: ldfld float32 MCCTest.VTypeF::f3 + IL_0082: conv.r8 + IL_0083: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_0088: throw + + IL_0089: ldarg.0 + IL_008a: ldfld char MCCTest.VTypeF::f6 + IL_008f: ldarga.s expected + IL_0091: ldfld char MCCTest.VTypeF::f6 + IL_0096: ceq + IL_0098: stloc.3 + IL_0099: ldloc.3 + IL_009a: brtrue.s IL_00b7 + + IL_009c: nop + IL_009d: ldstr "f6" + IL_00a2: ldarg.0 + IL_00a3: ldfld char MCCTest.VTypeF::f6 + IL_00a8: conv.u8 + IL_00a9: ldarga.s expected + IL_00ab: ldfld char MCCTest.VTypeF::f6 + IL_00b0: conv.u8 + IL_00b1: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_00b6: throw + + IL_00b7: ldarg.0 + IL_00b8: ldfld float32 MCCTest.VTypeF::f7 + IL_00bd: ldarga.s expected + IL_00bf: ldfld float32 MCCTest.VTypeF::f7 + IL_00c4: ceq + IL_00c6: stloc.3 + IL_00c7: ldloc.3 + IL_00c8: brtrue.s IL_00e5 + + IL_00ca: nop + IL_00cb: ldstr "f7" + IL_00d0: ldarg.0 + IL_00d1: ldfld float32 MCCTest.VTypeF::f7 + IL_00d6: conv.r8 + IL_00d7: ldarga.s expected + IL_00d9: ldfld float32 MCCTest.VTypeF::f7 + IL_00de: conv.r8 + IL_00df: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_00e4: throw + + IL_00e5: ldarg.0 + IL_00e6: ldfld int64 MCCTest.VTypeF::f12 + IL_00eb: ldarga.s expected + IL_00ed: ldfld int64 MCCTest.VTypeF::f12 + IL_00f2: ceq + IL_00f4: stloc.3 + IL_00f5: ldloc.3 + IL_00f6: brtrue.s IL_0111 + + IL_00f8: nop + IL_00f9: ldstr "f12" + IL_00fe: ldarg.0 + IL_00ff: ldfld int64 MCCTest.VTypeF::f12 + IL_0104: ldarga.s expected + IL_0106: ldfld int64 MCCTest.VTypeF::f12 + IL_010b: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_0110: throw + + IL_0111: ldarg.0 + IL_0112: ldfld uint16 MCCTest.VTypeF::f13 + IL_0117: ldarga.s expected + IL_0119: ldfld uint16 MCCTest.VTypeF::f13 + IL_011e: ceq + IL_0120: stloc.3 + IL_0121: ldloc.3 + IL_0122: brtrue.s IL_013f + + IL_0124: nop + IL_0125: ldstr "f13" + IL_012a: ldarg.0 + IL_012b: ldfld uint16 MCCTest.VTypeF::f13 + IL_0130: conv.u8 + IL_0131: ldarga.s expected + IL_0133: ldfld uint16 MCCTest.VTypeF::f13 + IL_0138: conv.u8 + IL_0139: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_013e: throw + + IL_013f: ldarg.0 + IL_0140: ldfld int32 MCCTest.VTypeF::f18 + IL_0145: ldarga.s expected + IL_0147: ldfld int32 MCCTest.VTypeF::f18 + IL_014c: ceq + IL_014e: stloc.3 + IL_014f: ldloc.3 + IL_0150: brtrue.s IL_016d + + IL_0152: nop + IL_0153: ldstr "f18" + IL_0158: ldarg.0 + IL_0159: ldfld int32 MCCTest.VTypeF::f18 + IL_015e: conv.i8 + IL_015f: ldarga.s expected + IL_0161: ldfld int32 MCCTest.VTypeF::f18 + IL_0166: conv.i8 + IL_0167: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_016c: throw + + IL_016d: ldarg.0 + IL_016e: ldfld float32 MCCTest.VTypeF::f26 + IL_0173: ldarga.s expected + IL_0175: ldfld float32 MCCTest.VTypeF::f26 + IL_017a: ceq + IL_017c: stloc.3 + IL_017d: ldloc.3 + IL_017e: brtrue.s IL_019b + + IL_0180: nop + IL_0181: ldstr "f26" + IL_0186: ldarg.0 + IL_0187: ldfld float32 MCCTest.VTypeF::f26 + IL_018c: conv.r8 + IL_018d: ldarga.s expected + IL_018f: ldfld float32 MCCTest.VTypeF::f26 + IL_0194: conv.r8 + IL_0195: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_019a: throw + + IL_019b: ldarg.0 + IL_019c: ldfld uint8 MCCTest.VTypeF::f27 + IL_01a1: ldarga.s expected + IL_01a3: ldfld uint8 MCCTest.VTypeF::f27 + IL_01a8: ceq + IL_01aa: stloc.3 + IL_01ab: ldloc.3 + IL_01ac: brtrue.s IL_01c9 + + IL_01ae: nop + IL_01af: ldstr "f27" + IL_01b4: ldarg.0 + IL_01b5: ldfld uint8 MCCTest.VTypeF::f27 + IL_01ba: conv.u8 + IL_01bb: ldarga.s expected + IL_01bd: ldfld uint8 MCCTest.VTypeF::f27 + IL_01c2: conv.u8 + IL_01c3: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_01c8: throw + + IL_01c9: ldarg.0 + IL_01ca: ldfld float64 MCCTest.VTypeF::f28 + IL_01cf: ldarga.s expected + IL_01d1: ldfld float64 MCCTest.VTypeF::f28 + IL_01d6: ceq + IL_01d8: stloc.3 + IL_01d9: ldloc.3 + IL_01da: brtrue.s IL_01f5 + + IL_01dc: nop + IL_01dd: ldstr "f28" + IL_01e2: ldarg.0 + IL_01e3: ldfld float64 MCCTest.VTypeF::f28 + IL_01e8: ldarga.s expected + IL_01ea: ldfld float64 MCCTest.VTypeF::f28 + IL_01ef: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_01f4: throw + + IL_01f5: ldarg.0 + IL_01f6: ldfld float64 MCCTest.VTypeF::f31 + IL_01fb: ldarga.s expected + IL_01fd: ldfld float64 MCCTest.VTypeF::f31 + IL_0202: ceq + IL_0204: stloc.3 + IL_0205: ldloc.3 + IL_0206: brtrue.s IL_0221 + + IL_0208: nop + IL_0209: ldstr "f31" + IL_020e: ldarg.0 + IL_020f: ldfld float64 MCCTest.VTypeF::f31 + IL_0214: ldarga.s expected + IL_0216: ldfld float64 MCCTest.VTypeF::f31 + IL_021b: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_0220: throw + + IL_0221: ldarg.0 + IL_0222: ldfld int8 MCCTest.VTypeF::f33 + IL_0227: ldarga.s expected + IL_0229: ldfld int8 MCCTest.VTypeF::f33 + IL_022e: ceq + IL_0230: stloc.3 + IL_0231: ldloc.3 + IL_0232: brtrue.s IL_024f + + IL_0234: nop + IL_0235: ldstr "f33" + IL_023a: ldarg.0 + IL_023b: ldfld int8 MCCTest.VTypeF::f33 + IL_0240: conv.i8 + IL_0241: ldarga.s expected + IL_0243: ldfld int8 MCCTest.VTypeF::f33 + IL_0248: conv.i8 + IL_0249: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_024e: throw + + IL_024f: ldarg.0 + IL_0250: ldfld float32 MCCTest.VTypeF::f40 + IL_0255: ldarga.s expected + IL_0257: ldfld float32 MCCTest.VTypeF::f40 + IL_025c: ceq + IL_025e: stloc.3 + IL_025f: ldloc.3 + IL_0260: brtrue.s IL_027d + + IL_0262: nop + IL_0263: ldstr "f40" + IL_0268: ldarg.0 + IL_0269: ldfld float32 MCCTest.VTypeF::f40 + IL_026e: conv.r8 + IL_026f: ldarga.s expected + IL_0271: ldfld float32 MCCTest.VTypeF::f40 + IL_0276: conv.r8 + IL_0277: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + float64, + float64) + IL_027c: throw + + IL_027d: ldarg.0 + IL_027e: ldfld int32 MCCTest.VTypeF::f41 + IL_0283: ldarga.s expected + IL_0285: ldfld int32 MCCTest.VTypeF::f41 + IL_028a: ceq + IL_028c: stloc.3 + IL_028d: ldloc.3 + IL_028e: brtrue.s IL_02ab + + IL_0290: nop + IL_0291: ldstr "f41" + IL_0296: ldarg.0 + IL_0297: ldfld int32 MCCTest.VTypeF::f41 + IL_029c: conv.i8 + IL_029d: ldarga.s expected + IL_029f: ldfld int32 MCCTest.VTypeF::f41 + IL_02a4: conv.i8 + IL_02a5: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_02aa: throw + + IL_02ab: ldarg.0 + IL_02ac: ldfld char MCCTest.VTypeF::f42 + IL_02b1: ldarga.s expected + IL_02b3: ldfld char MCCTest.VTypeF::f42 + IL_02b8: ceq + IL_02ba: stloc.3 + IL_02bb: ldloc.3 + IL_02bc: brtrue.s IL_02d9 + + IL_02be: nop + IL_02bf: ldstr "f42" + IL_02c4: ldarg.0 + IL_02c5: ldfld char MCCTest.VTypeF::f42 + IL_02ca: conv.u8 + IL_02cb: ldarga.s expected + IL_02cd: ldfld char MCCTest.VTypeF::f42 + IL_02d2: conv.u8 + IL_02d3: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + int64, + int64) + IL_02d8: throw + + IL_02d9: ldnull + IL_02da: stloc.0 + IL_02db: ldnull + IL_02dc: stloc.1 + .try + { + IL_02dd: nop + IL_02de: ldstr "f4" + IL_02e3: stloc.0 + IL_02e4: ldarg.0 + IL_02e5: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_02ea: box MCCTest.VType9 + IL_02ef: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02f4: stloc.1 + IL_02f5: ldarg.0 + IL_02f6: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_02fb: ldarga.s expected + IL_02fd: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_0302: call instance void MCCTest.VType9::Check(valuetype MCCTest.VType9) + IL_0307: nop + IL_0308: ldstr "f5" + IL_030d: stloc.0 + IL_030e: ldarg.0 + IL_030f: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_0314: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0319: stloc.1 + IL_031a: ldarg.0 + IL_031b: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_0320: ldarga.s expected + IL_0322: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_0327: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_032c: nop + IL_032d: ldstr "f8" + IL_0332: stloc.0 + IL_0333: ldarg.0 + IL_0334: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_0339: box MCCTest.VTypeE + IL_033e: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0343: stloc.1 + IL_0344: ldarg.0 + IL_0345: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_034a: ldarga.s expected + IL_034c: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_0351: call instance void MCCTest.VTypeE::Check(valuetype MCCTest.VTypeE) + IL_0356: nop + IL_0357: ldstr "f9" + IL_035c: stloc.0 + IL_035d: ldarg.0 + IL_035e: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_0363: box MCCTest.VType8 + IL_0368: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_036d: stloc.1 + IL_036e: ldarg.0 + IL_036f: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_0374: ldarga.s expected + IL_0376: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_037b: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_0380: nop + IL_0381: ldstr "f10" + IL_0386: stloc.0 + IL_0387: ldarg.0 + IL_0388: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_038d: box MCCTest.VTypeE + IL_0392: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0397: stloc.1 + IL_0398: ldarg.0 + IL_0399: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_039e: ldarga.s expected + IL_03a0: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_03a5: call instance void MCCTest.VTypeE::Check(valuetype MCCTest.VTypeE) + IL_03aa: nop + IL_03ab: ldstr "f11" + IL_03b0: stloc.0 + IL_03b1: ldarg.0 + IL_03b2: ldfld valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_03b7: box MCCTest.VType3 + IL_03bc: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03c1: stloc.1 + IL_03c2: ldarg.0 + IL_03c3: ldflda valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_03c8: ldarga.s expected + IL_03ca: ldfld valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_03cf: call instance void MCCTest.VType3::Check(valuetype MCCTest.VType3) + IL_03d4: nop + IL_03d5: ldstr "f14" + IL_03da: stloc.0 + IL_03db: ldarg.0 + IL_03dc: ldfld valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_03e1: box MCCTest.VType5 + IL_03e6: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03eb: stloc.1 + IL_03ec: ldarg.0 + IL_03ed: ldflda valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_03f2: ldarga.s expected + IL_03f4: ldfld valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_03f9: call instance void MCCTest.VType5::Check(valuetype MCCTest.VType5) + IL_03fe: nop + IL_03ff: ldstr "f15" + IL_0404: stloc.0 + IL_0405: ldarg.0 + IL_0406: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_040b: box MCCTest.VType9 + IL_0410: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0415: stloc.1 + IL_0416: ldarg.0 + IL_0417: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_041c: ldarga.s expected + IL_041e: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_0423: call instance void MCCTest.VType9::Check(valuetype MCCTest.VType9) + IL_0428: nop + IL_0429: ldstr "f16" + IL_042e: stloc.0 + IL_042f: ldarg.0 + IL_0430: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_0435: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_043a: stloc.1 + IL_043b: ldarg.0 + IL_043c: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_0441: ldarga.s expected + IL_0443: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_0448: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_044d: nop + IL_044e: ldstr "f17" + IL_0453: stloc.0 + IL_0454: ldarg.0 + IL_0455: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_045a: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_045f: stloc.1 + IL_0460: ldarg.0 + IL_0461: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_0466: ldarga.s expected + IL_0468: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_046d: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_0472: nop + IL_0473: ldstr "f19" + IL_0478: stloc.0 + IL_0479: ldarg.0 + IL_047a: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_047f: box MCCTest.VType8 + IL_0484: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0489: stloc.1 + IL_048a: ldarg.0 + IL_048b: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_0490: ldarga.s expected + IL_0492: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_0497: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_049c: nop + IL_049d: ldstr "f20" + IL_04a2: stloc.0 + IL_04a3: ldarg.0 + IL_04a4: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_04a9: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_04ae: stloc.1 + IL_04af: ldarg.0 + IL_04b0: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_04b5: ldarga.s expected + IL_04b7: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_04bc: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_04c1: nop + IL_04c2: ldstr "f21" + IL_04c7: stloc.0 + IL_04c8: ldarg.0 + IL_04c9: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_04ce: box MCCTest.VTypeA + IL_04d3: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_04d8: stloc.1 + IL_04d9: ldarg.0 + IL_04da: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_04df: ldarga.s expected + IL_04e1: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_04e6: call instance void MCCTest.VTypeA::Check(valuetype MCCTest.VTypeA) + IL_04eb: nop + IL_04ec: ldstr "f22" + IL_04f1: stloc.0 + IL_04f2: ldarg.0 + IL_04f3: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_04f8: box MCCTest.VTypeD + IL_04fd: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0502: stloc.1 + IL_0503: ldarg.0 + IL_0504: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_0509: ldarga.s expected + IL_050b: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_0510: call instance void MCCTest.VTypeD::Check(valuetype MCCTest.VTypeD) + IL_0515: nop + IL_0516: ldstr "f23" + IL_051b: stloc.0 + IL_051c: ldarg.0 + IL_051d: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_0522: box MCCTest.VType9 + IL_0527: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_052c: stloc.1 + IL_052d: ldarg.0 + IL_052e: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_0533: ldarga.s expected + IL_0535: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_053a: call instance void MCCTest.VType9::Check(valuetype MCCTest.VType9) + IL_053f: nop + IL_0540: ldstr "f24" + IL_0545: stloc.0 + IL_0546: ldarg.0 + IL_0547: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_054c: box MCCTest.VTypeE + IL_0551: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0556: stloc.1 + IL_0557: ldarg.0 + IL_0558: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_055d: ldarga.s expected + IL_055f: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_0564: call instance void MCCTest.VTypeE::Check(valuetype MCCTest.VTypeE) + IL_0569: nop + IL_056a: ldstr "f25" + IL_056f: stloc.0 + IL_0570: ldarg.0 + IL_0571: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_0576: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_057b: stloc.1 + IL_057c: ldarg.0 + IL_057d: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_0582: ldarga.s expected + IL_0584: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_0589: callvirt instance void MCCTest.RType4::Check(class MCCTest.RType4) + IL_058e: nop + IL_058f: ldstr "f29" + IL_0594: stloc.0 + IL_0595: ldarg.0 + IL_0596: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_059b: box MCCTest.VTypeC + IL_05a0: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_05a5: stloc.1 + IL_05a6: ldarg.0 + IL_05a7: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_05ac: ldarga.s expected + IL_05ae: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_05b3: call instance void MCCTest.VTypeC::Check(valuetype MCCTest.VTypeC) + IL_05b8: nop + IL_05b9: ldstr "f30" + IL_05be: stloc.0 + IL_05bf: ldarg.0 + IL_05c0: ldfld valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_05c5: box MCCTest.VType6 + IL_05ca: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_05cf: stloc.1 + IL_05d0: ldarg.0 + IL_05d1: ldflda valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_05d6: ldarga.s expected + IL_05d8: ldfld valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_05dd: call instance void MCCTest.VType6::Check(valuetype MCCTest.VType6) + IL_05e2: nop + IL_05e3: ldstr "f32" + IL_05e8: stloc.0 + IL_05e9: ldarg.0 + IL_05ea: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_05ef: box MCCTest.VType8 + IL_05f4: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_05f9: stloc.1 + IL_05fa: ldarg.0 + IL_05fb: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_0600: ldarga.s expected + IL_0602: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_0607: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_060c: nop + IL_060d: ldstr "f34" + IL_0612: stloc.0 + IL_0613: ldarg.0 + IL_0614: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_0619: box MCCTest.VTypeD + IL_061e: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0623: stloc.1 + IL_0624: ldarg.0 + IL_0625: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_062a: ldarga.s expected + IL_062c: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_0631: call instance void MCCTest.VTypeD::Check(valuetype MCCTest.VTypeD) + IL_0636: nop + IL_0637: ldstr "f35" + IL_063c: stloc.0 + IL_063d: ldarg.0 + IL_063e: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_0643: box MCCTest.VTypeE + IL_0648: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_064d: stloc.1 + IL_064e: ldarg.0 + IL_064f: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_0654: ldarga.s expected + IL_0656: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_065b: call instance void MCCTest.VTypeE::Check(valuetype MCCTest.VTypeE) + IL_0660: nop + IL_0661: ldstr "f36" + IL_0666: stloc.0 + IL_0667: ldarg.0 + IL_0668: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_066d: box MCCTest.VTypeE + IL_0672: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0677: stloc.1 + IL_0678: ldarg.0 + IL_0679: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_067e: ldarga.s expected + IL_0680: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_0685: call instance void MCCTest.VTypeE::Check(valuetype MCCTest.VTypeE) + IL_068a: nop + IL_068b: ldstr "f37" + IL_0690: stloc.0 + IL_0691: ldarg.0 + IL_0692: ldfld valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_0697: box MCCTest.VType7 + IL_069c: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_06a1: stloc.1 + IL_06a2: ldarg.0 + IL_06a3: ldflda valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_06a8: ldarga.s expected + IL_06aa: ldfld valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_06af: call instance void MCCTest.VType7::Check(valuetype MCCTest.VType7) + IL_06b4: nop + IL_06b5: ldstr "f38" + IL_06ba: stloc.0 + IL_06bb: ldarg.0 + IL_06bc: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_06c1: box MCCTest.VType9 + IL_06c6: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_06cb: stloc.1 + IL_06cc: ldarg.0 + IL_06cd: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_06d2: ldarga.s expected + IL_06d4: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_06d9: call instance void MCCTest.VType9::Check(valuetype MCCTest.VType9) + IL_06de: nop + IL_06df: ldstr "f39" + IL_06e4: stloc.0 + IL_06e5: ldarg.0 + IL_06e6: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_06eb: box MCCTest.VType8 + IL_06f0: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_06f5: stloc.1 + IL_06f6: ldarg.0 + IL_06f7: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_06fc: ldarga.s expected + IL_06fe: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_0703: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_0708: nop + IL_0709: ldstr "f43" + IL_070e: stloc.0 + IL_070f: ldarg.0 + IL_0710: ldfld valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_0715: box MCCTest.VTypeB + IL_071a: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_071f: stloc.1 + IL_0720: ldarg.0 + IL_0721: ldflda valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_0726: ldarga.s expected + IL_0728: ldfld valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_072d: call instance void MCCTest.VTypeB::Check(valuetype MCCTest.VTypeB) + IL_0732: nop + IL_0733: ldstr "f44" + IL_0738: stloc.0 + IL_0739: ldarg.0 + IL_073a: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_073f: box MCCTest.VType8 + IL_0744: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0749: stloc.1 + IL_074a: ldarg.0 + IL_074b: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_0750: ldarga.s expected + IL_0752: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_0757: call instance void MCCTest.VType8::Check(valuetype MCCTest.VType8) + IL_075c: nop + IL_075d: nop + IL_075e: leave.s IL_076b + + } // end .try + catch MCCTest.ResultVerificationException + { + IL_0760: stloc.2 + IL_0761: nop + IL_0762: ldloc.0 + IL_0763: ldloc.1 + IL_0764: ldloc.2 + IL_0765: newobj instance void MCCTest.ResultVerificationException::.ctor(string, + class [mscorlib]System.Type, + class MCCTest.ResultVerificationException) + IL_076a: throw + + } // end handler + IL_076b: nop + IL_076c: ret + } // end of method VTypeF::Check + + .method public hidebysig instance string + Dump(int32 level) cil managed + { + // Code size 2144 (0x860) + .maxstack 4 + .locals init (string V_0, + class [mscorlib]System.IO.StringWriter V_1, + string V_2) + IL_0000: nop + IL_0001: ldarg.1 + IL_0002: call string MCCTest.FormatUtils::GetPadding(int32) + IL_0007: stloc.0 + IL_0008: newobj instance void [mscorlib]System.IO.StringWriter::.ctor() + IL_000d: stloc.1 + IL_000e: ldloc.1 + IL_000f: ldloc.0 + IL_0010: ldstr "f1 = " + IL_0015: ldarg.0 + IL_0016: ldfld char MCCTest.VTypeF::f1 + IL_001b: box [mscorlib]System.Char + IL_0020: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0025: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_002a: nop + IL_002b: ldloc.1 + IL_002c: ldloc.0 + IL_002d: ldstr "f2 = " + IL_0032: ldarg.0 + IL_0033: ldfld float64 MCCTest.VTypeF::f2 + IL_0038: box [mscorlib]System.Double + IL_003d: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0042: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0047: nop + IL_0048: ldloc.1 + IL_0049: ldloc.0 + IL_004a: ldstr "f3 = " + IL_004f: ldarg.0 + IL_0050: ldfld float32 MCCTest.VTypeF::f3 + IL_0055: box [mscorlib]System.Single + IL_005a: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_005f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0064: nop + IL_0065: ldloc.1 + IL_0066: ldloc.0 + IL_0067: ldstr "[Field f4] [Type '{0}']" + IL_006c: call string [mscorlib]System.String::Concat(string, + string) + IL_0071: ldarg.0 + IL_0072: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_0077: box MCCTest.VType9 + IL_007c: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0081: callvirt instance string [mscorlib]System.Object::ToString() + IL_0086: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_008b: nop + IL_008c: ldloc.1 + IL_008d: ldarg.0 + IL_008e: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f4 + IL_0093: ldarg.1 + IL_0094: ldc.i4.1 + IL_0095: add + IL_0096: call instance string MCCTest.VType9::Dump(int32) + IL_009b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00a0: nop + IL_00a1: ldloc.1 + IL_00a2: ldloc.0 + IL_00a3: ldstr "[Field f5] [Type '{0}']" + IL_00a8: call string [mscorlib]System.String::Concat(string, + string) + IL_00ad: ldarg.0 + IL_00ae: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_00b3: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_00b8: callvirt instance string [mscorlib]System.Object::ToString() + IL_00bd: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_00c2: nop + IL_00c3: ldloc.1 + IL_00c4: ldarg.0 + IL_00c5: ldfld class MCCTest.RType4 MCCTest.VTypeF::f5 + IL_00ca: ldarg.1 + IL_00cb: ldc.i4.1 + IL_00cc: add + IL_00cd: callvirt instance string MCCTest.RType4::Dump(int32) + IL_00d2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00d7: nop + IL_00d8: ldloc.1 + IL_00d9: ldloc.0 + IL_00da: ldstr "f6 = " + IL_00df: ldarg.0 + IL_00e0: ldfld char MCCTest.VTypeF::f6 + IL_00e5: box [mscorlib]System.Char + IL_00ea: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_00ef: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_00f4: nop + IL_00f5: ldloc.1 + IL_00f6: ldloc.0 + IL_00f7: ldstr "f7 = " + IL_00fc: ldarg.0 + IL_00fd: ldfld float32 MCCTest.VTypeF::f7 + IL_0102: box [mscorlib]System.Single + IL_0107: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_010c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0111: nop + IL_0112: ldloc.1 + IL_0113: ldloc.0 + IL_0114: ldstr "[Field f8] [Type '{0}']" + IL_0119: call string [mscorlib]System.String::Concat(string, + string) + IL_011e: ldarg.0 + IL_011f: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_0124: box MCCTest.VTypeE + IL_0129: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_012e: callvirt instance string [mscorlib]System.Object::ToString() + IL_0133: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0138: nop + IL_0139: ldloc.1 + IL_013a: ldarg.0 + IL_013b: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f8 + IL_0140: ldarg.1 + IL_0141: ldc.i4.1 + IL_0142: add + IL_0143: call instance string MCCTest.VTypeE::Dump(int32) + IL_0148: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_014d: nop + IL_014e: ldloc.1 + IL_014f: ldloc.0 + IL_0150: ldstr "[Field f9] [Type '{0}']" + IL_0155: call string [mscorlib]System.String::Concat(string, + string) + IL_015a: ldarg.0 + IL_015b: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_0160: box MCCTest.VType8 + IL_0165: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_016a: callvirt instance string [mscorlib]System.Object::ToString() + IL_016f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0174: nop + IL_0175: ldloc.1 + IL_0176: ldarg.0 + IL_0177: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f9 + IL_017c: ldarg.1 + IL_017d: ldc.i4.1 + IL_017e: add + IL_017f: call instance string MCCTest.VType8::Dump(int32) + IL_0184: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0189: nop + IL_018a: ldloc.1 + IL_018b: ldloc.0 + IL_018c: ldstr "[Field f10] [Type '{0}']" + IL_0191: call string [mscorlib]System.String::Concat(string, + string) + IL_0196: ldarg.0 + IL_0197: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_019c: box MCCTest.VTypeE + IL_01a1: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01a6: callvirt instance string [mscorlib]System.Object::ToString() + IL_01ab: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01b0: nop + IL_01b1: ldloc.1 + IL_01b2: ldarg.0 + IL_01b3: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f10 + IL_01b8: ldarg.1 + IL_01b9: ldc.i4.1 + IL_01ba: add + IL_01bb: call instance string MCCTest.VTypeE::Dump(int32) + IL_01c0: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_01c5: nop + IL_01c6: ldloc.1 + IL_01c7: ldloc.0 + IL_01c8: ldstr "[Field f11] [Type '{0}']" + IL_01cd: call string [mscorlib]System.String::Concat(string, + string) + IL_01d2: ldarg.0 + IL_01d3: ldfld valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_01d8: box MCCTest.VType3 + IL_01dd: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_01e2: callvirt instance string [mscorlib]System.Object::ToString() + IL_01e7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_01ec: nop + IL_01ed: ldloc.1 + IL_01ee: ldarg.0 + IL_01ef: ldflda valuetype MCCTest.VType3 MCCTest.VTypeF::f11 + IL_01f4: ldarg.1 + IL_01f5: ldc.i4.1 + IL_01f6: add + IL_01f7: call instance string MCCTest.VType3::Dump(int32) + IL_01fc: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0201: nop + IL_0202: ldloc.1 + IL_0203: ldloc.0 + IL_0204: ldstr "f12 = " + IL_0209: ldarg.0 + IL_020a: ldfld int64 MCCTest.VTypeF::f12 + IL_020f: box [mscorlib]System.Int64 + IL_0214: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0219: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_021e: nop + IL_021f: ldloc.1 + IL_0220: ldloc.0 + IL_0221: ldstr "f13 = " + IL_0226: ldarg.0 + IL_0227: ldfld uint16 MCCTest.VTypeF::f13 + IL_022c: box [mscorlib]System.UInt16 + IL_0231: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0236: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_023b: nop + IL_023c: ldloc.1 + IL_023d: ldloc.0 + IL_023e: ldstr "[Field f14] [Type '{0}']" + IL_0243: call string [mscorlib]System.String::Concat(string, + string) + IL_0248: ldarg.0 + IL_0249: ldfld valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_024e: box MCCTest.VType5 + IL_0253: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0258: callvirt instance string [mscorlib]System.Object::ToString() + IL_025d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0262: nop + IL_0263: ldloc.1 + IL_0264: ldarg.0 + IL_0265: ldflda valuetype MCCTest.VType5 MCCTest.VTypeF::f14 + IL_026a: ldarg.1 + IL_026b: ldc.i4.1 + IL_026c: add + IL_026d: call instance string MCCTest.VType5::Dump(int32) + IL_0272: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0277: nop + IL_0278: ldloc.1 + IL_0279: ldloc.0 + IL_027a: ldstr "[Field f15] [Type '{0}']" + IL_027f: call string [mscorlib]System.String::Concat(string, + string) + IL_0284: ldarg.0 + IL_0285: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_028a: box MCCTest.VType9 + IL_028f: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0294: callvirt instance string [mscorlib]System.Object::ToString() + IL_0299: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_029e: nop + IL_029f: ldloc.1 + IL_02a0: ldarg.0 + IL_02a1: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f15 + IL_02a6: ldarg.1 + IL_02a7: ldc.i4.1 + IL_02a8: add + IL_02a9: call instance string MCCTest.VType9::Dump(int32) + IL_02ae: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02b3: nop + IL_02b4: ldloc.1 + IL_02b5: ldloc.0 + IL_02b6: ldstr "[Field f16] [Type '{0}']" + IL_02bb: call string [mscorlib]System.String::Concat(string, + string) + IL_02c0: ldarg.0 + IL_02c1: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_02c6: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_02cb: callvirt instance string [mscorlib]System.Object::ToString() + IL_02d0: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_02d5: nop + IL_02d6: ldloc.1 + IL_02d7: ldarg.0 + IL_02d8: ldfld class MCCTest.RType4 MCCTest.VTypeF::f16 + IL_02dd: ldarg.1 + IL_02de: ldc.i4.1 + IL_02df: add + IL_02e0: callvirt instance string MCCTest.RType4::Dump(int32) + IL_02e5: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_02ea: nop + IL_02eb: ldloc.1 + IL_02ec: ldloc.0 + IL_02ed: ldstr "[Field f17] [Type '{0}']" + IL_02f2: call string [mscorlib]System.String::Concat(string, + string) + IL_02f7: ldarg.0 + IL_02f8: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_02fd: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0302: callvirt instance string [mscorlib]System.Object::ToString() + IL_0307: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_030c: nop + IL_030d: ldloc.1 + IL_030e: ldarg.0 + IL_030f: ldfld class MCCTest.RType4 MCCTest.VTypeF::f17 + IL_0314: ldarg.1 + IL_0315: ldc.i4.1 + IL_0316: add + IL_0317: callvirt instance string MCCTest.RType4::Dump(int32) + IL_031c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0321: nop + IL_0322: ldloc.1 + IL_0323: ldloc.0 + IL_0324: ldstr "f18 = " + IL_0329: ldarg.0 + IL_032a: ldfld int32 MCCTest.VTypeF::f18 + IL_032f: box [mscorlib]System.Int32 + IL_0334: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0339: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_033e: nop + IL_033f: ldloc.1 + IL_0340: ldloc.0 + IL_0341: ldstr "[Field f19] [Type '{0}']" + IL_0346: call string [mscorlib]System.String::Concat(string, + string) + IL_034b: ldarg.0 + IL_034c: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_0351: box MCCTest.VType8 + IL_0356: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_035b: callvirt instance string [mscorlib]System.Object::ToString() + IL_0360: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0365: nop + IL_0366: ldloc.1 + IL_0367: ldarg.0 + IL_0368: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f19 + IL_036d: ldarg.1 + IL_036e: ldc.i4.1 + IL_036f: add + IL_0370: call instance string MCCTest.VType8::Dump(int32) + IL_0375: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_037a: nop + IL_037b: ldloc.1 + IL_037c: ldloc.0 + IL_037d: ldstr "[Field f20] [Type '{0}']" + IL_0382: call string [mscorlib]System.String::Concat(string, + string) + IL_0387: ldarg.0 + IL_0388: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_038d: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0392: callvirt instance string [mscorlib]System.Object::ToString() + IL_0397: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_039c: nop + IL_039d: ldloc.1 + IL_039e: ldarg.0 + IL_039f: ldfld class MCCTest.RType4 MCCTest.VTypeF::f20 + IL_03a4: ldarg.1 + IL_03a5: ldc.i4.1 + IL_03a6: add + IL_03a7: callvirt instance string MCCTest.RType4::Dump(int32) + IL_03ac: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_03b1: nop + IL_03b2: ldloc.1 + IL_03b3: ldloc.0 + IL_03b4: ldstr "[Field f21] [Type '{0}']" + IL_03b9: call string [mscorlib]System.String::Concat(string, + string) + IL_03be: ldarg.0 + IL_03bf: ldfld valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_03c4: box MCCTest.VTypeA + IL_03c9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_03ce: callvirt instance string [mscorlib]System.Object::ToString() + IL_03d3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_03d8: nop + IL_03d9: ldloc.1 + IL_03da: ldarg.0 + IL_03db: ldflda valuetype MCCTest.VTypeA MCCTest.VTypeF::f21 + IL_03e0: ldarg.1 + IL_03e1: ldc.i4.1 + IL_03e2: add + IL_03e3: call instance string MCCTest.VTypeA::Dump(int32) + IL_03e8: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_03ed: nop + IL_03ee: ldloc.1 + IL_03ef: ldloc.0 + IL_03f0: ldstr "[Field f22] [Type '{0}']" + IL_03f5: call string [mscorlib]System.String::Concat(string, + string) + IL_03fa: ldarg.0 + IL_03fb: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_0400: box MCCTest.VTypeD + IL_0405: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_040a: callvirt instance string [mscorlib]System.Object::ToString() + IL_040f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0414: nop + IL_0415: ldloc.1 + IL_0416: ldarg.0 + IL_0417: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f22 + IL_041c: ldarg.1 + IL_041d: ldc.i4.1 + IL_041e: add + IL_041f: call instance string MCCTest.VTypeD::Dump(int32) + IL_0424: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0429: nop + IL_042a: ldloc.1 + IL_042b: ldloc.0 + IL_042c: ldstr "[Field f23] [Type '{0}']" + IL_0431: call string [mscorlib]System.String::Concat(string, + string) + IL_0436: ldarg.0 + IL_0437: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_043c: box MCCTest.VType9 + IL_0441: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0446: callvirt instance string [mscorlib]System.Object::ToString() + IL_044b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0450: nop + IL_0451: ldloc.1 + IL_0452: ldarg.0 + IL_0453: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f23 + IL_0458: ldarg.1 + IL_0459: ldc.i4.1 + IL_045a: add + IL_045b: call instance string MCCTest.VType9::Dump(int32) + IL_0460: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0465: nop + IL_0466: ldloc.1 + IL_0467: ldloc.0 + IL_0468: ldstr "[Field f24] [Type '{0}']" + IL_046d: call string [mscorlib]System.String::Concat(string, + string) + IL_0472: ldarg.0 + IL_0473: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_0478: box MCCTest.VTypeE + IL_047d: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0482: callvirt instance string [mscorlib]System.Object::ToString() + IL_0487: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_048c: nop + IL_048d: ldloc.1 + IL_048e: ldarg.0 + IL_048f: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f24 + IL_0494: ldarg.1 + IL_0495: ldc.i4.1 + IL_0496: add + IL_0497: call instance string MCCTest.VTypeE::Dump(int32) + IL_049c: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_04a1: nop + IL_04a2: ldloc.1 + IL_04a3: ldloc.0 + IL_04a4: ldstr "[Field f25] [Type '{0}']" + IL_04a9: call string [mscorlib]System.String::Concat(string, + string) + IL_04ae: ldarg.0 + IL_04af: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_04b4: callvirt instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_04b9: callvirt instance string [mscorlib]System.Object::ToString() + IL_04be: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_04c3: nop + IL_04c4: ldloc.1 + IL_04c5: ldarg.0 + IL_04c6: ldfld class MCCTest.RType4 MCCTest.VTypeF::f25 + IL_04cb: ldarg.1 + IL_04cc: ldc.i4.1 + IL_04cd: add + IL_04ce: callvirt instance string MCCTest.RType4::Dump(int32) + IL_04d3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_04d8: nop + IL_04d9: ldloc.1 + IL_04da: ldloc.0 + IL_04db: ldstr "f26 = " + IL_04e0: ldarg.0 + IL_04e1: ldfld float32 MCCTest.VTypeF::f26 + IL_04e6: box [mscorlib]System.Single + IL_04eb: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_04f0: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_04f5: nop + IL_04f6: ldloc.1 + IL_04f7: ldloc.0 + IL_04f8: ldstr "f27 = " + IL_04fd: ldarg.0 + IL_04fe: ldfld uint8 MCCTest.VTypeF::f27 + IL_0503: box [mscorlib]System.Byte + IL_0508: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_050d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0512: nop + IL_0513: ldloc.1 + IL_0514: ldloc.0 + IL_0515: ldstr "f28 = " + IL_051a: ldarg.0 + IL_051b: ldfld float64 MCCTest.VTypeF::f28 + IL_0520: box [mscorlib]System.Double + IL_0525: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_052a: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_052f: nop + IL_0530: ldloc.1 + IL_0531: ldloc.0 + IL_0532: ldstr "[Field f29] [Type '{0}']" + IL_0537: call string [mscorlib]System.String::Concat(string, + string) + IL_053c: ldarg.0 + IL_053d: ldfld valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_0542: box MCCTest.VTypeC + IL_0547: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_054c: callvirt instance string [mscorlib]System.Object::ToString() + IL_0551: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0556: nop + IL_0557: ldloc.1 + IL_0558: ldarg.0 + IL_0559: ldflda valuetype MCCTest.VTypeC MCCTest.VTypeF::f29 + IL_055e: ldarg.1 + IL_055f: ldc.i4.1 + IL_0560: add + IL_0561: call instance string MCCTest.VTypeC::Dump(int32) + IL_0566: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_056b: nop + IL_056c: ldloc.1 + IL_056d: ldloc.0 + IL_056e: ldstr "[Field f30] [Type '{0}']" + IL_0573: call string [mscorlib]System.String::Concat(string, + string) + IL_0578: ldarg.0 + IL_0579: ldfld valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_057e: box MCCTest.VType6 + IL_0583: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0588: callvirt instance string [mscorlib]System.Object::ToString() + IL_058d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0592: nop + IL_0593: ldloc.1 + IL_0594: ldarg.0 + IL_0595: ldflda valuetype MCCTest.VType6 MCCTest.VTypeF::f30 + IL_059a: ldarg.1 + IL_059b: ldc.i4.1 + IL_059c: add + IL_059d: call instance string MCCTest.VType6::Dump(int32) + IL_05a2: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_05a7: nop + IL_05a8: ldloc.1 + IL_05a9: ldloc.0 + IL_05aa: ldstr "f31 = " + IL_05af: ldarg.0 + IL_05b0: ldfld float64 MCCTest.VTypeF::f31 + IL_05b5: box [mscorlib]System.Double + IL_05ba: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_05bf: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_05c4: nop + IL_05c5: ldloc.1 + IL_05c6: ldloc.0 + IL_05c7: ldstr "[Field f32] [Type '{0}']" + IL_05cc: call string [mscorlib]System.String::Concat(string, + string) + IL_05d1: ldarg.0 + IL_05d2: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_05d7: box MCCTest.VType8 + IL_05dc: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_05e1: callvirt instance string [mscorlib]System.Object::ToString() + IL_05e6: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_05eb: nop + IL_05ec: ldloc.1 + IL_05ed: ldarg.0 + IL_05ee: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f32 + IL_05f3: ldarg.1 + IL_05f4: ldc.i4.1 + IL_05f5: add + IL_05f6: call instance string MCCTest.VType8::Dump(int32) + IL_05fb: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0600: nop + IL_0601: ldloc.1 + IL_0602: ldloc.0 + IL_0603: ldstr "f33 = " + IL_0608: ldarg.0 + IL_0609: ldfld int8 MCCTest.VTypeF::f33 + IL_060e: box [mscorlib]System.SByte + IL_0613: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_0618: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_061d: nop + IL_061e: ldloc.1 + IL_061f: ldloc.0 + IL_0620: ldstr "[Field f34] [Type '{0}']" + IL_0625: call string [mscorlib]System.String::Concat(string, + string) + IL_062a: ldarg.0 + IL_062b: ldfld valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_0630: box MCCTest.VTypeD + IL_0635: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_063a: callvirt instance string [mscorlib]System.Object::ToString() + IL_063f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0644: nop + IL_0645: ldloc.1 + IL_0646: ldarg.0 + IL_0647: ldflda valuetype MCCTest.VTypeD MCCTest.VTypeF::f34 + IL_064c: ldarg.1 + IL_064d: ldc.i4.1 + IL_064e: add + IL_064f: call instance string MCCTest.VTypeD::Dump(int32) + IL_0654: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0659: nop + IL_065a: ldloc.1 + IL_065b: ldloc.0 + IL_065c: ldstr "[Field f35] [Type '{0}']" + IL_0661: call string [mscorlib]System.String::Concat(string, + string) + IL_0666: ldarg.0 + IL_0667: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_066c: box MCCTest.VTypeE + IL_0671: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0676: callvirt instance string [mscorlib]System.Object::ToString() + IL_067b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0680: nop + IL_0681: ldloc.1 + IL_0682: ldarg.0 + IL_0683: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f35 + IL_0688: ldarg.1 + IL_0689: ldc.i4.1 + IL_068a: add + IL_068b: call instance string MCCTest.VTypeE::Dump(int32) + IL_0690: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0695: nop + IL_0696: ldloc.1 + IL_0697: ldloc.0 + IL_0698: ldstr "[Field f36] [Type '{0}']" + IL_069d: call string [mscorlib]System.String::Concat(string, + string) + IL_06a2: ldarg.0 + IL_06a3: ldfld valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_06a8: box MCCTest.VTypeE + IL_06ad: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_06b2: callvirt instance string [mscorlib]System.Object::ToString() + IL_06b7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_06bc: nop + IL_06bd: ldloc.1 + IL_06be: ldarg.0 + IL_06bf: ldflda valuetype MCCTest.VTypeE MCCTest.VTypeF::f36 + IL_06c4: ldarg.1 + IL_06c5: ldc.i4.1 + IL_06c6: add + IL_06c7: call instance string MCCTest.VTypeE::Dump(int32) + IL_06cc: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_06d1: nop + IL_06d2: ldloc.1 + IL_06d3: ldloc.0 + IL_06d4: ldstr "[Field f37] [Type '{0}']" + IL_06d9: call string [mscorlib]System.String::Concat(string, + string) + IL_06de: ldarg.0 + IL_06df: ldfld valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_06e4: box MCCTest.VType7 + IL_06e9: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_06ee: callvirt instance string [mscorlib]System.Object::ToString() + IL_06f3: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_06f8: nop + IL_06f9: ldloc.1 + IL_06fa: ldarg.0 + IL_06fb: ldflda valuetype MCCTest.VType7 MCCTest.VTypeF::f37 + IL_0700: ldarg.1 + IL_0701: ldc.i4.1 + IL_0702: add + IL_0703: call instance string MCCTest.VType7::Dump(int32) + IL_0708: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_070d: nop + IL_070e: ldloc.1 + IL_070f: ldloc.0 + IL_0710: ldstr "[Field f38] [Type '{0}']" + IL_0715: call string [mscorlib]System.String::Concat(string, + string) + IL_071a: ldarg.0 + IL_071b: ldfld valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_0720: box MCCTest.VType9 + IL_0725: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_072a: callvirt instance string [mscorlib]System.Object::ToString() + IL_072f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0734: nop + IL_0735: ldloc.1 + IL_0736: ldarg.0 + IL_0737: ldflda valuetype MCCTest.VType9 MCCTest.VTypeF::f38 + IL_073c: ldarg.1 + IL_073d: ldc.i4.1 + IL_073e: add + IL_073f: call instance string MCCTest.VType9::Dump(int32) + IL_0744: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0749: nop + IL_074a: ldloc.1 + IL_074b: ldloc.0 + IL_074c: ldstr "[Field f39] [Type '{0}']" + IL_0751: call string [mscorlib]System.String::Concat(string, + string) + IL_0756: ldarg.0 + IL_0757: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_075c: box MCCTest.VType8 + IL_0761: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0766: callvirt instance string [mscorlib]System.Object::ToString() + IL_076b: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0770: nop + IL_0771: ldloc.1 + IL_0772: ldarg.0 + IL_0773: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f39 + IL_0778: ldarg.1 + IL_0779: ldc.i4.1 + IL_077a: add + IL_077b: call instance string MCCTest.VType8::Dump(int32) + IL_0780: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0785: nop + IL_0786: ldloc.1 + IL_0787: ldloc.0 + IL_0788: ldstr "f40 = " + IL_078d: ldarg.0 + IL_078e: ldfld float32 MCCTest.VTypeF::f40 + IL_0793: box [mscorlib]System.Single + IL_0798: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_079d: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_07a2: nop + IL_07a3: ldloc.1 + IL_07a4: ldloc.0 + IL_07a5: ldstr "f41 = " + IL_07aa: ldarg.0 + IL_07ab: ldfld int32 MCCTest.VTypeF::f41 + IL_07b0: box [mscorlib]System.Int32 + IL_07b5: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_07ba: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_07bf: nop + IL_07c0: ldloc.1 + IL_07c1: ldloc.0 + IL_07c2: ldstr "f42 = " + IL_07c7: ldarg.0 + IL_07c8: ldfld char MCCTest.VTypeF::f42 + IL_07cd: box [mscorlib]System.Char + IL_07d2: call string [mscorlib]System.String::Concat(object, + object, + object) + IL_07d7: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_07dc: nop + IL_07dd: ldloc.1 + IL_07de: ldloc.0 + IL_07df: ldstr "[Field f43] [Type '{0}']" + IL_07e4: call string [mscorlib]System.String::Concat(string, + string) + IL_07e9: ldarg.0 + IL_07ea: ldfld valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_07ef: box MCCTest.VTypeB + IL_07f4: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_07f9: callvirt instance string [mscorlib]System.Object::ToString() + IL_07fe: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_0803: nop + IL_0804: ldloc.1 + IL_0805: ldarg.0 + IL_0806: ldflda valuetype MCCTest.VTypeB MCCTest.VTypeF::f43 + IL_080b: ldarg.1 + IL_080c: ldc.i4.1 + IL_080d: add + IL_080e: call instance string MCCTest.VTypeB::Dump(int32) + IL_0813: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0818: nop + IL_0819: ldloc.1 + IL_081a: ldloc.0 + IL_081b: ldstr "[Field f44] [Type '{0}']" + IL_0820: call string [mscorlib]System.String::Concat(string, + string) + IL_0825: ldarg.0 + IL_0826: ldfld valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_082b: box MCCTest.VType8 + IL_0830: call instance class [mscorlib]System.Type [mscorlib]System.Object::GetType() + IL_0835: callvirt instance string [mscorlib]System.Object::ToString() + IL_083a: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string, + object) + IL_083f: nop + IL_0840: ldloc.1 + IL_0841: ldarg.0 + IL_0842: ldflda valuetype MCCTest.VType8 MCCTest.VTypeF::f44 + IL_0847: ldarg.1 + IL_0848: ldc.i4.1 + IL_0849: add + IL_084a: call instance string MCCTest.VType8::Dump(int32) + IL_084f: callvirt instance void [mscorlib]System.IO.TextWriter::WriteLine(string) + IL_0854: nop + IL_0855: ldloc.1 + IL_0856: callvirt instance string [mscorlib]System.Object::ToString() + IL_085b: stloc.2 + IL_085c: br.s IL_085e + + IL_085e: ldloc.2 + IL_085f: ret + } // end of method VTypeF::Dump + + .method public hidebysig instance string + Dump() cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (string V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldc.i4.0 + IL_0003: call instance string MCCTest.VTypeF::Dump(int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method VTypeF::Dump + +} // end of class MCCTest.VTypeF + .class public auto ansi beforefieldinit MCCTest.Common extends [mscorlib]System.Object { @@ -16372,6 +23285,132 @@ IL_000c: ret } // end of method Common::CheckResult + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VType9 actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VTypeA actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VTypeB actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VTypeC actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VTypeD actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VTypeE actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + + .method public hidebysig static int32 CheckResult(valuetype MCCTest.VTypeF actual, + int32 count) cil managed + { + // Code size 13 (0xd) + .maxstack 2 + .locals init (int32 V_0) + IL_0000: nop + IL_0001: ldarg.0 + IL_0002: ldarg.1 + IL_0003: call int32 class MCCTest.Common2`1::CheckResult(!0, + int32) + IL_0008: stloc.0 + IL_0009: br.s IL_000b + + IL_000b: ldloc.0 + IL_000c: ret + } // end of method Common::CheckResult + .method public hidebysig specialname rtspecialname instance void .ctor() cil managed { diff --git a/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs b/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs deleted file mode 100644 index 0d18e7bf535145..00000000000000 --- a/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Runtime.CompilerServices; -using Xunit; - -public unsafe class MemsetMemcpyNullref -{ - [Fact] - public static void MemsetMemcpyThrowNullRefonNull() - { - Assert.Throws(() => MemoryInit(null)); - Assert.Throws(() => MemoryCopy(null, null)); - Assert.Throws(() => - { - // Check when only src is null - HugeStruct hs = default; - MemoryCopy(&hs, null); - }); - Assert.Throws(() => - { - // Check when only dst is null - HugeStruct hs = default; - MemoryCopy(null, &hs); - }); - - // Check various lengths - uint[] lengths = [1, 10, 100, 1000, 10000, 100000, 1000000]; - foreach (uint length in lengths) - { - Assert.Throws(() => MemoryInitByref(ref Unsafe.NullRef(), length)); - Assert.Throws(() => MemoryCopyByref(ref Unsafe.NullRef(), ref Unsafe.NullRef(), length)); - } - - // These APIs are not expected to fail/throw on zero length, even if pointers are not valid - byte valid = 0; - MemoryInitByref(ref Unsafe.NullRef(), 0); - MemoryCopyByref(ref Unsafe.NullRef(), ref valid, 0); - MemoryCopyByref(ref valid, ref Unsafe.NullRef(), 0); - MemoryCopyByref(ref Unsafe.NullRef(), ref Unsafe.NullRef(), 0); - - byte valid2 = 0; - MemoryInitByrefZeroLen(ref valid); - MemoryInitByrefZeroLen(ref Unsafe.NullRef()); - MemoryCopyByrefZeroLen(ref valid, ref valid2); - MemoryCopyByrefZeroLen(ref valid, ref Unsafe.NullRef()); - MemoryCopyByrefZeroLen(ref Unsafe.NullRef(), ref valid2); - MemoryCopyByrefZeroLen(ref Unsafe.NullRef(), ref Unsafe.NullRef()); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemoryCopy(HugeStruct* dst, HugeStruct* src) => - *dst = *src; - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemoryCopyByref(ref byte dst, ref byte src, uint len) => - Unsafe.CopyBlockUnaligned(ref dst, ref src, len); - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemoryInit(HugeStruct* dst) => - *dst = default; - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemoryInitByref(ref byte dst, uint len) => - Unsafe.InitBlockUnaligned(ref dst, 42, len); - - private struct HugeStruct - { - public fixed byte Data[20_000]; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemoryCopyByrefZeroLen(ref byte dst, ref byte src) => - Unsafe.CopyBlockUnaligned(ref dst, ref src, 0); - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void MemoryInitByrefZeroLen(ref byte dst) => - Unsafe.InitBlockUnaligned(ref dst, 42, 0); -} diff --git a/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.csproj b/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.csproj deleted file mode 100644 index 23d7b90be5361c..00000000000000 --- a/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.csproj +++ /dev/null @@ -1,10 +0,0 @@ - - - true - None - True - - - - - diff --git a/src/tests/JIT/opt/Vectorization/BufferMemmoveTailCall.il b/src/tests/JIT/opt/Vectorization/BufferMemmoveTailCall.il new file mode 100644 index 00000000000000..068f11ad7b6176 --- /dev/null +++ b/src/tests/JIT/opt/Vectorization/BufferMemmoveTailCall.il @@ -0,0 +1,99 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +.assembly extern System.Runtime { .publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A ) } +.assembly extern xunit.core {} +.assembly extern System.Runtime.Extensions {} +.assembly BufferMemmoveTailCall { + // Allow access to private members of System.Private.CoreLib + .custom instance void System.Runtime.CompilerServices.IgnoresAccessChecksToAttribute::.ctor(string) = ( + 01 00 16 53 79 73 74 65 6d 2e 50 72 69 76 61 74 + 65 2e 43 6f 72 65 4c 69 62 00 00 + ) +} + +.class public abstract auto ansi sealed beforefieldinit TailCallBufferMemmove + extends [System.Runtime]System.Object +{ + .method public hidebysig static int32 Main() cil managed + { + .custom instance void [xunit.core]Xunit.FactAttribute::.ctor() = ( + 01 00 00 00 + ) + .maxstack 8 + .entrypoint + + // C#: + // byte[] src = new byte[32]; + // Test(ref src[0]); + + ldc.i4.s 32 + newarr [System.Runtime]System.Byte + ldc.i4.0 + ldelema [System.Runtime]System.Byte + call void TailCallBufferMemmove::Test(uint8&) + + // return 100; + ldc.i4.s 100 + ret + } + + .method private hidebysig static void Test (uint8& src) cil managed noinlining + { + .maxstack 3 + + // C#: + // byte* data = stackalloc byte[64]; // to trigger slow helper-based tail calls + // Buffer.Memmove(ref Unsafe.AsRef(data), ref src, 64); + + ldc.i4.s 64 + conv.u + localloc + call !!0& [System.Runtime]System.Runtime.CompilerServices.Unsafe::AsRef(void*) + ldarg.0 + ldc.i4.s 64 + conv.i + tail. call void [System.Runtime]System.Buffer::Memmove(uint8&, uint8&, native uint) + ret + } +} + +// C#: +// namespace System.Runtime.CompilerServices +// { +// public class IgnoresAccessChecksToAttribute : Attribute +// { +// public IgnoresAccessChecksToAttribute(string assemblyName) +// { +// AssemblyName = assemblyName; +// } +// public string AssemblyName { get; } +// } +// } +// +.class public auto ansi beforefieldinit System.Runtime.CompilerServices.IgnoresAccessChecksToAttribute + extends [System.Runtime]System.Attribute +{ + .field private initonly string 'k__BackingField' + .method public hidebysig specialname rtspecialname instance void .ctor (string assemblyName) cil managed + { + .maxstack 8 + ldarg.0 + call instance void [System.Runtime]System.Attribute::.ctor() + ldarg.0 + ldarg.1 + stfld string System.Runtime.CompilerServices.IgnoresAccessChecksToAttribute::'k__BackingField' + ret + } + .method public hidebysig specialname instance string get_AssemblyName () cil managed + { + .maxstack 8 + ldarg.0 + ldfld string System.Runtime.CompilerServices.IgnoresAccessChecksToAttribute::'k__BackingField' + ret + } + .property instance string AssemblyName() + { + .get instance string System.Runtime.CompilerServices.IgnoresAccessChecksToAttribute::get_AssemblyName() + } +} diff --git a/src/tests/JIT/opt/Vectorization/BufferMemmoveTailCall.ilproj b/src/tests/JIT/opt/Vectorization/BufferMemmoveTailCall.ilproj new file mode 100644 index 00000000000000..5fa250452852d2 --- /dev/null +++ b/src/tests/JIT/opt/Vectorization/BufferMemmoveTailCall.ilproj @@ -0,0 +1,8 @@ + + + True + + + + + diff --git a/src/tests/Loader/classloader/DefaultInterfaceMethods/constrainedcall/constrained2_brl.il b/src/tests/Loader/classloader/DefaultInterfaceMethods/constrainedcall/constrained2_brl.il deleted file mode 100644 index 9382ff66730c2b..00000000000000 --- a/src/tests/Loader/classloader/DefaultInterfaceMethods/constrainedcall/constrained2_brl.il +++ /dev/null @@ -1,153 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -.assembly extern System.Console { .publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A ) } -.assembly extern System.Runtime { .publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A ) } -.assembly extern Microsoft.DotNet.XUnitExtensions {} -.assembly extern xunit.core {} -.assembly constrained2_brl { } - -.class interface private auto ansi abstract - IAdder -{ - .method public hidebysig newslot virtual - instance int32 Add(int32) cil managed - { - ldstr "Calling DIM from ByRefLike type is invalid" - newobj instance void [System.Runtime]System.Exception::.ctor(string) - throw - } -} - -.class private sequential ansi sealed beforefieldinit Adder - extends [System.Runtime]System.ValueType - implements IAdder -{ - .custom instance void [System.Runtime]System.Runtime.CompilerServices.IsByRefLikeAttribute::.ctor() = ( - 01 00 00 00 - ) - - .field private int32 _field - - .method public hidebysig specialname rtspecialname - instance void .ctor (int32) cil managed - { - ldarg.0 - ldarg.1 - stfld int32 Adder::_field - ret - } - - .method public hidebysig newslot virtual - instance int32 Add(int32) cil managed - { - // Load field and add with argument - ldarg.0 - dup - ldfld int32 valuetype Adder::_field - ldarg.1 - add - - // Update the field - stfld int32 valuetype Adder::_field - - // Return the field value - ldarg.0 - ldfld int32 valuetype Adder::_field - ret - } -} - -.class private sequential ansi sealed beforefieldinit Adder_Invalid - extends [System.Runtime]System.ValueType - implements IAdder -{ - .custom instance void [System.Runtime]System.Runtime.CompilerServices.IsByRefLikeAttribute::.ctor() = ( - 01 00 00 00 - ) - - .method public hidebysig specialname rtspecialname - instance void .ctor (int32) cil managed - { - ret - } - - // - // Deferring to the DIM on IAdder - // -} - -.method public hidebysig static int32 Check(!!0, int32) -{ - ldarga.s 0 - ldarg.1 - constrained. !!0 - callvirt instance int32 IAdder::Add(int32) - ret -} - -.class public auto ansi abstract sealed beforefieldinit constrained2_brl - extends [System.Runtime]System.Object -{ - .method public hidebysig static int32 Main() - { - .custom instance void [xunit.core]Xunit.FactAttribute::.ctor() = ( - 01 00 00 00 - ) - .custom instance void [Microsoft.DotNet.XUnitExtensions]Xunit.SkipOnMonoAttribute::.ctor(string, valuetype [Microsoft.DotNet.XUnitExtensions]Xunit.TestPlatforms) = ( - 01 00 2c 4d 6f 6e 6f 20 64 6f 65 73 20 6e 6f 74 - 20 73 75 70 70 6f 72 74 20 42 79 52 65 66 4c 69 - 6b 65 20 67 65 6e 65 72 69 63 73 20 79 65 74 ff - ff ff ff 00 00 - ) - .entrypoint - - .locals init ( - valuetype Adder, - valuetype Adder_Invalid - ) - - // Initialize Adder instance - ldloca.s 0 - ldc.i4 10 - call instance void Adder::.ctor(int32) - - ldstr "Validate constrained call of ByRefLike interface method passes" - call void [System.Console]System.Console::WriteLine(string) - ldloc.0 - ldc.i4 20 - call int32 Check(!!0, int32) - ldc.i4 30 - ceq - brfalse FAIL - - // Initialize Adder_Invalid instance - ldloca.s 1 - ldc.i4 10 - call instance void Adder_Invalid::.ctor(int32) - - .try - { - ldstr "Validate constrained call of ByRefLike interface DIM fails" - call void [System.Console]System.Console::WriteLine(string) - - ldloc.1 - ldc.i4 20 - call int32 Check(!!0, int32) - leave FAIL - } - catch [System.Runtime]System.Exception - { - pop - leave ExpectedFailure - } - - ExpectedFailure: - ldc.i4 100 - ret - - FAIL: - ldc.i4 101 - ret - } -} diff --git a/src/tests/Loader/classloader/DefaultInterfaceMethods/constrainedcall/constrained2_brl.ilproj b/src/tests/Loader/classloader/DefaultInterfaceMethods/constrainedcall/constrained2_brl.ilproj deleted file mode 100644 index 8aceddffe8f5ef..00000000000000 --- a/src/tests/Loader/classloader/DefaultInterfaceMethods/constrainedcall/constrained2_brl.ilproj +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/src/tests/Loader/classloader/generics/ByRefLike/GenericTypeSubstitution.cs b/src/tests/Loader/classloader/generics/ByRefLike/GenericTypeSubstitution.cs index 75b22b974792c3..3453b2a1d3e5ee 100644 --- a/src/tests/Loader/classloader/generics/ByRefLike/GenericTypeSubstitution.cs +++ b/src/tests/Loader/classloader/generics/ByRefLike/GenericTypeSubstitution.cs @@ -12,29 +12,32 @@ public class GenericTypeSubstitution { [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void AllowByRefLike_Substituted_For_AllowByRefLike() { Console.WriteLine($"{nameof(AllowByRefLike_Substituted_For_AllowByRefLike)}..."); - + Console.WriteLine($" -- Instantiate: {Exec.TypeSubstitutionInterfaceImplementationAllowByRefLike()}"); Console.WriteLine($" -- Instantiate: {Exec.TypeSubstitutionInheritanceAllowByRefLike()}"); Console.WriteLine($" -- Instantiate: {Exec.TypeSubstitutionFieldAllowByRefLike()}"); } [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void NonByRefLike_Substituted_For_AllowByRefLike() { - Console.WriteLine($"{nameof(NonByRefLike_Substituted_For_AllowByRefLike)}..."); - Console.WriteLine($" -- Instantiate: {Exec.TypeSubstitutionInterfaceImplementationNonByRefLike()}"); Console.WriteLine($" -- Instantiate: {Exec.TypeSubstitutionInheritanceNonByRefLike()}"); Console.WriteLine($" -- Instantiate: {Exec.TypeSubstitutionFieldNonByRefLike()}"); } [Fact] - public static void AllowByRefLike_Substituted_For_NonByRefLike() + [ActiveIssue("To be created", TestRuntimes.CoreCLR)] + [SkipOnMono("Mono does not support ByRefLike generics yet")] + public static void AllowByRefLike_Substituted_For_NonByRefLike_Invalid() { - Console.WriteLine($"{nameof(AllowByRefLike_Substituted_For_NonByRefLike)}..."); - Exec.TypeSubstitutionFieldAllowNonByRefLikeIntoNonByRefLike(); + Assert.Throws(() => { Exec.TypeSubstitutionInterfaceImplementationAllowByRefLikeIntoNonByRefLike(); }); + Assert.Throws(() => { Exec.TypeSubstitutionInheritanceAllowByRefLikeIntoNonByRefLike(); }); + Assert.Throws(() => { Exec.TypeSubstitutionFieldAllowByRefLikeIntoNonByRefLike(); }); } } \ No newline at end of file diff --git a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.il b/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.il index 72f0dc67f69e86..d81452d398af9d 100644 --- a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.il +++ b/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.il @@ -344,22 +344,9 @@ ret } - .field public static class InvalidCSharp.GenericClass_Over`1 StaticField1 .field public static !T StaticField } -.class public sequential ansi sealed beforefieldinit InvalidCSharp.GenericClass_IndependentConstraints`2 - extends [System.Runtime]System.Object -{ - .method public hidebysig specialname rtspecialname - instance void .ctor () cil managed - { - ldarg.0 - call instance void [System.Runtime]System.Object::.ctor() - ret - } -} - .class public sequential ansi sealed beforefieldinit ByRefLikeType extends [System.Runtime]System.ValueType { @@ -368,19 +355,6 @@ ) } -.class interface public auto ansi abstract InvalidCSharp.EmptyInterface -{ -} - -.class public sequential ansi sealed beforefieldinit InvalidCSharp.ByRefLikeTypeWithInterface - extends [System.Runtime]System.ValueType - implements InvalidCSharp.EmptyInterface -{ - .custom instance void [System.Runtime]System.Runtime.CompilerServices.IsByRefLikeAttribute::.ctor() = ( - 01 00 00 00 - ) -} - .class public sequential ansi sealed beforefieldinit RegularValueType extends [System.Runtime]System.ValueType { @@ -465,6 +439,15 @@ } // Invalid generic substitution of non-allow-byreflike with allow-byreflike +.class interface public auto ansi abstract InvalidCSharp.GenericDerivedInterface_Invalid`1 + implements class InvalidCSharp.GenericInterface_Invalid`1 +{ +} + +.class public sequential ansi sealed beforefieldinit InvalidCSharp.GenericDerivedClass_Invalid`1 + extends class InvalidCSharp.GenericClass_Invalid`1 +{ +} .class public sequential ansi sealed beforefieldinit InvalidCSharp.GenericValueTypeWrapper_Invalid`1 extends [System.Runtime]System.ValueType @@ -542,27 +525,6 @@ callvirt instance string [System.Runtime]System.Object::ToString() ret } - .method public hidebysig static - class [System.Runtime]System.Type GenericByRefLike_ConstraintsAreIndependent_Int32_Int32() cil managed - { - newobj instance void class InvalidCSharp.GenericClass_IndependentConstraints`2::.ctor() - callvirt instance class [System.Runtime]System.Type [System.Runtime]System.Object::GetType() - ret - } - .method public hidebysig static - class [System.Runtime]System.Type GenericByRefLike_ConstraintsAreIndependent_Interface_ByRefLike_Invalid() cil managed - { - newobj instance void class InvalidCSharp.GenericClass_IndependentConstraints`2::.ctor() - callvirt instance class [System.Runtime]System.Type [System.Runtime]System.Object::GetType() - ret - } - .method public hidebysig static - class [System.Runtime]System.Type GenericByRefLike_ConstraintsAreIndependent_ByRefLike_ByRefLike_Invalid() cil managed - { - newobj instance void class InvalidCSharp.GenericClass_IndependentConstraints`2::.ctor() - callvirt instance class [System.Runtime]System.Type [System.Runtime]System.Object::GetType() - ret - } .method public hidebysig static object BoxAsObject() cil managed @@ -836,9 +798,27 @@ callvirt instance string [System.Runtime]System.Object::ToString() ret } + + .method public hidebysig static + string TypeSubstitutionInterfaceImplementationAllowByRefLikeIntoNonByRefLike() cil managed + { + ldtoken class InvalidCSharp.GenericDerivedInterface_Invalid`1 + call class [System.Runtime]System.Type [System.Runtime]System.Type::GetTypeFromHandle(valuetype [System.Runtime]System.RuntimeTypeHandle) + callvirt instance string [System.Runtime]System.Object::ToString() + ret + } + + .method public hidebysig static + string TypeSubstitutionInheritanceAllowByRefLikeIntoNonByRefLike() cil managed + { + ldtoken class InvalidCSharp.GenericDerivedClass_Invalid`1 + call class [System.Runtime]System.Type [System.Runtime]System.Type::GetTypeFromHandle(valuetype [System.Runtime]System.RuntimeTypeHandle) + callvirt instance string [System.Runtime]System.Object::ToString() + ret + } .method public hidebysig static - string TypeSubstitutionFieldAllowNonByRefLikeIntoNonByRefLike() cil managed + string TypeSubstitutionFieldAllowByRefLikeIntoNonByRefLike() cil managed { ldtoken valuetype InvalidCSharp.GenericValueTypeWrapper_Invalid`1 call class [System.Runtime]System.Type [System.Runtime]System.Type::GetTypeFromHandle(valuetype [System.Runtime]System.RuntimeTypeHandle) diff --git a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.ilproj b/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.ilproj index 8e167b20da08e5..58dc9527f80461 100644 --- a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.ilproj +++ b/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharp.ilproj @@ -2,6 +2,7 @@ library true + true diff --git a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharpNegative.il b/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharpNegative.il deleted file mode 100644 index 08767e372ea460..00000000000000 --- a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharpNegative.il +++ /dev/null @@ -1,143 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -.assembly extern System.Console { .publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A ) } -.assembly extern System.Runtime { .publickeytoken = (B0 3F 5F 7F 11 D5 0A 3A ) } - -.assembly InvalidCSharpNegative { } - -.class public sequential ansi sealed beforefieldinit ByRefLikeType - extends [System.Runtime]System.ValueType -{ - .custom instance void [System.Runtime]System.Runtime.CompilerServices.IsByRefLikeAttribute::.ctor() = ( - 01 00 00 00 - ) -} - -// -// Begin invalid -// - -.class public sequential ansi sealed beforefieldinit InvalidCSharpNegative.GenericClass_Invalid`1 - extends [System.Runtime]System.Object -{ - .method public hidebysig specialname rtspecialname - instance void .ctor () cil managed - { - ldarg.0 - call instance void [System.Runtime]System.Object::.ctor() - ret - } -} - -.class interface public auto ansi abstract InvalidCSharpNegative.GenericInterface_Invalid`1 -{ -} - -.class public sequential ansi sealed beforefieldinit InvalidCSharpNegative.GenericValueType_Invalid`1 - extends [System.Runtime]System.ValueType -{ -} - -// Invalid generic substitution of non-allow-byreflike with allow-byreflike -.class interface public auto ansi abstract InvalidCSharpNegative.GenericDerivedInterface_Invalid`1 - implements class InvalidCSharpNegative.GenericInterface_Invalid`1 -{ -} - -.class public sequential ansi sealed beforefieldinit InvalidCSharpNegative.GenericDerivedClass_Invalid`1 - extends class InvalidCSharpNegative.GenericClass_Invalid`1 -{ -} - -.class public sequential ansi sealed beforefieldinit InvalidCSharpNegative.GenericValueTypeWrapper_Invalid`1 - extends [System.Runtime]System.ValueType -{ - .field public valuetype InvalidCSharpNegative.GenericValueType_Invalid`1 fld; -} - -.class public auto ansi beforefieldinit InvalidCSharpNegative.BaseClassWithGenericMethod - extends [System.Runtime]System.Object -{ - .method public hidebysig newslot virtual - instance void AcceptsByRefLike () cil managed - { - ret - } - .method public hidebysig specialname rtspecialname - instance void .ctor () cil managed - { - ldarg.0 - call instance void [System.Runtime]System.Object::.ctor() - ret - } -} - -.class public auto ansi beforefieldinit InvalidCSharpNegative.DerivedClassWithGenericMethod_Invalid - extends InvalidCSharpNegative.BaseClassWithGenericMethod -{ - .method public hidebysig static - class InvalidCSharpNegative.BaseClassWithGenericMethod Create () cil managed noinlining - { - newobj instance void InvalidCSharpNegative.DerivedClassWithGenericMethod_Invalid::.ctor() - ret - } - .method public hidebysig virtual - instance void AcceptsByRefLike () cil managed // Missing constraint - { - ret - } - .method private hidebysig specialname rtspecialname - instance void .ctor () cil managed - { - ldarg.0 - call instance void InvalidCSharpNegative.BaseClassWithGenericMethod::.ctor() - ret - } -} - -// Entry points - -.class public auto ansi abstract sealed beforefieldinit Exec - extends [System.Runtime]System.Object -{ - .method public hidebysig static - string TypeSubstitutionInterfaceImplementationAllowByRefLikeIntoNonByRefLike() cil managed - { - ldtoken class InvalidCSharpNegative.GenericDerivedInterface_Invalid`1 - call class [System.Runtime]System.Type [System.Runtime]System.Type::GetTypeFromHandle(valuetype [System.Runtime]System.RuntimeTypeHandle) - callvirt instance string [System.Runtime]System.Object::ToString() - ret - } - - .method public hidebysig static - string TypeSubstitutionInheritanceAllowByRefLikeIntoNonByRefLike() cil managed - { - ldtoken class InvalidCSharpNegative.GenericDerivedClass_Invalid`1 - call class [System.Runtime]System.Type [System.Runtime]System.Type::GetTypeFromHandle(valuetype [System.Runtime]System.RuntimeTypeHandle) - callvirt instance string [System.Runtime]System.Object::ToString() - ret - } - - .method public hidebysig static - string TypeSubstitutionFieldAllowByRefLikeIntoNonByRefLike() cil managed - { - ldtoken valuetype InvalidCSharpNegative.GenericValueTypeWrapper_Invalid`1 - call class [System.Runtime]System.Type [System.Runtime]System.Type::GetTypeFromHandle(valuetype [System.Runtime]System.RuntimeTypeHandle) - callvirt instance string [System.Runtime]System.Object::ToString() - ret - } - - .method public hidebysig static - void OverrideMethodNotByRefLike() cil managed - { - .locals init ( - [0] class InvalidCSharpNegative.BaseClassWithGenericMethod - ) - call class InvalidCSharpNegative.BaseClassWithGenericMethod InvalidCSharpNegative.DerivedClassWithGenericMethod_Invalid::Create() - stloc.0 - ldloc.0 - callvirt instance void InvalidCSharpNegative.BaseClassWithGenericMethod::AcceptsByRefLike() - ret - } -} \ No newline at end of file diff --git a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharpNegative.ilproj b/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharpNegative.ilproj deleted file mode 100644 index 6b8ac61f3da1e4..00000000000000 --- a/src/tests/Loader/classloader/generics/ByRefLike/InvalidCSharpNegative.ilproj +++ /dev/null @@ -1,10 +0,0 @@ - - - library - true - true - - - - - \ No newline at end of file diff --git a/src/tests/Loader/classloader/generics/ByRefLike/Validate.cs b/src/tests/Loader/classloader/generics/ByRefLike/Validate.cs index 4d7c05d6db9628..4d70945b4c7465 100644 --- a/src/tests/Loader/classloader/generics/ByRefLike/Validate.cs +++ b/src/tests/Loader/classloader/generics/ByRefLike/Validate.cs @@ -12,6 +12,7 @@ public class Validate { [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void Validate_TypeLoad() { Console.WriteLine($"{nameof(Validate_TypeLoad)}..."); @@ -20,17 +21,15 @@ public static void Validate_TypeLoad() Console.WriteLine($" -- Instantiate: {Exec.GenericInterface()}"); Console.WriteLine($" -- Instantiate: {Exec.GenericValueType()}"); Console.WriteLine($" -- Instantiate: {Exec.GenericByRefLike()}"); - Console.WriteLine($" -- Instantiate: {Exec.GenericByRefLike_ConstraintsAreIndependent_Int32_Int32()}"); Assert.Throws(() => { Exec.GenericClass_Invalid(); }); Assert.Throws(() => { Exec.GenericInterface_Invalid(); }); Assert.Throws(() => { Exec.GenericValueType_Invalid(); }); Assert.Throws(() => { Exec.GenericByRefLike_Invalid(); }); - Assert.Throws(() => { Exec.GenericByRefLike_ConstraintsAreIndependent_Interface_ByRefLike_Invalid(); }); - Assert.Throws(() => { Exec.GenericByRefLike_ConstraintsAreIndependent_ByRefLike_ByRefLike_Invalid(); }); } [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void Validate_Casting_Scenarios() { Console.WriteLine($"{nameof(Validate_Casting_Scenarios)}..."); @@ -45,7 +44,7 @@ public static void Validate_Casting_Scenarios() } [Fact] - [SkipOnMono("https://github.com/dotnet/runtime/issues/99379")] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void Validate_RecognizedOpCodeSequences_Scenarios() { Console.WriteLine($"{nameof(Validate_RecognizedOpCodeSequences_Scenarios)}..."); @@ -57,6 +56,7 @@ public static void Validate_RecognizedOpCodeSequences_Scenarios() } [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void Validate_InvalidOpCode_Scenarios() { Console.WriteLine($"{nameof(Validate_InvalidOpCode_Scenarios)}..."); @@ -73,6 +73,7 @@ public static void Validate_InvalidOpCode_Scenarios() } [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void Validate_Inlining_Behavior() { Console.WriteLine($"{nameof(Validate_Inlining_Behavior)}..."); @@ -84,6 +85,7 @@ public static void Validate_Inlining_Behavior() } // [Fact] + [SkipOnMono("Mono does not support ByRefLike generics yet")] public static void Validate_MemberDiscoveryViaReflection_ForSpanReadOnlySpan() { Console.WriteLine($"{nameof(Validate_MemberDiscoveryViaReflection_ForSpanReadOnlySpan)}..."); diff --git a/src/tests/Loader/classloader/generics/ByRefLike/Validate.csproj b/src/tests/Loader/classloader/generics/ByRefLike/Validate.csproj index 83580f549743d0..8bb4ee77df067e 100644 --- a/src/tests/Loader/classloader/generics/ByRefLike/Validate.csproj +++ b/src/tests/Loader/classloader/generics/ByRefLike/Validate.csproj @@ -2,6 +2,7 @@ true + true diff --git a/src/tests/Loader/classloader/generics/ByRefLike/ValidateNegative.cs b/src/tests/Loader/classloader/generics/ByRefLike/ValidateNegative.cs deleted file mode 100644 index 7660130f253d6a..00000000000000 --- a/src/tests/Loader/classloader/generics/ByRefLike/ValidateNegative.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using InvalidCSharpNegative; - -using Xunit; - -public class ValidateNegative -{ - [Fact] - [SkipOnMono("Mono does not support ByRefLike generics yet")] - public static void AllowByRefLike_Substituted_For_NonByRefLike_Invalid() - { - Console.WriteLine($"{nameof(AllowByRefLike_Substituted_For_NonByRefLike_Invalid)}..."); - - Assert.Throws(() => { Exec.TypeSubstitutionInterfaceImplementationAllowByRefLikeIntoNonByRefLike(); }); - Assert.Throws(() => { Exec.TypeSubstitutionInheritanceAllowByRefLikeIntoNonByRefLike(); }); - Assert.Throws(() => { Exec.TypeSubstitutionFieldAllowByRefLikeIntoNonByRefLike(); }); - Assert.Throws(() => { Exec.OverrideMethodNotByRefLike(); }); - } -} \ No newline at end of file diff --git a/src/tests/Loader/classloader/generics/ByRefLike/ValidateNegative.csproj b/src/tests/Loader/classloader/generics/ByRefLike/ValidateNegative.csproj deleted file mode 100644 index 433b007bbca07e..00000000000000 --- a/src/tests/Loader/classloader/generics/ByRefLike/ValidateNegative.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - true - true - - - false - true - - - - - - - - - diff --git a/src/tests/Loader/classloader/generics/Pointers/Pointers.cs b/src/tests/Loader/classloader/generics/Pointers/Pointers.cs index 8017887d710ca3..f8c1e6116dbff4 100644 --- a/src/tests/Loader/classloader/generics/Pointers/Pointers.cs +++ b/src/tests/Loader/classloader/generics/Pointers/Pointers.cs @@ -106,20 +106,18 @@ public static void PointerArray() [MethodImpl(MethodImplOptions.NoInlining)] private static void PointerArrayImpl() { - int intAllocationSize = 5; - int*[] intPtrArray = new int*[intAllocationSize]; - int* intArray = stackalloc int[intAllocationSize]; - Span intSpan = new Span(intArray, intAllocationSize); + int*[] intPtrArray = new int*[5]; + Span intSpan = stackalloc int[intPtrArray.Length]; + int* intArray = (int*)Unsafe.AsPointer(ref intSpan.GetPinnableReference()); for (int i = 0; i < intPtrArray.Length; i++) { intArray[i] = i; intPtrArray[i] = &intArray[i]; } - int structAllocationSize = 5; - Struct*[] structPtrArray = new Struct*[structAllocationSize]; - Struct* structArray = stackalloc Struct[structAllocationSize]; - Span structSpan = new Span(structArray, structAllocationSize); + Struct*[] structPtrArray = new Struct*[5]; + Span structSpan = stackalloc Struct[structPtrArray.Length]; + Struct* structArray = (Struct*)Unsafe.AsPointer(ref structSpan.GetPinnableReference()); for (int i = 0; i < structPtrArray.Length; i++) { structArray[i] = new Struct() { Num = i }; diff --git a/src/tests/baseservices/exceptions/simple/ParallelCrashTester.csproj b/src/tests/baseservices/exceptions/simple/ParallelCrashTester.csproj index f60f1f053aaa5e..30bcef8c9c8b50 100644 --- a/src/tests/baseservices/exceptions/simple/ParallelCrashTester.csproj +++ b/src/tests/baseservices/exceptions/simple/ParallelCrashTester.csproj @@ -1,8 +1,6 @@ 1 - - true true diff --git a/src/tests/baseservices/exceptions/unhandled/dependencytodelete.cs b/src/tests/baseservices/exceptions/unhandled/dependencytodelete.cs deleted file mode 100644 index 3897a8779df75c..00000000000000 --- a/src/tests/baseservices/exceptions/unhandled/dependencytodelete.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System; - -namespace Dependency -{ - public class DependencyClass - { - public static void Hello() - { - Console.WriteLine("Hello"); - } - } -} diff --git a/src/tests/baseservices/exceptions/unhandled/dependencytodelete.csproj b/src/tests/baseservices/exceptions/unhandled/dependencytodelete.csproj deleted file mode 100644 index fa1f2d01f80e57..00000000000000 --- a/src/tests/baseservices/exceptions/unhandled/dependencytodelete.csproj +++ /dev/null @@ -1,9 +0,0 @@ - - - BuildOnly - Library - - - - - diff --git a/src/tests/baseservices/exceptions/unhandled/unhandledTester.cs b/src/tests/baseservices/exceptions/unhandled/unhandledTester.cs index 2c30f593dfd90f..151c8b63516606 100644 --- a/src/tests/baseservices/exceptions/unhandled/unhandledTester.cs +++ b/src/tests/baseservices/exceptions/unhandled/unhandledTester.cs @@ -14,14 +14,14 @@ namespace TestUnhandledExceptionTester { public class Program { - static void RunExternalProcess(string unhandledType, string assembly) + static void RunExternalProcess(string unhandledType) { List lines = new List(); Process testProcess = new Process(); testProcess.StartInfo.FileName = Path.Combine(Environment.GetEnvironmentVariable("CORE_ROOT"), "corerun"); - testProcess.StartInfo.Arguments = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), assembly) + " " + unhandledType; + testProcess.StartInfo.Arguments = Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "unhandled.dll") + " " + unhandledType; testProcess.StartInfo.RedirectStandardError = true; // Disable creating dump since the target process is expected to fail with an unhandled exception testProcess.StartInfo.Environment.Remove("DOTNET_DbgEnableMiniDump"); @@ -116,10 +116,8 @@ static void RunExternalProcess(string unhandledType, string assembly) [Fact] public static void TestEntryPoint() { - RunExternalProcess("main", "unhandled.dll"); - RunExternalProcess("foreign", "unhandled.dll"); - File.Delete(Path.Combine(Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location), "dependencytodelete.dll")); - RunExternalProcess("missingdependency", "unhandledmissingdependency.dll"); + RunExternalProcess("main"); + RunExternalProcess("foreign"); } } } diff --git a/src/tests/baseservices/exceptions/unhandled/unhandledTester.csproj b/src/tests/baseservices/exceptions/unhandled/unhandledTester.csproj index fff804f1924b85..99e154e74cd6de 100644 --- a/src/tests/baseservices/exceptions/unhandled/unhandledTester.csproj +++ b/src/tests/baseservices/exceptions/unhandled/unhandledTester.csproj @@ -17,10 +17,5 @@ Content Always - - false - Content - Always - diff --git a/src/tests/baseservices/exceptions/unhandled/unhandledmissingdependency.cs b/src/tests/baseservices/exceptions/unhandled/unhandledmissingdependency.cs deleted file mode 100644 index 4863cafae875f5..00000000000000 --- a/src/tests/baseservices/exceptions/unhandled/unhandledmissingdependency.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Dependency; - -namespace DependencyTest -{ - internal class Program - { - static void Main(string[] args) - { - DependencyClass.Hello(); - } - } -} diff --git a/src/tests/baseservices/exceptions/unhandled/unhandledmissingdependency.csproj b/src/tests/baseservices/exceptions/unhandled/unhandledmissingdependency.csproj deleted file mode 100644 index d388263ca3bcb2..00000000000000 --- a/src/tests/baseservices/exceptions/unhandled/unhandledmissingdependency.csproj +++ /dev/null @@ -1,19 +0,0 @@ - - - - true - false - BuildOnly - true - true - - - - - - - - - - - diff --git a/src/tests/build.proj b/src/tests/build.proj index d41ed76b98276a..3592d7c0761313 100644 --- a/src/tests/build.proj +++ b/src/tests/build.proj @@ -543,7 +543,6 @@ <_ConfigurationProperties>/p:TargetOS=$(TargetOS) /p:TargetArchitecture=$(TargetArchitecture) /p:Configuration=$(Configuration) /p:CrossBuild=$(CrossBuild) - <_ConfigurationProperties Condition="'$(UseLocalAppHostPack)' == 'true'">$(_ConfigurationProperties) -p:EnableAppHostPackDownload=false -p:EnableTargetingPackDownload=false -p:EnableRuntimePackDownload=false "$(DotNetTool)" restore $(RestoreProj) $(PackageVersionArg) /p:SetTFMForRestore=true $(_ConfigurationProperties) diff --git a/src/tests/issues.targets b/src/tests/issues.targets index 01b2f9aab129d4..9efbe150a7cd1c 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -75,7 +75,7 @@ https://github.com/dotnet/runtime/issues/88586 - + https://github.com/dotnet/runtime/issues/93631 @@ -1158,6 +1158,10 @@ Dynamic code generation is not supported on this platform + + + https://github.com/dotnet/runtime/issues/93631 + diff --git a/src/tests/nativeaot/CustomMain/CustomMain.cs b/src/tests/nativeaot/CustomMain/CustomMain.cs index 97db548fe54d17..914809961d778d 100644 --- a/src/tests/nativeaot/CustomMain/CustomMain.cs +++ b/src/tests/nativeaot/CustomMain/CustomMain.cs @@ -24,7 +24,7 @@ static Program() Console.WriteLine("hello from static constructor"); } - [UnmanagedCallersOnly(EntryPoint = "IncrementExitCode")] + [UnmanagedCallersOnly(EntryPoint = "IncrementExitCode", CallConvs = new Type[] { typeof(CallConvCdecl) })] static void IncrementExitCode(int amount) { s_exitCode += amount; diff --git a/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.csproj b/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.csproj index 1fd766631b4a98..0ea909b29cb70b 100644 --- a/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.csproj +++ b/src/tests/nativeaot/SmokeTests/ControlFlowGuard/ControlFlowGuard.csproj @@ -5,8 +5,7 @@ true Guard true - - true + true diff --git a/src/tests/nativeaot/SmokeTests/DynamicGenerics/universal_generics.cs b/src/tests/nativeaot/SmokeTests/DynamicGenerics/universal_generics.cs index 85fc88fb9058df..12415d8ec581ad 100644 --- a/src/tests/nativeaot/SmokeTests/DynamicGenerics/universal_generics.cs +++ b/src/tests/nativeaot/SmokeTests/DynamicGenerics/universal_generics.cs @@ -461,9 +461,7 @@ public class UnmanagedByRef : Base where T : struct, IGetValue [MethodImpl(MethodImplOptions.NoInlining)] public unsafe void TestAsPointer(T x) { -#pragma warning disable CS8500 // takes address of managed type - IntPtr unsafeValuePtr = (IntPtr)&x; -#pragma warning restore CS8500 + IntPtr unsafeValuePtr = (IntPtr)Unsafe.AsPointer(ref x); GC.Collect(); GC.Collect(); GC.Collect(); @@ -489,7 +487,6 @@ public unsafe void TestGeneralFunction(T x) [MethodImpl(MethodImplOptions.NoInlining)] unsafe IntPtr someFuncWithByRefArgs(ref T x) { - // Unsafe.AsPointer is safe since the reference is expected to be pinned return (IntPtr)Unsafe.AsPointer(ref x); } diff --git a/src/tests/nativeaot/SmokeTests/Preinitialization/Preinitialization.cs b/src/tests/nativeaot/SmokeTests/Preinitialization/Preinitialization.cs index 78223dca967fe4..d08e87bbaa76e3 100644 --- a/src/tests/nativeaot/SmokeTests/Preinitialization/Preinitialization.cs +++ b/src/tests/nativeaot/SmokeTests/Preinitialization/Preinitialization.cs @@ -404,15 +404,7 @@ public ReferenceType(int intValue, double doubleValue) public static void Run() { - if (RuntimeInformation.ProcessArchitecture is Architecture.Arm or Architecture.Wasm) - { - // Because of the double field, this is not preinitialized - Assert.IsLazyInitialized(typeof(TestReferenceTypeAllocation)); - } - else - { - Assert.IsPreinitialized(typeof(TestReferenceTypeAllocation)); - } + Assert.IsPreinitialized(typeof(TestReferenceTypeAllocation)); Assert.AreEqual(12345, s_referenceType.IntValue); Assert.AreEqual(3.14159, s_referenceType.DoubleValue); } @@ -1059,13 +1051,11 @@ public static void Run() int val = AccessCookie(); Assert.AreEqual(42, val); - val = (int)typeof(ClassWithTemplate<>).MakeGenericType(GetC2()).GetField("Cookie").GetValue(null); + val = (int)typeof(ClassWithTemplate<>).MakeGenericType(typeof(C2)).GetField("Cookie").GetValue(null); Assert.AreEqual(42, val); - static Type GetC2() => typeof(C2); - val = (int)typeof(TestSharedCode).GetMethod(nameof(AccessCookie)).MakeGenericMethod(GetC3()).Invoke(null, Array.Empty()); + val = (int)typeof(TestSharedCode).GetMethod(nameof(AccessCookie)).MakeGenericMethod(typeof(C3)).Invoke(null, Array.Empty()); Assert.AreEqual(42, val); - static Type GetC3() => typeof(C3); } { @@ -1073,15 +1063,13 @@ public static void Run() object val = AccessArray(); Assert.AreEqual(int.MaxValue, GC.GetGeneration(val)); - val = typeof(ClassWithTemplate<>).MakeGenericType(GetC4()).GetField("Array").GetValue(null); + val = typeof(ClassWithTemplate<>).MakeGenericType(typeof(C4)).GetField("Array").GetValue(null); Assert.AreEqual(0, GC.GetGeneration(val)); Assert.AreEqual(nameof(C4), val.GetType().GetElementType().Name); - static Type GetC4() => typeof(C4); - val = typeof(TestSharedCode).GetMethod(nameof(AccessArray)).MakeGenericMethod(GetC5()).Invoke(null, Array.Empty()); + val = typeof(TestSharedCode).GetMethod(nameof(AccessArray)).MakeGenericMethod(typeof(C5)).Invoke(null, Array.Empty()); Assert.AreEqual(0, GC.GetGeneration(val)); Assert.AreEqual(nameof(C5), val.GetType().GetElementType().Name); - static Type GetC5() => typeof(C5); } } } diff --git a/src/tests/nativeaot/SmokeTests/Reflection/Reflection.cs b/src/tests/nativeaot/SmokeTests/Reflection/Reflection.cs index 27f332cbdd14c6..c000d2687b57fd 100644 --- a/src/tests/nativeaot/SmokeTests/Reflection/Reflection.cs +++ b/src/tests/nativeaot/SmokeTests/Reflection/Reflection.cs @@ -1509,7 +1509,6 @@ public static void Run() try { Type.GetType("System.Span`1[[System.Byte, System.Runtime]][], System.Runtime"); - Type.GetType("System.Collections.Generic.Dictionary`2[System.String]"); } catch { } @@ -1824,10 +1823,9 @@ public static void Run() typeof(GenericType<>).MakeGenericType(typeof(object)).GetMethod("Gimme"); } - var t = (Type)s_type.MakeGenericType(GetDouble()).GetMethod("Gimme").Invoke(null, Array.Empty()); + var t = (Type)s_type.MakeGenericType(typeof(double)).GetMethod("Gimme").Invoke(null, Array.Empty()); if (t != typeof(double)) throw new Exception(); - static Type GetDouble() => typeof(double); } } @@ -2433,10 +2431,9 @@ class Atom { } public static void Run() { - var mi = typeof(TestMdArrayLoad).GetMethod(nameof(MakeMdArray)).MakeGenericMethod(GetAtom()); + var mi = typeof(TestMdArrayLoad).GetMethod(nameof(MakeMdArray)).MakeGenericMethod(typeof(Atom)); if ((Type)mi.Invoke(null, Array.Empty()) != typeof(Atom[,,])) throw new Exception(); - static Type GetAtom() => typeof(Atom); } } @@ -2448,10 +2445,9 @@ class Atom { } public static void Run() { - var mi = typeof(TestByRefTypeLoad).GetMethod(nameof(MakeFnPtrType)).MakeGenericMethod(GetAtom()); + var mi = typeof(TestByRefTypeLoad).GetMethod(nameof(MakeFnPtrType)).MakeGenericMethod(typeof(Atom)); if ((Type)mi.Invoke(null, Array.Empty()) != typeof(delegate*)) throw new Exception(); - static Type GetAtom() => typeof(Atom); } } diff --git a/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/Dataflow.cs b/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/Dataflow.cs index a75ca2bcd0c4f0..5efcd99d9ca643 100644 --- a/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/Dataflow.cs +++ b/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/Dataflow.cs @@ -26,8 +26,6 @@ public static int Run() TestDynamicDependency.Run(); TestDynamicDependencyWithGenerics.Run(); TestObjectGetTypeDataflow.Run(); - TestMakeGenericDataflow.Run(); - TestMakeGenericDataflowInvalid.Run(); TestMarshalIntrinsics.Run(); Regression97758.Run(); @@ -590,117 +588,6 @@ public static void Run() } } - class TestMakeGenericDataflow - { - class Gen1 - { - public static void Bridge() { } - } - - class Gen1 - { - public static void Bridge() { } - } - - - class Gen2 - { - public static void Bridge() { } - } - - class Gen2 - { - public static void Bridge() { } - } - - struct MyStruct { } - - static void DoBridgeT1() => typeof(Gen1<,>).MakeGenericType([typeof(T), typeof(U)]).GetMethod(nameof(Gen1.Bridge)).Invoke(null, []); - - static void DoBridgeT2() => typeof(Gen2<>).MakeGenericType([typeof(MyStruct)]).GetMethod(nameof(Gen2.Bridge)).Invoke(null, []); - - static void DoBridgeM1() => typeof(Gen1).GetMethod(nameof(Gen1.Bridge)).MakeGenericMethod([typeof(T), typeof(U)]).Invoke(null, []); - - static void DoBridgeM2() => typeof(Gen2).GetMethod(nameof(Gen2.Bridge)).MakeGenericMethod([typeof(MyStruct)]).Invoke(null, []); - - public static void Run() - { - DoBridgeT1(); - DoBridgeT1(); - DoBridgeT1(); - - DoBridgeT2(); - DoBridgeT2(); - - DoBridgeM1(); - DoBridgeM1(); - DoBridgeM1(); - - DoBridgeM2(); - DoBridgeM2(); - - typeof(Gen1<,>).MakeGenericType([typeof(float), typeof(string)]).GetMethod(nameof(Gen1.Bridge)).Invoke(null, []); - typeof(Gen2<>).MakeGenericType([typeof(MyStruct)]).GetMethod(nameof(Gen2.Bridge)).Invoke(null, []); - typeof(Gen1).GetMethod(nameof(Gen1.Bridge)).MakeGenericMethod([typeof(float), typeof(string)]).Invoke(null, []); - typeof(Gen2).GetMethod(nameof(Gen2.Bridge)).MakeGenericMethod([typeof(MyStruct)]).Invoke(null, []); - } - } - - class TestMakeGenericDataflowInvalid - { - class Gen { } - - class Gen - { - public static void Bridge() { } - } - - public static void Run() - { - try - { - typeof(Gen<>).MakeGenericType(null); - } - catch (ArgumentException) { } - - try - { - typeof(Gen<>).MakeGenericType([]); - } - catch (ArgumentException) { } - - try - { - typeof(Gen<>).MakeGenericType([typeof(float), typeof(double)]); - } - catch (ArgumentException) { } - - try - { - typeof(Gen<>).MakeGenericType([typeof(Gen<>)]); - } - catch (ArgumentException) { } - - try - { - typeof(Gen).GetMethod("Bridge").MakeGenericMethod(null); - } - catch (ArgumentException) { } - - try - { - typeof(Gen).GetMethod("Bridge").MakeGenericMethod([]); - } - catch (ArgumentException) { } - - try - { - typeof(Gen).GetMethod("Bridge").MakeGenericMethod([typeof(float), typeof(double)]); - } - catch (ArgumentException) { } - } - } - class TestMarshalIntrinsics { [StructLayout(LayoutKind.Sequential)] diff --git a/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/DeadCodeElimination.cs b/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/DeadCodeElimination.cs index bf4b8639cefa0c..d54c801ccee3e9 100644 --- a/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/DeadCodeElimination.cs +++ b/src/tests/nativeaot/SmokeTests/TrimmingBehaviors/DeadCodeElimination.cs @@ -340,8 +340,6 @@ public static void Run() class TestTypeEquals { - sealed class Gen { } - sealed class Never { } static Type s_type = null; @@ -352,9 +350,6 @@ public static void Run() // despite the typeof Console.WriteLine(s_type == typeof(Never)); - // This was a compiler crash - Console.WriteLine(typeof(object) == typeof(Gen<>)); - #if !DEBUG ThrowIfPresent(typeof(TestTypeEquals), nameof(Never)); #endif diff --git a/src/tests/nativeaot/SmokeTests/UnitTests/Devirtualization.cs b/src/tests/nativeaot/SmokeTests/UnitTests/Devirtualization.cs index c69c013150941b..f2c70d7b1ee60a 100644 --- a/src/tests/nativeaot/SmokeTests/UnitTests/Devirtualization.cs +++ b/src/tests/nativeaot/SmokeTests/UnitTests/Devirtualization.cs @@ -111,8 +111,7 @@ public static void Run() TestIntf1((IIntf1)new Intf1CastableImpl(), 456); TestIntf2(new Intf2Impl1(), 123); - TestIntf2((IIntf2)Activator.CreateInstance(typeof(Intf2Impl2<>).MakeGenericType(GetObject())), 456); - static Type GetObject() => typeof(object); + TestIntf2((IIntf2)Activator.CreateInstance(typeof(Intf2Impl2<>).MakeGenericType(typeof(object))), 456); } } diff --git a/src/tests/nativeaot/SmokeTests/UnitTests/Generics.cs b/src/tests/nativeaot/SmokeTests/UnitTests/Generics.cs index a9d3d44c758dde..145d7b3d0c3ea9 100644 --- a/src/tests/nativeaot/SmokeTests/UnitTests/Generics.cs +++ b/src/tests/nativeaot/SmokeTests/UnitTests/Generics.cs @@ -1976,10 +1976,8 @@ private static void TestDynamicStaticFields() Foo.s_floatField = 12.34f; Foo.s_longField1 = 0x1111; - var fooDynamicOfClassType = typeof(Foo<>).MakeGenericType(GetClassType()).GetTypeInfo(); - static Type GetClassType() => typeof(ClassType); - var fooDynamicOfClassType2 = typeof(Foo<>).MakeGenericType(GetClassType2()).GetTypeInfo(); - static Type GetClassType2() => typeof(ClassType2); + var fooDynamicOfClassType = typeof(Foo<>).MakeGenericType(typeof(ClassType)).GetTypeInfo(); + var fooDynamicOfClassType2 = typeof(Foo<>).MakeGenericType(typeof(ClassType2)).GetTypeInfo(); FieldInfo fi = fooDynamicOfClassType.GetDeclaredField("s_intField"); FieldInfo fi2 = fooDynamicOfClassType2.GetDeclaredField("s_intField"); @@ -2033,8 +2031,7 @@ private static void TestDynamicInvokeStubs() heh2.GenericVirtualMethod(new Program(), "ayy"); // Simple method invocation - var dynamicBaseOfString = typeof(DynamicBase<>).MakeGenericType(GetString()); - static Type GetString() => typeof(string); + var dynamicBaseOfString = typeof(DynamicBase<>).MakeGenericType(typeof(string)); object obj = Activator.CreateInstance(dynamicBaseOfString); { var simpleMethod = dynamicBaseOfString.GetTypeInfo().GetDeclaredMethod("SimpleMethod"); @@ -2057,7 +2054,7 @@ private static void TestDynamicInvokeStubs() } { - var dynamicDerivedOfString = typeof(DynamicDerived<>).MakeGenericType(GetString()); + var dynamicDerivedOfString = typeof(DynamicDerived<>).MakeGenericType(typeof(string)); object dynamicDerivedObj = Activator.CreateInstance(dynamicDerivedOfString); var virtualMethodDynamicDerived = dynamicDerivedOfString.GetTypeInfo().GetDeclaredMethod("VirtualMethod"); string result = (string)virtualMethodDynamicDerived.Invoke(dynamicDerivedObj, new[] { "fad" }); @@ -2066,7 +2063,7 @@ private static void TestDynamicInvokeStubs() // Test generic method invocation { - var genericMethod = dynamicBaseOfString.GetTypeInfo().GetDeclaredMethod("GenericMethod").MakeGenericMethod(new[] { GetString() }); + var genericMethod = dynamicBaseOfString.GetTypeInfo().GetDeclaredMethod("GenericMethod").MakeGenericMethod(new[] { typeof(string) }); string result = (string)genericMethod.Invoke(obj, new[] { "hey", "hello" }); Verify("System.Stringhello", result); @@ -2075,15 +2072,15 @@ private static void TestDynamicInvokeStubs() // Test GVM invocation { var genericMethod = dynamicBaseOfString.GetTypeInfo().GetDeclaredMethod("GenericVirtualMethod"); - genericMethod = genericMethod.MakeGenericMethod(new[] { GetString() }); + genericMethod = genericMethod.MakeGenericMethod(new[] { typeof(string) }); string result = (string)genericMethod.Invoke(obj, new[] { "hey", "hello" }); Verify("DynamicBaseSystem.Stringhello", result); } { - var dynamicDerivedOfString = typeof(DynamicDerived<>).MakeGenericType(GetString()); + var dynamicDerivedOfString = typeof(DynamicDerived<>).MakeGenericType(typeof(string)); object dynamicDerivedObj = Activator.CreateInstance(dynamicDerivedOfString); - var virtualMethodDynamicDerived = dynamicDerivedOfString.GetTypeInfo().GetDeclaredMethod("GenericVirtualMethod").MakeGenericMethod(new[] { GetString() }); + var virtualMethodDynamicDerived = dynamicDerivedOfString.GetTypeInfo().GetDeclaredMethod("GenericVirtualMethod").MakeGenericMethod(new[] { typeof(string) }); string result = (string)virtualMethodDynamicDerived.Invoke(dynamicDerivedObj, new[] { "hey", "fad" }); Verify("DynamicDerivedSystem.Stringfad", result); } diff --git a/src/tests/nativeaot/SmokeTests/UnitTests/Interfaces.cs b/src/tests/nativeaot/SmokeTests/UnitTests/Interfaces.cs index b5023e8c84163e..4f34959be84b3b 100644 --- a/src/tests/nativeaot/SmokeTests/UnitTests/Interfaces.cs +++ b/src/tests/nativeaot/SmokeTests/UnitTests/Interfaces.cs @@ -1583,13 +1583,11 @@ class Gen where T : IFoo [UnconditionalSuppressMessage("AOT", "IL3050", Justification = "MakeGenericType - Intentional")] public static void Run() { - var r = (string)typeof(Gen<>).MakeGenericType(GetBaz()).GetMethod("GrabCookie").Invoke(null, Array.Empty()); - static Type GetBaz() => typeof(Baz); + var r = (string)typeof(Gen<>).MakeGenericType(typeof(Baz)).GetMethod("GrabCookie").Invoke(null, Array.Empty()); if (r != "IBar") throw new Exception(r); - r = (string)typeof(Gen<>).MakeGenericType(GetIBar()).GetMethod("GrabCookie").Invoke(null, Array.Empty()); - static Type GetIBar() => typeof(IBar); + r = (string)typeof(Gen<>).MakeGenericType(typeof(IBar)).GetMethod("GrabCookie").Invoke(null, Array.Empty()); if (r != "IBar") throw new Exception(r); } @@ -1622,18 +1620,15 @@ class Gen where T : IFoo [UnconditionalSuppressMessage("AOT", "IL3050", Justification = "MakeGenericType - Intentional")] public static void Run() { - Activator.CreateInstance(typeof(Baz<>).MakeGenericType(GetAtom1())); + Activator.CreateInstance(typeof(Baz<>).MakeGenericType(typeof(Atom1))); - var r = (string)typeof(Gen<>).MakeGenericType(typeof(Baz<>).MakeGenericType(GetAtom1())).GetMethod("GrabCookie").Invoke(null, Array.Empty()); + var r = (string)typeof(Gen<>).MakeGenericType(typeof(Baz<>).MakeGenericType(typeof(Atom1))).GetMethod("GrabCookie").Invoke(null, Array.Empty()); if (r != "IBar") throw new Exception(r); - r = (string)typeof(Gen<>).MakeGenericType(typeof(IBar<>).MakeGenericType(GetAtom2())).GetMethod("GrabCookie").Invoke(null, Array.Empty()); + r = (string)typeof(Gen<>).MakeGenericType(typeof(IBar<>).MakeGenericType(typeof(Atom2))).GetMethod("GrabCookie").Invoke(null, Array.Empty()); if (r != "IBar") throw new Exception(r); - - static Type GetAtom1() => typeof(Atom1); - static Type GetAtom2() => typeof(Atom2); } } diff --git a/src/tests/profiler/multiple/multiple.cs b/src/tests/profiler/multiple/multiple.cs index 0d686ae0691a94..aa0388fa0eb293 100644 --- a/src/tests/profiler/multiple/multiple.cs +++ b/src/tests/profiler/multiple/multiple.cs @@ -35,9 +35,9 @@ public static int RunTest(String[] args) } Console.WriteLine("Waiting for profilers to all detach"); - if (!_profilerDone.WaitOne(TimeSpan.FromMinutes(10))) + if (!_profilerDone.WaitOne(TimeSpan.FromMinutes(5))) { - throw new Exception("Test timed out waiting for the profilers to set the callback, test will fail."); + Console.WriteLine("Profiler did not set the callback, test will fail."); } return 100; diff --git a/src/tests/profiler/native/multiple/multiple.cpp b/src/tests/profiler/native/multiple/multiple.cpp index c36e4a8efa491f..6b8fba571858aa 100644 --- a/src/tests/profiler/native/multiple/multiple.cpp +++ b/src/tests/profiler/native/multiple/multiple.cpp @@ -24,6 +24,7 @@ HRESULT MultiplyLoaded::InitializeCommon(IUnknown* pICorProfilerInfoUnk) Profiler::Initialize(pICorProfilerInfoUnk); HRESULT hr = S_OK; + printf("Setting exception mask\n"); if (FAILED(hr = pCorProfilerInfo->SetEventMask2(COR_PRF_MONITOR_EXCEPTIONS, 0))) { _failures++; @@ -36,13 +37,11 @@ HRESULT MultiplyLoaded::InitializeCommon(IUnknown* pICorProfilerInfoUnk) HRESULT MultiplyLoaded::Initialize(IUnknown* pICorProfilerInfoUnk) { - printf("MultiplyLoaded::Initialize\n"); return InitializeCommon(pICorProfilerInfoUnk); } HRESULT MultiplyLoaded::InitializeForAttach(IUnknown* pICorProfilerInfoUnk, void* pvClientData, UINT cbClientData) { - printf("MultiplyLoaded::InitializeForAttach\n"); return InitializeCommon(pICorProfilerInfoUnk); } @@ -57,8 +56,8 @@ HRESULT MultiplyLoaded::ProfilerDetachSucceeded() ++_detachCount; printf("ProfilerDetachSucceeded _detachCount=%d\n", _detachCount.load()); - if (_detachCount == MAX_PROFILERS - && _exceptionThrownSeenCount >= MAX_PROFILERS + if (_detachCount == (MAX_PROFILERS - 1) + && _exceptionThrownSeenCount >= (MAX_PROFILERS - 1) && _failures == 0) { printf("PROFILER TEST PASSES\n"); @@ -70,7 +69,9 @@ HRESULT MultiplyLoaded::ProfilerDetachSucceeded() HRESULT MultiplyLoaded::ExceptionThrown(ObjectID thrownObjectId) { - printf("MultiplyLoaded::ExceptionThrown, number seen = %d\n", ++_exceptionThrownSeenCount); + int seen = _exceptionThrownSeenCount++; + + printf("MultiplyLoaded::ExceptionThrown, number seen = %d\n", seen); thread detachThread([&]() { diff --git a/src/tests/readytorun/tests/genericsload/usegenericfield.csproj b/src/tests/readytorun/tests/genericsload/usegenericfield.csproj index d543cd9e47699d..67b266b8658faa 100644 --- a/src/tests/readytorun/tests/genericsload/usegenericfield.csproj +++ b/src/tests/readytorun/tests/genericsload/usegenericfield.csproj @@ -2,6 +2,7 @@ true + 1 1 diff --git a/src/tools/illink/illink.sln b/src/tools/illink/illink.sln index 9b8d904942baaf..87e99d208c7fb6 100644 --- a/src/tools/illink/illink.sln +++ b/src/tools/illink/illink.sln @@ -223,7 +223,6 @@ Global SolutionGuid = {E43A3901-42B0-48CA-BB36-5CD40A99A6EE} EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution - test\Trimming.Tests.Shared\Trimming.Tests.Shared.projitems*{400a1561-b6b6-482d-9e4c-3ddaede5bd07}*SharedItemsImports = 5 src\ILLink.Shared\ILLink.Shared.projitems*{dd28e2b1-057b-4b4d-a04d-b2ebd9e76e46}*SharedItemsImports = 5 src\ILLink.Shared\ILLink.Shared.projitems*{f1a44a78-34ee-408b-8285-9a26f0e7d4f2}*SharedItemsImports = 5 src\ILLink.Shared\ILLink.Shared.projitems*{ff598e93-8e9e-4091-9f50-61a7572663ae}*SharedItemsImports = 13 diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksVisitor.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureCheckVisitor.cs similarity index 60% rename from src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksVisitor.cs rename to src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureCheckVisitor.cs index 6a927f74fd15b7..707294718fda32 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksVisitor.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureCheckVisitor.cs @@ -24,7 +24,7 @@ namespace ILLink.RoslynAnalyzer.DataFlow // (a set features that are checked to be enabled or disabled). // The visitor takes a LocalDataFlowState as an argument, allowing for checks that // depend on the current dataflow state. - public class FeatureChecksVisitor : OperationVisitor + public class FeatureChecksVisitor : OperationVisitor { DataFlowAnalyzerContext _dataFlowAnalyzerContext; @@ -33,48 +33,32 @@ public FeatureChecksVisitor (DataFlowAnalyzerContext dataFlowAnalyzerContext) _dataFlowAnalyzerContext = dataFlowAnalyzerContext; } - public override FeatureChecksValue DefaultVisit (IOperation operation, StateValue state) - { - // Visiting a non-understood pattern should return the empty set of features, which will - // prevent this check from acting as a guard for any feature. - return FeatureChecksValue.None; - } - - public override FeatureChecksValue VisitArgument (IArgumentOperation operation, StateValue state) + public override FeatureChecksValue? VisitArgument (IArgumentOperation operation, StateValue state) { return Visit (operation.Value, state); } - public override FeatureChecksValue VisitPropertyReference (IPropertyReferenceOperation operation, StateValue state) + public override FeatureChecksValue? VisitPropertyReference (IPropertyReferenceOperation operation, StateValue state) { - // A single property may serve as a feature check for multiple features. - FeatureChecksValue featureChecks = FeatureChecksValue.None; foreach (var analyzer in _dataFlowAnalyzerContext.EnabledRequiresAnalyzers) { - if (analyzer.IsFeatureGuard (operation.Property, _dataFlowAnalyzerContext.Compilation)) { - var featureCheck = new FeatureChecksValue (analyzer.RequiresAttributeFullyQualifiedName); - featureChecks = featureChecks.And (featureCheck); + if (analyzer.IsRequiresCheck (_dataFlowAnalyzerContext.Compilation, operation.Property)) { + return new FeatureChecksValue (analyzer.FeatureName); } } - return featureChecks; + return null; } - public override FeatureChecksValue VisitUnaryOperator (IUnaryOperation operation, StateValue state) + public override FeatureChecksValue? VisitUnaryOperator (IUnaryOperation operation, StateValue state) { if (operation.OperatorKind is not UnaryOperatorKind.Not) - return FeatureChecksValue.None; - - FeatureChecksValue context = Visit (operation.Operand, state); - return context.Negate (); - } + return null; - public override FeatureChecksValue VisitLiteral (ILiteralOperation operation, StateValue state) - { - // 'false' can guard any feature - if (GetConstantBool (operation.ConstantValue) is false) - return FeatureChecksValue.All; + FeatureChecksValue? context = Visit (operation.Operand, state); + if (context == null) + return null; - return FeatureChecksValue.None; + return context.Value.Negate (); } public bool? GetLiteralBool (IOperation operation) @@ -93,7 +77,7 @@ public override FeatureChecksValue VisitLiteral (ILiteralOperation operation, St return value; } - public override FeatureChecksValue VisitBinaryOperator (IBinaryOperation operation, StateValue state) + public override FeatureChecksValue? VisitBinaryOperator (IBinaryOperation operation, StateValue state) { bool expectEqual; switch (operation.OperatorKind) { @@ -104,32 +88,36 @@ public override FeatureChecksValue VisitBinaryOperator (IBinaryOperation operati expectEqual = false; break; default: - return FeatureChecksValue.None; + return null; } if (GetLiteralBool (operation.LeftOperand) is bool leftBool) { - FeatureChecksValue rightValue = Visit (operation.RightOperand, state); + if (Visit (operation.RightOperand, state) is not FeatureChecksValue rightValue) + return null; return leftBool == expectEqual ? rightValue : rightValue.Negate (); } if (GetLiteralBool (operation.RightOperand) is bool rightBool) { - FeatureChecksValue leftValue = Visit (operation.LeftOperand, state); + if (Visit (operation.LeftOperand, state) is not FeatureChecksValue leftValue) + return null; return rightBool == expectEqual ? leftValue : leftValue.Negate (); } - return FeatureChecksValue.None; + return null; } - public override FeatureChecksValue VisitIsPattern (IIsPatternOperation operation, StateValue state) + public override FeatureChecksValue? VisitIsPattern (IIsPatternOperation operation, StateValue state) { if (GetExpectedValueFromPattern (operation.Pattern) is not bool patternValue) - return FeatureChecksValue.None; + return null; + + if (Visit (operation.Value, state) is not FeatureChecksValue value) + return null; - FeatureChecksValue value = Visit (operation.Value, state); return patternValue ? value : value.Negate (); diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksValue.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksValue.cs index 028628f2dd5934..268833431274a0 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksValue.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/FeatureChecksValue.cs @@ -13,15 +13,11 @@ namespace ILLink.RoslynAnalyzer.DataFlow // For now, this is only designed to track the built-in "features"/"capabilities" // like RuntimeFeatures.IsDynamicCodeSupported, where a true return value // indicates that a feature/capability is available. - public record struct FeatureChecksValue : INegate, IDeepCopyValue + public record struct FeatureChecksValue : INegate { public ValueSet EnabledFeatures; public ValueSet DisabledFeatures; - public static readonly FeatureChecksValue All = new FeatureChecksValue (ValueSet.Unknown, ValueSet.Empty); - - public static readonly FeatureChecksValue None = new FeatureChecksValue (ValueSet.Empty, ValueSet.Empty); - public FeatureChecksValue (string enabledFeature) { EnabledFeatures = new ValueSet (enabledFeature); @@ -52,10 +48,5 @@ public FeatureChecksValue Negate () { return new FeatureChecksValue (DisabledFeatures.DeepCopy (), EnabledFeatures.DeepCopy ()); } - - public FeatureChecksValue DeepCopy () - { - return new FeatureChecksValue (EnabledFeatures.DeepCopy (), DisabledFeatures.DeepCopy ()); - } } } diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/LocalDataFlowVisitor.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/LocalDataFlowVisitor.cs index 84065ecc25fe8f..bbaeff53f0c591 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/LocalDataFlowVisitor.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlow/LocalDataFlowVisitor.cs @@ -88,12 +88,12 @@ public LocalDataFlowVisitor ( return null; var branchValue = Visit (branchValueOperation, state); - TConditionValue conditionValue = GetConditionValue (branchValueOperation, state); + if (block.Block.ConditionKind != ControlFlowConditionKind.None) { // BranchValue may represent a value used in a conditional branch to the ConditionalSuccessor. // If so, give the analysis an opportunity to model the checked condition, and return the model // of the condition back to the generic analysis. It will be applied to the state of each outgoing branch. - return conditionValue; + return GetConditionValue (branchValueOperation, state); } // If not, the BranchValue represents a return or throw value associated with the FallThroughSuccessor of this block. @@ -118,13 +118,10 @@ public LocalDataFlowVisitor ( // We don't want the return operation because this might have multiple possible return values in general. var current = state.Current; HandleReturnValue (branchValue, branchValueOperation, in current.Context); - // Must be called for every return value even if it did not return an understood condition, - // because the non-understood conditions will produce warnings for FeatureGuard properties. - HandleReturnConditionValue (conditionValue, branchValueOperation); return null; } - public abstract TConditionValue GetConditionValue ( + public abstract TConditionValue? GetConditionValue ( IOperation branchValueOperation, LocalDataFlowState state); @@ -149,10 +146,6 @@ public abstract void HandleReturnValue ( IOperation operation, in TContext context); - public abstract void HandleReturnConditionValue ( - TConditionValue returnConditionValue, - IOperation branchValueOperation); - // This is called for any method call, which includes: // - Normal invocation operation // - Accessing property value - which is treated as a call to the getter @@ -783,7 +776,9 @@ TValue HandleMethodCallHelper ( // Get the condition value that is being asserted. If the attribute is DoesNotReturnIf(true), // the condition value needs to be negated so that we can assert the false condition. - TConditionValue conditionValue = GetConditionValue (argumentOperation, state); + if (GetConditionValue (argumentOperation, state) is not TConditionValue conditionValue) + continue; + var current = state.Current; ApplyCondition ( doesNotReturnIfConditionValue == false diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlowAnalyzerContext.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/DataflowAnalyzerContext.cs similarity index 100% rename from src/tools/illink/src/ILLink.RoslynAnalyzer/DataFlowAnalyzerContext.cs rename to src/tools/illink/src/ILLink.RoslynAnalyzer/DataflowAnalyzerContext.cs diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/DynamicallyAccessedMembersAnalyzer.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/DynamicallyAccessedMembersAnalyzer.cs index c7f7eb2eb6aa0b..173bb667a4d178 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/DynamicallyAccessedMembersAnalyzer.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/DynamicallyAccessedMembersAnalyzer.cs @@ -21,7 +21,6 @@ public class DynamicallyAccessedMembersAnalyzer : DiagnosticAnalyzer internal const string DynamicallyAccessedMembersAttribute = nameof (DynamicallyAccessedMembersAttribute); public const string attributeArgument = "attributeArgument"; public const string FullyQualifiedDynamicallyAccessedMembersAttribute = "System.Diagnostics.CodeAnalysis." + DynamicallyAccessedMembersAttribute; - public const string FullyQualifiedFeatureGuardAttribute = "System.Diagnostics.CodeAnalysis.FeatureGuardAttribute"; public static Lazy> RequiresAnalyzers { get; } = new Lazy> (GetRequiresAnalyzers); static ImmutableArray GetRequiresAnalyzers () => ImmutableArray.Create ( @@ -52,8 +51,6 @@ public static ImmutableArray GetSupportedDiagnostics () diagDescriptorsArrayBuilder.Add (DiagnosticDescriptors.GetDiagnosticDescriptor (DiagnosticId.UnrecognizedTypeNameInTypeGetType)); diagDescriptorsArrayBuilder.Add (DiagnosticDescriptors.GetDiagnosticDescriptor (DiagnosticId.UnrecognizedParameterInMethodCreateInstance)); diagDescriptorsArrayBuilder.Add (DiagnosticDescriptors.GetDiagnosticDescriptor (DiagnosticId.ParametersOfAssemblyCreateInstanceCannotBeAnalyzed)); - diagDescriptorsArrayBuilder.Add (DiagnosticDescriptors.GetDiagnosticDescriptor (DiagnosticId.ReturnValueDoesNotMatchFeatureGuards)); - diagDescriptorsArrayBuilder.Add (DiagnosticDescriptors.GetDiagnosticDescriptor (DiagnosticId.InvalidFeatureGuard)); foreach (var requiresAnalyzer in RequiresAnalyzers.Value) { foreach (var diagnosticDescriptor in requiresAnalyzer.SupportedDiagnostics) diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj b/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj index bc410523d5d716..7db05c1cde9afe 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj @@ -1,4 +1,4 @@ - + netstandard2.0 @@ -9,12 +9,6 @@ Latest $(NoWarn);CS8524 cs - - true diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/ISymbolExtensions.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/ISymbolExtensions.cs index 3f1903ba12ceec..7e830f7c6ecd4c 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/ISymbolExtensions.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/ISymbolExtensions.cs @@ -1,14 +1,9 @@ // Copyright (c) .NET Foundation and contributors. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. -using System.Collections.Immutable; -using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Text; using Microsoft.CodeAnalysis; -using ILLink.RoslynAnalyzer.DataFlow; -using ILLink.Shared.DataFlow; namespace ILLink.RoslynAnalyzer { @@ -39,14 +34,6 @@ internal static bool TryGetAttribute (this ISymbol member, string attributeName, return false; } - internal static IEnumerable GetAttributes (this ISymbol member, string attributeName) - { - foreach (var attr in member.GetAttributes ()) { - if (attr.AttributeClass is { } attrClass && attrClass.HasName (attributeName)) - yield return attr; - } - } - internal static DynamicallyAccessedMemberTypes GetDynamicallyAccessedMemberTypes (this ISymbol symbol) { if (!TryGetAttribute (symbol, DynamicallyAccessedMembersAnalyzer.DynamicallyAccessedMembersAttribute, out var dynamicallyAccessedMembers)) @@ -71,16 +58,6 @@ internal static DynamicallyAccessedMemberTypes GetDynamicallyAccessedMemberTypes return (DynamicallyAccessedMemberTypes) dynamicallyAccessedMembers.ConstructorArguments[0].Value!; } - internal static ValueSet GetFeatureGuardAnnotations (this IPropertySymbol propertySymbol) - { - HashSet featureSet = new (); - foreach (var featureGuardAttribute in propertySymbol.GetAttributes (DynamicallyAccessedMembersAnalyzer.FullyQualifiedFeatureGuardAttribute)) { - if (featureGuardAttribute.ConstructorArguments is [TypedConstant { Value: INamedTypeSymbol featureType }]) - featureSet.Add (featureType.GetDisplayName ()); - } - return featureSet.Count == 0 ? ValueSet.Empty : new ValueSet (featureSet); - } - internal static bool TryGetReturnAttribute (this IMethodSymbol member, string attributeName, [NotNullWhen (returnValue: true)] out AttributeData? attribute) { attribute = null; diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAnalyzerBase.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAnalyzerBase.cs index c31a69a24d24b1..d951404845cd41 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAnalyzerBase.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAnalyzerBase.cs @@ -5,9 +5,8 @@ using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Linq; -using ILLink.RoslynAnalyzer.DataFlow; using ILLink.Shared; -using ILLink.Shared.DataFlow; +using ILLink.RoslynAnalyzer.DataFlow; using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; @@ -20,7 +19,9 @@ public abstract class RequiresAnalyzerBase : DiagnosticAnalyzer { private protected abstract string RequiresAttributeName { get; } - internal abstract string RequiresAttributeFullyQualifiedName { get; } + internal abstract string FeatureName { get; } + + private protected abstract string RequiresAttributeFullyQualifiedName { get; } private protected abstract DiagnosticTargets AnalyzerDiagnosticTargets { get; } @@ -300,23 +301,7 @@ protected virtual bool CreateSpecialIncompatibleMembersDiagnostic ( // - false return value indicating that a feature is supported // - feature settings supplied by the project // - custom feature checks defined in library code - private protected virtual bool IsRequiresCheck (IPropertySymbol propertySymbol, Compilation compilation) => false; - - internal static bool IsAnnotatedFeatureGuard (IPropertySymbol propertySymbol, string featureName) - { - // Only respect FeatureGuardAttribute on static boolean properties. - if (!propertySymbol.IsStatic || propertySymbol.Type.SpecialType != SpecialType.System_Boolean || propertySymbol.SetMethod != null) - return false; - - ValueSet featureCheckAnnotations = propertySymbol.GetFeatureGuardAnnotations (); - return featureCheckAnnotations.Contains (featureName); - } - - internal bool IsFeatureGuard (IPropertySymbol propertySymbol, Compilation compilation) - { - return IsAnnotatedFeatureGuard (propertySymbol, RequiresAttributeFullyQualifiedName) - || IsRequiresCheck (propertySymbol, compilation); - } + internal virtual bool IsRequiresCheck (Compilation compilation, IPropertySymbol propertySymbol) => false; internal bool CheckAndCreateRequiresDiagnostic ( IOperation operation, @@ -327,7 +312,7 @@ internal bool CheckAndCreateRequiresDiagnostic ( [NotNullWhen (true)] out Diagnostic? diagnostic) { // Warnings are not emitted if the featureContext says the feature is available. - if (featureContext.IsEnabled (RequiresAttributeFullyQualifiedName)) { + if (featureContext.IsEnabled (FeatureName)) { diagnostic = null; return false; } diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAssemblyFilesAnalyzer.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAssemblyFilesAnalyzer.cs index 8949b249b35ecb..e8807896d93735 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAssemblyFilesAnalyzer.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresAssemblyFilesAnalyzer.cs @@ -36,7 +36,9 @@ public sealed class RequiresAssemblyFilesAnalyzer : RequiresAnalyzerBase private protected override string RequiresAttributeName => RequiresAssemblyFilesAttribute; - internal override string RequiresAttributeFullyQualifiedName => RequiresAssemblyFilesAttributeFullyQualifiedName; + internal override string FeatureName => "AssemblyFiles"; + + private protected override string RequiresAttributeFullyQualifiedName => RequiresAssemblyFilesAttributeFullyQualifiedName; private protected override DiagnosticTargets AnalyzerDiagnosticTargets => DiagnosticTargets.MethodOrConstructor | DiagnosticTargets.Property | DiagnosticTargets.Event; @@ -59,7 +61,7 @@ internal override bool IsAnalyzerEnabled (AnalyzerOptions options) return true; } - private protected override bool IsRequiresCheck (IPropertySymbol propertySymbol, Compilation compilation) + internal override bool IsRequiresCheck (Compilation compilation, IPropertySymbol propertySymbol) { // "IsAssemblyFilesSupported" is treated as a requires check for testing purposes only, and // is not officially-supported product behavior. diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresDynamicCodeAnalyzer.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresDynamicCodeAnalyzer.cs index 34bb7808d20314..5232ca9a9854e3 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresDynamicCodeAnalyzer.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresDynamicCodeAnalyzer.cs @@ -25,7 +25,9 @@ public sealed class RequiresDynamicCodeAnalyzer : RequiresAnalyzerBase private protected override string RequiresAttributeName => RequiresDynamicCodeAttribute; - internal override string RequiresAttributeFullyQualifiedName => FullyQualifiedRequiresDynamicCodeAttribute; + internal override string FeatureName => "DynamicCode"; + + private protected override string RequiresAttributeFullyQualifiedName => FullyQualifiedRequiresDynamicCodeAttribute; private protected override DiagnosticTargets AnalyzerDiagnosticTargets => DiagnosticTargets.MethodOrConstructor | DiagnosticTargets.Class; @@ -38,7 +40,7 @@ public sealed class RequiresDynamicCodeAnalyzer : RequiresAnalyzerBase internal override bool IsAnalyzerEnabled (AnalyzerOptions options) => options.IsMSBuildPropertyValueTrue (MSBuildPropertyOptionNames.EnableAotAnalyzer); - private protected override bool IsRequiresCheck (IPropertySymbol propertySymbol, Compilation compilation) { + internal override bool IsRequiresCheck (Compilation compilation, IPropertySymbol propertySymbol) { var runtimeFeaturesType = compilation.GetTypeByMetadataName ("System.Runtime.CompilerServices.RuntimeFeature"); if (runtimeFeaturesType == null) return false; diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresUnreferencedCodeAnalyzer.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresUnreferencedCodeAnalyzer.cs index 69c38629c43e17..3623150b752057 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresUnreferencedCodeAnalyzer.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/RequiresUnreferencedCodeAnalyzer.cs @@ -49,7 +49,11 @@ private Action typeDerivesFromRucBase { private protected override string RequiresAttributeName => RequiresUnreferencedCodeAttribute; - internal override string RequiresAttributeFullyQualifiedName => FullyQualifiedRequiresUnreferencedCodeAttribute; + public const string UnreferencedCode = nameof (UnreferencedCode); + + internal override string FeatureName => UnreferencedCode; + + private protected override string RequiresAttributeFullyQualifiedName => FullyQualifiedRequiresUnreferencedCodeAttribute; private protected override DiagnosticTargets AnalyzerDiagnosticTargets => DiagnosticTargets.MethodOrConstructor | DiagnosticTargets.Class; @@ -62,7 +66,7 @@ private Action typeDerivesFromRucBase { internal override bool IsAnalyzerEnabled (AnalyzerOptions options) => options.IsMSBuildPropertyValueTrue (MSBuildPropertyOptionNames.EnableTrimAnalyzer); - private protected override bool IsRequiresCheck (IPropertySymbol propertySymbol, Compilation compilation) + internal override bool IsRequiresCheck (Compilation compilation, IPropertySymbol propertySymbol) { // "IsUnreferencedCodeSupported" is treated as a requires check for testing purposes only, and // is not officially-supported product behavior. diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/FeatureCheckReturnValuePattern.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/FeatureCheckReturnValuePattern.cs deleted file mode 100644 index fe81dbb79ec392..00000000000000 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/FeatureCheckReturnValuePattern.cs +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; -using ILLink.Shared; -using ILLink.Shared.DataFlow; -using ILLink.Shared.TrimAnalysis; -using ILLink.RoslynAnalyzer.DataFlow; -using Microsoft.CodeAnalysis; - -namespace ILLink.RoslynAnalyzer.TrimAnalysis -{ - public readonly record struct FeatureCheckReturnValuePattern - { - public FeatureChecksValue ReturnValue { get; init; } - public ValueSet FeatureCheckAnnotations { get; init; } - public IOperation Operation { get; init; } - public IPropertySymbol OwningSymbol { get; init; } - - public FeatureCheckReturnValuePattern ( - FeatureChecksValue returnValue, - ValueSet featureCheckAnnotations, - IOperation operation, - IPropertySymbol owningSymbol) - { - ReturnValue = returnValue.DeepCopy (); - FeatureCheckAnnotations = featureCheckAnnotations.DeepCopy (); - Operation = operation; - OwningSymbol = owningSymbol; - } - - public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext context) - { - var diagnosticContext = new DiagnosticContext (Operation.Syntax.GetLocation ()); - // For now, feature check validation is enabled only when trim analysis is enabled. - if (!context.EnableTrimAnalyzer) - return diagnosticContext.Diagnostics; - - if (!OwningSymbol.IsStatic || OwningSymbol.Type.SpecialType != SpecialType.System_Boolean || OwningSymbol.SetMethod != null) { - // Warn about invalid feature checks (non-static or non-bool properties or properties with setter) - diagnosticContext.AddDiagnostic ( - DiagnosticId.InvalidFeatureGuard); - return diagnosticContext.Diagnostics; - } - - if (ReturnValue == FeatureChecksValue.All) - return diagnosticContext.Diagnostics; - - ValueSet returnValueFeatures = ReturnValue.EnabledFeatures; - // For any analyzer-supported feature that this property is declared to guard, - // the abstract return value must include that feature - // (indicating it is known to be enabled when the return value is true). - foreach (string feature in FeatureCheckAnnotations.GetKnownValues ()) { - foreach (var analyzer in context.EnabledRequiresAnalyzers) { - if (feature != analyzer.RequiresAttributeFullyQualifiedName) - continue; - - if (!returnValueFeatures.Contains (feature)) { - diagnosticContext.AddDiagnostic ( - DiagnosticId.ReturnValueDoesNotMatchFeatureGuards, - OwningSymbol.GetDisplayName (), - feature); - } - } - } - - return diagnosticContext.Diagnostics; - } - } -} diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisAssignmentPattern.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisAssignmentPattern.cs index 2ffcbc43ae8846..5dfc31db3486f9 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisAssignmentPattern.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisAssignmentPattern.cs @@ -58,7 +58,7 @@ public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext conte var diagnosticContext = new DiagnosticContext (Operation.Syntax.GetLocation ()); if (context.EnableTrimAnalyzer && !OwningSymbol.IsInRequiresUnreferencedCodeAttributeScope (out _) && - !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.FullyQualifiedRequiresUnreferencedCodeAttribute)) { + !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.UnreferencedCode)) { foreach (var sourceValue in Source.AsEnumerable ()) { foreach (var targetValue in Target.AsEnumerable ()) { // The target should always be an annotated value, but the visitor design currently prevents diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisGenericInstantiationPattern.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisGenericInstantiationPattern.cs index 26f275085fa8bf..8d484e66036b9a 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisGenericInstantiationPattern.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisGenericInstantiationPattern.cs @@ -48,7 +48,7 @@ public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext conte DiagnosticContext diagnosticContext = new (Operation.Syntax.GetLocation ()); if (context.EnableTrimAnalyzer && !OwningSymbol.IsInRequiresUnreferencedCodeAttributeScope (out _) && - !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.FullyQualifiedRequiresUnreferencedCodeAttribute)) { + !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.UnreferencedCode)) { switch (GenericInstantiation) { case INamedTypeSymbol type: GenericArgumentDataFlow.ProcessGenericArgumentDataFlow (diagnosticContext, type); diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisMethodCallPattern.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisMethodCallPattern.cs index 3dfd7fa285521a..8341afa2ea5f24 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisMethodCallPattern.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisMethodCallPattern.cs @@ -77,7 +77,7 @@ public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext conte DiagnosticContext diagnosticContext = new (Operation.Syntax.GetLocation ()); if (context.EnableTrimAnalyzer && !OwningSymbol.IsInRequiresUnreferencedCodeAttributeScope(out _) && - !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.FullyQualifiedRequiresUnreferencedCodeAttribute)) + !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.UnreferencedCode)) { TrimAnalysisVisitor.HandleCall(Operation, OwningSymbol, CalledMethod, Instance, Arguments, diagnosticContext, default, out var _); } diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisPatternStore.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisPatternStore.cs index dd66d802934be6..78de8fdf4235ce 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisPatternStore.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisPatternStore.cs @@ -17,20 +17,16 @@ public readonly struct TrimAnalysisPatternStore readonly Dictionary GenericInstantiationPatterns; readonly Dictionary MethodCallPatterns; readonly Dictionary ReflectionAccessPatterns; - readonly Dictionary FeatureCheckReturnValuePatterns; readonly ValueSetLattice Lattice; readonly FeatureContextLattice FeatureContextLattice; - public TrimAnalysisPatternStore ( - ValueSetLattice lattice, - FeatureContextLattice featureContextLattice) + public TrimAnalysisPatternStore (ValueSetLattice lattice, FeatureContextLattice featureContextLattice) { AssignmentPatterns = new Dictionary<(IOperation, bool), TrimAnalysisAssignmentPattern> (); FieldAccessPatterns = new Dictionary (); GenericInstantiationPatterns = new Dictionary (); MethodCallPatterns = new Dictionary (); ReflectionAccessPatterns = new Dictionary (); - FeatureCheckReturnValuePatterns = new Dictionary (); Lattice = lattice; FeatureContextLattice = featureContextLattice; } @@ -93,16 +89,6 @@ public void Add (TrimAnalysisReflectionAccessPattern pattern) ReflectionAccessPatterns[pattern.Operation] = pattern.Merge (Lattice, FeatureContextLattice, existingPattern); } - public void Add (FeatureCheckReturnValuePattern pattern) - { - if (!FeatureCheckReturnValuePatterns.TryGetValue (pattern.Operation, out var existingPattern)) { - FeatureCheckReturnValuePatterns.Add (pattern.Operation, pattern); - return; - } - - Debug.Assert (existingPattern == pattern, "Return values should be identical"); - } - public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext context) { foreach (var assignmentPattern in AssignmentPatterns.Values) { @@ -129,11 +115,6 @@ public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext conte foreach (var diagnostic in reflectionAccessPattern.CollectDiagnostics (context)) yield return diagnostic; } - - foreach (var returnValuePattern in FeatureCheckReturnValuePatterns.Values) { - foreach (var diagnostic in returnValuePattern.CollectDiagnostics (context)) - yield return diagnostic; - } } } } diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisReflectionAccessPattern.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisReflectionAccessPattern.cs index 85897420596f49..0e4c45a9f011fc 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisReflectionAccessPattern.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisReflectionAccessPattern.cs @@ -50,7 +50,7 @@ public IEnumerable CollectDiagnostics (DataFlowAnalyzerContext conte DiagnosticContext diagnosticContext = new (Operation.Syntax.GetLocation ()); if (context.EnableTrimAnalyzer && !OwningSymbol.IsInRequiresUnreferencedCodeAttributeScope (out _) && - !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.FullyQualifiedRequiresUnreferencedCodeAttribute)) { + !FeatureContext.IsEnabled (RequiresUnreferencedCodeAnalyzer.UnreferencedCode)) { foreach (var diagnostic in ReflectionAccessAnalyzer.GetDiagnosticsForReflectionAccessToDAMOnMethod (diagnosticContext, ReferencedMethod)) diagnosticContext.AddDiagnostic (diagnostic); } diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisVisitor.cs b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisVisitor.cs index 9db61498b28ceb..6f118ac8c47922 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisVisitor.cs +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/TrimAnalysis/TrimAnalysisVisitor.cs @@ -59,12 +59,12 @@ public TrimAnalysisVisitor ( _featureChecksVisitor = new FeatureChecksVisitor (dataFlowAnalyzerContext); } - public override FeatureChecksValue GetConditionValue (IOperation branchValueOperation, StateValue state) + public override FeatureChecksValue? GetConditionValue (IOperation branchValueOperation, StateValue state) { return _featureChecksVisitor.Visit (branchValueOperation, state); } - public override void ApplyCondition (FeatureChecksValue featureChecksValue, ref LocalStateAndContext currentState) + public override void ApplyCondition (FeatureChecksValue featureChecksValue, ref LocalStateAndContext currentState) { currentState.Context = currentState.Context.Union (new FeatureContext (featureChecksValue.EnabledFeatures)); } @@ -426,29 +426,6 @@ public override void HandleReturnValue (MultiValue returnValue, IOperation opera } } - public override void HandleReturnConditionValue (FeatureChecksValue returnConditionValue, IOperation operation) - { - // Return statements should only happen inside of method bodies. - Debug.Assert (OwningSymbol is IMethodSymbol); - if (OwningSymbol is not IMethodSymbol method) - return; - - // FeatureGuard validation needs to happen only for property getters. - // Include properties with setters here because they will get validated later. - if (method.MethodKind != MethodKind.PropertyGet) - return; - - IPropertySymbol propertySymbol = (IPropertySymbol) method.AssociatedSymbol!; - var featureCheckAnnotations = propertySymbol.GetFeatureGuardAnnotations (); - - // If there are no feature checks, there is nothing to validate. - if (featureCheckAnnotations.IsEmpty()) - return; - - TrimAnalysisPatterns.Add ( - new FeatureCheckReturnValuePattern (returnConditionValue, featureCheckAnnotations, operation, propertySymbol)); - } - public override MultiValue HandleDelegateCreation (IMethodSymbol method, IOperation operation, in FeatureContext featureContext) { TrimAnalysisPatterns.Add (new TrimAnalysisReflectionAccessPattern ( diff --git a/src/tools/illink/src/ILLink.Shared/DiagnosticId.cs b/src/tools/illink/src/ILLink.Shared/DiagnosticId.cs index 5d40efbbaaf994..1c33bb084a04af 100644 --- a/src/tools/illink/src/ILLink.Shared/DiagnosticId.cs +++ b/src/tools/illink/src/ILLink.Shared/DiagnosticId.cs @@ -202,10 +202,6 @@ public enum DiagnosticId GenericRecursionCycle = 3054, CorrectnessOfAbstractDelegatesCannotBeGuaranteed = 3055, RequiresDynamicCodeOnStaticConstructor = 3056, - - // Feature guard diagnostic ids. - ReturnValueDoesNotMatchFeatureGuards = 4000, - InvalidFeatureGuard = 4001 } public static class DiagnosticIdExtensions diff --git a/src/tools/illink/src/ILLink.Shared/SharedStrings.resx b/src/tools/illink/src/ILLink.Shared/SharedStrings.resx index c9bb62de5263c7..111c1c5877de7f 100644 --- a/src/tools/illink/src/ILLink.Shared/SharedStrings.resx +++ b/src/tools/illink/src/ILLink.Shared/SharedStrings.resx @@ -1197,16 +1197,4 @@ Unused 'UnconditionalSuppressMessageAttribute' found. Consider removing the unused warning suppression. - - Return value does not match FeatureGuardAttribute '{1}'. - - - Return value does not match FeatureGuard annotations of the property. The check should return false whenever any of the features referenced in the FeatureGuard annotations is disabled. - - - Invalid FeatureGuardAttribute. The attribute must be placed on a static boolean property with only a 'get' accessor. - - - Invalid FeatureGuardAttribute. - - + \ No newline at end of file diff --git a/src/tools/illink/src/linker/CompatibilitySuppressions.xml b/src/tools/illink/src/linker/CompatibilitySuppressions.xml index 7bf0a1e0ce6974..4a0a6296c4e2f3 100644 --- a/src/tools/illink/src/linker/CompatibilitySuppressions.xml +++ b/src/tools/illink/src/linker/CompatibilitySuppressions.xml @@ -253,10 +253,6 @@ CP0001 T:Mono.Linker.ILogger - - CP0001 - T:Mono.Linker.InterfaceImplementor - CP0001 T:Mono.Linker.InternalErrorException @@ -1485,6 +1481,10 @@ CP0002 M:Mono.Linker.OverrideInformation.get_IsOverrideOfInterfaceMember + + CP0002 + M:Mono.Linker.OverrideInformation.get_IsStaticInterfaceMethodPair + CP0002 M:Mono.Linker.Steps.BaseStep.get_MarkingHelpers diff --git a/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs b/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs index 32fd98cbe039ca..c7eb071913f59a 100644 --- a/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs +++ b/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs @@ -701,16 +701,17 @@ void ProcessVirtualMethod (MethodDefinition method) var defaultImplementations = Annotations.GetDefaultInterfaceImplementations (method); if (defaultImplementations is not null) { foreach (var dimInfo in defaultImplementations) { - ProcessDefaultImplementation (dimInfo); + ProcessDefaultImplementation (dimInfo.ImplementingType, dimInfo.InterfaceImpl, dimInfo.DefaultInterfaceMethod); - if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (dimInfo)) - MarkMethod (dimInfo.Override, new DependencyInfo (DependencyKind.Override, dimInfo.Base), ScopeStack.CurrentScope.Origin); + var ov = new OverrideInformation (method, dimInfo.DefaultInterfaceMethod, Context); + if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (ov, dimInfo.ImplementingType)) + MarkMethod (ov.Override, new DependencyInfo (DependencyKind.Override, ov.Base), ScopeStack.CurrentScope.Origin); } } var overridingMethods = Annotations.GetOverrides (method); if (overridingMethods is not null) { - foreach (OverrideInformation ov in overridingMethods) { - if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (ov)) + foreach (var ov in overridingMethods) { + if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (ov, ov.Override.DeclaringType)) MarkMethod (ov.Override, new DependencyInfo (DependencyKind.Override, ov.Base), ScopeStack.CurrentScope.Origin); } } @@ -818,14 +819,13 @@ bool RequiresInterfaceRecursively (TypeDefinition typeToExamine, TypeDefinition return false; } - void ProcessDefaultImplementation (OverrideInformation ov) + void ProcessDefaultImplementation (TypeDefinition typeWithDefaultImplementedInterfaceMethod, InterfaceImplementation implementation, MethodDefinition implementationMethod) { - Debug.Assert (ov.IsOverrideOfInterfaceMember); - if ((!ov.Override.IsStatic && !Annotations.IsInstantiated (ov.InterfaceImplementor.Implementor)) - || ov.Override.IsStatic && !Annotations.IsRelevantToVariantCasting (ov.InterfaceImplementor.Implementor)) + if ((!implementationMethod.IsStatic && !Annotations.IsInstantiated (typeWithDefaultImplementedInterfaceMethod)) + || implementationMethod.IsStatic && !Annotations.IsRelevantToVariantCasting (typeWithDefaultImplementedInterfaceMethod)) return; - MarkInterfaceImplementation (ov.InterfaceImplementor.InterfaceImplementation); + MarkInterfaceImplementation (implementation); } void MarkMarshalSpec (IMarshalInfoProvider spec, in DependencyInfo reason) @@ -2549,11 +2549,11 @@ bool IsMethodNeededByTypeDueToPreservedScope (MethodDefinition method) /// /// Returns true if the override method is required due to the interface that the base method is declared on. See doc at for explanation of logic. /// - bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformation overrideInformation) + bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformation overrideInformation, TypeDefinition typeThatImplsInterface) { var @base = overrideInformation.Base; var method = overrideInformation.Override; - Debug.Assert (overrideInformation.IsOverrideOfInterfaceMember); + Debug.Assert (@base.DeclaringType.IsInterface); if (@base is null || method is null || @base.DeclaringType is null) return false; @@ -2562,7 +2562,7 @@ bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformat // If the interface implementation is not marked, do not mark the implementation method // A type that doesn't implement the interface isn't required to have methods that implement the interface. - InterfaceImplementation? iface = overrideInformation.InterfaceImplementor.InterfaceImplementation; + InterfaceImplementation? iface = overrideInformation.MatchingInterfaceImplementation; if (!((iface is not null && Annotations.IsMarked (iface)) || IsInterfaceImplementationMarkedRecursively (method.DeclaringType, @base.DeclaringType))) return false; @@ -2580,12 +2580,12 @@ bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformat // If the method is static and the implementing type is relevant to variant casting, mark the implementation method. // A static method may only be called through a constrained call if the type is relevant to variant casting. if (@base.IsStatic) - return Annotations.IsRelevantToVariantCasting (overrideInformation.InterfaceImplementor.Implementor) + return Annotations.IsRelevantToVariantCasting (typeThatImplsInterface) || IgnoreScope (@base.DeclaringType.Scope); // If the implementing type is marked as instantiated, mark the implementation method. // If the type is not instantiated, do not mark the implementation method - return Annotations.IsInstantiated (overrideInformation.InterfaceImplementor.Implementor); + return Annotations.IsInstantiated (typeThatImplsInterface); } static bool IsSpecialSerializationConstructor (MethodDefinition method) @@ -3256,7 +3256,7 @@ protected virtual void ProcessMethod (MethodDefinition method, in DependencyInfo // Only if the interface method is referenced, then all the methods which implemented must be kept, but not the other way round. if (!markAllOverrides && Context.Resolve (@base) is MethodDefinition baseDefinition - && baseDefinition.DeclaringType.IsInterface && baseDefinition.IsStatic && method.IsStatic) + && new OverrideInformation.OverridePair (baseDefinition, method).IsStaticInterfaceMethodPair ()) continue; MarkMethod (@base, new DependencyInfo (DependencyKind.MethodImplOverride, method), ScopeStack.CurrentScope.Origin); MarkExplicitInterfaceImplementation (method, @base); diff --git a/src/tools/illink/src/linker/Linker.Steps/RootAssemblyInputStep.cs b/src/tools/illink/src/linker/Linker.Steps/RootAssemblyInputStep.cs index b167ed5f58be20..d29432e8b60cec 100644 --- a/src/tools/illink/src/linker/Linker.Steps/RootAssemblyInputStep.cs +++ b/src/tools/illink/src/linker/Linker.Steps/RootAssemblyInputStep.cs @@ -78,8 +78,7 @@ protected override void Process () CodeOptimizations.RemoveLinkAttributes | CodeOptimizations.RemoveSubstitutions | CodeOptimizations.RemoveDynamicDependencyAttribute | - CodeOptimizations.OptimizeTypeHierarchyAnnotations | - CodeOptimizations.SubstituteFeatureGuards, assembly.Name.Name); + CodeOptimizations.OptimizeTypeHierarchyAnnotations, assembly.Name.Name); // Enable EventSource special handling Context.DisableEventSourceSpecialHandling = false; diff --git a/src/tools/illink/src/linker/Linker/Annotations.cs b/src/tools/illink/src/linker/Linker/Annotations.cs index a7b3198265e812..8f7747cba3543a 100644 --- a/src/tools/illink/src/linker/Linker/Annotations.cs +++ b/src/tools/illink/src/linker/Linker/Annotations.cs @@ -462,7 +462,7 @@ public bool IsPublic (IMetadataTokenProvider provider) /// DefaultInterfaceMethod is the method that implements . /// /// The interface method to find default implementations for - public IEnumerable? GetDefaultInterfaceImplementations (MethodDefinition method) + public IEnumerable<(TypeDefinition ImplementingType, InterfaceImplementation InterfaceImpl, MethodDefinition DefaultInterfaceMethod)>? GetDefaultInterfaceImplementations (MethodDefinition method) { return TypeMapInfo.GetDefaultInterfaceImplementations (method); } diff --git a/src/tools/illink/src/linker/Linker/CustomAttributeSource.cs b/src/tools/illink/src/linker/Linker/CustomAttributeSource.cs index f32e93a078579d..f11419de4b4b2e 100644 --- a/src/tools/illink/src/linker/Linker/CustomAttributeSource.cs +++ b/src/tools/illink/src/linker/Linker/CustomAttributeSource.cs @@ -52,14 +52,6 @@ public bool TryGetEmbeddedXmlInfo (ICustomAttributeProvider provider, [NotNullWh return xmlInfo != null; } - public IEnumerable GetCustomAttributes (ICustomAttributeProvider provider, string attributeNamespace, string attributeName) - { - foreach (var attr in GetCustomAttributes (provider)) { - if (attr.AttributeType.Namespace == attributeNamespace && attr.AttributeType.Name == attributeName) - yield return attr; - } - } - public IEnumerable GetCustomAttributes (ICustomAttributeProvider provider) { if (provider.HasCustomAttributes) { diff --git a/src/tools/illink/src/linker/Linker/DictionaryExtensions.cs b/src/tools/illink/src/linker/Linker/DictionaryExtensions.cs deleted file mode 100644 index 9cf8945fe480c5..00000000000000 --- a/src/tools/illink/src/linker/Linker/DictionaryExtensions.cs +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System.Collections.Generic; - -namespace Mono.Linker -{ - internal static class DictionaryExtensions - { - public static void AddToList (this Dictionary> me, TKey key, TElement value) - where TKey : notnull - { - if (!me.TryGetValue (key, out List? valueList)) { - valueList = new (); - me[key] = valueList; - } - valueList.Add (value); - } - } -} diff --git a/src/tools/illink/src/linker/Linker/Driver.cs b/src/tools/illink/src/linker/Linker/Driver.cs index de99303f12b7fe..742ee2140b9ce0 100644 --- a/src/tools/illink/src/linker/Linker/Driver.cs +++ b/src/tools/illink/src/linker/Linker/Driver.cs @@ -1179,9 +1179,6 @@ protected bool GetOptimizationName (string text, out CodeOptimizations optimizat case "sealer": optimization = CodeOptimizations.Sealer; return true; - case "substitutefeatureguards": - optimization = CodeOptimizations.SubstituteFeatureGuards; - return true; } Context.LogError (null, DiagnosticId.InvalidOptimizationValue, text); @@ -1364,7 +1361,6 @@ static void Usage () Console.WriteLine (" unreachablebodies: Instance methods that are marked but not executed are converted to throws"); Console.WriteLine (" unusedinterfaces: Removes interface types from declaration when not used"); Console.WriteLine (" unusedtypechecks: Inlines never successful type checks"); - Console.WriteLine (" substitutefeatureguards: Substitutes properties annotated as FeatureGuard(typeof(RequiresUnreferencedCodeAttribute)) to false"); Console.WriteLine (" --enable-opt NAME [ASM] Enable one of the additional optimizations globaly or for a specific assembly name"); Console.WriteLine (" sealer: Any method or type which does not have override is marked as sealed"); Console.WriteLine (" --explicit-reflection Adds to members never used through reflection DisablePrivateReflection attribute. Defaults to false"); diff --git a/src/tools/illink/src/linker/Linker/InterfaceImplementor.cs b/src/tools/illink/src/linker/Linker/InterfaceImplementor.cs deleted file mode 100644 index e981ce872703f7..00000000000000 --- a/src/tools/illink/src/linker/Linker/InterfaceImplementor.cs +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using Mono.Cecil; - -namespace Mono.Linker -{ - public class InterfaceImplementor - { - /// - /// The type that implements . - /// - public TypeDefinition Implementor { get; } - /// - /// The .interfaceimpl on that points to - /// - public InterfaceImplementation InterfaceImplementation { get; } - /// - /// The type of the interface that is implemented by - /// - public TypeDefinition InterfaceType { get; } - - public InterfaceImplementor (TypeDefinition implementor, InterfaceImplementation interfaceImplementation, TypeDefinition interfaceType, IMetadataResolver resolver) - { - Implementor = implementor; - InterfaceImplementation = interfaceImplementation; - InterfaceType = interfaceType; - Debug.Assert(resolver.Resolve (interfaceImplementation.InterfaceType) == interfaceType); - } - - public static InterfaceImplementor Create(TypeDefinition implementor, TypeDefinition interfaceType, IMetadataResolver resolver) - { - foreach(InterfaceImplementation iface in implementor.Interfaces) { - if (resolver.Resolve(iface.InterfaceType) == interfaceType) { - return new InterfaceImplementor(implementor, iface, interfaceType, resolver); - } - } - - Queue ifacesToCheck = new (); - ifacesToCheck.Enqueue(implementor); - while (ifacesToCheck.Count > 0) { - var currentIface = ifacesToCheck.Dequeue (); - - foreach(InterfaceImplementation ifaceImpl in currentIface.Interfaces) { - var iface = resolver.Resolve (ifaceImpl.InterfaceType); - if (iface == interfaceType) { - return new InterfaceImplementor(implementor, ifaceImpl, interfaceType, resolver); - } - ifacesToCheck.Enqueue (iface); - } - } - throw new InvalidOperationException ($"Type '{implementor.FullName}' does not implement interface '{interfaceType.FullName}' directly or through any interfaces"); - } - } -} diff --git a/src/tools/illink/src/linker/Linker/LinkContext.cs b/src/tools/illink/src/linker/Linker/LinkContext.cs index 43d6f994fd0754..41fcba1a6f05b2 100644 --- a/src/tools/illink/src/linker/Linker/LinkContext.cs +++ b/src/tools/illink/src/linker/Linker/LinkContext.cs @@ -246,8 +246,7 @@ protected LinkContext (Pipeline pipeline, ILogger logger, string outputDirectory CodeOptimizations.RemoveLinkAttributes | CodeOptimizations.RemoveSubstitutions | CodeOptimizations.RemoveDynamicDependencyAttribute | - CodeOptimizations.OptimizeTypeHierarchyAnnotations | - CodeOptimizations.SubstituteFeatureGuards; + CodeOptimizations.OptimizeTypeHierarchyAnnotations; DisableEventSourceSpecialHandling = true; @@ -1145,10 +1144,5 @@ public enum CodeOptimizations /// Otherwise, type annotation will only be applied with calls to object.GetType() /// OptimizeTypeHierarchyAnnotations = 1 << 24, - - /// - /// Option to substitute properties annotated as FeatureGuard(typeof(RequiresUnreferencedCodeAttribute)) with false - /// - SubstituteFeatureGuards = 1 << 25, } } diff --git a/src/tools/illink/src/linker/Linker/MemberActionStore.cs b/src/tools/illink/src/linker/Linker/MemberActionStore.cs index 51d792fc45b40d..84f2cfd7fb4675 100644 --- a/src/tools/illink/src/linker/Linker/MemberActionStore.cs +++ b/src/tools/illink/src/linker/Linker/MemberActionStore.cs @@ -2,9 +2,7 @@ // Licensed under the MIT license. See LICENSE file in the project root for full license information. using System.Collections.Generic; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using ILLink.Shared; using Mono.Cecil; namespace Mono.Linker @@ -22,7 +20,7 @@ public MemberActionStore (LinkContext context) _context = context; } - private bool TryGetSubstitutionInfo (MemberReference member, [NotNullWhen (true)] out SubstitutionInfo? xmlInfo) + public bool TryGetSubstitutionInfo (MemberReference member, [NotNullWhen (true)] out SubstitutionInfo? xmlInfo) { var assembly = member.Module.Assembly; if (!_embeddedXmlInfos.TryGetValue (assembly, out xmlInfo)) { @@ -43,9 +41,6 @@ public MethodAction GetAction (MethodDefinition method) return action; } - if (TryGetFeatureCheckValue (method, out _)) - return MethodAction.ConvertToStub; - return MethodAction.Nothing; } @@ -54,78 +49,10 @@ public bool TryGetMethodStubValue (MethodDefinition method, out object? value) if (PrimarySubstitutionInfo.MethodStubValues.TryGetValue (method, out value)) return true; - if (TryGetSubstitutionInfo (method, out var embeddedXml) - && embeddedXml.MethodStubValues.TryGetValue (method, out value)) - return true; - - if (TryGetFeatureCheckValue (method, out bool bValue)) { - value = bValue ? 1 : 0; - return true; - } - - return false; - } - - internal bool TryGetFeatureCheckValue (MethodDefinition method, out bool value) - { - value = false; - - if (!method.IsStatic) - return false; - - if (method.ReturnType.MetadataType != MetadataType.Boolean) - return false; - - if (FindProperty (method) is not PropertyDefinition property) + if (!TryGetSubstitutionInfo (method, out var embeddedXml)) return false; - if (property.SetMethod != null) - return false; - - foreach (var featureSwitchDefinitionAttribute in _context.CustomAttributes.GetCustomAttributes (property, "System.Diagnostics.CodeAnalysis", "FeatureSwitchDefinitionAttribute")) { - if (featureSwitchDefinitionAttribute.ConstructorArguments is not [CustomAttributeArgument { Value: string switchName }]) - continue; - - // If there's a FeatureSwitchDefinition, don't continue looking for FeatureGuard. - // We don't want to infer feature switch settings from FeatureGuard. - return _context.FeatureSettings.TryGetValue (switchName, out value); - } - - if (!_context.IsOptimizationEnabled (CodeOptimizations.SubstituteFeatureGuards, method)) - return false; - - foreach (var featureGuardAttribute in _context.CustomAttributes.GetCustomAttributes (property, "System.Diagnostics.CodeAnalysis", "FeatureGuardAttribute")) { - if (featureGuardAttribute.ConstructorArguments is not [CustomAttributeArgument { Value: TypeReference featureType }]) - continue; - - if (featureType.Namespace == "System.Diagnostics.CodeAnalysis") { - switch (featureType.Name) { - case "RequiresUnreferencedCodeAttribute": - return true; - case "RequiresDynamicCodeAttribute": - if (_context.FeatureSettings.TryGetValue ( - "System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeSupported", - out bool isDynamicCodeSupported) - && !isDynamicCodeSupported) - return true; - break; - } - } - } - - return false; - - static PropertyDefinition? FindProperty (MethodDefinition method) { - if (!method.IsGetter) - return null; - - foreach (var property in method.DeclaringType.Properties) { - if (property.GetMethod == method) - return property; - } - - return null; - } + return embeddedXml.MethodStubValues.TryGetValue (method, out value); } public bool TryGetFieldUserValue (FieldDefinition field, out object? value) diff --git a/src/tools/illink/src/linker/Linker/OverrideInformation.cs b/src/tools/illink/src/linker/Linker/OverrideInformation.cs index 0727d5d25c19a0..077353eb2ee7b0 100644 --- a/src/tools/illink/src/linker/Linker/OverrideInformation.cs +++ b/src/tools/illink/src/linker/Linker/OverrideInformation.cs @@ -3,39 +3,71 @@ using System.Diagnostics; using Mono.Cecil; -using System.Diagnostics.CodeAnalysis; namespace Mono.Linker { [DebuggerDisplay ("{Override}")] public class OverrideInformation { - public MethodDefinition Base { get; } + readonly ITryResolveMetadata resolver; + readonly OverridePair _pair; + private InterfaceImplementation? _matchingInterfaceImplementation; - public MethodDefinition Override { get; } + public OverrideInformation (MethodDefinition @base, MethodDefinition @override, ITryResolveMetadata resolver, InterfaceImplementation? matchingInterfaceImplementation = null) + { + _pair = new OverridePair (@base, @override); + _matchingInterfaceImplementation = matchingInterfaceImplementation; + this.resolver = resolver; + } + public readonly record struct OverridePair (MethodDefinition Base, MethodDefinition Override) + { + public bool IsStaticInterfaceMethodPair () => Base.DeclaringType.IsInterface && Base.IsStatic && Override.IsStatic; + public InterfaceImplementation? GetMatchingInterfaceImplementation (ITryResolveMetadata resolver) + { + if (!Base.DeclaringType.IsInterface) + return null; + var interfaceType = Base.DeclaringType; + foreach (var @interface in Override.DeclaringType.Interfaces) { + if (resolver.TryResolve (@interface.InterfaceType)?.Equals (interfaceType) == true) { + return @interface; + } + } + return null; + } + } - internal InterfaceImplementor? InterfaceImplementor { get; } + public MethodDefinition Base { get => _pair.Base; } + public MethodDefinition Override { get => _pair.Override; } + public InterfaceImplementation? MatchingInterfaceImplementation { + get { + if (_matchingInterfaceImplementation is not null) + return _matchingInterfaceImplementation; + _matchingInterfaceImplementation = _pair.GetMatchingInterfaceImplementation (resolver); + return _matchingInterfaceImplementation; + } + } - internal OverrideInformation (MethodDefinition @base, MethodDefinition @override, InterfaceImplementor? interfaceImplementor = null) - { - Base = @base; - Override = @override; - InterfaceImplementor = interfaceImplementor; - // Ensure we have an interface implementation if the base method is from an interface and the override method is on a class - Debug.Assert(@base.DeclaringType.IsInterface && interfaceImplementor != null - || !@base.DeclaringType.IsInterface && interfaceImplementor == null); - // Ensure the interfaceImplementor is for the interface we expect - Debug.Assert (@base.DeclaringType.IsInterface ? interfaceImplementor!.InterfaceType == @base.DeclaringType : true); + public bool IsOverrideOfInterfaceMember { + get { + if (MatchingInterfaceImplementation != null) + return true; + + return Base.DeclaringType.IsInterface; + } } - public InterfaceImplementation? MatchingInterfaceImplementation - => InterfaceImplementor?.InterfaceImplementation; + public TypeDefinition? InterfaceType { + get { + if (!IsOverrideOfInterfaceMember) + return null; - public TypeDefinition? InterfaceType - => InterfaceImplementor?.InterfaceType; + if (MatchingInterfaceImplementation != null) + return resolver.TryResolve (MatchingInterfaceImplementation.InterfaceType); + + return Base.DeclaringType; + } + } - [MemberNotNullWhen (true, nameof (InterfaceImplementor), nameof (MatchingInterfaceImplementation))] - public bool IsOverrideOfInterfaceMember - => InterfaceImplementor != null; + public bool IsStaticInterfaceMethodPair => _pair.IsStaticInterfaceMethodPair (); } } diff --git a/src/tools/illink/src/linker/Linker/TypeMapInfo.cs b/src/tools/illink/src/linker/Linker/TypeMapInfo.cs index 7e68b71bcb7d08..a2f118adf9fb78 100644 --- a/src/tools/illink/src/linker/Linker/TypeMapInfo.cs +++ b/src/tools/illink/src/linker/Linker/TypeMapInfo.cs @@ -29,11 +29,9 @@ // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // -using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Linq; using Mono.Cecil; namespace Mono.Linker @@ -45,7 +43,7 @@ public class TypeMapInfo readonly LinkContext context; protected readonly Dictionary> base_methods = new Dictionary> (); protected readonly Dictionary> override_methods = new Dictionary> (); - protected readonly Dictionary> default_interface_implementations = new Dictionary> (); + protected readonly Dictionary> default_interface_implementations = new Dictionary> (); public TypeMapInfo (LinkContext context) { @@ -94,26 +92,41 @@ public void EnsureProcessed (AssemblyDefinition assembly) /// DefaultInterfaceMethod is the method that implements . /// /// The interface method to find default implementations for - public IEnumerable? GetDefaultInterfaceImplementations (MethodDefinition baseMethod) + public IEnumerable<(TypeDefinition ImplementingType, InterfaceImplementation InterfaceImpl, MethodDefinition DefaultImplementationMethod)>? GetDefaultInterfaceImplementations (MethodDefinition baseMethod) { default_interface_implementations.TryGetValue (baseMethod, out var ret); return ret; } - public void AddBaseMethod (MethodDefinition method, MethodDefinition @base, InterfaceImplementor? interfaceImplementor) + public void AddBaseMethod (MethodDefinition method, MethodDefinition @base, InterfaceImplementation? matchingInterfaceImplementation) { - base_methods.AddToList (method, new OverrideInformation (@base, method, interfaceImplementor)); + if (!base_methods.TryGetValue (method, out List? methods)) { + methods = new List (); + base_methods[method] = methods; + } + + methods.Add (new OverrideInformation (@base, method, context, matchingInterfaceImplementation)); } - public void AddOverride (MethodDefinition @base, MethodDefinition @override, InterfaceImplementor? interfaceImplementor = null) + public void AddOverride (MethodDefinition @base, MethodDefinition @override, InterfaceImplementation? matchingInterfaceImplementation = null) { - override_methods.AddToList (@base, new OverrideInformation (@base, @override, interfaceImplementor)); + if (!override_methods.TryGetValue (@base, out List? methods)) { + methods = new List (); + override_methods.Add (@base, methods); + } + + methods.Add (new OverrideInformation (@base, @override, context, matchingInterfaceImplementation)); } - public void AddDefaultInterfaceImplementation (MethodDefinition @base, InterfaceImplementor interfaceImplementor, MethodDefinition defaultImplementationMethod) + public void AddDefaultInterfaceImplementation (MethodDefinition @base, TypeDefinition implementingType, (InterfaceImplementation, MethodDefinition) matchingInterfaceImplementation) { Debug.Assert(@base.DeclaringType.IsInterface); - default_interface_implementations.AddToList (@base, new OverrideInformation (@base, defaultImplementationMethod, interfaceImplementor)); + if (!default_interface_implementations.TryGetValue (@base, out var implementations)) { + implementations = new List<(TypeDefinition, InterfaceImplementation, MethodDefinition)> (); + default_interface_implementations.Add (@base, implementations); + } + + implementations.Add ((implementingType, matchingInterfaceImplementation.Item1, matchingInterfaceImplementation.Item2)); } protected virtual void MapType (TypeDefinition type) @@ -155,20 +168,20 @@ void MapInterfaceMethodsInTypeHierarchy (TypeDefinition type) // Try to find an implementation with a name/sig match on the current type MethodDefinition? exactMatchOnType = TryMatchMethod (type, interfaceMethod); if (exactMatchOnType != null) { - AnnotateMethods (resolvedInterfaceMethod, exactMatchOnType, new (type, interfaceImpl.OriginalImpl, resolvedInterfaceMethod.DeclaringType, context)); + AnnotateMethods (resolvedInterfaceMethod, exactMatchOnType); continue; } // Next try to find an implementation with a name/sig match in the base hierarchy var @base = GetBaseMethodInTypeHierarchy (type, interfaceMethod); if (@base != null) { - AnnotateMethods (resolvedInterfaceMethod, @base, new (type, interfaceImpl.OriginalImpl, resolvedInterfaceMethod.DeclaringType, context)); + AnnotateMethods (resolvedInterfaceMethod, @base, interfaceImpl.OriginalImpl); continue; } } // Look for a default implementation last. - FindAndAddDefaultInterfaceImplementations (type, type, resolvedInterfaceMethod, interfaceImpl.OriginalImpl); + FindAndAddDefaultInterfaceImplementations (type, resolvedInterfaceMethod); } } } @@ -198,29 +211,24 @@ void MapVirtualMethod (MethodDefinition method) if (@base == null) return; - Debug.Assert(!@base.DeclaringType.IsInterface); - AnnotateMethods (@base, method); } void MapOverrides (MethodDefinition method) { - foreach (MethodReference baseMethodRef in method.Overrides) { - MethodDefinition? baseMethod = context.TryResolve (baseMethodRef); - if (baseMethod == null) + foreach (MethodReference override_ref in method.Overrides) { + MethodDefinition? @override = context.TryResolve (override_ref); + if (@override == null) continue; - if (baseMethod.DeclaringType.IsInterface) { - AnnotateMethods (baseMethod, method, InterfaceImplementor.Create (method.DeclaringType, baseMethod.DeclaringType, context)); - } else { - AnnotateMethods (baseMethod, method); - } + + AnnotateMethods (@override, method); } } - void AnnotateMethods (MethodDefinition @base, MethodDefinition @override, InterfaceImplementor? interfaceImplementor = null) + void AnnotateMethods (MethodDefinition @base, MethodDefinition @override, InterfaceImplementation? matchingInterfaceImplementation = null) { - AddBaseMethod (@override, @base, interfaceImplementor); - AddOverride (@base, @override, interfaceImplementor); + AddBaseMethod (@override, @base, matchingInterfaceImplementation); + AddOverride (@base, @override, matchingInterfaceImplementation); } MethodDefinition? GetBaseMethodInTypeHierarchy (MethodDefinition method) @@ -271,23 +279,16 @@ void AnnotateMethods (MethodDefinition @base, MethodDefinition @override, Interf return context.TryResolve (type)?.BaseType; } - /// - /// Returns a list of default implementations of the given interface method on this type. - /// Note that this returns a list to potentially cover the diamond case (more than one - /// most specific implementation of the given interface methods). ILLink needs to preserve - /// all the implementations so that the proper exception can be thrown at runtime. - /// - /// The type that implements (directly or via a base interface) the declaring interface of - /// The method to find a default implementation for - /// - /// The InterfaceImplementation on that points to the DeclaringType of . - /// - void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplementsInterface, TypeDefinition typeThatMayHaveDIM, MethodDefinition interfaceMethodToBeImplemented, InterfaceImplementation originalInterfaceImpl) + // Returns a list of default implementations of the given interface method on this type. + // Note that this returns a list to potentially cover the diamond case (more than one + // most specific implementation of the given interface methods). ILLink needs to preserve + // all the implementations so that the proper exception can be thrown at runtime. + void FindAndAddDefaultInterfaceImplementations (TypeDefinition type, MethodDefinition interfaceMethod) { // Go over all interfaces, trying to find a method that is an explicit MethodImpl of the // interface method in question. - foreach (var interfaceImpl in typeThatMayHaveDIM.Interfaces) { + foreach (var interfaceImpl in type.Interfaces) { var potentialImplInterface = context.TryResolve (interfaceImpl.InterfaceType); if (potentialImplInterface == null) continue; @@ -295,9 +296,9 @@ void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplement bool foundImpl = false; foreach (var potentialImplMethod in potentialImplInterface.Methods) { - if (potentialImplMethod == interfaceMethodToBeImplemented && + if (potentialImplMethod == interfaceMethod && !potentialImplMethod.IsAbstract) { - AddDefaultInterfaceImplementation (interfaceMethodToBeImplemented, new (typeThatImplementsInterface, originalInterfaceImpl, interfaceMethodToBeImplemented.DeclaringType, context), potentialImplMethod); + AddDefaultInterfaceImplementation (interfaceMethod, type, (interfaceImpl, potentialImplMethod)); foundImpl = true; break; } @@ -306,9 +307,9 @@ void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplement continue; // This method is an override of something. Let's see if it's the method we are looking for. - foreach (var baseMethod in potentialImplMethod.Overrides) { - if (context.TryResolve (baseMethod) == interfaceMethodToBeImplemented) { - AddDefaultInterfaceImplementation (interfaceMethodToBeImplemented, new (typeThatImplementsInterface, originalInterfaceImpl, interfaceMethodToBeImplemented.DeclaringType, context), @potentialImplMethod); + foreach (var @override in potentialImplMethod.Overrides) { + if (context.TryResolve (@override) == interfaceMethod) { + AddDefaultInterfaceImplementation (interfaceMethod, type, (interfaceImpl, @potentialImplMethod)); foundImpl = true; break; } @@ -322,7 +323,7 @@ void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplement // We haven't found a MethodImpl on the current interface, but one of the interfaces // this interface requires could still provide it. if (!foundImpl) { - FindAndAddDefaultInterfaceImplementations (typeThatImplementsInterface, potentialImplInterface, interfaceMethodToBeImplemented, originalInterfaceImpl); + FindAndAddDefaultInterfaceImplementations (potentialImplInterface, interfaceMethod); } } } diff --git a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/DataFlowTests.cs b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/DataFlowTests.cs index 087c7afea46a5a..2b1cf973d8a234 100644 --- a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/DataFlowTests.cs +++ b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/DataFlowTests.cs @@ -149,12 +149,6 @@ public Task FeatureCheckDataFlow () return RunTest (); } - [Fact] - public Task FeatureGuardAttributeDataFlow () - { - return RunTest (); - } - [Fact] public Task FieldDataFlow () { diff --git a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/SubstitutionsTests.cs b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/SubstitutionsTests.cs deleted file mode 100644 index 551284ef38a7be..00000000000000 --- a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/SubstitutionsTests.cs +++ /dev/null @@ -1,17 +0,0 @@ -using System; -using System.Threading.Tasks; -using Xunit; - -namespace ILLink.RoslynAnalyzer.Tests -{ - public sealed partial class SubstitutionsTests : LinkerTestBase - { - protected override string TestSuiteName => "Substitutions"; - - [Fact] - public Task FeatureGuardSubstitutions () - { - return RunTest (); - } - } -} diff --git a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs index 649b8449527f75..2e1a2bbcb3454d 100644 --- a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs +++ b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs @@ -15,12 +15,6 @@ public Task CanDisableUnusedInterfaces () return RunTest (allowMissingWarnings: true); } - [Fact] - public Task InterfaceImplementedThroughBaseInterface () - { - return RunTest (allowMissingWarnings: true); - } - [Fact] public Task InterfaceOnUninstantiatedTypeRemoved () { diff --git a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/SubstitutionsTests.g.cs b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/SubstitutionsTests.g.cs index 2e3c9ca3884163..1dd1a52a1a993f 100644 --- a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/SubstitutionsTests.g.cs +++ b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/SubstitutionsTests.g.cs @@ -7,6 +7,8 @@ namespace ILLink.RoslynAnalyzer.Tests public sealed partial class SubstitutionsTests : LinkerTestBase { + protected override string TestSuiteName => "Substitutions"; + [Fact] public Task EmbeddedFieldSubstitutionsInReferencedAssembly () { @@ -43,12 +45,6 @@ public Task EmbeddedSubstitutionsNotProcessedWithIgnoreSubstitutionsAndRemoved ( return RunTest (allowMissingWarnings: true); } - [Fact] - public Task FeatureGuardSubstitutionsDisabled () - { - return RunTest (allowMissingWarnings: true); - } - [Fact] public Task InitField () { diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Support/FeatureGuardAttribute.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Support/FeatureGuardAttribute.cs deleted file mode 100644 index a4351d0fa8ef88..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Support/FeatureGuardAttribute.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Diagnostics.CodeAnalysis -{ - // Allow AttributeTargets.Method for testing invalid usages of a custom FeatureGuardAttribute - [AttributeUsage (AttributeTargets.Property | AttributeTargets.Method, Inherited = false, AllowMultiple = true)] - public sealed class FeatureGuardAttribute : Attribute - { - public Type FeatureType { get; } - - public FeatureGuardAttribute (Type featureType) - { - FeatureType = featureType; - } - } -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Support/FeatureSwitchDefinitionAttribute.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Support/FeatureSwitchDefinitionAttribute.cs deleted file mode 100644 index 71b030ab299f69..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases.Expectations/Support/FeatureSwitchDefinitionAttribute.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Diagnostics.CodeAnalysis -{ - [AttributeUsage(AttributeTargets.Property, Inherited = false)] - public sealed class FeatureSwitchDefinitionAttribute : Attribute - { - public string SwitchName { get; } - - public FeatureSwitchDefinitionAttribute (string switchName) - { - SwitchName = switchName; - } - } -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/AttributePropertyDataflow.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/AttributePropertyDataflow.cs index f953784dc37537..15523ffb02d6fe 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/AttributePropertyDataflow.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/AttributePropertyDataflow.cs @@ -224,7 +224,9 @@ public static void Test () // where the owning symbol is not a method. [Kept] [KeptAttributeAttribute (typeof (KeepsPublicMethodsAttribute))] - [ExpectedWarning ("IL2026", "--ClassWithKeptPublicMethods--", ProducedBy = Tool.Trimmer | Tool.NativeAot)] + // NativeAot doesn't handle the type name on fields: https://github.com/dotnet/runtime/issues/92259 + [ExpectedWarning ("IL2105", "Mono.Linker.Tests.Cases.DataFlow.AttributePropertyDataflow+AttributeWithConditionalExpression+ClassWithKeptPublicMethods", ProducedBy = Tool.NativeAot)] + [ExpectedWarning ("IL2026", "--ClassWithKeptPublicMethods--", ProducedBy = Tool.Trimmer)] [KeepsPublicMethods (TypeName = 1 + 1 == 2 ? "Mono.Linker.Tests.Cases.DataFlow.AttributePropertyDataflow+AttributeWithConditionalExpression+ClassWithKeptPublicMethods" : null)] public static int field; diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/Dependencies/TestFeatures.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/Dependencies/TestFeatures.cs deleted file mode 100644 index 942c9f3586dd34..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/Dependencies/TestFeatures.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace ILLink.RoslynAnalyzer -{ - public class TestFeatures - { - public static bool IsUnreferencedCodeSupported => true; - - public static bool IsAssemblyFilesSupported => true; - } -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/ExponentialDataFlow.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/ExponentialDataFlow.cs index 2f749beae0f813..55d07efff70c16 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/ExponentialDataFlow.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/ExponentialDataFlow.cs @@ -58,7 +58,7 @@ class GenericTypeWithRequires<[DynamicallyAccessedMembers (DynamicallyAccessedMe { } - [ExpectedWarning ("IL3050", ProducedBy = Tool.Analyzer)] + [ExpectedWarning ("IL3050", ProducedBy = Tool.Analyzer | Tool.NativeAot)] [ExpectedWarning ("IL2090", "'T'")] public static void Test () { diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlow.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlow.cs index d0d236997445ed..29f18ea7063890 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlow.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlow.cs @@ -20,8 +20,7 @@ namespace Mono.Linker.Tests.Cases.DataFlow // Note: the XML must be passed as an embedded resource named ILLink.Substitutions.xml, // not as a separate substitution file, for it to work with NativeAot. // Related: https://github.com/dotnet/runtime/issues/88647 - [SetupCompileBefore ("TestFeatures.dll", new[] { "Dependencies/TestFeatures.cs" }, - resources: new object[] { new [] { "FeatureCheckDataFlowTestSubstitutions.xml", "ILLink.Substitutions.xml" } })] + [SetupCompileResource ("FeatureCheckDataFlowTestSubstitutions.xml", "ILLink.Substitutions.xml")] [IgnoreSubstitutions (false)] public class FeatureCheckDataFlow { @@ -526,14 +525,14 @@ static void CallTestUnreferencedCodeUnguarded () RequiresUnreferencedCode (); } - static void CallTestDynamicCodeGuarded () + static void CallTestRequiresDynamicCodeGuarded () { if (RuntimeFeature.IsDynamicCodeSupported) RequiresDynamicCode (); } [ExpectedWarning ("IL3050", nameof (RequiresDynamicCode), ProducedBy = Tool.Analyzer | Tool.NativeAot)] - static void CallTestDynamicCodeUnguarded () + static void CallTestRequiresDynamicCodeUnguarded () { RequiresDynamicCode (); } @@ -555,8 +554,8 @@ public static void Test () { CallTestUnreferencedCodeGuarded (); CallTestUnreferencedCodeUnguarded (); - CallTestDynamicCodeGuarded (); - CallTestDynamicCodeUnguarded (); + CallTestRequiresDynamicCodeGuarded (); + CallTestRequiresDynamicCodeUnguarded (); CallTestAssemblyFilesGuarded (); CallTestAssemblyFilesUnguarded (); } @@ -1221,3 +1220,12 @@ class ClassWithRequires class RequiresAllGeneric<[DynamicallyAccessedMembers (DynamicallyAccessedMemberTypes.All)] T> {} } } + +namespace ILLink.RoslynAnalyzer +{ + class TestFeatures + { + public static bool IsUnreferencedCodeSupported => true; + public static bool IsAssemblyFilesSupported => true; + } +} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlowTestSubstitutions.xml b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlowTestSubstitutions.xml index db0bf370336765..c096cf07d6e7c1 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlowTestSubstitutions.xml +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureCheckDataFlowTestSubstitutions.xml @@ -1,5 +1,5 @@ - + diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureGuardAttributeDataFlow.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureGuardAttributeDataFlow.cs deleted file mode 100644 index 60aa4f18a0662b..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/DataFlow/FeatureGuardAttributeDataFlow.cs +++ /dev/null @@ -1,518 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Runtime.CompilerServices; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using ILLink.RoslynAnalyzer; -using Mono.Linker.Tests.Cases.Expectations.Assertions; -using Mono.Linker.Tests.Cases.Expectations.Helpers; -using Mono.Linker.Tests.Cases.Expectations.Metadata; - -namespace Mono.Linker.Tests.Cases.DataFlow -{ - [SkipKeptItemsValidation] - [ExpectedNoWarnings] - [SetupCompileBefore ("TestFeatures.dll", new[] { "Dependencies/TestFeatures.cs" })] - public class FeatureGuardAttributeDataFlow - { - public static void Main () - { - ValidGuardBodies.Test (); - InvalidGuardBodies.Test (); - InvalidFeatureGuards.Test (); - } - - class ValidGuardBodies { - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool ReturnFalseGuard => false; - - static void TestReturnFalseGuard () - { - if (ReturnFalseGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool DirectGuard => TestFeatures.IsUnreferencedCodeSupported; - - static void TestDirectGuard () - { - if (DirectGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IndirectGuard => DirectGuard; - - static void TestIndirectGuard () - { - if (IndirectGuard) - RequiresUnreferencedCode (); - } - - // Analyzer doesn't understand this pattern because it compiles into a CFG that effectively - // looks like this: - // - // bool tmp; - // if (TestFeatures.IsUnreferencedCodeSupported) - // tmp = OtherCondition (); - // else - // tmp = false; - // return tmp; - // - // The analyzer doesn't do constant propagation of the boolean, so it doesn't know that - // the return value is always false when TestFeatures.IsUnreferencedCodeSupported is false. - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool AndGuard => TestFeatures.IsUnreferencedCodeSupported && OtherCondition (); - - static void TestAndGuard () - { - if (AndGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool NotNotGuard => !!TestFeatures.IsUnreferencedCodeSupported; - - static void TestNotNotGuard () - { - if (NotNotGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool EqualsTrueGuard => TestFeatures.IsUnreferencedCodeSupported == true; - - static void TestEqualsTrueGuard () - { - if (EqualsTrueGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool TrueEqualsGuard => true == TestFeatures.IsUnreferencedCodeSupported; - - static void TestTrueEqualsGuard () - { - if (TrueEqualsGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool NotEqualsFalseGuard => TestFeatures.IsUnreferencedCodeSupported != false; - - static void TestNotEqualsFalseGuard () - { - if (NotEqualsFalseGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool FalseNotEqualsGuard => false != TestFeatures.IsUnreferencedCodeSupported; - - static void TestFalseNotEqualsGuard () - { - if (FalseNotEqualsGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IsTrueGuard => TestFeatures.IsUnreferencedCodeSupported is true; - - static void TestIsTrueGuard () - { - if (IsTrueGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IsNotFalseGuard => TestFeatures.IsUnreferencedCodeSupported is not false; - - static void TestIsNotFalseGuard () - { - if (IsNotFalseGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IfReturnTrueGuard { - get { - if (TestFeatures.IsUnreferencedCodeSupported) - return true; - return false; - } - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool ElseReturnTrueGuard { - get { - if (!TestFeatures.IsUnreferencedCodeSupported) - return false; - else - return true; - } - } - - static void TestElseReturnTrueGuard () - { - if (ElseReturnTrueGuard) - RequiresUnreferencedCode (); - } - - static void TestIfReturnTrueGuard () - { - if (IfReturnTrueGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool AssertReturnFalseGuard { - get { - Debug.Assert (TestFeatures.IsUnreferencedCodeSupported); - return false; - } - } - - static void TestAssertReturnFalseGuard () - { - if (AssertReturnFalseGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool AssertNotReturnFalseGuard { - get { - Debug.Assert (!TestFeatures.IsUnreferencedCodeSupported); - return false; - } - } - - static void TestAssertNotReturnFalseGuard () - { - if (AssertNotReturnFalseGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool AssertReturnTrueGuard { - get { - Debug.Assert (TestFeatures.IsUnreferencedCodeSupported); - return true; - } - } - - static void TestAssertReturnTrueGuard () - { - if (AssertReturnTrueGuard) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool ThrowGuard { - get { - if (!TestFeatures.IsUnreferencedCodeSupported) - throw new Exception (); - return false; - } - } - - static void TestThrowGuard () - { - if (ThrowGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool TernaryIfGuard => TestFeatures.IsUnreferencedCodeSupported ? true : false; - - static void TestTernaryIfGuard () - { - if (TernaryIfGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool TernaryElseGuard => !TestFeatures.IsUnreferencedCodeSupported ? false : true; - - static void TestTernaryElseGuard () - { - if (TernaryElseGuard) - RequiresUnreferencedCode (); - } - - public static void Test () - { - TestDirectGuard (); - TestIndirectGuard (); - - TestReturnFalseGuard (); - TestAndGuard (); - TestNotNotGuard (); - TestEqualsTrueGuard (); - TestTrueEqualsGuard (); - TestNotEqualsFalseGuard (); - TestFalseNotEqualsGuard (); - TestIsTrueGuard (); - TestIsNotFalseGuard (); - TestIfReturnTrueGuard (); - TestElseReturnTrueGuard (); - TestAssertReturnFalseGuard (); - TestAssertNotReturnFalseGuard (); - TestAssertReturnTrueGuard (); - TestThrowGuard (); - TestTernaryIfGuard (); - TestTernaryElseGuard (); - } - } - - class InvalidGuardBodies { - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool ReturnTrueGuard => true; - - static void TestReturnTrueGuard () - { - if (ReturnTrueGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool OtherConditionGuard => OtherCondition (); - - static void TestOtherConditionGuard () - { - if (OtherConditionGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool OrGuard => TestFeatures.IsUnreferencedCodeSupported || OtherCondition (); - - static void TestOrGuard () - { - if (OrGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool NotGuard => !TestFeatures.IsUnreferencedCodeSupported; - - static void TestNotGuard () - { - if (NotGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool EqualsFalseGuard => TestFeatures.IsUnreferencedCodeSupported == false; - - static void TestEqualsFalseGuard () - { - if (EqualsFalseGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool FalseEqualsGuard => false == TestFeatures.IsUnreferencedCodeSupported; - - static void TestFalseEqualsGuard () - { - if (FalseEqualsGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool NotEqualsTrueGuard => TestFeatures.IsUnreferencedCodeSupported != true; - - static void TestNotEqualsTrueGuard () - { - if (NotEqualsTrueGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool TrueNotEqualsGuard => true != TestFeatures.IsUnreferencedCodeSupported; - - static void TestTrueNotEqualsGuard () - { - if (TrueNotEqualsGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IsNotTrueGuard => TestFeatures.IsUnreferencedCodeSupported is not true; - - static void TestIsNotTrueGuard () - { - if (IsNotTrueGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IsFalseGuard => TestFeatures.IsUnreferencedCodeSupported is false; - - static void TestIsFalseGuard () - { - if (IsFalseGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool IfReturnFalseGuard { - get { - if (TestFeatures.IsUnreferencedCodeSupported) - return false; - return true; - } - } - - static void TestIfReturnFalseGuard () - { - if (IfReturnFalseGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool ElseReturnFalseGuard { - get { - if (!TestFeatures.IsUnreferencedCodeSupported) - return true; - else - return false; - } - } - - static void TestElseReturnFalseGuard () - { - if (ElseReturnFalseGuard) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool AssertNotReturnTrueGuard { - get { - Debug.Assert (!TestFeatures.IsUnreferencedCodeSupported); - return true; - } - } - - static void TestAssertNotReturnTrueGuard () - { - if (AssertNotReturnTrueGuard) - RequiresUnreferencedCode (); - } - - public static void Test () - { - TestOtherConditionGuard (); - - TestReturnTrueGuard (); - TestOrGuard (); - TestNotGuard (); - TestEqualsFalseGuard (); - TestFalseEqualsGuard (); - TestNotEqualsTrueGuard (); - TestTrueNotEqualsGuard (); - TestIsNotTrueGuard (); - TestIsFalseGuard (); - TestIfReturnFalseGuard (); - TestElseReturnFalseGuard (); - TestAssertNotReturnTrueGuard (); - } - } - - class InvalidFeatureGuards { - [ExpectedWarning ("IL4001", ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static int NonBooleanProperty => 0; - - [ExpectedWarning ("IL2026", nameof (RequiresUnreferencedCodeAttribute))] - static void TestNonBooleanProperty () - { - if (NonBooleanProperty == 0) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4001", ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - bool NonStaticProperty => true; - - [ExpectedWarning ("IL2026", nameof (RequiresUnreferencedCodeAttribute))] - static void TestNonStaticProperty () - { - var instance = new InvalidFeatureGuards (); - if (instance.NonStaticProperty) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool SetOnlyProperty { set => throw null; } - - [ExpectedWarning ("IL2026", nameof (RequiresUnreferencedCodeAttribute))] - static void TestSetOnlyProperty () - { - if (SetOnlyProperty = true) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4001", ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool GetAndSetProperty { get => true; set => throw null; } - - [ExpectedWarning ("IL2026", nameof (RequiresUnreferencedCodeAttribute))] - static void TestGetAndSetProperty () - { - if (GetAndSetProperty) - RequiresUnreferencedCode (); - } - - // No warning for this case because we don't validate that the attribute usage matches - // the expected AttributeUsage.Property for assemblies that define their own version - // of FeatureGuardAttribute. - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool Method () => true; - - [ExpectedWarning ("IL2026", nameof (RequiresUnreferencedCodeAttribute))] - static void TestMethod () - { - if (Method ()) - RequiresUnreferencedCode (); - } - - public static void Test () - { - TestNonBooleanProperty (); - TestNonStaticProperty (); - TestSetOnlyProperty (); - TestGetAndSetProperty (); - TestMethod (); - } - } - - [RequiresUnreferencedCode (nameof (RequiresUnreferencedCode))] - static void RequiresUnreferencedCode () { } - - static bool OtherCondition () => true; - } -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/Dependencies/InterfaceImplementedThroughBaseInterface.il b/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/Dependencies/InterfaceImplementedThroughBaseInterface.il deleted file mode 100644 index 61080f8b7d066d..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/Dependencies/InterfaceImplementedThroughBaseInterface.il +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -.assembly extern mscorlib { } - -.assembly 'library' { } - -.class interface public auto ansi abstract beforefieldinit IBase -{ - // Methods - .method public hidebysig newslot abstract virtual - instance void M () cil managed - { - } // end of method IBase::M - -} // end of class IBase - -.class interface public auto ansi abstract beforefieldinit IDerived - implements IBase -{ -} // end of class IDerived - -.class public auto ansi beforefieldinit C - extends [System.Runtime]System.Object - implements IDerived -{ - // Methods - .method private final hidebysig newslot virtual - instance void IBase.M () cil managed - { - .override method instance void IBase::M() - // Method begins at RVA 0x2050 - // Code size 2 (0x2) - .maxstack 8 - - IL_0001: ret - } // end of method C::IBase.M - - .method public hidebysig specialname rtspecialname - instance void .ctor () cil managed - { - // Method begins at RVA 0x2053 - // Code size 8 (0x8) - .maxstack 8 - - IL_0007: ret - } // end of method C::.ctor -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/InterfaceImplementedThroughBaseInterface.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/InterfaceImplementedThroughBaseInterface.cs deleted file mode 100644 index e701fb9c28ba61..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/InterfaceImplementedThroughBaseInterface.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Mono.Linker.Tests.Cases.Expectations.Assertions; -using Mono.Linker.Tests.Cases.Expectations.Metadata; - -namespace Mono.Linker.Tests.Cases.Inheritance.Interfaces -{ - [SetupLinkerArgument ("--skip-unresolved", "true")] - [SetupLinkerArgument ("-a", "test.exe", "library")] - [SetupLinkerArgument ("-a", "library.dll", "library")] - [TestCaseRequirements (TestRunCharacteristics.SupportsDefaultInterfaceMethods, "Requires support for default interface methods")] - [Define ("IL_ASSEMBLY_AVAILABLE")] - [SetupCompileBefore ("library.dll", new[] { "Dependencies/InterfaceImplementedThroughBaseInterface.il" })] - [SkipILVerify] - -#if IL_ASSEMBLY_AVAILABLE - [KeptMemberInAssembly ("library.dll", typeof(C), "IBase.M()")] -#endif - [KeptMember(".ctor()")] - public class InterfaceImplementedThroughBaseInterface - { - public static void Main () - { - } - } -} - - diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Libraries/RootLibrary.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Libraries/RootLibrary.cs index 9e8154e1a4084b..34c38504d088b0 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Libraries/RootLibrary.cs +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Libraries/RootLibrary.cs @@ -18,7 +18,6 @@ namespace Mono.Linker.Tests.Cases.Libraries [SetupLinkerArgument ("-a", "test.exe", "library")] [SetupLinkerArgument ("--enable-opt", "ipconstprop")] [VerifyMetadataNames] - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Libraries.RootLibrary.FeatureGuardSubstitutionsTest.FeatureSwitch", "false")] public class RootLibrary { private int field; @@ -162,48 +161,6 @@ private void LocalMethod () } } - [Kept] - public class FeatureGuardSubstitutionsTest - { - [Kept] - [KeptAttributeAttribute (typeof (FeatureGuardAttribute))] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - private static bool GuardUnreferencedCode { - [Kept] - get => throw null; - } - - [Kept] - // Body is not modified because feature guard substitutions are disabled in library mode - private static void TestGuard () { - if (GuardUnreferencedCode) - RequiresUnreferencedCode (); - } - - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Libraries.RootLibrary.FeatureGuardSubstitutionsTest.FeatureSwitch")] - private static bool FeatureSwitch => throw null; - - [Kept] - // Feature switches are still substituted in library mode if explicitly passed on the command-line - [ExpectBodyModified] - private static void TestFeatureSwitch () { - if (FeatureSwitch) - RequiresUnreferencedCode (); - } - - [Kept] - public FeatureGuardSubstitutionsTest () - { - TestGuard (); - TestFeatureSwitch (); - } - - [Kept] - [KeptAttributeAttribute (typeof (RequiresUnreferencedCodeAttribute))] - [RequiresUnreferencedCode (nameof (RequiresUnreferencedCode))] - private static void RequiresUnreferencedCode () { } - } - [Kept] [KeptInterface (typeof (I))] public class IfaceClass : I diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Mono.Linker.Tests.Cases.csproj b/src/tools/illink/test/Mono.Linker.Tests.Cases/Mono.Linker.Tests.Cases.csproj index 2b025d9861a116..b30d39a672b44d 100644 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Mono.Linker.Tests.Cases.csproj +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Mono.Linker.Tests.Cases.csproj @@ -8,7 +8,6 @@ - diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/Dependencies/TestFeatures.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/Dependencies/TestFeatures.cs deleted file mode 100644 index 942c9f3586dd34..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/Dependencies/TestFeatures.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace ILLink.RoslynAnalyzer -{ - public class TestFeatures - { - public static bool IsUnreferencedCodeSupported => true; - - public static bool IsAssemblyFilesSupported => true; - } -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutions.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutions.cs deleted file mode 100644 index e34f2b4bbfd3fc..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutions.cs +++ /dev/null @@ -1,405 +0,0 @@ - // Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Runtime.CompilerServices; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using ILLink.RoslynAnalyzer; -using Mono.Linker.Tests.Cases.Expectations.Assertions; -using Mono.Linker.Tests.Cases.Expectations.Helpers; -using Mono.Linker.Tests.Cases.Expectations.Metadata; - -namespace Mono.Linker.Tests.Cases.Substitutions -{ - [ExpectedNoWarnings] - [SetupCompileBefore ("TestFeatures.dll", new[] { "Dependencies/TestFeatures.cs" })] - [SetupCompileResource ("FeatureGuardSubstitutions.xml", "ILLink.Substitutions.xml")] - [IgnoreSubstitutions (false)] -#if NATIVEAOT - // ILC has different constant propagation behavior than ILLink, and we don't have - // the test infrastructure to check for different IL sequences between ILLink/ILC. - // Just validate the warning behavior instead. - [SkipKeptItemsValidation] -#else - // Tell linker to treat RequiresDynamicCodeAttribute as a disabled feature: - [SetupLinkerArgument ("--feature", "System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeSupported", "false")] -#endif - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.DefineFeatureGuard.FeatureSwitch", "false")] - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.DefineFeatureGuard.FeatureSwitchAndGuard", "false")] - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitch", "true")] - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.SwitchWithXml", "false")] - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitchWithXml", "false")] - public class FeatureGuardSubstitutions - { - public static void Main () - { - DefineFeatureGuard.Test (); - FeatureGuardPrecedence.Test (); - } - - [Kept] - class DefineFeatureGuard { - [FeatureGuard (typeof(RequiresDynamicCodeAttribute))] - static bool GuardDynamicCode => RuntimeFeature.IsDynamicCodeSupported; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - static void TestGuardDynamicCode () - { - if (GuardDynamicCode) - RequiresDynamicCode (); - } - - [FeatureGuard (typeof(RequiresUnreferencedCodeAttribute))] - static bool GuardUnreferencedCode => TestFeatures.IsUnreferencedCodeSupported; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - - static void TestGuardUnreferencedCode () - { - if (GuardUnreferencedCode) - RequiresUnreferencedCode (); - } - - [Kept] - [KeptAttributeAttribute (typeof (FeatureGuardAttribute))] - [FeatureGuard (typeof(RequiresAssemblyFilesAttribute))] - static bool GuardAssemblyFiles { - [Kept] - get => TestFeatures.IsAssemblyFilesSupported; - } - - [Kept] - // Linker doesn't treat RequiresAssemblyFilesAttribute as a disabled feature, so it's not removed. - static void TestGuardAssemblyFiles () - { - if (GuardAssemblyFiles) - RequiresAssemblyFiles (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresDynamicCodeAttribute), ProducedBy = Tool.Analyzer)] - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureGuard (typeof (RequiresDynamicCodeAttribute))] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardDynamicCodeAndUnreferencedCode => RuntimeFeature.IsDynamicCodeSupported && TestFeatures.IsUnreferencedCodeSupported; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - - static void TestMultipleGuards () - { - if (GuardDynamicCodeAndUnreferencedCode) { - RequiresDynamicCode (); - RequiresUnreferencedCode (); - } - } - - static class UnreferencedCode { - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - public static bool GuardUnreferencedCode => TestFeatures.IsUnreferencedCodeSupported; - } - - static class UnreferencedCodeIndirect { - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - public static bool GuardUnreferencedCode => UnreferencedCode.GuardUnreferencedCode; - } - - // Currently there is no way to annotate a feature type as depending on another feature, - // so indirect guards are expressed the same way as direct guards, by using - // FeatureGuardAttribute that references the underlying feature type. - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardUnreferencedCodeIndirect => UnreferencedCodeIndirect.GuardUnreferencedCode; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - - static void TestIndirectGuard () - { - if (GuardUnreferencedCodeIndirect) - RequiresUnreferencedCode (); - } - - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.DefineFeatureGuard.FeatureSwitch")] - static bool FeatureSwitch => AppContext.TryGetSwitch ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.DefineFeatureGuard.FeatureSwitch", out bool isEnabled) && isEnabled; - - [ExpectedWarning ("IL2026", ProducedBy = Tool.Analyzer)] // Analyzer doesn't respect FeatureSwitchDefinition or feature settings - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - - [Kept] - static void TestFeatureSwitch () - { - if (FeatureSwitch) - RequiresUnreferencedCode (); - } - - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.DefineFeatureGuard.FeatureSwitchAndGuard")] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool FeatureSwitchAndGuard => AppContext.TryGetSwitch ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.DefineFeatureGuard.FeatureSwitchAndGuard", out bool isEnabled) && isEnabled; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - - static void TestFeatureSwitchAndGuard () - { - if (FeatureSwitchAndGuard) - RequiresUnreferencedCode (); - } - - static class UnreferencedCodeCycle { - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - public static bool IsSupported => UnreferencedCodeCycle.IsSupported; - } - - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardUnreferencedCodeCycle => TestFeatures.IsUnreferencedCodeSupported; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - - static void TestFeatureDependencyCycle1 () - { - if (GuardUnreferencedCodeCycle) - RequiresUnreferencedCode (); - } - - static class UnreferencedCodeCycle2_A { - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - public static bool IsSupported => UnreferencedCodeCycle2_A.IsSupported; - } - - static class UnreferencedCodeCycle2_B { - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - public static bool IsSupported => UnreferencedCodeCycle2_B.IsSupported; - } - - static class UnreferencedCodeCycle2 { - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - public static bool IsSupported => UnreferencedCodeCycle2_A.IsSupported; - } - - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardUnreferencedCodeCycle2 => TestFeatures.IsUnreferencedCodeSupported; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.0", - "stloc.0", - "ldloc.0", - "brfalse.s il_6", - "ret" - })] - static void TestFeatureDependencyCycle2 () - { - if (GuardUnreferencedCodeCycle2) - RequiresUnreferencedCode (); - } - - [Kept] - public static void Test () - { - TestGuardDynamicCode (); - TestGuardUnreferencedCode (); - TestGuardAssemblyFiles (); - TestMultipleGuards (); - TestIndirectGuard (); - TestFeatureDependencyCycle1 (); - TestFeatureDependencyCycle2 (); - TestFeatureSwitch (); - TestFeatureSwitchAndGuard (); - } - } - - [Kept] - class FeatureGuardPrecedence { - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitch")] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardAndSwitch => AppContext.TryGetSwitch ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitch", out bool isEnabled) && isEnabled; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.1", - "stloc.0", - "ldloc.0", - "pop", - "call System.Void Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions::RequiresUnreferencedCode()", - "nop", - "ret" - })] - // ILLink/ILCompiler ignore FeatureGuard on properties that also have FeatureSwitchDefinition - [ExpectedWarning ("IL2026", ProducedBy = Tool.Trimmer | Tool.NativeAot)] - static void TestSwitchWinsOverGuard () - { - if (GuardAndSwitch) - RequiresUnreferencedCode (); - } - - [Kept] - [KeptAttributeAttribute (typeof (FeatureSwitchDefinitionAttribute))] - [KeptAttributeAttribute (typeof (FeatureGuardAttribute))] - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitchNotSet")] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardAndSwitchNotSet { - [Kept] - get => AppContext.TryGetSwitch ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitchNotSet", out bool isEnabled) && isEnabled; - } - - [Kept] - // No IL modifications because feature is not set, and FeatureGuard is ignored due to FeatureSwitchDefinition. - [ExpectedWarning ("IL2026", ProducedBy = Tool.Trimmer | Tool.NativeAot)] - static void TestSwitchNotSetWinsOverGuard () - { - if (GuardAndSwitchNotSet) - RequiresUnreferencedCode (); - } - - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardWithXml => TestFeatures.IsUnreferencedCodeSupported; - - [Kept] - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.1", - "stloc.0", - "ldloc.0", - "pop", - "call System.Void Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions::RequiresUnreferencedCode()", - "nop", - "ret" - })] - [ExpectedWarning ("IL2026", ProducedBy = Tool.Trimmer | Tool.NativeAot)] - static void TestXmlWinsOverGuard () - { - if (GuardWithXml) - RequiresUnreferencedCode (); - } - - [KeptAttributeAttribute (typeof (FeatureSwitchDefinitionAttribute))] - [KeptAttributeAttribute (typeof (FeatureGuardAttribute))] - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.SwitchWithXml")] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool SwitchWithXml => AppContext.TryGetSwitch ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.SwitchWithXml", out bool isEnabled) && isEnabled; - - [Kept] - // XML substitutions win despite FeatureSwitchDefinition and feature settings. - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.1", - "stloc.0", - "ldloc.0", - "pop", - "call System.Void Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions::RequiresUnreferencedCode()", - "nop", - "ret" - })] - [ExpectedWarning ("IL2026", ProducedBy = Tool.Trimmer | Tool.NativeAot)] - static void TestXmlWinsOverSwitch () { - if (SwitchWithXml) - RequiresUnreferencedCode (); - } - - [KeptAttributeAttribute (typeof (FeatureSwitchDefinitionAttribute))] - [KeptAttributeAttribute (typeof (FeatureGuardAttribute))] - [ExpectedWarning ("IL4000", nameof (RequiresUnreferencedCodeAttribute), ProducedBy = Tool.Analyzer)] - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardPrecedence.GuardAndSwitchWithXml")] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardAndSwitchWithXml => AppContext.TryGetSwitch ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions.FeatureGuardPrecedence.GuardAndSwitchWithXml", out bool isEnabled) && isEnabled; - - [Kept] - // XML substitutions win despite FeatureSwitchDefinition and feature settings. - [ExpectedInstructionSequence (new[] { - "nop", - "ldc.i4.1", - "stloc.0", - "ldloc.0", - "pop", - "call System.Void Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutions::RequiresUnreferencedCode()", - "nop", - "ret" - })] - [ExpectedWarning ("IL2026", ProducedBy = Tool.Trimmer | Tool.NativeAot)] - static void TestXmlWinsOverGuardAndSwitch () { - if (GuardAndSwitchWithXml) - RequiresUnreferencedCode (); - } - - [Kept] - public static void Test () { - TestSwitchWinsOverGuard (); - TestSwitchNotSetWinsOverGuard (); - TestXmlWinsOverGuard (); - TestXmlWinsOverSwitch (); - TestXmlWinsOverGuardAndSwitch (); - } - } - - [RequiresDynamicCode (nameof (RequiresDynamicCode))] - static void RequiresDynamicCode () { } - - [Kept] - [KeptAttributeAttribute (typeof (RequiresUnreferencedCodeAttribute))] - [RequiresUnreferencedCode (nameof (RequiresUnreferencedCode))] - static void RequiresUnreferencedCode () { } - - [Kept] - [KeptAttributeAttribute (typeof (RequiresAssemblyFilesAttribute))] - [RequiresAssemblyFiles (nameof (RequiresAssemblyFiles))] - static void RequiresAssemblyFiles () { } - } -} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutions.xml b/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutions.xml deleted file mode 100644 index ab5947a1a6b4ca..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutions.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutionsDisabled.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutionsDisabled.cs deleted file mode 100644 index 198595d79394df..00000000000000 --- a/src/tools/illink/test/Mono.Linker.Tests.Cases/Substitutions/FeatureGuardSubstitutionsDisabled.cs +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) .NET Foundation and contributors. All rights reserved. -// Licensed under the MIT license. See LICENSE file in the project root for full license information. - -using System; -using System.Runtime.CompilerServices; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; -using ILLink.RoslynAnalyzer; -using Mono.Linker.Tests.Cases.Expectations.Assertions; -using Mono.Linker.Tests.Cases.Expectations.Helpers; -using Mono.Linker.Tests.Cases.Expectations.Metadata; - -namespace Mono.Linker.Tests.Cases.Substitutions -{ - [ExpectedNoWarnings] - [SetupCompileBefore ("TestFeatures.dll", new[] { "Dependencies/TestFeatures.cs" })] - [SetupLinkerArgument ("--disable-opt", "substitutefeatureguards")] - [SetupLinkerArgument ("--feature", "Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutionsDisabled.FeatureSwitch", "false")] - public class FeatureGuardSubstitutionsDisabled - { - public static void Main () - { - TestGuard (); - TestFeatureSwitch (); - } - - [Kept] - [ExpectedWarning ("IL4000", ProducedBy = Tool.Analyzer)] - [KeptAttributeAttribute (typeof (FeatureGuardAttribute))] - [FeatureGuard (typeof (RequiresUnreferencedCodeAttribute))] - static bool GuardUnreferencedCode { - [Kept] - get => throw null; - } - - [Kept] - // Body is not modified because feature guard substitutions are disabled in this test - [ExpectedWarning ("IL2026")] - static void TestGuard () - { - if (GuardUnreferencedCode) - RequiresUnreferencedCode (); - } - - [FeatureSwitchDefinition ("Mono.Linker.Tests.Cases.Substitutions.FeatureGuardSubstitutionsDisabled.FeatureSwitch")] - static bool FeatureSwitch => throw null; - - [Kept] - [ExpectedWarning ("IL2026", ProducedBy = Tool.Analyzer)] - // Feature switches are still substituted when feature guard substitutions are disabled - [ExpectBodyModified] - static void TestFeatureSwitch () - { - if (FeatureSwitch) - RequiresUnreferencedCode (); - } - - [Kept] - [KeptAttributeAttribute (typeof (RequiresUnreferencedCodeAttribute))] - [RequiresUnreferencedCode (nameof (RequiresUnreferencedCode))] - static void RequiresUnreferencedCode () { } - } -}