diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5a697ac088194..d76e325e8b6ca 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -25,4 +25,5 @@ RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ libssl-dev \ libkrb5-dev \ zlib1g-dev \ - ninja-build + ninja-build \ + tzdata diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b042cebfc6b92..ecc2cc3d3867c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -21,7 +21,7 @@ /src/mono @marek-safar -/src/mono/llvm @vargaz @SamMonoRT +/src/mono/llvm @vargaz @steveisok /src/mono/mono/arch @vargaz /src/mono/mono/eglib @vargaz @lambdageek @@ -36,7 +36,7 @@ /src/mono/mono/eventpipe @lateralusX @lambdageek -/src/mono/mono/mini @vargaz @lambdageek @SamMonoRT +/src/mono/mono/mini @vargaz @lambdageek @steveisok /src/mono/mono/mini/*cfgdump* @vargaz /src/mono/mono/mini/*exceptions* @vargaz @BrzVlad /src/mono/mono/mini/*llvm* @vargaz @fanyang-mono @@ -50,7 +50,7 @@ /src/mono/mono/mini/*simd* @fanyang-mono /src/mono/mono/profiler @BrzVlad @lambdageek -/src/mono/mono/sgen @BrzVlad @lambdageek @SamMonoRT +/src/mono/mono/sgen @BrzVlad @lambdageek /src/mono/mono/utils @vargaz @lambdageek /src/mono/mono/utils/*-win* @lateralusX @lambdageek diff --git a/.github/ISSUE_TEMPLATE/05_blank_issue.md b/.github/ISSUE_TEMPLATE/04_blank_issue.md similarity index 100% rename from .github/ISSUE_TEMPLATE/05_blank_issue.md rename to .github/ISSUE_TEMPLATE/04_blank_issue.md diff --git a/.github/ISSUE_TEMPLATE/04_ci_known_issue.yml b/.github/ISSUE_TEMPLATE/04_ci_known_issue.yml deleted file mode 100644 index 17ec4e5e5ec93..0000000000000 --- a/.github/ISSUE_TEMPLATE/04_ci_known_issue.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: CI Known Issue Report -description: Create a known issue directly -labels: ["blocking-clean-ci","Known Build Error"] -body: - - type: markdown - attributes: - value: | - Use this template to report issues currently affecting PR stability, be it build or test failures. - - type: textarea - id: background - attributes: - label: Error Blob - description: Please identify a clear error string that can help identify future instances of this issue. For more information on how to fill this check our issue triage guidelines at [Failure Analysis](/dotnet/runtime/blob/main/docs/workflow/ci/failure-analysis.md#what-to-do-if-you-determine-the-failure-is-unrelated) - value: | - ```json - { - "ErrorMessage": "", - "BuildRetry": false, - "ErrorPattern": "", - "ExcludeConsoleLog": true - } - ``` - validations: - required: true - - type: textarea - id: repro-steps - attributes: - label: Reproduction Steps - description: | - If possible describe where you observe the issue with links and any other relevant details. - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 54d8c5740bad6..b14edd954edee 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -18,3 +18,6 @@ contact_links: - name: Issue with WPF url: https://github.com/dotnet/wpf/issues/new/choose about: Please open issues relating to WPF in dotnet/wpf. + - name: CI Known Issue Report + url: https://helix.dot.net/BuildAnalysis/CreateKnownIssues + about: Use the helper to create a Known Issue in CI if failures in your runs are unrelated to your change. See [Failure Analysis](https://github.com/dotnet/runtime/blob/main/docs/workflow/ci/failure-analysis.md#what-to-do-if-you-determine-the-failure-is-unrelated) for triage instructions. diff --git a/Directory.Build.props b/Directory.Build.props index 26e112fab56e1..1969e3e16a2a9 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -313,6 +313,8 @@ '$(OfficialBuildId)' == ''">true true + + ClrFullNativeBuild;ClrRuntimeSubset;ClrJitSubset;ClrPalTestsSubset;ClrAllJitsSubset;ClrILToolsSubset;ClrNativeAotSubset;ClrSpmiSubset;ClrCrossComponentsSubset;ClrDebugSubset;HostArchitecture;PgoInstrument;NativeOptimizationDataSupported;CMakeArgs diff --git a/docs/area-owners.md b/docs/area-owners.md index 52cb16d8d8d72..6795bebb817f3 100644 --- a/docs/area-owners.md +++ b/docs/area-owners.md @@ -73,9 +73,9 @@ Note: Editing this file doesn't update the mapping used by `@msftbot` for area-s | area-System.Composition | @ericstj | @dotnet/area-system-composition | | | area-System.Configuration | @ericstj | @dotnet/area-system-configuration | | | area-System.Console | @jeffhandley | @dotnet/area-system-console | | -| area-System.Data | @ajcvickers | @ajcvickers @davoudeshtehari @david-engel @roji | | -| area-System.Data.Odbc | @ajcvickers | @ajcvickers @roji | | -| area-System.Data.OleDB | @ajcvickers | @ajcvickers @roji | | +| area-System.Data | @sammonort | @ajcvickers @davoudeshtehari @david-engel @roji | | +| area-System.Data.Odbc | @sammonort | @ajcvickers @roji | | +| area-System.Data.OleDB | @sammonort | @ajcvickers @roji | | | area-System.Data.SqlClient | @David-Engel | @davoudeshtehari @david-engel @jrahnama | Archived component - limited churn/contributions (see https://devblogs.microsoft.com/dotnet/introducing-the-new-microsoftdatasqlclient/) | | area-System.DateTime | @ericstj | @dotnet/area-system-datetime | System namespace APIs related to dates and times, including DateOnly, DateTime, DateTimeKind, DateTimeOffset, DayOfWeek, TimeOnly, TimeSpan, TimeZone, and TimeZoneInfo | | area-System.Diagnostics | @tommcdon | @dotnet/area-system-diagnostics | | @@ -135,7 +135,7 @@ Note: Editing this file doesn't update the mapping used by `@msftbot` for area-s | area-System.Threading.Channels | @ericstj | @dotnet/area-system-threading-channels | Consultants: @stephentoub | | area-System.Threading.RateLimiting | @rafikiassumani-msft | @BrennanConroy @halter73 | | | area-System.Threading.Tasks | @ericstj | @dotnet/area-system-threading-tasks | Consultants: @stephentoub | -| area-System.Transactions | @ajcvickers | @roji | | +| area-System.Transactions | @sammonort | @roji | | | area-System.Xml | @jeffhandley | @dotnet/area-system-xml | | | area-TieredCompilation-coreclr | @mangod9 | @kouvel | | | area-Tools-ILLink | @agocke | @dotnet/illink | | diff --git a/docs/design/coreclr/jit/first-class-structs.md b/docs/design/coreclr/jit/first-class-structs.md index dc017aee75f2e..4211f75ff745f 100644 --- a/docs/design/coreclr/jit/first-class-structs.md +++ b/docs/design/coreclr/jit/first-class-structs.md @@ -94,10 +94,6 @@ encountered by most phases of the JIT: [#21705](https://github.com/dotnet/coreclr/pull/21705) they are no longer large nodes. * `GT_STORE_OBJ` and `GT_STORE_BLK` have the same structure as `GT_OBJ` and `GT_BLK`, respectively * `Data()` is op2 - * `GT_STORE_DYN_BLK` (GenTreeStoreDynBlk extends GenTreeBlk) - * Additional child `gtDynamicSize` - * Note that these aren't really struct stores; they represent dynamically sized blocks - of arbitrary data. * For `GT_LCL_FLD` nodes, we store a pointer to `ClassLayout` in the node. * For `GT_LCL_VAR` nodes, the `ClassLayout` is obtained from the `LclVarDsc`. diff --git a/docs/design/coreclr/jit/ryujit-overview.md b/docs/design/coreclr/jit/ryujit-overview.md index cdb17002ee197..5e63d38e98f66 100644 --- a/docs/design/coreclr/jit/ryujit-overview.md +++ b/docs/design/coreclr/jit/ryujit-overview.md @@ -222,6 +222,7 @@ The top-level function of interest is `Compiler::compCompile`. It invokes the fo | [Common Subexpression Elimination (CSE)](#cse) | Elimination of redundant subexressions based on value numbers. | | [Assertion Propagation](#assertion-propagation) | Utilizes value numbers to propagate and transform based on properties such as non-nullness. | | [Range analysis](#range-analysis) | Eliminate array index range checks based on value numbers and assertions | +| [Induction variable optimization](#iv-opts) | Optimize induction variables used inside natural loops based on scalar evolution analysis | | [VN-based dead store elimination](#vn-based-dead-store-elimination) | Eliminate stores that do not change the value of a local. | | [If conversion](#if-conversion) | Transform conditional definitions into `GT_SELECT` operators. | | [Rationalization](#rationalization) | Flowgraph order changes from `FGOrderTree` to `FGOrderLinear`. All `GT_COMMA` nodes are transformed. | @@ -347,6 +348,11 @@ reused. Utilizes value numbers to propagate and transform based on properties such as non-nullness. +### Induction variable optimization + +Performs scalar evolution analysis and utilized it to optimize induction variables inside loops. +Currently this entails IV widening which is done on x64 only. + ### Range analysis Optimize array index range checks based on value numbers and assertions. diff --git a/docs/design/coreclr/jit/ryujit-tutorial.md b/docs/design/coreclr/jit/ryujit-tutorial.md index 34466e45afbcd..ec900ccc8cd93 100644 --- a/docs/design/coreclr/jit/ryujit-tutorial.md +++ b/docs/design/coreclr/jit/ryujit-tutorial.md @@ -447,6 +447,10 @@ This is the same diagram as before, but with additional links to indicate execut - Determine initial value for dependent phis - Eliminate checks where the range of the index is within the check range +### Induction Variable Optimization +- Perform scalar evolution analysis to describe values of IR nodes inside loops +- Perform IV widening on x64 to avoid unnecessary zero extensions for array/span indexing + ## RyuJIT Back-End ### Rationalization diff --git a/docs/workflow/ci/failure-analysis.md b/docs/workflow/ci/failure-analysis.md index 57917c841316a..58a11c06bdfa4 100644 --- a/docs/workflow/ci/failure-analysis.md +++ b/docs/workflow/ci/failure-analysis.md @@ -12,6 +12,19 @@ ## Triaging errors seen in CI +## Summary + +**Passing Build Analysis is required to merge into the runtime repo**. + +To resolve failures, do the following, in order: + +1. Fix the problem if your PR is the cause. +2. For all failures not in the "Known test errors" section, [try to file a Known Build Error issue](#what-to-do-if-you-determine-the-failure-is-unrelated). +3. If all else fails, perform a [manual bypass](#bypassing-build-analysis). + + +## Details + In case of failure, any PR on the runtime will have a failed GitHub check - PR Build Analysis - which has a summary of all failures, including a list of matching known issues as well as any regressions introduced to the build or the tests. This tab should be your first stop for analyzing the PR failures. ![Build analysis check](analysis-check.png) @@ -78,6 +91,7 @@ If you have considered all the diagnostic artifacts and determined the failure i ```` It already contains most of the essential information, but *it is very important that you fill out the json blob*. + - You can now use the [Build Analysis Known Issue Helper](https://helix.dot.net/BuildAnalysis/CreateKnownIssues) to create an issue. It assists in adding the right set of labels, fill the necessary paths in the json blob, and it will validate that it matches the text presented for the issue found in the logs. - You can add into the `ErrorMessage` field the string that you found uniquely identifies the issue. In case you need to use a regex, use the `ErrorPattern` field instead. This is a limited to a single-line, non-backtracking regex as described [here](https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssues.md#regex-matching). This regex also needs to be appropriately escaped. Check the [arcade known issues](https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/KnownIssues.md#filling-out-known-issues-json-blob) documentation for a good guide on proper regex and JSON escaping. - The field `ExcludeConsoleLog` describes if the execution logs should be considered on top of the individual test results. **For most cases, this should be set to `true` as the failure will happen within a single test**. Setting it to `false` will mean all failures within an xUnit set of tests will also get attributed to this particular error, since there's one log describing all the problems. Due to limitations in Known Issues around rate limiting and xUnit resiliency, setting `ExcludeConsoleLog=false` is necessary in two scenarios: + Nested tests as reported to Azure DevOps. Essentially this means theory failures, which look like this when reported in Azure DevOps: ![xUnit theory seen in azure devops](theory-azdo.png). @@ -95,6 +109,16 @@ After you do this, if the failure is occurring frequently as per the data captur There are plenty of intermittent failures that won't manifest again on a retry. Therefore these steps should be followed for every iteration of the PR build, e.g. before retrying/rebuilding. +### Bypassing build analysis + +To unconditionally bypass the build analysis check (turn it green), you can add a comment to your PR with the following text: + +``` +/ba-g +``` + +For more information, see https://github.com/dotnet/arcade/blob/main/Documentation/Projects/Build%20Analysis/EscapeMechanismforBuildAnalysis.md + ### Examples of Build Analysis #### Good usage examples diff --git a/eng/DotNetBuild.props b/eng/DotNetBuild.props index 53d03c7f4dd1d..a6350c7fea93f 100644 --- a/eng/DotNetBuild.props +++ b/eng/DotNetBuild.props @@ -21,6 +21,10 @@ <_hostArch>$(_hostRid.Substring($(_hostRidPlatformIndex)).TrimStart('-')) minimal + + + true diff --git a/eng/Subsets.props b/eng/Subsets.props index dd284ea6d9977..29d7467e6b43e 100644 --- a/eng/Subsets.props +++ b/eng/Subsets.props @@ -255,7 +255,7 @@ - + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 3ee91b259e46d..327d959b892e8 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,8 +1,8 @@ - + https://github.com/dotnet/icu - c15a038f408fef6814e5f9c0bf8882bcdf53a290 + 9c4c9995bc756a01597b5efb0e452ef879a76d99 https://github.com/dotnet/msquic @@ -12,9 +12,9 @@ https://github.com/dotnet/wcf 7f504aabb1988e9a093c1e74d8040bd52feb2f01 - + https://github.com/dotnet/emsdk - 2d3f1fe4807a21879cedba9d3fde8cd329fb17f2 + 0f3e462442af5fe65271e3185d5b645ad40a6041 https://github.com/dotnet/llvm-project @@ -90,121 +90,121 @@ a045dd54a4c44723c215d992288160eb1401bb7f - + https://github.com/dotnet/cecil - 61250b0ed403b3f9b69a33f7d8f66f311338d6a1 + 0d0bc8e0f47fdae9834e1eac678f364c50946133 - + https://github.com/dotnet/cecil - 61250b0ed403b3f9b69a33f7d8f66f311338d6a1 + 0d0bc8e0f47fdae9834e1eac678f364c50946133 - + https://github.com/dotnet/emsdk - 2d3f1fe4807a21879cedba9d3fde8cd329fb17f2 + 0f3e462442af5fe65271e3185d5b645ad40a6041 - + https://github.com/dotnet/emsdk - 2d3f1fe4807a21879cedba9d3fde8cd329fb17f2 + 0f3e462442af5fe65271e3185d5b645ad40a6041 - + https://github.com/dotnet/source-build-reference-packages - 2f79f97b7a6a0cf2ca3297a8fa526e6f4ea98ce2 + 62fb9a85e5c4af657b0014fd6d6588c139d0bb4f - + https://github.com/dotnet/source-build-externals - ddfb60463c966af55fd0e222c2266170e83d1324 + 88f13afba58a6c455039d71bbdd2cff3d847b236 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 https://github.com/dotnet/runtime-assets @@ -314,43 +314,43 @@ https://github.com/dotnet/llvm-project 9885e5aecc176ca701fc3527877d608bf7ccfb7d - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 - + https://github.com/dotnet/runtime - d972a19c077e899d0b3fff97d955968e50906396 + c55c4d50793c878cc73ae6ca3335f2b6b3ccc8a4 https://github.com/dotnet/xharness @@ -364,9 +364,9 @@ https://github.com/dotnet/xharness 8aa2a4cb80000ebb46ee61cd6ac29b2e63ebe87c - + https://github.com/dotnet/arcade - c3f5cbfb2829795294f5c2d9fa5a0522f47e91fb + 9aa3f2e68b30ac51823dd444e8cb962e058c5699 https://dev.azure.com/dnceng/internal/_git/dotnet-optimization @@ -384,9 +384,9 @@ https://dev.azure.com/dnceng/internal/_git/dotnet-optimization db9f1c2362565f3ef41c8e8feb5ed49ab11a6459 - + https://github.com/dotnet/hotreload-utils - ec73ebf54c4ae98ac1450fcf95998180d4160f31 + 81cdd6568c7360cf337b1ab6a2dcf2ce84530a7f https://github.com/dotnet/runtime-assets @@ -399,19 +399,18 @@ https://github.com/dotnet/roslyn 77372c66fd54927312b5b0a2e399e192f74445c9 - https://github.com/dotnet/roslyn 77372c66fd54927312b5b0a2e399e192f74445c9 - + https://github.com/dotnet/roslyn-analyzers - 4195460a822168a75aa3d31b4a8d0fa88c42855c + ba8b7f2c3ae092d0301f0c5e49bd30340af553c8 - + https://github.com/dotnet/roslyn-analyzers - 4195460a822168a75aa3d31b4a8d0fa88c42855c + ba8b7f2c3ae092d0301f0c5e49bd30340af553c8 @@ -419,14 +418,14 @@ 77372c66fd54927312b5b0a2e399e192f74445c9 - + https://github.com/dotnet/sdk - b834bb25bdf308c4971d00cef6b726dfaa828c66 + 7900db19bd7d1a384490909f085cec371ec696d2 - + https://github.com/dotnet/sdk - b834bb25bdf308c4971d00cef6b726dfaa828c66 + 7900db19bd7d1a384490909f085cec371ec696d2 @@ -443,9 +442,9 @@ https://github.com/NuGet/NuGet.Client 8fef55f5a55a3b4f2c96cd1a9b5ddc51d4b927f8 - + https://github.com/dotnet/installer - e6b3ff2dff85b43bd3a323e7c0bac4f1f58ccd62 + d070660282eb5f78497310f77093638744112e03 diff --git a/eng/Versions.props b/eng/Versions.props index 5c7aeded43391..256e6e83a22f8 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,7 +11,7 @@ 7.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet8)').Build),14)) 6.0.$([MSBuild]::Add($([System.Version]::Parse('$(PackageVersionNet7)').Build),11)) preview - 2 + 3 false release @@ -34,8 +34,8 @@ - 3.11.0-beta1.24121.1 - 9.0.0-preview.24121.1 + 3.11.0-beta1.24122.2 + 9.0.0-preview.24122.2 - 9.0.100-preview.2.24118.3 + 9.0.100-preview.3.24126.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 2.6.7-beta.24112.1 - 9.0.0-beta.24112.1 - 2.6.7-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 - 9.0.0-beta.24112.1 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 2.6.7-beta.24127.4 + 9.0.0-beta.24127.4 + 2.6.7-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 + 9.0.0-beta.24127.4 1.4.0 6.0.0-preview.1.102 - 9.0.0-preview.2.24116.2 + 9.0.0-preview.3.24126.1 6.0.0 - 9.0.0-preview.2.24116.2 + 9.0.0-preview.3.24126.1 16.0.5-alpha.1.24112.1 16.0.5-alpha.1.24112.1 @@ -128,19 +128,19 @@ 8.0.0 5.0.0 4.5.5 - 9.0.0-preview.2.24116.2 - 9.0.0-preview.2.24116.2 + 9.0.0-preview.3.24126.1 + 9.0.0-preview.3.24126.1 6.0.0 5.0.0 5.0.0 5.0.0 7.0.0 - 9.0.0-preview.2.24116.2 + 9.0.0-preview.3.24126.1 6.0.0 7.0.0 4.5.4 4.5.0 - 9.0.0-preview.2.24116.2 + 9.0.0-preview.3.24126.1 8.0.0 8.0.0 @@ -190,7 +190,7 @@ 9.0.0-prerelease.24119.1 9.0.0-prerelease.24119.1 9.0.0-prerelease.24119.1 - 9.0.0-alpha.0.24119.1 + 9.0.0-alpha.0.24120.1 3.12.0 4.5.0 6.0.0 @@ -216,11 +216,11 @@ 8.0.0-preview-20230918.1 - 0.11.4-alpha.24119.1 + 0.11.4-alpha.24120.1 - 9.0.0-preview.2.24116.2 + 9.0.0-preview.3.24126.1 - 9.0.0-preview.2.24119.1 + 9.0.0-preview.3.24123.1 2.2.3 9.0.0-alpha.1.24067.1 @@ -243,9 +243,9 @@ Note: when the name is updated, make sure to update dependency name in eng/pipelines/common/xplat-setup.yml like - DarcDependenciesChanged.Microsoft_NET_Workload_Emscripten_Current_Manifest-9_0_100_Transport --> - 9.0.0-preview.2.24121.1 + 9.0.0-preview.3.24126.1 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100TransportVersion) - 9.0.0-preview.2.24121.1 + 9.0.0-preview.3.24126.1 1.1.87-gba258badda 1.0.0-v3.14.0.5722 @@ -262,7 +262,7 @@ 3.1.7 1.0.406601 - 9.0.100-preview.2.24116.21 + 9.0.100-preview.3.24126.2 $(MicrosoftDotnetSdkInternalVersion) diff --git a/eng/build.ps1 b/eng/build.ps1 index db18267f33e1c..be88dcb263e8d 100644 --- a/eng/build.ps1 +++ b/eng/build.ps1 @@ -325,6 +325,9 @@ if ($env:TreatWarningsAsErrors -eq 'false') { $arguments += " -warnAsError 0" } +# disable terminal logger for now: https://github.com/dotnet/runtime/issues/97211 +$arguments += " /tl:false" + # Disable targeting pack caching as we reference a partially constructed targeting pack and update it later. # The later changes are ignored when using the cache. $env:DOTNETSDK_ALLOW_TARGETING_PACK_CACHING=0 diff --git a/eng/build.sh b/eng/build.sh index 67f3cfeea4727..75fe2cdc39c5d 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -553,6 +553,9 @@ if [[ "${TreatWarningsAsErrors:-}" == "false" ]]; then arguments="$arguments -warnAsError 0" fi +# disable terminal logger for now: https://github.com/dotnet/runtime/issues/97211 +arguments="$arguments -tl:false" + initDistroRid "$os" "$arch" "$crossBuild" # Disable targeting pack caching as we reference a partially constructed targeting pack and update it later. diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 7d8dc89b919bc..269fdb9420da9 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -65,6 +65,9 @@ $ErrorActionPreference = 'Stop' # Base-64 encoded SAS token that has permission to storage container described by $runtimeSourceFeed [string]$runtimeSourceFeedKey = if (Test-Path variable:runtimeSourceFeedKey) { $runtimeSourceFeedKey } else { $null } +# True if the build is a product build +[bool]$productBuild = if (Test-Path variable:productBuild) { $productBuild } else { $false } + function Create-Directory ([string[]] $path) { New-Item -Path $path -Force -ItemType 'Directory' | Out-Null } @@ -850,7 +853,8 @@ function MSBuild-Core() { } # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null) { + # Skip this when the build is a child of the VMR orchestrator build. + if ($ci -and $env:SYSTEM_TEAMPROJECT -ne $null -and !$productBuild -and $properties -notlike "*DotNetBuildRepo=true*") { Write-PipelineSetResult -Result "Failed" -Message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/common/tools.sh b/eng/common/tools.sh index ece4b73079536..6f0141e7d513f 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -68,6 +68,9 @@ fi runtime_source_feed=${runtime_source_feed:-''} runtime_source_feed_key=${runtime_source_feed_key:-''} +# True if the build is a product build +product_build=${product_build:-false} + # Resolve any symlinks in the given path. function ResolvePath { local path=$1 @@ -141,7 +144,7 @@ function InitializeDotNetCli { if [[ $global_json_has_runtimes == false && -n "${DOTNET_INSTALL_DIR:-}" && -d "$DOTNET_INSTALL_DIR/sdk/$dotnet_sdk_version" ]]; then dotnet_root="$DOTNET_INSTALL_DIR" else - dotnet_root="$repo_root/.dotnet" + dotnet_root="${repo_root}.dotnet" export DOTNET_INSTALL_DIR="$dotnet_root" @@ -503,7 +506,8 @@ function MSBuild-Core { echo "Build failed with exit code $exit_code. Check errors above." # When running on Azure Pipelines, override the returned exit code to avoid double logging. - if [[ "$ci" == "true" && -n ${SYSTEM_TEAMPROJECT:-} ]]; then + # Skip this when the build is a child of the VMR orchestrator build. + if [[ "$ci" == true && -n ${SYSTEM_TEAMPROJECT:-} && "$product_build" != true && $properties != *"DotNetBuildRepo=true"* ]]; then Write-PipelineSetResult -result "Failed" -message "msbuild execution failed." # Exiting with an exit code causes the azure pipelines task to log yet another "noise" error # The above Write-PipelineSetResult will cause the task to be marked as failure without adding yet another error diff --git a/eng/pipelines/common/xplat-setup.yml b/eng/pipelines/common/xplat-setup.yml index c66fec22bead5..dfb8952b423e4 100644 --- a/eng/pipelines/common/xplat-setup.yml +++ b/eng/pipelines/common/xplat-setup.yml @@ -171,9 +171,13 @@ jobs: demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 # OSX Build Pool (we don't have on-prem OSX BuildPool). - ${{ if in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator') }}: + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), eq(variables['System.TeamProject'], 'public')) }}: vmImage: 'macos-12' + # Official build OSX pool + ${{ if and(in(parameters.osGroup, 'osx', 'maccatalyst', 'ios', 'iossimulator', 'tvos', 'tvossimulator'), ne(variables['System.TeamProject'], 'public')) }}: + vmImage: 'macos-13-arm64' + # Official Build Windows Pool ${{ if and(or(eq(parameters.osGroup, 'windows'), eq(parameters.jobParameters.hostedOs, 'windows')), ne(variables['System.TeamProject'], 'public')) }}: name: $(DncEngInternalBuildPool) diff --git a/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml b/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml index b348c636ffb2e..de23519f9c62e 100644 --- a/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml +++ b/eng/pipelines/coreclr/runtime-nativeaot-outerloop.yml @@ -68,7 +68,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Libs - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false /p:RunAnalyzers=false timeoutInMinutes: 300 # doesn't normally take this long, but I've seen Helix queues backed up for 160 minutes includeAllPlatforms: true # extra steps, run tests @@ -94,7 +94,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Checked_Libs - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:IlcUseServerGc=false /p:RunAnalyzers=false timeoutInMinutes: 360 # extra steps, run tests postBuildSteps: @@ -119,7 +119,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Checked_Libs_SizeOpt - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Size /p:IlcUseServerGc=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Size /p:IlcUseServerGc=false /p:RunAnalyzers=false timeoutInMinutes: 240 # extra steps, run tests postBuildSteps: @@ -144,7 +144,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Checked_Libs_SpeedOpt - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Speed /p:IlcUseServerGc=false + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) -rc Checked /p:TestNativeAot=true /p:ArchiveTests=true /p:OptimizationPreference=Speed /p:IlcUseServerGc=false /p:RunAnalyzers=false timeoutInMinutes: 240 # extra steps, run tests postBuildSteps: @@ -174,7 +174,7 @@ extends: jobParameters: timeoutInMinutes: 240 nameSuffix: NativeAOT_Pri0 - buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release + buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 7b4ce6c6c7f43..9339194111448 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -86,9 +86,9 @@ jobs: # Linux musl arm32 - ${{ if eq(parameters.platform, 'linux_musl_arm') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Alpine.316.Arm32.Open)Ubuntu.2004.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.16-helix-arm32v7 + - (Alpine.316.Arm32.Open)Ubuntu.2004.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm32v7 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.316.Arm32)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.16-helix-arm32v7 + - (Alpine.316.Arm32)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.17-helix-arm32v7 # Linux musl arm64 - ${{ if eq(parameters.platform, 'linux_musl_arm64') }}: diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index be2870611e617..d7a1f1847eb11 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -254,7 +254,7 @@ extends: jobParameters: timeoutInMinutes: 120 nameSuffix: NativeAOT - buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release + buildArgs: -s clr.aot+host.native+libs -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: @@ -293,7 +293,7 @@ extends: jobParameters: timeoutInMinutes: 180 nameSuffix: NativeAOT - buildArgs: -s clr.aot+host.native+libs.native+libs.sfx -rc $(_BuildConfig) -lc Release -hc Release + buildArgs: -s clr.aot+host.native+libs.native+libs.sfx -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: @@ -338,7 +338,7 @@ extends: testGroup: innerloop timeoutInMinutes: 120 nameSuffix: NativeAOT - buildArgs: -s clr.aot+host.native+libs+tools.illink -c $(_BuildConfig) -rc $(_BuildConfig) -lc Release -hc Release + buildArgs: -s clr.aot+host.native+libs+tools.illink -c $(_BuildConfig) -rc $(_BuildConfig) -lc Release -hc Release /p:RunAnalyzers=false postBuildSteps: - template: /eng/pipelines/coreclr/nativeaot-post-build-steps.yml parameters: @@ -375,7 +375,7 @@ extends: testGroup: innerloop isSingleFile: true nameSuffix: NativeAOT_Libraries - buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true /p:ArchiveTests=true + buildArgs: -s clr.aot+host.native+libs+libs.tests -c $(_BuildConfig) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true /p:ArchiveTests=true /p:RunAnalyzers=false timeoutInMinutes: 240 # Doesn't actually take long, but we've seen the ARM64 Helix queue often get backlogged for 2+ hours # extra steps, run tests postBuildSteps: diff --git a/eng/testing/ChromeVersions.props b/eng/testing/ChromeVersions.props index 8549c7646a516..ecf01665114dd 100644 --- a/eng/testing/ChromeVersions.props +++ b/eng/testing/ChromeVersions.props @@ -1,12 +1,12 @@ - 121.0.6167.184 - 1233107 - https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/1233114 - 12.1.285 - 121.0.6167.185 - 1233107 - https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/1233136 - 12.1.285 + 122.0.6261.69 + 1250580 + https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/1250580 + 12.2.281 + 122.0.6261.69 + 1250580 + https://storage.googleapis.com/chromium-browser-snapshots/Win_x64/1250586 + 12.2.281 \ No newline at end of file diff --git a/global.json b/global.json index 1e159da9c5464..6fe08725d44cf 100644 --- a/global.json +++ b/global.json @@ -1,18 +1,18 @@ { "sdk": { - "version": "9.0.100-alpha.1.23615.4", + "version": "9.0.100-preview.1.24101.2", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "9.0.100-alpha.1.23615.4" + "dotnet": "9.0.100-preview.1.24101.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24112.1", - "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24112.1", - "Microsoft.DotNet.SharedFramework.Sdk": "9.0.0-beta.24112.1", + "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.24127.4", + "Microsoft.DotNet.Helix.Sdk": "9.0.0-beta.24127.4", + "Microsoft.DotNet.SharedFramework.Sdk": "9.0.0-beta.24127.4", "Microsoft.Build.NoTargets": "3.7.0", "Microsoft.Build.Traversal": "3.4.0", - "Microsoft.NET.Sdk.IL": "9.0.0-preview.2.24116.2" + "Microsoft.NET.Sdk.IL": "9.0.0-preview.3.24126.1" } } diff --git a/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj index 6b3ddff0cc868..d005341dae6b1 100644 --- a/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/coreclr/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -203,7 +203,6 @@ - @@ -231,7 +230,6 @@ - diff --git a/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs b/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs index db8d4ead4659b..77f64abd1b42c 100644 --- a/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs +++ b/src/coreclr/System.Private.CoreLib/src/Internal/Runtime/InteropServices/ComActivator.cs @@ -203,7 +203,6 @@ private static void ClassRegistrationScenarioForType(ComActivationContext cxt, b // Finally validate signature ReadOnlySpan methParams = method.GetParametersAsSpan(); if (method.ReturnType != typeof(void) - || methParams == null || methParams.Length != 1 || (methParams[0].ParameterType != typeof(string) && methParams[0].ParameterType != typeof(Type))) { diff --git a/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs index 88c929dbe74cb..940d1622bad18 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Object.CoreCLR.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; using System.Runtime.CompilerServices; namespace System @@ -19,7 +20,9 @@ public partial class Object [Intrinsic] protected internal unsafe object MemberwiseClone() { - object clone = RuntimeHelpers.AllocateUninitializedClone(this); + object clone = this; + RuntimeHelpers.AllocateUninitializedClone(ObjectHandleOnStack.Create(ref clone)); + Debug.Assert(clone != this); // copy contents of "this" to the clone diff --git a/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs b/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs index 2b695f1baf5b0..327113c63f9a3 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Reflection/Emit/DynamicILGenerator.cs @@ -417,7 +417,7 @@ private int GetMemberRefToken(MethodInfo methodInfo, Type[]? optionalParameterTy throw new ArgumentException(SR.Argument_MustBeRuntimeMethodInfo, nameof(methodInfo)); ReadOnlySpan paramInfo = methodInfo.GetParametersAsSpan(); - if (paramInfo != null && paramInfo.Length != 0) + if (paramInfo.Length != 0) { parameterTypes = new Type[paramInfo.Length]; requiredCustomModifiers = new Type[parameterTypes.Length][]; diff --git a/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs b/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs index b5cff2f1e42ec..53f2690948df4 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs @@ -645,27 +645,29 @@ public override Assembly GetSatelliteAssembly(CultureInfo culture, Version? vers { ArgumentNullException.ThrowIfNull(culture); - return InternalGetSatelliteAssembly(culture, version, throwOnFileNotFound: true)!; + return InternalGetSatelliteAssembly(this, culture, version, throwOnFileNotFound: true)!; } [DynamicSecurityMethod] // Methods containing StackCrawlMark local var has to be marked DynamicSecurityMethod - internal Assembly? InternalGetSatelliteAssembly(CultureInfo culture, + internal static Assembly? InternalGetSatelliteAssembly(Assembly assembly, + CultureInfo culture, Version? version, bool throwOnFileNotFound) { var an = new AssemblyName(); - an.SetPublicKey(GetPublicKey()); - an.Flags = GetFlags() | AssemblyNameFlags.PublicKey; - an.Version = version ?? GetVersion(); + RuntimeAssembly runtimeAssembly = (RuntimeAssembly)assembly; + an.SetPublicKey(runtimeAssembly.GetPublicKey()); + an.Flags = runtimeAssembly.GetFlags() | AssemblyNameFlags.PublicKey; + an.Version = version ?? runtimeAssembly.GetVersion(); an.CultureInfo = culture; - an.Name = GetSimpleName() + ".resources"; + an.Name = runtimeAssembly.GetSimpleName() + ".resources"; // This stack crawl mark is never used because the requesting assembly is explicitly specified, // so the value could be anything. StackCrawlMark unused = default; - RuntimeAssembly? retAssembly = InternalLoad(an, ref unused, requestingAssembly: this, throwOnFileNotFound: throwOnFileNotFound); + RuntimeAssembly? retAssembly = InternalLoad(an, ref unused, requestingAssembly: runtimeAssembly, throwOnFileNotFound: throwOnFileNotFound); - if (retAssembly == this) + if (retAssembly == runtimeAssembly) { retAssembly = null; } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.CoreCLR.cs deleted file mode 100644 index 05805072cd7cf..0000000000000 --- a/src/coreclr/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.CoreCLR.cs +++ /dev/null @@ -1,19 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Globalization; -using System.Reflection; - -namespace System.Resources -{ - internal sealed partial class ManifestBasedResourceGroveler - { - // Internal version of GetSatelliteAssembly that avoids throwing FileNotFoundException - private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, - CultureInfo culture, - Version? version) - { - return ((RuntimeAssembly)mainAssembly).InternalGetSatelliteAssembly(culture, version, throwOnFileNotFound: false); - } - } -} diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 02ecf96568927..733e3a664bcc5 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -139,8 +139,32 @@ public static unsafe void PrepareMethod(RuntimeMethodHandle method, RuntimeTypeH [MethodImpl(MethodImplOptions.InternalCall)] internal static extern int TryGetHashCode(object o); + public static new unsafe bool Equals(object? o1, object? o2) + { + // Compare by ref for normal classes, by value for value types. + + if (ReferenceEquals(o1, o2)) + return true; + + if (o1 is null || o2 is null) + return false; + + MethodTable* pMT = GetMethodTable(o1); + + // If it's not a value class, don't compare by value + if (!pMT->IsValueType) + return false; + + // Make sure they are the same type. + if (pMT != GetMethodTable(o2)) + return false; + + // Compare the contents + return ContentEquals(o1, o2); + } + [MethodImpl(MethodImplOptions.InternalCall)] - public static extern new bool Equals(object? o1, object? o2); + private static extern unsafe bool ContentEquals(object o1, object o2); [Obsolete("OffsetToStringData has been deprecated. Use string.GetPinnableReference() instead.")] public static int OffsetToStringData @@ -194,8 +218,8 @@ public static object GetUninitializedObject( return rt.GetUninitializedObject(); } - [MethodImpl(MethodImplOptions.InternalCall)] - internal static extern object AllocateUninitializedClone(object obj); + [LibraryImport(QCall, EntryPoint = "ObjectNative_AllocateUninitializedClone")] + internal static partial void AllocateUninitializedClone(ObjectHandleOnStack objHandle); /// true if given type is reference type or value type that contains references [Intrinsic] diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs index 4ae608fc17d23..228f58c0ea4d0 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/ExceptionServices/InternalCalls.cs @@ -42,7 +42,7 @@ internal static unsafe partial bool RhpCallFilterFunclet( [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "EHEnumInitFromStackFrameIterator")] [return: MarshalAs(UnmanagedType.Bool)] - internal static unsafe partial bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, byte** pMethodStartAddress, void* pEHEnum); + internal static unsafe partial bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, out EH.MethodRegionInfo pMethodRegionInfo, void* pEHEnum); [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "EHEnumNext")] [return: MarshalAs(UnmanagedType.Bool)] diff --git a/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs index c74d76388b91a..00a8d78685d4d 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/RuntimeType.CoreCLR.cs @@ -2757,7 +2757,12 @@ public override InterfaceMapping GetInterfaceMap([DynamicallyAccessedMembers(Dyn MethodBase? rtTypeMethodBase = GetMethodBase(reflectedType, classRtMethodHandle); // a class may not implement all the methods of an interface (abstract class) so null is a valid value Debug.Assert(rtTypeMethodBase is null || rtTypeMethodBase is RuntimeMethodInfo); - im.TargetMethods[i] = (MethodInfo)rtTypeMethodBase!; + RuntimeMethodInfo? targetMethod = (RuntimeMethodInfo?)rtTypeMethodBase; + // the TargetMethod provided to us by runtime internals may be a generic method instance, + // potentially with invalid arguments. TargetMethods in the InterfaceMap should never be + // instances, only definitions. + im.TargetMethods[i] = (targetMethod is { IsGenericMethod: true, IsGenericMethodDefinition: false }) + ? targetMethod.GetGenericMethodDefinition() : targetMethod!; } return im; diff --git a/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs b/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs index 78301866c36dc..f4c3acb31adf8 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/ValueType.cs @@ -120,7 +120,7 @@ public override unsafe int GetHashCode() else { object thisRef = this; - switch (GetHashCodeStrategy(pMT, ObjectHandleOnStack.Create(ref thisRef), out uint fieldOffset, out uint fieldSize)) + switch (GetHashCodeStrategy(pMT, ObjectHandleOnStack.Create(ref thisRef), out uint fieldOffset, out uint fieldSize, out MethodTable* fieldMT)) { case ValueTypeHashCodeStrategy.ReferenceField: hashCode.Add(Unsafe.As(ref Unsafe.AddByteOffset(ref rawData, fieldOffset)).GetHashCode()); @@ -138,6 +138,12 @@ public override unsafe int GetHashCode() Debug.Assert(fieldSize != 0); hashCode.AddBytes(MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AddByteOffset(ref rawData, fieldOffset), (int)fieldSize)); break; + + case ValueTypeHashCodeStrategy.ValueTypeOverride: + Debug.Assert(fieldMT != null); + // Box the field to handle complicated cases like mutable method and shared generic + hashCode.Add(RuntimeHelpers.Box(fieldMT, ref Unsafe.AddByteOffset(ref rawData, fieldOffset))?.GetHashCode() ?? 0); + break; } } @@ -152,11 +158,12 @@ private enum ValueTypeHashCodeStrategy DoubleField, SingleField, FastGetHashCode, + ValueTypeOverride, } [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ValueType_GetHashCodeStrategy")] private static unsafe partial ValueTypeHashCodeStrategy GetHashCodeStrategy( - MethodTable* pMT, ObjectHandleOnStack objHandle, out uint fieldOffset, out uint fieldSize); + MethodTable* pMT, ObjectHandleOnStack objHandle, out uint fieldOffset, out uint fieldSize, out MethodTable* fieldMT); public override string? ToString() { diff --git a/src/coreclr/classlibnative/bcltype/objectnative.cpp b/src/coreclr/classlibnative/bcltype/objectnative.cpp index 4622955b44adf..afbda5fad9916 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.cpp +++ b/src/coreclr/classlibnative/bcltype/objectnative.cpp @@ -123,48 +123,22 @@ FCIMPL1(INT32, ObjectNative::TryGetHashCode, Object* obj) { } FCIMPLEND -// -// Compare by ref for normal classes, by value for value types. -// -// @todo: it would be nice to customize this method based on the -// defining class rather than doing a runtime check whether it is -// a value type. -// - -FCIMPL2(FC_BOOL_RET, ObjectNative::Equals, Object *pThisRef, Object *pCompareRef) +FCIMPL2(FC_BOOL_RET, ObjectNative::ContentEquals, Object *pThisRef, Object *pCompareRef) { - CONTRACTL - { - FCALL_CHECK; - INJECT_FAULT(FCThrow(kOutOfMemoryException);); - } - CONTRACTL_END; - - if (pThisRef == pCompareRef) - FC_RETURN_BOOL(TRUE); + FCALL_CONTRACT; - // Since we are in FCALL, we must handle NULL specially. - if (pThisRef == NULL || pCompareRef == NULL) - FC_RETURN_BOOL(FALSE); + // Should be ensured by caller + _ASSERTE(pThisRef != NULL); + _ASSERTE(pCompareRef != NULL); + _ASSERTE(pThisRef->GetMethodTable() == pCompareRef->GetMethodTable()); MethodTable *pThisMT = pThisRef->GetMethodTable(); - // If it's not a value class, don't compare by value - if (!pThisMT->IsValueType()) - FC_RETURN_BOOL(FALSE); - - // Make sure they are the same type. - if (pThisMT != pCompareRef->GetMethodTable()) - FC_RETURN_BOOL(FALSE); - - // Compare the contents (size - vtable - sync block index). - DWORD dwBaseSize = pThisMT->GetBaseSize(); - if(pThisMT == g_pStringClass) - dwBaseSize -= sizeof(WCHAR); + // Compare the contents BOOL ret = memcmp( - (void *) (pThisRef+1), - (void *) (pCompareRef+1), - dwBaseSize - sizeof(Object) - sizeof(int)) == 0; + pThisRef->GetData(), + pCompareRef->GetData(), + pThisMT->GetNumInstanceFieldBytes()) == 0; FC_GC_POLL_RET(); @@ -215,36 +189,34 @@ FCIMPL1(Object*, ObjectNative::GetClass, Object* pThis) } FCIMPLEND -FCIMPL1(Object*, ObjectNative::AllocateUninitializedClone, Object* pObjUNSAFE) +extern "C" void QCALLTYPE ObjectNative_AllocateUninitializedClone(QCall::ObjectHandleOnStack objHandle) { - FCALL_CONTRACT; - - // Delegate error handling to managed side (it will throw NullReferenceException) - if (pObjUNSAFE == NULL) - return NULL; + QCALL_CONTRACT; - OBJECTREF refClone = ObjectToOBJECTREF(pObjUNSAFE); + BEGIN_QCALL; - HELPER_METHOD_FRAME_BEGIN_RET_1(refClone); + GCX_COOP(); + OBJECTREF refClone = objHandle.Get(); + _ASSERTE(refClone != NULL); // Should be handled at managed side MethodTable* pMT = refClone->GetMethodTable(); - + // assert that String has overloaded the Clone() method _ASSERTE(pMT != g_pStringClass); - - if (pMT->IsArray()) { - refClone = DupArrayForCloning((BASEARRAYREF)refClone); - } else { + + if (pMT->IsArray()) + { + objHandle.Set(DupArrayForCloning((BASEARRAYREF)refClone)); + } + else + { // We don't need to call the because we know // that it has been called....(It was called before this was created) - refClone = AllocateObject(pMT); + objHandle.Set(AllocateObject(pMT)); } - HELPER_METHOD_FRAME_END(); - - return OBJECTREFToObject(refClone); + END_QCALL; } -FCIMPLEND extern "C" BOOL QCALLTYPE Monitor_Wait(QCall::ObjectHandleOnStack pThis, INT32 Timeout) { diff --git a/src/coreclr/classlibnative/bcltype/objectnative.h b/src/coreclr/classlibnative/bcltype/objectnative.h index d8948922dd0b7..418fd2561d7cc 100644 --- a/src/coreclr/classlibnative/bcltype/objectnative.h +++ b/src/coreclr/classlibnative/bcltype/objectnative.h @@ -27,12 +27,12 @@ class ObjectNative static FCDECL1(INT32, GetHashCode, Object* vThisRef); static FCDECL1(INT32, TryGetHashCode, Object* vThisRef); - static FCDECL2(FC_BOOL_RET, Equals, Object *pThisRef, Object *pCompareRef); - static FCDECL1(Object*, AllocateUninitializedClone, Object* pObjUNSAFE); + static FCDECL2(FC_BOOL_RET, ContentEquals, Object *pThisRef, Object *pCompareRef); static FCDECL1(Object*, GetClass, Object* pThis); static FCDECL1(FC_BOOL_RET, IsLockHeld, Object* pThisUNSAFE); }; +extern "C" void QCALLTYPE ObjectNative_AllocateUninitializedClone(QCall::ObjectHandleOnStack objHandle); extern "C" BOOL QCALLTYPE Monitor_Wait(QCall::ObjectHandleOnStack pThis, INT32 Timeout); extern "C" void QCALLTYPE Monitor_Pulse(QCall::ObjectHandleOnStack pThis); extern "C" void QCALLTYPE Monitor_PulseAll(QCall::ObjectHandleOnStack pThis); diff --git a/src/coreclr/classlibnative/bcltype/system.cpp b/src/coreclr/classlibnative/bcltype/system.cpp index ef02743b36696..6e9a9a9ee956a 100644 --- a/src/coreclr/classlibnative/bcltype/system.cpp +++ b/src/coreclr/classlibnative/bcltype/system.cpp @@ -145,7 +145,7 @@ WCHAR *g_pFailFastBuffer = g_szFailFastBuffer; // This is the common code for FailFast processing that is wrapped by the two // FailFast FCalls below. -void SystemNative::GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExceptionForWatsonBucketing, UINT_PTR retAddress, UINT exitCode, STRINGREF refErrorSourceString) +void SystemNative::GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExceptionForWatsonBucketing, UINT_PTR retAddress, STRINGREF refErrorSourceString) { CONTRACTL { @@ -282,7 +282,7 @@ void SystemNative::GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExce if (gc.refExceptionForWatsonBucketing != NULL) pThread->SetLastThrownObject(gc.refExceptionForWatsonBucketing); - EEPolicy::HandleFatalError(exitCode, retAddress, pszMessage, NULL, errorSourceString, argExceptionString); + EEPolicy::HandleFatalError(COR_E_FAILFAST, retAddress, pszMessage, NULL, errorSourceString, argExceptionString); GCPROTECT_END(); } @@ -301,25 +301,7 @@ FCIMPL1(VOID, SystemNative::FailFast, StringObject* refMessageUNSAFE) UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); // Call the actual worker to perform failfast - GenericFailFast(refMessage, NULL, retaddr, COR_E_FAILFAST, NULL); - - HELPER_METHOD_FRAME_END(); -} -FCIMPLEND - -FCIMPL2(VOID, SystemNative::FailFastWithExitCode, StringObject* refMessageUNSAFE, UINT exitCode) -{ - FCALL_CONTRACT; - - STRINGREF refMessage = (STRINGREF)refMessageUNSAFE; - - HELPER_METHOD_FRAME_BEGIN_1(refMessage); - - // The HelperMethodFrame knows how to get the return address. - UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); - - // Call the actual worker to perform failfast - GenericFailFast(refMessage, NULL, retaddr, exitCode, NULL); + GenericFailFast(refMessage, NULL, retaddr, NULL); HELPER_METHOD_FRAME_END(); } @@ -338,7 +320,7 @@ FCIMPL2(VOID, SystemNative::FailFastWithException, StringObject* refMessageUNSAF UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); // Call the actual worker to perform failfast - GenericFailFast(refMessage, refException, retaddr, COR_E_FAILFAST, NULL); + GenericFailFast(refMessage, refException, retaddr, NULL); HELPER_METHOD_FRAME_END(); } @@ -358,7 +340,7 @@ FCIMPL3(VOID, SystemNative::FailFastWithExceptionAndSource, StringObject* refMes UINT_PTR retaddr = HELPER_METHOD_FRAME_GET_RETURN_ADDRESS(); // Call the actual worker to perform failfast - GenericFailFast(refMessage, refException, retaddr, COR_E_FAILFAST, errorSource); + GenericFailFast(refMessage, refException, retaddr, errorSource); HELPER_METHOD_FRAME_END(); } diff --git a/src/coreclr/classlibnative/bcltype/system.h b/src/coreclr/classlibnative/bcltype/system.h index b4a773a847c39..e440f1fa8b067 100644 --- a/src/coreclr/classlibnative/bcltype/system.h +++ b/src/coreclr/classlibnative/bcltype/system.h @@ -44,7 +44,6 @@ class SystemNative static FCDECL0(INT32, GetExitCode); static FCDECL1(VOID, FailFast, StringObject* refMessageUNSAFE); - static FCDECL2(VOID, FailFastWithExitCode, StringObject* refMessageUNSAFE, UINT exitCode); static FCDECL2(VOID, FailFastWithException, StringObject* refMessageUNSAFE, ExceptionObject* refExceptionUNSAFE); static FCDECL3(VOID, FailFastWithExceptionAndSource, StringObject* refMessageUNSAFE, ExceptionObject* refExceptionUNSAFE, StringObject* errorSourceUNSAFE); @@ -55,7 +54,7 @@ class SystemNative private: // Common processing code for FailFast - static void GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExceptionForWatsonBucketing, UINT_PTR retAddress, UINT exitCode, STRINGREF errorSource); + static void GenericFailFast(STRINGREF refMesgString, EXCEPTIONREF refExceptionForWatsonBucketing, UINT_PTR retAddress, STRINGREF errorSource); }; extern "C" void QCALLTYPE Environment_Exit(INT32 exitcode); diff --git a/src/coreclr/clrdefinitions.cmake b/src/coreclr/clrdefinitions.cmake index fb8d095b5606d..2097ef5360ef0 100644 --- a/src/coreclr/clrdefinitions.cmake +++ b/src/coreclr/clrdefinitions.cmake @@ -258,7 +258,7 @@ function(set_target_definitions_to_custom_os_and_arch) if (TARGETDETAILS_OS STREQUAL "unix_anyos") target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_UNIX_ANYOS) endif() - elseif (TARGETDETAILS_OS STREQUAL "win") + elseif (TARGETDETAILS_OS MATCHES "^win") target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE TARGET_WINDOWS) endif((TARGETDETAILS_OS MATCHES "^unix")) @@ -287,7 +287,7 @@ function(set_target_definitions_to_custom_os_and_arch) target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE ARM_SOFTFP) endif() - if (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix")) + if (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix") OR (TARGETDETAILS_OS MATCHES "win_aot")) target_compile_definitions(${TARGETDETAILS_TARGET} PRIVATE FEATURE_EH_FUNCLETS) - endif (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix")) + endif (NOT (TARGETDETAILS_ARCH STREQUAL "x86") OR (TARGETDETAILS_OS MATCHES "^unix") OR (TARGETDETAILS_OS MATCHES "win_aot")) endfunction() diff --git a/src/coreclr/crosscomponents.cmake b/src/coreclr/crosscomponents.cmake index 11e923805a6ea..b06b706070489 100644 --- a/src/coreclr/crosscomponents.cmake +++ b/src/coreclr/crosscomponents.cmake @@ -25,6 +25,13 @@ if (CLR_CMAKE_HOST_OS STREQUAL CLR_CMAKE_TARGET_OS OR CLR_CMAKE_TARGET_IOS OR CL DESTINATIONS . COMPONENT crosscomponents ) + if (CLR_CMAKE_TARGET_ARCH_I386) + install_clr (TARGETS + clrjit_win_aot_${ARCH_TARGET_NAME}_${ARCH_HOST_NAME} + DESTINATIONS . + COMPONENT crosscomponents + ) + endif() endif() endif() endif() diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 45674a77c3539..d9571f0776456 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -259,7 +259,7 @@ RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_legacyCorruptedStateExceptionsPolicy, W("le CONFIG_DWORD_INFO(INTERNAL_SuppressLostExceptionTypeAssert, W("SuppressLostExceptionTypeAssert"), 0, "") RETAIL_CONFIG_DWORD_INFO(INTERNAL_UseEntryPointFilter, W("UseEntryPointFilter"), 0, "") RETAIL_CONFIG_DWORD_INFO(INTERNAL_Corhost_Swallow_Uncaught_Exceptions, W("Corhost_Swallow_Uncaught_Exceptions"), 0, "") -RETAIL_CONFIG_DWORD_INFO(EXTERNAL_LegacyExceptionHandling, W("LegacyExceptionHandling"), 0, "Enable legacy exception handling."); +RETAIL_CONFIG_DWORD_INFO(EXTERNAL_LegacyExceptionHandling, W("LegacyExceptionHandling"), 1, "Enable legacy exception handling."); /// diff --git a/src/coreclr/inc/clrnt.h b/src/coreclr/inc/clrnt.h index 56245ea46f25e..1373753862804 100644 --- a/src/coreclr/inc/clrnt.h +++ b/src/coreclr/inc/clrnt.h @@ -184,12 +184,17 @@ RtlVirtualUnwind_Unsafe( #ifdef HOST_X86 typedef struct _RUNTIME_FUNCTION { DWORD BeginAddress; + // NOTE: R2R doesn't include EndAddress (see docs/design/coreclr/botr/readytorun-format.md). + // NativeAOT does include the EndAddress because the Microsoft linker expects it. In NativeAOT + // the info is generated in the managed ObjectWriter, so the structures don't have to match. + // DWORD EndAddress; DWORD UnwindData; } RUNTIME_FUNCTION, *PRUNTIME_FUNCTION; typedef struct _DISPATCHER_CONTEXT { _EXCEPTION_REGISTRATION_RECORD* RegistrationPointer; } DISPATCHER_CONTEXT, *PDISPATCHER_CONTEXT; + #endif // HOST_X86 #endif // !HOST_UNIX @@ -207,7 +212,7 @@ RtlpGetFunctionEndAddress ( _In_ TADDR ImageBase ) { - PTR_UNWIND_INFO pUnwindInfo = (PTR_UNWIND_INFO)(ImageBase + FunctionEntry->UnwindData); + PUNWIND_INFO pUnwindInfo = (PUNWIND_INFO)(ImageBase + FunctionEntry->UnwindData); return FunctionEntry->BeginAddress + pUnwindInfo->FunctionLength; } diff --git a/src/coreclr/jit/CMakeLists.txt b/src/coreclr/jit/CMakeLists.txt index ae08a27e4c00a..5ba50306d1b72 100644 --- a/src/coreclr/jit/CMakeLists.txt +++ b/src/coreclr/jit/CMakeLists.txt @@ -23,6 +23,8 @@ function(create_standalone_jit) if(TARGETDETAILS_OS STREQUAL "unix_osx" OR TARGETDETAILS_OS STREQUAL "unix_anyos") set(JIT_ARCH_LINK_LIBRARIES gcinfo_unix_${TARGETDETAILS_ARCH}) + elseif(TARGETDETAILS_OS STREQUAL "win_aot") + set(JIT_ARCH_LINK_LIBRARIES gcinfo_win_${TARGETDETAILS_ARCH}) else() set(JIT_ARCH_LINK_LIBRARIES gcinfo_${TARGETDETAILS_OS}_${TARGETDETAILS_ARCH}) endif() @@ -94,7 +96,6 @@ set( JIT_SOURCES bitset.cpp block.cpp buildstring.cpp - layout.cpp codegencommon.cpp codegenlinear.cpp compiler.cpp @@ -123,14 +124,15 @@ set( JIT_SOURCES gentree.cpp gschecks.cpp hashbv.cpp - hwintrinsic.cpp + helperexpansion.cpp hostallocator.cpp + hwintrinsic.cpp ifconversion.cpp - helperexpansion.cpp - indirectcalltransformer.cpp - importercalls.cpp importer.cpp + importercalls.cpp importervectorization.cpp + indirectcalltransformer.cpp + inductionvariableopts.cpp inline.cpp inlinepolicy.cpp instr.cpp @@ -138,6 +140,7 @@ set( JIT_SOURCES jiteh.cpp jithashtable.cpp jitmetadata.cpp + layout.cpp lclmorph.cpp lclvars.cpp likelyclass.cpp @@ -152,7 +155,6 @@ set( JIT_SOURCES objectalloc.cpp optcse.cpp optimizebools.cpp - switchrecognition.cpp optimizer.cpp patchpoint.cpp phase.cpp @@ -165,6 +167,7 @@ set( JIT_SOURCES regalloc.cpp registerargconvention.cpp regset.cpp + scev.cpp scopeinfo.cpp sideeffects.cpp sm.cpp @@ -173,6 +176,7 @@ set( JIT_SOURCES ssabuilder.cpp ssarenamestate.cpp stacklevelsetter.cpp + switchrecognition.cpp treelifeupdater.cpp unwind.cpp utils.cpp @@ -359,6 +363,7 @@ set( JIT_HEADERS registerargconvention.h register.h regset.h + scev.h sideeffects.h simd.h simdashwintrinsic.h @@ -649,6 +654,7 @@ else() create_standalone_jit(TARGET clrjit_universal_arm_${ARCH_HOST_NAME} OS universal ARCH arm DESTINATIONS .) target_compile_definitions(clrjit_universal_arm_${ARCH_HOST_NAME} PRIVATE ARM_SOFTFP CONFIGURABLE_ARM_ABI) create_standalone_jit(TARGET clrjit_win_x86_${ARCH_HOST_NAME} OS win ARCH x86 DESTINATIONS .) + create_standalone_jit(TARGET clrjit_win_aot_x86_${ARCH_HOST_NAME} OS win_aot ARCH x86 DESTINATIONS .) endif (CLR_CMAKE_TARGET_ARCH_RISCV64) if (CLR_CMAKE_TARGET_ARCH_I386 AND CLR_CMAKE_TARGET_UNIX) diff --git a/src/coreclr/jit/assertionprop.cpp b/src/coreclr/jit/assertionprop.cpp index ba25a60af8edb..5d69c8d1bdd94 100644 --- a/src/coreclr/jit/assertionprop.cpp +++ b/src/coreclr/jit/assertionprop.cpp @@ -5493,7 +5493,6 @@ GenTree* Compiler::optAssertionProp(ASSERT_VALARG_TP assertions, GenTree* tree, case GT_IND: case GT_STOREIND: case GT_NULLCHECK: - case GT_STORE_DYN_BLK: return optAssertionProp_Ind(assertions, tree, stmt); case GT_BOUNDS_CHECK: @@ -6147,7 +6146,7 @@ ASSERT_TP* Compiler::optComputeAssertionGen() AssertionIndex valueAssertionIndex; AssertionIndex jumpDestAssertionIndex; - if (info.IsNextEdgeAssertion()) + if (info.AssertionHoldsOnFalseEdge()) { valueAssertionIndex = info.GetAssertionIndex(); jumpDestAssertionIndex = optFindComplementary(info.GetAssertionIndex()); diff --git a/src/coreclr/jit/block.cpp b/src/coreclr/jit/block.cpp index 273c6f045123d..bd9abbe7fef30 100644 --- a/src/coreclr/jit/block.cpp +++ b/src/coreclr/jit/block.cpp @@ -68,6 +68,24 @@ unsigned SsaStressHashHelper() } #endif +//------------------------------------------------------------------------ +// setLikelihood: set the likelihood of aflow edge +// +// Arguments: +// likelihood -- value in range [0.0, 1.0] indicating how likely +// the source block is to transfer control along this edge. +// +void FlowEdge::setLikelihood(weight_t likelihood) +{ + assert(likelihood >= 0.0); + assert(likelihood <= 1.0); + m_likelihoodSet = true; + m_likelihood = likelihood; + + JITDUMP("setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT "\n", m_sourceBlock->bbNum, m_destBlock->bbNum, + m_likelihood); +} + //------------------------------------------------------------------------ // AllSuccessorEnumerator: Construct an instance of the enumerator. // @@ -676,11 +694,11 @@ void BasicBlock::dspKind() const break; case BBJ_EHFILTERRET: - printf(" -> %s (fltret)", dspBlockNum(bbTarget)); + printf(" -> %s (fltret)", dspBlockNum(GetTarget())); break; case BBJ_EHCATCHRET: - printf(" -> %s (cret)", dspBlockNum(bbTarget)); + printf(" -> %s (cret)", dspBlockNum(GetTarget())); break; case BBJ_THROW: @@ -694,28 +712,28 @@ void BasicBlock::dspKind() const case BBJ_ALWAYS: if (HasFlag(BBF_KEEP_BBJ_ALWAYS)) { - printf(" -> %s (ALWAYS)", dspBlockNum(bbTarget)); + printf(" -> %s (ALWAYS)", dspBlockNum(GetTarget())); } else { - printf(" -> %s (always)", dspBlockNum(bbTarget)); + printf(" -> %s (always)", dspBlockNum(GetTarget())); } break; case BBJ_LEAVE: - printf(" -> %s (leave)", dspBlockNum(bbTarget)); + printf(" -> %s (leave)", dspBlockNum(GetTarget())); break; case BBJ_CALLFINALLY: - printf(" -> %s (callf)", dspBlockNum(bbTarget)); + printf(" -> %s (callf)", dspBlockNum(GetTarget())); break; case BBJ_CALLFINALLYRET: - printf(" -> %s (callfr)", dspBlockNum(bbTarget)); + printf(" -> %s (callfr)", dspBlockNum(GetTarget())); break; case BBJ_COND: - printf(" -> %s,%s (cond)", dspBlockNum(bbTrueTarget), dspBlockNum(bbFalseTarget)); + printf(" -> %s,%s (cond)", dspBlockNum(GetTrueTarget()), dspBlockNum(GetFalseTarget())); break; case BBJ_SWITCH: @@ -857,11 +875,16 @@ void BasicBlock::TransferTarget(BasicBlock* from) SetEhf(from->GetEhfTargets()); from->bbEhfTargets = nullptr; // Make sure nobody uses the descriptor after this. break; + + // TransferTarget may be called after setting the source block of `from`'s + // successor edges to this block. + // This means calling GetTarget/GetTrueTarget/GetFalseTarget would trigger asserts. + // Avoid this by accessing the edges directly. case BBJ_COND: - SetCond(from->GetTrueTarget(), from->GetFalseTarget()); + SetCond(from->bbTrueEdge, from->bbFalseEdge); break; case BBJ_ALWAYS: - SetKindAndTarget(from->GetKind(), from->GetTarget()); + SetKindAndTargetEdge(BBJ_ALWAYS, from->bbTargetEdge); CopyFlags(from, BBF_NONE_QUIRK); break; case BBJ_CALLFINALLY: @@ -869,10 +892,10 @@ void BasicBlock::TransferTarget(BasicBlock* from) case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: - SetKindAndTarget(from->GetKind(), from->GetTarget()); + SetKindAndTargetEdge(from->GetKind(), from->bbTargetEdge); break; default: - SetKindAndTarget(from->GetKind()); // Clear the target + SetKindAndTargetEdge(from->GetKind()); // Clear the target break; } assert(KindIs(from->GetKind())); @@ -985,7 +1008,7 @@ BasicBlock* BasicBlock::GetUniquePred(Compiler* compiler) const // BasicBlock* BasicBlock::GetUniqueSucc() const { - return KindIs(BBJ_ALWAYS) ? bbTarget : nullptr; + return KindIs(BBJ_ALWAYS) ? GetTarget() : nullptr; } // Static vars. @@ -1145,7 +1168,7 @@ unsigned BasicBlock::NumSucc() const return 1; case BBJ_COND: - if (bbTrueTarget == bbFalseTarget) + if (bbTrueEdge == bbFalseEdge) { return 1; } @@ -1180,15 +1203,15 @@ unsigned BasicBlock::NumSucc() const } //------------------------------------------------------------------------ -// GetSucc: Returns the requested block successor. See the declaration comment for details. +// GetSucc: Returns the requested successor edge. See the declaration comment for details. // // Arguments: // i - index of successor to return. 0 <= i <= NumSucc(). // // Return Value: -// Requested successor block +// Requested successor edge // -BasicBlock* BasicBlock::GetSucc(unsigned i) const +FlowEdge* BasicBlock::GetSuccEdge(unsigned i) const { assert(i < NumSucc()); // Index bounds check. switch (bbKind) @@ -1199,31 +1222,45 @@ BasicBlock* BasicBlock::GetSucc(unsigned i) const case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: - return bbTarget; + return GetTargetEdge(); case BBJ_COND: if (i == 0) { - return bbFalseTarget; + return GetFalseEdge(); } else { assert(i == 1); - assert(bbFalseTarget != bbTrueTarget); - return bbTrueTarget; + assert(bbTrueEdge != bbFalseEdge); + return GetTrueEdge(); } case BBJ_EHFINALLYRET: - return bbEhfTargets->bbeSuccs[i]->getDestinationBlock(); + return bbEhfTargets->bbeSuccs[i]; case BBJ_SWITCH: - return bbSwtTargets->bbsDstTab[i]->getDestinationBlock(); + return bbSwtTargets->bbsDstTab[i]; default: unreached(); } } +//------------------------------------------------------------------------ +// GetSucc: Returns the requested block successor. See the declaration comment for details. +// +// Arguments: +// i - index of successor to return. 0 <= i <= NumSucc(). +// +// Return Value: +// Requested successor block +// +BasicBlock* BasicBlock::GetSucc(unsigned i) const +{ + return GetSuccEdge(i)->getDestinationBlock(); +} + //------------------------------------------------------------------------ // NumSucc: Returns the count of block successors. See the declaration comment for details. // @@ -1270,7 +1307,7 @@ unsigned BasicBlock::NumSucc(Compiler* comp) return 1; case BBJ_COND: - if (bbTrueTarget == bbFalseTarget) + if (bbTrueEdge == bbFalseEdge) { return 1; } @@ -1291,16 +1328,16 @@ unsigned BasicBlock::NumSucc(Compiler* comp) } //------------------------------------------------------------------------ -// GetSucc: Returns the requested block successor. See the declaration comment for details. +// GetSucc: Returns the requested successor edge. See the declaration comment for details. // // Arguments: // i - index of successor to return. 0 <= i <= NumSucc(comp). // comp - Compiler instance // // Return Value: -// Requested successor block +// Requested successor edge // -BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) +FlowEdge* BasicBlock::GetSuccEdge(unsigned i, Compiler* comp) { assert(comp != nullptr); @@ -1309,31 +1346,31 @@ BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) { case BBJ_EHFILTERRET: // Handler is the (sole) normal successor of the filter. - assert(comp->fgFirstBlockOfHandler(this) == bbTarget); - return bbTarget; + assert(comp->fgFirstBlockOfHandler(this) == GetTarget()); + return GetTargetEdge(); case BBJ_EHFINALLYRET: assert(bbEhfTargets != nullptr); assert(i < bbEhfTargets->bbeCount); - return bbEhfTargets->bbeSuccs[i]->getDestinationBlock(); + return bbEhfTargets->bbeSuccs[i]; case BBJ_CALLFINALLY: case BBJ_CALLFINALLYRET: case BBJ_ALWAYS: case BBJ_EHCATCHRET: case BBJ_LEAVE: - return bbTarget; + return GetTargetEdge(); case BBJ_COND: if (i == 0) { - return bbFalseTarget; + return GetFalseEdge(); } else { assert(i == 1); - assert(bbFalseTarget != bbTrueTarget); - return bbTrueTarget; + assert(bbTrueEdge != bbFalseEdge); + return GetTrueEdge(); } case BBJ_SWITCH: @@ -1348,6 +1385,21 @@ BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) } } +//------------------------------------------------------------------------ +// GetSucc: Returns the requested block successor. See the declaration comment for details. +// +// Arguments: +// i - index of successor to return. 0 <= i <= NumSucc(comp). +// comp - Compiler instance +// +// Return Value: +// Requested successor block +// +BasicBlock* BasicBlock::GetSucc(unsigned i, Compiler* comp) +{ + return GetSuccEdge(i, comp)->getDestinationBlock(); +} + void BasicBlock::InitVarSets(Compiler* comp) { VarSetOps::AssignNoCopy(comp, bbVarUse, VarSetOps::MakeEmpty(comp)); @@ -1585,15 +1637,10 @@ BasicBlock* BasicBlock::New(Compiler* compiler) return block; } -BasicBlock* BasicBlock::New(Compiler* compiler, BBKinds kind, BasicBlock* target /* = nullptr */) +BasicBlock* BasicBlock::New(Compiler* compiler, BBKinds kind) { BasicBlock* block = BasicBlock::New(compiler); - - // In some cases, we don't know a block's jump target during initialization, so don't check the jump kind/target - // yet. - // The checks will be done any time the jump kind/target is read or written to after initialization. - block->bbKind = kind; - block->bbTarget = target; + block->bbKind = kind; if (block->KindIs(BBJ_THROW)) { diff --git a/src/coreclr/jit/block.h b/src/coreclr/jit/block.h index 208eb07e583fa..e4b6b9d0daacd 100644 --- a/src/coreclr/jit/block.h +++ b/src/coreclr/jit/block.h @@ -307,29 +307,17 @@ class PredBlockList } }; -// BBArrayIterator: forward iterator for an array of BasicBlock*, such as the BBswtDesc->bbsDstTab. +// BBArrayIterator: forward iterator for an array of BasicBlock*. // It is an error (with assert) to yield a nullptr BasicBlock* in this array. -// `m_bbEntry` can be nullptr, but it only makes sense if both the begin and end of an iteration range are nullptr +// `m_edgeEntry` can be nullptr, but it only makes sense if both the begin and end of an iteration range are nullptr // (meaning, no actual iteration will happen). // class BBArrayIterator { - // Quirk: Some BasicBlock kinds refer to their successors with BasicBlock pointers, - // while others use FlowEdge pointers. Eventually, every type will use FlowEdge pointers. - // For now, support iterating with both types. - union { - BasicBlock* const* m_bbEntry; - FlowEdge* const* m_edgeEntry; - }; - - bool iterateEdges; + FlowEdge* const* m_edgeEntry; public: - BBArrayIterator(BasicBlock* const* bbEntry) : m_bbEntry(bbEntry), iterateEdges(false) - { - } - - BBArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry), iterateEdges(true) + BBArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry) { } @@ -337,14 +325,49 @@ class BBArrayIterator BBArrayIterator& operator++() { - assert(m_bbEntry != nullptr); - ++m_bbEntry; + assert(m_edgeEntry != nullptr); + ++m_edgeEntry; return *this; } bool operator!=(const BBArrayIterator& i) const { - return m_bbEntry != i.m_bbEntry; + return m_edgeEntry != i.m_edgeEntry; + } +}; + +// FlowEdgeArrayIterator: forward iterator for an array of FlowEdge*, such as the BBswtDesc->bbsDstTab. +// It is an error (with assert) to yield a nullptr FlowEdge* in this array. +// `m_edgeEntry` can be nullptr, but it only makes sense if both the begin and end of an iteration range are nullptr +// (meaning, no actual iteration will happen). +// +class FlowEdgeArrayIterator +{ + FlowEdge* const* m_edgeEntry; + +public: + FlowEdgeArrayIterator(FlowEdge* const* edgeEntry) : m_edgeEntry(edgeEntry) + { + } + + FlowEdge* operator*() const + { + assert(m_edgeEntry != nullptr); + FlowEdge* const edge = *m_edgeEntry; + assert(edge != nullptr); + return edge; + } + + FlowEdgeArrayIterator& operator++() + { + assert(m_edgeEntry != nullptr); + ++m_edgeEntry; + return *this; + } + + bool operator!=(const FlowEdgeArrayIterator& i) const + { + return m_edgeEntry != i.m_edgeEntry; } }; @@ -506,6 +529,179 @@ enum class BasicBlockVisit // clang-format on +//------------------------------------------------------------------------- +// FlowEdge -- control flow edge +// +// In compiler terminology the control flow between two BasicBlocks +// is typically referred to as an "edge". Most well known are the +// backward branches for loops, which are often called "back-edges". +// +// "struct FlowEdge" is the type that represents our control flow edges. +// This type is a linked list of zero or more "edges". +// (The list of zero edges is represented by NULL.) +// Every BasicBlock has a field called bbPreds of this type. This field +// represents the list of "edges" that flow into this BasicBlock. +// The FlowEdge type only stores the BasicBlock* of the source for the +// control flow edge. The destination block for the control flow edge +// is implied to be the block which contained the bbPreds field. +// +// For a switch branch target there may be multiple "edges" that have +// the same source block (and destination block). We need to count the +// number of these edges so that during optimization we will know when +// we have zero of them. Rather than have extra FlowEdge entries we +// track this via the DupCount property. +// +// When we have Profile weight for the BasicBlocks we can usually compute +// the number of times each edge was executed by examining the adjacent +// BasicBlock weights. As we are doing for BasicBlocks, we call the number +// of times that a control flow edge was executed the "edge weight". +// In order to compute the edge weights we need to use a bounded range +// for every edge weight. These two fields, 'flEdgeWeightMin' and 'flEdgeWeightMax' +// are used to hold a bounded range. Most often these will converge such +// that both values are the same and that value is the exact edge weight. +// Sometimes we are left with a rage of possible values between [Min..Max] +// which represents an inexact edge weight. +// +// The bbPreds list is initially created by Compiler::fgLinkBasicBlocks() +// and is incrementally kept up to date. +// +// The edge weight are computed by Compiler::fgComputeEdgeWeights() +// the edge weights are used to straighten conditional branches +// by Compiler::fgReorderBlocks() +// +struct FlowEdge +{ +private: + // The next predecessor edge in the list, nullptr for end of list. + FlowEdge* m_nextPredEdge; + + // The source of the control flow + BasicBlock* m_sourceBlock; + + // The destination of the control flow + BasicBlock* m_destBlock; + + // Edge weights + weight_t m_edgeWeightMin; + weight_t m_edgeWeightMax; + + // Likelihood that m_sourceBlock transfers control along this edge. + // Values in range [0..1] + weight_t m_likelihood; + + // The count of duplicate "edges" (used for switch stmts or degenerate branches) + unsigned m_dupCount; + + // True if likelihood has been set + bool m_likelihoodSet; + +public: + FlowEdge(BasicBlock* sourceBlock, BasicBlock* destBlock, FlowEdge* rest) + : m_nextPredEdge(rest) + , m_sourceBlock(sourceBlock) + , m_destBlock(destBlock) + , m_edgeWeightMin(0) + , m_edgeWeightMax(0) + , m_likelihood(0) + , m_dupCount(0) + , m_likelihoodSet(false) + { + } + + FlowEdge* getNextPredEdge() const + { + return m_nextPredEdge; + } + + FlowEdge** getNextPredEdgeRef() + { + return &m_nextPredEdge; + } + + void setNextPredEdge(FlowEdge* newEdge) + { + m_nextPredEdge = newEdge; + } + + BasicBlock* getSourceBlock() const + { + assert(m_sourceBlock != nullptr); + return m_sourceBlock; + } + + void setSourceBlock(BasicBlock* newBlock) + { + assert(newBlock != nullptr); + m_sourceBlock = newBlock; + } + + BasicBlock* getDestinationBlock() const + { + assert(m_destBlock != nullptr); + return m_destBlock; + } + + void setDestinationBlock(BasicBlock* newBlock) + { + assert(newBlock != nullptr); + m_destBlock = newBlock; + } + + weight_t edgeWeightMin() const + { + return m_edgeWeightMin; + } + + weight_t edgeWeightMax() const + { + return m_edgeWeightMax; + } + + // These two methods are used to set new values for edge weights. + // They return false if the newWeight is not between the current [min..max] + // when slop is non-zero we allow for the case where our weights might be off by 'slop' + // + bool setEdgeWeightMinChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); + bool setEdgeWeightMaxChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); + void setEdgeWeights(weight_t newMinWeight, weight_t newMaxWeight, BasicBlock* bDst); + + weight_t getLikelihood() const + { + return m_likelihood; + } + + void setLikelihood(weight_t likelihood); + + void clearLikelihood() + { + m_likelihood = 0.0; + m_likelihoodSet = false; + } + + bool hasLikelihood() const + { + return m_likelihoodSet; + } + + weight_t getLikelyWeight() const; + + unsigned getDupCount() const + { + return m_dupCount; + } + + void incrementDupCount() + { + m_dupCount++; + } + + void decrementDupCount() + { + assert(m_dupCount >= 1); + m_dupCount--; + } +}; + //------------------------------------------------------------------------ // BasicBlock: describes a basic block in the flowgraph. // @@ -525,19 +721,19 @@ struct BasicBlock : private LIR::Range /* The following union describes the jump target(s) of this block */ union { - unsigned bbTargetOffs; // PC offset (temporary only) - BasicBlock* bbTarget; // basic block - BasicBlock* bbTrueTarget; // BBJ_COND jump target when its condition is true (alias for bbTarget) - BBswtDesc* bbSwtTargets; // switch descriptor - BBehfDesc* bbEhfTargets; // BBJ_EHFINALLYRET descriptor + unsigned bbTargetOffs; // PC offset (temporary only) + FlowEdge* bbTargetEdge; // successor edge for block kinds with only one successor (BBJ_ALWAYS, etc) + FlowEdge* bbTrueEdge; // BBJ_COND successor edge when its condition is true (alias for bbTargetEdge) + BBswtDesc* bbSwtTargets; // switch descriptor + BBehfDesc* bbEhfTargets; // BBJ_EHFINALLYRET descriptor }; - // Points to the successor of a BBJ_COND block if bbTrueTarget is not taken - BasicBlock* bbFalseTarget; + // Successor edge of a BBJ_COND block if bbTrueEdge is not taken + FlowEdge* bbFalseEdge; public: static BasicBlock* New(Compiler* compiler); - static BasicBlock* New(Compiler* compiler, BBKinds kind, BasicBlock* target = nullptr); + static BasicBlock* New(Compiler* compiler, BBKinds kind); static BasicBlock* New(Compiler* compiler, BBehfDesc* ehfTargets); static BasicBlock* New(Compiler* compiler, BBswtDesc* swtTargets); static BasicBlock* New(Compiler* compiler, BBKinds kind, unsigned targetOffs); @@ -623,100 +819,135 @@ struct BasicBlock : private LIR::Range return bbTargetOffs; } - void SetKindAndTarget(BBKinds kind, unsigned targetOffs) - { - bbKind = kind; - bbTargetOffs = targetOffs; - assert(KindIs(BBJ_ALWAYS, BBJ_COND, BBJ_LEAVE)); - } - bool HasTarget() const { - // These block types should always have bbTarget set + // These block types should always have bbTargetEdge set return KindIs(BBJ_ALWAYS, BBJ_CALLFINALLY, BBJ_CALLFINALLYRET, BBJ_EHCATCHRET, BBJ_EHFILTERRET, BBJ_LEAVE); } BasicBlock* GetTarget() const { - // Only block kinds that use `bbTarget` can access it, and it must be non-null. + return GetTargetEdge()->getDestinationBlock(); + } + + FlowEdge* GetTargetEdge() const + { + // Only block kinds that use `bbTargetEdge` can access it, and it must be non-null. assert(HasInitializedTarget()); - return bbTarget; + assert(bbTargetEdge->getSourceBlock() == this); + assert(bbTargetEdge->getDestinationBlock() != nullptr); + return bbTargetEdge; } - void SetTarget(BasicBlock* target) + void SetTargetEdge(FlowEdge* targetEdge) { // SetKindAndTarget() nulls target for non-jump kinds, - // so don't use SetTarget() to null bbTarget without updating bbKind. - bbTarget = target; + // so don't use SetTargetEdge() to null bbTargetEdge without updating bbKind. + bbTargetEdge = targetEdge; assert(HasInitializedTarget()); + assert(bbTargetEdge->getSourceBlock() == this); + assert(bbTargetEdge->getDestinationBlock() != nullptr); } BasicBlock* GetTrueTarget() const + { + return GetTrueEdge()->getDestinationBlock(); + } + + FlowEdge* GetTrueEdge() const { assert(KindIs(BBJ_COND)); - assert(bbTrueTarget != nullptr); - return bbTrueTarget; + assert(bbTrueEdge != nullptr); + assert(bbTrueEdge->getSourceBlock() == this); + assert(bbTrueEdge->getDestinationBlock() != nullptr); + return bbTrueEdge; } - void SetTrueTarget(BasicBlock* target) + void SetTrueEdge(FlowEdge* trueEdge) { assert(KindIs(BBJ_COND)); - assert(target != nullptr); - bbTrueTarget = target; + bbTrueEdge = trueEdge; + assert(bbTrueEdge != nullptr); + assert(bbTrueEdge->getSourceBlock() == this); + assert(bbTrueEdge->getDestinationBlock() != nullptr); } bool TrueTargetIs(const BasicBlock* target) const { - assert(KindIs(BBJ_COND)); - assert(bbTrueTarget != nullptr); - return (bbTrueTarget == target); + return (GetTrueTarget() == target); + } + + bool TrueEdgeIs(const FlowEdge* targetEdge) const + { + return (GetTrueEdge() == targetEdge); } BasicBlock* GetFalseTarget() const + { + return GetFalseEdge()->getDestinationBlock(); + } + + FlowEdge* GetFalseEdge() const { assert(KindIs(BBJ_COND)); - assert(bbFalseTarget != nullptr); - return bbFalseTarget; + assert(bbFalseEdge != nullptr); + assert(bbFalseEdge->getSourceBlock() == this); + assert(bbFalseEdge->getDestinationBlock() != nullptr); + return bbFalseEdge; } - void SetFalseTarget(BasicBlock* target) + void SetFalseEdge(FlowEdge* falseEdge) { assert(KindIs(BBJ_COND)); - assert(target != nullptr); - bbFalseTarget = target; + bbFalseEdge = falseEdge; + assert(bbFalseEdge != nullptr); + assert(bbFalseEdge->getSourceBlock() == this); + assert(bbFalseEdge->getDestinationBlock() != nullptr); } bool FalseTargetIs(const BasicBlock* target) const { - assert(KindIs(BBJ_COND)); - assert(bbFalseTarget != nullptr); - return (bbFalseTarget == target); + return (GetFalseTarget() == target); } - void SetCond(BasicBlock* trueTarget, BasicBlock* falseTarget) + bool FalseEdgeIs(const FlowEdge* targetEdge) const { - assert(trueTarget != nullptr); - bbKind = BBJ_COND; - bbTrueTarget = trueTarget; - bbFalseTarget = falseTarget; + return (GetFalseEdge() == targetEdge); } - // Set both the block kind and target. This can clear `bbTarget` when setting - // block kinds that don't use `bbTarget`. - void SetKindAndTarget(BBKinds kind, BasicBlock* target = nullptr) + void SetCond(FlowEdge* trueEdge, FlowEdge* falseEdge) { - bbKind = kind; - bbTarget = target; + bbKind = BBJ_COND; + SetTrueEdge(trueEdge); + SetFalseEdge(falseEdge); + } + + // In most cases, a block's true and false targets are known by the time SetCond is called. + // To simplify the few cases where the false target isn't available until later, + // overload SetCond to initialize only the true target. + // This simplifies, for example, lowering switch blocks into jump sequences. + void SetCond(FlowEdge* trueEdge) + { + bbKind = BBJ_COND; + SetTrueEdge(trueEdge); + } - // If bbKind indicates this block has a jump, bbTarget cannot be null. + // Set both the block kind and target edge. This can clear `bbTargetEdge` when setting + // block kinds that don't use `bbTargetEdge`. + void SetKindAndTargetEdge(BBKinds kind, FlowEdge* targetEdge = nullptr) + { + bbKind = kind; + bbTargetEdge = targetEdge; + + // If bbKind indicates this block has a jump, bbTargetEdge cannot be null. // You shouldn't use this to set a BBJ_COND, BBJ_SWITCH, or BBJ_EHFINALLYRET. - assert(HasTarget() ? HasInitializedTarget() : (bbTarget == nullptr)); + assert(HasTarget() ? HasInitializedTarget() : (bbTargetEdge == nullptr)); } bool HasInitializedTarget() const { assert(HasTarget()); - return (bbTarget != nullptr); + return (bbTargetEdge != nullptr); } bool TargetIs(const BasicBlock* target) const @@ -762,19 +993,13 @@ struct BasicBlock : private LIR::Range bbEhfTargets = ehfTarget; } - // BBJ_CALLFINALLYRET uses the `bbTarget` field. However, also treat it specially: + // BBJ_CALLFINALLYRET uses the `bbTargetEdge` field. However, also treat it specially: // for callers that know they want a continuation, use this function instead of the // general `GetTarget()` to allow asserting on the block kind. BasicBlock* GetFinallyContinuation() const { assert(KindIs(BBJ_CALLFINALLYRET)); - return bbTarget; - } - - void SetFinallyContinuation(BasicBlock* finallyContinuation) - { - assert(KindIs(BBJ_CALLFINALLYRET)); - bbTarget = finallyContinuation; + return GetTarget(); } #ifdef DEBUG @@ -783,21 +1008,21 @@ struct BasicBlock : private LIR::Range BasicBlock* GetTargetRaw() const { assert(HasTarget()); - return bbTarget; + return (bbTargetEdge == nullptr) ? nullptr : bbTargetEdge->getDestinationBlock(); } // Return the BBJ_COND true target; it might be null. Only used during dumping. BasicBlock* GetTrueTargetRaw() const { assert(KindIs(BBJ_COND)); - return bbTrueTarget; + return (bbTrueEdge == nullptr) ? nullptr : bbTrueEdge->getDestinationBlock(); } // Return the BBJ_COND false target; it might be null. Only used during dumping. BasicBlock* GetFalseTargetRaw() const { assert(KindIs(BBJ_COND)); - return bbFalseTarget; + return (bbFalseEdge == nullptr) ? nullptr : bbFalseEdge->getDestinationBlock(); } #endif // DEBUG @@ -1087,7 +1312,11 @@ struct BasicBlock : private LIR::Range unsigned NumSucc() const; unsigned NumSucc(Compiler* comp); - // GetSucc: Returns the "i"th successor. Requires (0 <= i < NumSucc()). + // GetSuccEdge: Returns the "i"th successor edge. Requires (0 <= i < NumSucc()). + FlowEdge* GetSuccEdge(unsigned i) const; + FlowEdge* GetSuccEdge(unsigned i, Compiler* comp); + + // GetSucc: Returns the "i"th successor block. Requires (0 <= i < NumSucc()). BasicBlock* GetSucc(unsigned i) const; BasicBlock* GetSucc(unsigned i, Compiler* comp); @@ -1566,37 +1795,64 @@ struct BasicBlock : private LIR::Range bool HasPotentialEHSuccs(Compiler* comp); - // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, - // normally used via BasicBlock::Succs(), e.g.: - // for (BasicBlock* const target : block->Succs()) ... + // Base class for Successor block/edge iterators. // - class BBSuccList + class SuccList { + protected: // For one or two successors, pre-compute and stash the successors inline, in m_succs[], so we don't // need to call a function or execute another `switch` to get them. Also, pre-compute the begin and end // points of the iteration, for use by BBArrayIterator. `m_begin` and `m_end` will either point at // `m_succs` or at the switch table successor array. - BasicBlock* m_succs[2]; - - // Quirk: Some BasicBlock kinds refer to their successors with BasicBlock pointers, - // while others use FlowEdge pointers. Eventually, every type will use FlowEdge pointers. - // For now, support iterating with both types. - union { - BasicBlock* const* m_begin; - FlowEdge* const* m_beginEdge; - }; + FlowEdge* m_succs[2]; + FlowEdge* const* m_begin; + FlowEdge* const* m_end; - union { - BasicBlock* const* m_end; - FlowEdge* const* m_endEdge; - }; + SuccList(const BasicBlock* block); + }; - bool iterateEdges; + // BBSuccList: adapter class for forward iteration of block successors, using range-based `for`, + // normally used via BasicBlock::Succs(), e.g.: + // for (BasicBlock* const target : block->Succs()) ... + // + class BBSuccList : private SuccList + { + public: + BBSuccList(const BasicBlock* block) : SuccList(block) + { + } + BBArrayIterator begin() const + { + return BBArrayIterator(m_begin); + } + + BBArrayIterator end() const + { + return BBArrayIterator(m_end); + } + }; + + // BBSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, + // normally used via BasicBlock::SuccEdges(), e.g.: + // for (FlowEdge* const succEdge : block->SuccEdges()) ... + // + class BBSuccEdgeList : private SuccList + { public: - BBSuccList(const BasicBlock* block); - BBArrayIterator begin() const; - BBArrayIterator end() const; + BBSuccEdgeList(const BasicBlock* block) : SuccList(block) + { + } + + FlowEdgeArrayIterator begin() const + { + return FlowEdgeArrayIterator(m_begin); + } + + FlowEdgeArrayIterator end() const + { + return FlowEdgeArrayIterator(m_end); + } }; // BBCompilerSuccList: adapter class for forward iteration of block successors, using range-based `for`, @@ -1610,7 +1866,7 @@ struct BasicBlock : private LIR::Range Compiler* m_comp; BasicBlock* m_block; - // iterator: forward iterator for an array of BasicBlock*, such as the BBswtDesc->bbsDstTab. + // iterator: forward iterator for an array of BasicBlock* // class iterator { @@ -1660,6 +1916,67 @@ struct BasicBlock : private LIR::Range } }; + // BBCompilerSuccEdgeList: adapter class for forward iteration of block successors edges, using range-based `for`, + // normally used via BasicBlock::SuccEdges(), e.g.: + // for (FlowEdge* const succEdge : block->SuccEdges(compiler)) ... + // + // This version uses NumSucc(Compiler*)/GetSucc(Compiler*). See the documentation there for the explanation + // of the implications of this versus the version that does not take `Compiler*`. + class BBCompilerSuccEdgeList + { + Compiler* m_comp; + BasicBlock* m_block; + + // iterator: forward iterator for an array of BasicBlock* + // + class iterator + { + Compiler* m_comp; + BasicBlock* m_block; + unsigned m_succNum; + + public: + iterator(Compiler* comp, BasicBlock* block, unsigned succNum) + : m_comp(comp), m_block(block), m_succNum(succNum) + { + } + + FlowEdge* operator*() const + { + assert(m_block != nullptr); + FlowEdge* succEdge = m_block->GetSuccEdge(m_succNum, m_comp); + assert(succEdge != nullptr); + return succEdge; + } + + iterator& operator++() + { + ++m_succNum; + return *this; + } + + bool operator!=(const iterator& i) const + { + return m_succNum != i.m_succNum; + } + }; + + public: + BBCompilerSuccEdgeList(Compiler* comp, BasicBlock* block) : m_comp(comp), m_block(block) + { + } + + iterator begin() const + { + return iterator(m_comp, m_block, 0); + } + + iterator end() const + { + return iterator(m_comp, m_block, m_block->NumSucc(m_comp)); + } + }; + // Succs: convenience methods for enabling range-based `for` iteration over a block's successors, e.g.: // for (BasicBlock* const succ : block->Succs()) ... // @@ -1676,6 +1993,16 @@ struct BasicBlock : private LIR::Range return BBCompilerSuccList(comp, this); } + BBSuccEdgeList SuccEdges() + { + return BBSuccEdgeList(this); + } + + BBCompilerSuccEdgeList SuccEdges(Compiler* comp) + { + return BBCompilerSuccEdgeList(comp, this); + } + // Clone block state and statements from `from` block to `to` block (which must be new/empty) static void CloneBlockState(Compiler* compiler, BasicBlock* to, const BasicBlock* from); @@ -1927,12 +2254,11 @@ inline BBArrayIterator BBEhfSuccList::end() const return BBArrayIterator(m_bbeDesc->bbeSuccs + m_bbeDesc->bbeCount); } -// BBSuccList out-of-class-declaration implementations +// SuccList out-of-class-declaration implementations // -inline BasicBlock::BBSuccList::BBSuccList(const BasicBlock* block) +inline BasicBlock::SuccList::SuccList(const BasicBlock* block) { assert(block != nullptr); - iterateEdges = false; switch (block->bbKind) { @@ -1950,24 +2276,24 @@ inline BasicBlock::BBSuccList::BBSuccList(const BasicBlock* block) case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: - m_succs[0] = block->bbTarget; + m_succs[0] = block->GetTargetEdge(); m_begin = &m_succs[0]; m_end = &m_succs[1]; break; case BBJ_COND: - m_succs[0] = block->bbFalseTarget; + m_succs[0] = block->GetFalseEdge(); m_begin = &m_succs[0]; // If both fall-through and branch successors are identical, then only include // them once in the iteration (this is the same behavior as NumSucc()/GetSucc()). - if (block->TrueTargetIs(block->GetFalseTarget())) + if (block->TrueEdgeIs(block->GetFalseEdge())) { m_end = &m_succs[1]; } else { - m_succs[1] = block->bbTrueTarget; + m_succs[1] = block->GetTrueEdge(); m_end = &m_succs[2]; } break; @@ -1978,26 +2304,22 @@ inline BasicBlock::BBSuccList::BBSuccList(const BasicBlock* block) // been computed. if (block->GetEhfTargets() == nullptr) { - m_beginEdge = nullptr; - m_endEdge = nullptr; + m_begin = nullptr; + m_end = nullptr; } else { - m_beginEdge = block->GetEhfTargets()->bbeSuccs; - m_endEdge = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; + m_begin = block->GetEhfTargets()->bbeSuccs; + m_end = block->GetEhfTargets()->bbeSuccs + block->GetEhfTargets()->bbeCount; } - - iterateEdges = true; break; case BBJ_SWITCH: // We don't use the m_succs in-line data for switches; use the existing jump table in the block. assert(block->bbSwtTargets != nullptr); assert(block->bbSwtTargets->bbsDstTab != nullptr); - m_beginEdge = block->bbSwtTargets->bbsDstTab; - m_endEdge = block->bbSwtTargets->bbsDstTab + block->bbSwtTargets->bbsCount; - - iterateEdges = true; + m_begin = block->bbSwtTargets->bbsDstTab; + m_end = block->bbSwtTargets->bbsDstTab + block->bbSwtTargets->bbsCount; break; default: @@ -2007,16 +2329,6 @@ inline BasicBlock::BBSuccList::BBSuccList(const BasicBlock* block) assert(m_end >= m_begin); } -inline BBArrayIterator BasicBlock::BBSuccList::begin() const -{ - return (iterateEdges ? BBArrayIterator(m_beginEdge) : BBArrayIterator(m_begin)); -} - -inline BBArrayIterator BasicBlock::BBSuccList::end() const -{ - return (iterateEdges ? BBArrayIterator(m_endEdge) : BBArrayIterator(m_end)); -} - // We have a simpler struct, BasicBlockList, which is simply a singly-linked // list of blocks. @@ -2034,206 +2346,23 @@ struct BasicBlockList } }; -//------------------------------------------------------------------------- -// FlowEdge -- control flow edge -// -// In compiler terminology the control flow between two BasicBlocks -// is typically referred to as an "edge". Most well known are the -// backward branches for loops, which are often called "back-edges". -// -// "struct FlowEdge" is the type that represents our control flow edges. -// This type is a linked list of zero or more "edges". -// (The list of zero edges is represented by NULL.) -// Every BasicBlock has a field called bbPreds of this type. This field -// represents the list of "edges" that flow into this BasicBlock. -// The FlowEdge type only stores the BasicBlock* of the source for the -// control flow edge. The destination block for the control flow edge -// is implied to be the block which contained the bbPreds field. -// -// For a switch branch target there may be multiple "edges" that have -// the same source block (and destination block). We need to count the -// number of these edges so that during optimization we will know when -// we have zero of them. Rather than have extra FlowEdge entries we -// track this via the DupCount property. -// -// When we have Profile weight for the BasicBlocks we can usually compute -// the number of times each edge was executed by examining the adjacent -// BasicBlock weights. As we are doing for BasicBlocks, we call the number -// of times that a control flow edge was executed the "edge weight". -// In order to compute the edge weights we need to use a bounded range -// for every edge weight. These two fields, 'flEdgeWeightMin' and 'flEdgeWeightMax' -// are used to hold a bounded range. Most often these will converge such -// that both values are the same and that value is the exact edge weight. -// Sometimes we are left with a rage of possible values between [Min..Max] -// which represents an inexact edge weight. -// -// The bbPreds list is initially created by Compiler::fgLinkBasicBlocks() -// and is incrementally kept up to date. -// -// The edge weight are computed by Compiler::fgComputeEdgeWeights() -// the edge weights are used to straighten conditional branches -// by Compiler::fgReorderBlocks() -// -struct FlowEdge -{ -private: - // The next predecessor edge in the list, nullptr for end of list. - FlowEdge* m_nextPredEdge; - - // The source of the control flow - BasicBlock* m_sourceBlock; - - // The destination of the control flow - BasicBlock* m_destBlock; - - // Edge weights - weight_t m_edgeWeightMin; - weight_t m_edgeWeightMax; - - // Likelihood that m_sourceBlock transfers control along this edge. - // Values in range [0..1] - weight_t m_likelihood; - - // The count of duplicate "edges" (used for switch stmts or degenerate branches) - unsigned m_dupCount; - - // True if likelihood has been set - bool m_likelihoodSet; - -public: - FlowEdge(BasicBlock* sourceBlock, BasicBlock* destBlock, FlowEdge* rest) - : m_nextPredEdge(rest) - , m_sourceBlock(sourceBlock) - , m_destBlock(destBlock) - , m_edgeWeightMin(0) - , m_edgeWeightMax(0) - , m_likelihood(0) - , m_dupCount(0) - , m_likelihoodSet(false) - { - } - - FlowEdge* getNextPredEdge() const - { - return m_nextPredEdge; - } - - FlowEdge** getNextPredEdgeRef() - { - return &m_nextPredEdge; - } - - void setNextPredEdge(FlowEdge* newEdge) - { - m_nextPredEdge = newEdge; - } - - BasicBlock* getSourceBlock() const - { - assert(m_sourceBlock != nullptr); - return m_sourceBlock; - } - - void setSourceBlock(BasicBlock* newBlock) - { - assert(newBlock != nullptr); - m_sourceBlock = newBlock; - } - - BasicBlock* getDestinationBlock() const - { - assert(m_destBlock != nullptr); - return m_destBlock; - } - - void setDestinationBlock(BasicBlock* newBlock) - { - assert(newBlock != nullptr); - m_destBlock = newBlock; - } - - weight_t edgeWeightMin() const - { - return m_edgeWeightMin; - } - - weight_t edgeWeightMax() const - { - return m_edgeWeightMax; - } - - // These two methods are used to set new values for edge weights. - // They return false if the newWeight is not between the current [min..max] - // when slop is non-zero we allow for the case where our weights might be off by 'slop' - // - bool setEdgeWeightMinChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); - bool setEdgeWeightMaxChecked(weight_t newWeight, BasicBlock* bDst, weight_t slop, bool* wbUsedSlop); - void setEdgeWeights(weight_t newMinWeight, weight_t newMaxWeight, BasicBlock* bDst); - - weight_t getLikelihood() const - { - return m_likelihood; - } - - void setLikelihood(weight_t likelihood) - { - assert(likelihood >= 0.0); - assert(likelihood <= 1.0); - m_likelihoodSet = true; - m_likelihood = likelihood; - } +// FlowEdge implementations (that are required to be defined after the declaration of BasicBlock) - void clearLikelihood() - { - m_likelihood = 0.0; - m_likelihoodSet = false; - } - - bool hasLikelihood() const - { - return m_likelihoodSet; - } - - weight_t getLikelyWeight() const - { - assert(m_likelihoodSet); - return m_likelihood * m_sourceBlock->bbWeight; - } - - unsigned getDupCount() const - { - return m_dupCount; - } - - void incrementDupCount() - { - m_dupCount++; - } - - void decrementDupCount() - { - assert(m_dupCount >= 1); - m_dupCount--; - } -}; +inline weight_t FlowEdge::getLikelyWeight() const +{ + assert(m_likelihoodSet); + return m_likelihood * m_sourceBlock->bbWeight; +} // BasicBlock iterator implementations (that are required to be defined after the declaration of FlowEdge) inline BasicBlock* BBArrayIterator::operator*() const { - if (iterateEdges) - { - assert(m_edgeEntry != nullptr); - FlowEdge* edgeTarget = *m_edgeEntry; - assert(edgeTarget != nullptr); - assert(edgeTarget->getDestinationBlock() != nullptr); - return edgeTarget->getDestinationBlock(); - } - - assert(m_bbEntry != nullptr); - BasicBlock* bTarget = *m_bbEntry; - assert(bTarget != nullptr); - return bTarget; + assert(m_edgeEntry != nullptr); + FlowEdge* edgeTarget = *m_edgeEntry; + assert(edgeTarget != nullptr); + assert(edgeTarget->getDestinationBlock() != nullptr); + return edgeTarget->getDestinationBlock(); } // Pred list iterator implementations (that are required to be defined after the declaration of BasicBlock and FlowEdge) diff --git a/src/coreclr/jit/clrjit.natvis b/src/coreclr/jit/clrjit.natvis index 95dd3dc305689..e8ac7e8f7c4a3 100644 --- a/src/coreclr/jit/clrjit.natvis +++ b/src/coreclr/jit/clrjit.natvis @@ -21,7 +21,7 @@ Documentation for VS debugger format specifiers: https://docs.microsoft.com/en-u - BB{bbNum,d}->BB{bbTarget->bbNum,d}; {bbKind,en} + BB{bbNum,d}->BB{bbTargetEdge->m_destBlock->bbNum,d}; {bbKind,en} BB{bbNum,d}; {bbKind,en}; {bbSwtTargets->bbsCount} cases BB{bbNum,d}; {bbKind,en}; {bbEhfTargets->bbeCount} succs BB{bbNum,d}; {bbKind,en} @@ -86,6 +86,11 @@ Documentation for VS debugger format specifiers: https://docs.microsoft.com/en-u {gtTreeID, d}: [{gtOper,en}, {gtType,en} V{((GenTreeLclFld*)this)->_gtLclNum,u}[+{((GenTreeLclFld*)this)->m_lclOffs,u}]] + + + [{Oper,en}, {Type,en}] + + LinearScan diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index c36a6776a8cd9..4d17e291d0c8a 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1183,6 +1183,9 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCodeForCpBlkRepMovs(GenTreeBlk* cpBlkNode); void genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode); void genCodeForPhysReg(GenTreePhysReg* tree); +#ifdef SWIFT_SUPPORT + void genCodeForSwiftErrorReg(GenTree* tree); +#endif // SWIFT_SUPPORT void genCodeForNullCheck(GenTreeIndir* tree); void genCodeForCmpXchg(GenTreeCmpXchg* tree); void genCodeForReuseVal(GenTree* treeNode); diff --git a/src/coreclr/jit/codegenarm64test.cpp b/src/coreclr/jit/codegenarm64test.cpp index 1a7ae0ef7d032..ae5c6b0487c67 100644 --- a/src/coreclr/jit/codegenarm64test.cpp +++ b/src/coreclr/jit/codegenarm64test.cpp @@ -4581,6 +4581,14 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_subr, EA_SCALABLE, REG_V2, REG_P0, REG_V13, INS_OPTS_SCALABLE_S); // SUBR ., /M, ., . +#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_AB_3B + theEmitter->emitIns_R_R_R(INS_sve_addpt, EA_SCALABLE, REG_V0, REG_P1, REG_V2, + INS_OPTS_SCALABLE_D); // ADDPT .D, /M, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_subpt, EA_SCALABLE, REG_V0, REG_P1, REG_V2, + INS_OPTS_SCALABLE_D); // SUBPT .D, /M, .D, .D +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_AC_3A theEmitter->emitIns_R_R_R(INS_sve_sdiv, EA_SCALABLE, REG_V3, REG_P2, REG_V9, INS_OPTS_SCALABLE_S); // SDIV ., /M, ., . @@ -4725,6 +4733,54 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_P0, REG_V0, INS_OPTS_SCALABLE_S, INS_SCALABLE_OPTS_WIDE); // LSR ., /M, ., .D + // IF_SVE_CE_2A + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P2, REG_V12, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .B, + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V2, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [0] + + // IF_SVE_CE_2B + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P15, REG_V7, 7, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .D, [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P7, REG_V16, 0, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .D, [] + + // IF_SVE_CE_2C + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P0, REG_V31, 1, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V1, REG_P1, 0, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .H, [] + + // IF_SVE_CE_2D + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P3, REG_V9, 3, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .S, [] + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_P10, REG_V4, 0, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_TO_PREDICATE); // PMOV .S, [] + + // IF_SVE_CF_2A + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V11, REG_P12, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV , .B + theEmitter->emitIns_R_R(INS_sve_pmov, EA_SCALABLE, REG_V2, REG_P7, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [0], .S + + // IF_SVE_CF_2B + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V6, REG_P8, 7, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .D + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V9, REG_P7, 0, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .D + + // IF_SVE_CF_2C + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V8, REG_P4, 1, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .H + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V5, REG_P9, 0, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .H + + // IF_SVE_CF_2D + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V14, REG_P2, 3, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .S + theEmitter->emitIns_R_R_I(INS_sve_pmov, EA_SCALABLE, REG_V3, REG_P15, 0, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_TO_VECTOR); // PMOV [], .S + // IF_SVE_CJ_2A theEmitter->emitIns_R_R(INS_sve_rev, EA_SCALABLE, REG_P1, REG_P2, INS_OPTS_SCALABLE_B); // REV ., . @@ -5101,6 +5157,22 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_fsubr, EA_SCALABLE, REG_V6, REG_P4, REG_V29, INS_OPTS_SCALABLE_D); // FSUBR ., /M, ., . + // IF_SVE_HL_3B + theEmitter->emitIns_R_R_R(INS_sve_bfadd, EA_SCALABLE, REG_V0, REG_P0, REG_V1, + INS_OPTS_SCALABLE_H); // BFADD .H, /M, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmax, EA_SCALABLE, REG_V2, REG_P1, REG_V3, + INS_OPTS_SCALABLE_H); // BFMAX .H, /M, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmaxnm, EA_SCALABLE, REG_V4, REG_P2, REG_V5, + INS_OPTS_SCALABLE_H); // BFMAXNM .H, /M, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmin, EA_SCALABLE, REG_V6, REG_P3, REG_V7, + INS_OPTS_SCALABLE_H); // BFMIN .H, /M, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfminnm, EA_SCALABLE, REG_V8, REG_P4, REG_V9, + INS_OPTS_SCALABLE_H); // BFMINNM .H, /M, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmul, EA_SCALABLE, REG_V10, REG_P5, REG_V11, + INS_OPTS_SCALABLE_H); // BFMUL .H, /M, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfsub, EA_SCALABLE, REG_V12, REG_P6, REG_V13, + INS_OPTS_SCALABLE_H); // BFSUB .H, /M, .H, .H + // IF_SVE_HT_4A theEmitter->emitIns_R_R_R_R(INS_sve_facge, EA_SCALABLE, REG_P0, REG_P0, REG_V10, REG_V31, INS_OPTS_SCALABLE_H); // FACGE ., /Z, ., . @@ -5125,6 +5197,16 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R_R(INS_sve_fcmuo, EA_SCALABLE, REG_P5, REG_P2, REG_V31, REG_V20, INS_OPTS_SCALABLE_S); // FCMUO ., /Z, ., . + // IF_SVE_HU_4A + theEmitter->emitIns_R_R_R_R(INS_sve_fmla, EA_SCALABLE, REG_V0, REG_P0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // FMLA ., /M, ., . + theEmitter->emitIns_R_R_R_R(INS_sve_fmls, EA_SCALABLE, REG_V3, REG_P2, REG_V4, REG_V5, + INS_OPTS_SCALABLE_S); // FMLS ., /M, ., . + theEmitter->emitIns_R_R_R_R(INS_sve_fnmla, EA_SCALABLE, REG_V6, REG_P4, REG_V7, REG_V8, + INS_OPTS_SCALABLE_D); // FNMLA ., /M, ., . + theEmitter->emitIns_R_R_R_R(INS_sve_fnmls, EA_SCALABLE, REG_V9, REG_P6, REG_V10, REG_V11, + INS_OPTS_SCALABLE_H); // FNMLS ., /M, ., . + // IF_SVE_AF_3A theEmitter->emitIns_R_R_R(INS_sve_andv, EA_1BYTE, REG_V0, REG_P0, REG_V0, INS_OPTS_SCALABLE_B); // ANDV , , . @@ -5269,6 +5351,10 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_umulh, EA_SCALABLE, REG_V31, REG_V5, REG_V0, INS_OPTS_SCALABLE_D, INS_SCALABLE_OPTS_UNPREDICATED); // UMULH ., ., . + // IF_SVE_BD_3B + theEmitter->emitIns_R_R_R(INS_sve_pmul, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // PMUL .B, .B, .B + // IF_SVE_BE_3A theEmitter->emitIns_R_R_R(INS_sve_sqdmulh, EA_SCALABLE, REG_V7, REG_V28, REG_V0, INS_OPTS_SCALABLE_B); // SQDMULH ., ., . @@ -5283,6 +5369,24 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_lsr, EA_SCALABLE, REG_V29, REG_V10, REG_V22, INS_OPTS_SCALABLE_S, INS_SCALABLE_OPTS_UNPREDICATED_WIDE); // LSR ., ., .D + // IF_SVE_BH_3A + theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V4, REG_V2, REG_V0, 0, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_LSL_N); // ADR ., [., .{, }] + theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V29, REG_V1, REG_V10, 1, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_LSL_N); // ADR ., [., .{, }] + + // IF_SVE_BH_3B + theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V9, REG_V7, REG_V9, 0, + INS_OPTS_SCALABLE_D_SXTW); // ADR .D, [.D, .D, SXTW{}] + theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V12, REG_V3, REG_V5, 2, + INS_OPTS_SCALABLE_D_SXTW); // ADR .D, [.D, .D, SXTW{}] + + // IF_SVE_BH_3B_A + theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V9, REG_V10, REG_V14, 0, + INS_OPTS_SCALABLE_D_UXTW); // ADR .D, [.D, .D, UXTW{}] + theEmitter->emitInsSve_R_R_R_I(INS_sve_adr, EA_SCALABLE, REG_V3, REG_V15, REG_V11, 3, + INS_OPTS_SCALABLE_D_UXTW); // ADR .D, [.D, .D, UXTW{}] + // IF_SVE_BK_3A theEmitter->emitIns_R_R_R(INS_sve_ftssel, EA_SCALABLE, REG_V17, REG_V16, REG_V15, INS_OPTS_SCALABLE_D); // FTSSEL ., ., . @@ -5559,6 +5663,111 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_uabalt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_H); // UABALT ., ., . +#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_GC_3A + theEmitter->emitIns_R_R_R(INS_sve_addhnb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // ADDHNB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_addhnt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_H); // ADDHNT ., ., . + theEmitter->emitIns_R_R_R(INS_sve_raddhnb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_S); // RADDHNB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_raddhnt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_B); // RADDHNT ., ., . + theEmitter->emitIns_R_R_R(INS_sve_rsubhnb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_H); // RSUBHNB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_rsubhnt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, + INS_OPTS_SCALABLE_S); // RSUBHNT ., ., . + theEmitter->emitIns_R_R_R(INS_sve_subhnb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, + INS_OPTS_SCALABLE_B); // SUBHNB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_subhnt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, + INS_OPTS_SCALABLE_H); // SUBHNT ., ., . +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + + // IF_SVE_GF_3A + theEmitter->emitIns_R_R_R(INS_sve_histseg, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // HISTSEG .B, .B, .B + + // IF_SVE_GW_3A + theEmitter->emitIns_R_R_R(INS_sve_fclamp, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // FCLAMP ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fclamp, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_S); // FCLAMP ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fclamp, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_D); // FCLAMP ., ., . + + // IF_SVE_GW_3B + theEmitter->emitIns_R_R_R(INS_sve_bfclamp, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // BFCLAMP .H, .H, .H + + // IF_SVE_HK_3A + theEmitter->emitIns_R_R_R(INS_sve_fadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); // FADD ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); // FMUL ., ., . + theEmitter->emitIns_R_R_R(INS_sve_frecps, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // FRECPS ., ., . + theEmitter->emitIns_R_R_R(INS_sve_frsqrts, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); // FRSQRTS ., ., . + theEmitter->emitIns_R_R_R(INS_sve_fsub, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); // FSUB ., ., . + theEmitter->emitIns_R_R_R(INS_sve_ftsmul, EA_SCALABLE, REG_V15, REG_V16, REG_V17, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // FTSMUL ., ., . + + // IF_SVE_HK_3B + theEmitter->emitIns_R_R_R(INS_sve_bfadd, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); // BFADD .H, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmul, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); // BFMUL .H, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfsub, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); // BFSUB .H, .H, .H + +#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_AT_3B + theEmitter->emitIns_R_R_R(INS_sve_addpt, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // ADDPT .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_subpt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // SUBPT .D, .D, .D +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + + // IF_SVE_AU_3A + theEmitter->emitIns_R_R_R(INS_sve_and, EA_SCALABLE, REG_V0, REG_V1, REG_V2, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // AND .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_bic, EA_SCALABLE, REG_V3, REG_V4, REG_V5, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // BIC .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_eor, EA_SCALABLE, REG_V6, REG_V7, REG_V8, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // EOR .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V9, REG_V10, REG_V11, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // MOV .D, .D + theEmitter->emitIns_R_R_R(INS_sve_orr, EA_SCALABLE, REG_V12, REG_V13, REG_V14, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // ORR .D, .D, .D + + // IF_SVE_AV_3A + theEmitter->emitIns_R_R_R(INS_sve_bcax, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_D); // BCAX .D, .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_bsl, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_D); // BSL .D, .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_bsl1n, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_D); // BSL1N .D, .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_bsl2n, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_D); // BSL2N .D, .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_eor3, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_D); // EOR3 .D, .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_nbsl, EA_SCALABLE, REG_V15, REG_V16, REG_V17, + INS_OPTS_SCALABLE_D); // NBSL .D, .D, .D, .D + + // IF_SVE_BB_2A + theEmitter->emitIns_R_R_I(INS_sve_addpl, EA_8BYTE, REG_R0, REG_R1, -32); // ADDPL , , # + theEmitter->emitIns_R_R_I(INS_sve_addpl, EA_8BYTE, REG_R2, REG_SP, 0); // ADDPL , , # + theEmitter->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_R3, REG_R4, 5); // ADDVL , , # + theEmitter->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_SP, REG_R5, 31); // ADDVL , , # + theEmitter->emitIns_R_R_I(INS_sve_addvl, EA_8BYTE, REG_SP, REG_SP, 0); // ADDVL , , # + + // IF_SVE_BC_1A + theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R0, -32); // RDVL , # + theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R5, 0); // RDVL , # + theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R10, 5); // RDVL , # + theEmitter->emitIns_R_I(INS_sve_rdvl, EA_8BYTE, REG_R15, 31); // RDVL , # + // IF_SVE_BL_1A theEmitter->emitIns_R_PATTERN_I(INS_sve_cntb, EA_8BYTE, REG_R0, SVE_PATTERN_POW2, 1); // CNTB {, {, MUL #}} @@ -5671,6 +5880,14 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R(INS_sve_splice, EA_SCALABLE, REG_V2, REG_P6, REG_V28, INS_OPTS_SCALABLE_S); // SPLICE ., , ., . + // IF_SVE_CW_4A + theEmitter->emitIns_R_R_R(INS_sve_mov, EA_SCALABLE, REG_V0, REG_P0, REG_V30, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_PREDICATE_MERGE); // MOV ., /M, . + theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V29, REG_P15, REG_V28, REG_V4, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); // SEL ., , ., . + theEmitter->emitIns_R_R_R_R(INS_sve_sel, EA_SCALABLE, REG_V5, REG_P13, REG_V27, REG_V5, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); // SEL ., , ., . + // IF_SVE_EQ_3A // Note: Scalable size is the size of the destination , not the source . theEmitter->emitIns_R_R_R(INS_sve_sadalp, EA_SCALABLE, REG_V26, REG_P3, REG_V8, @@ -5940,8 +6157,24 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R(INS_sve_aesimc, EA_SCALABLE, REG_V0); // AESIMC .B, .B theEmitter->emitIns_R(INS_sve_aesmc, EA_SCALABLE, REG_V5); // AESMC .B, .B -// IF_SVE_GS_3A #ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_GN_3A + theEmitter->emitIns_R_R_R(INS_sve_fmlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // FMLALB .H, .B, .B + theEmitter->emitIns_R_R_R(INS_sve_fmlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_B); // FMLALT .H, .B, .B + + // IF_SVE_GO_3A + theEmitter->emitIns_R_R_R(INS_sve_fmlallbb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // FMLALLBB .S, .B, .B + theEmitter->emitIns_R_R_R(INS_sve_fmlallbt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_B); // FMLALLBT .S, .B, .B + theEmitter->emitIns_R_R_R(INS_sve_fmlalltb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_B); // FMLALLTB .S, .B, .B + theEmitter->emitIns_R_R_R(INS_sve_fmlalltt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_B); // FMLALLTT .S, .B, .B + + // IF_SVE_GS_3A theEmitter->emitIns_R_R_R(INS_sve_faddqv, EA_8BYTE, REG_V16, REG_P0, REG_V12, INS_OPTS_SCALABLE_H); // FADDQV ., , . theEmitter->emitIns_R_R_R(INS_sve_fmaxnmqv, EA_8BYTE, REG_V17, REG_P1, REG_V11, @@ -6074,6 +6307,49 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R_I(INS_sve_fmlslt, EA_SCALABLE, REG_V14, REG_V15, REG_V7, 7, INS_OPTS_SCALABLE_H); // FMLSLT .S, .H, .H[] + // IF_SVE_HA_3A + theEmitter->emitIns_R_R_R(INS_sve_bfdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // BFDOT .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_fdot, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_H); // FDOT .S, .H, .H + +#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_HA_3A_E + theEmitter->emitIns_R_R_R(INS_sve_fdot, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_B); // FDOT .H, .B, .B + + // IF_SVE_HA_3A_F + theEmitter->emitIns_R_R_R(INS_sve_fdot, EA_SCALABLE, REG_V9, REG_V10, REG_V11); // FDOT .S, .B, .B +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + + // IF_SVE_HB_3A + theEmitter->emitIns_R_R_R(INS_sve_bfmlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // BFMLALB .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_H); // BFMLALT .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmlslb, EA_SCALABLE, REG_V6, REG_V7, REG_V8, + INS_OPTS_SCALABLE_H); // BFMLSLB .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_bfmlslt, EA_SCALABLE, REG_V9, REG_V10, REG_V11, + INS_OPTS_SCALABLE_H); // BFMLSLT .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_fmlalb, EA_SCALABLE, REG_V12, REG_V13, REG_V14, + INS_OPTS_SCALABLE_H); // FMLALB .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_fmlalt, EA_SCALABLE, REG_V15, REG_V16, REG_V17, + INS_OPTS_SCALABLE_H); // FMLALT .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_fmlslb, EA_SCALABLE, REG_V18, REG_V19, REG_V20, + INS_OPTS_SCALABLE_H); // FMLSLB .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_fmlslt, EA_SCALABLE, REG_V21, REG_V22, REG_V23, + INS_OPTS_SCALABLE_H); // FMLSLT .S, .H, .H + + // IF_SVE_HD_3A + theEmitter->emitIns_R_R_R(INS_sve_bfmmla, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // BFMMLA .S, .H, .H + +#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_HD_3A_A + theEmitter->emitIns_R_R_R(INS_sve_fmmla, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_D); // FMMLA .D, .D, .D +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_HE_3A theEmitter->emitIns_R_R_R(INS_sve_faddv, EA_2BYTE, REG_V21, REG_P7, REG_V7, INS_OPTS_SCALABLE_H); // FADDV , , . @@ -6697,6 +6973,16 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_I(INS_sve_mul, EA_SCALABLE, REG_V3, 127, INS_OPTS_SCALABLE_D); // MUL ., ., # + // IF_SVE_EF_3A + theEmitter->emitIns_R_R_R(INS_sve_sdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_H); // SDOT .S, .H, .H + theEmitter->emitIns_R_R_R(INS_sve_udot, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_H); // UDOT .S, .H, .H + + // IF_SVE_EI_3A + theEmitter->emitIns_R_R_R(INS_sve_usdot, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_B); // USDOT .S, .B, .B + // IF_SVE_FA_3A theEmitter->emitIns_R_R_R_I_I(INS_sve_cdot, EA_SCALABLE, REG_V0, REG_V7, REG_V1, 3, 0, INS_OPTS_SCALABLE_B); // CDOT .S, .B, .B[], @@ -7754,6 +8040,18 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_R_R_I(INS_sve_fcmla, EA_SCALABLE, REG_V2, REG_P3, REG_V0, REG_V6, 270, INS_OPTS_SCALABLE_D); // FCMLA ., /M, ., ., + // IF_SVE_GI_4A + theEmitter->emitIns_R_R_R_R(INS_sve_histcnt, EA_SCALABLE, REG_V0, REG_P0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_S); // HISTCNT ., /Z, ., . + theEmitter->emitIns_R_R_R_R(INS_sve_histcnt, EA_SCALABLE, REG_V3, REG_P7, REG_V4, REG_V5, + INS_OPTS_SCALABLE_D); // HISTCNT ., /Z, ., . + + // IF_SVE_GJ_3A + theEmitter->emitIns_R_R_R(INS_sve_rax1, EA_SCALABLE, REG_V0, REG_V1, REG_V2, + INS_OPTS_SCALABLE_D); // RAX1 .D, .D, .D + theEmitter->emitIns_R_R_R(INS_sve_sm4ekey, EA_SCALABLE, REG_V3, REG_V4, REG_V5, + INS_OPTS_SCALABLE_S); // SM4EKEY .S, .S, .S + // IF_SVE_HI_3A theEmitter->emitIns_R_R_R(INS_sve_fcmeq, EA_SCALABLE, REG_P2, REG_P3, REG_V4, INS_OPTS_SCALABLE_H); // FCMEQ ., /Z, ., #0.0 @@ -7871,6 +8169,78 @@ void CodeGen::genArm64EmitterUnitTestsSve() theEmitter->emitIns_R_R_I(INS_sve_str, EA_SCALABLE, REG_V2, REG_R3, 255, INS_OPTS_NONE, INS_SCALABLE_OPTS_UNPREDICATED); +#ifdef ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_GG_3A + // LUTI2 .B, {.B }, [] + // luti2 z0.b, {z0.b}, z0[0] // 01000101-00100000-10110000-00000000 + // CHECK-INST: luti2 z0.b, { z0.b }, z0[0] + // CHECK-ENCODING: [0x00,0xb0,0x20,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); + // luti2 z21.b, {z10.b}, z21[1] // 01000101-01110101-10110001-01010101 + // CHECK-INST: luti2 z21.b, { z10.b }, z21[1] + // CHECK-ENCODING: [0x55,0xb1,0x75,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 1, INS_OPTS_SCALABLE_B); + + // IF_SVE_GH_3B + // LUTI4 .H, {.H, .H }, [] + // luti4 z0.h, {z0.h, z1.h}, z0[0] // 01000101-00100000-10110100-00000000 + // CHECK-INST: luti4 z0.h, { z0.h, z1.h }, z0[0] + // CHECK-ENCODING: [0x00,0xb4,0x20,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H, EA_UNKNOWN, + INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); + // luti4 z21.h, {z10.h, z11.h}, z21[1] // 01000101-01110101-10110101-01010101 + // CHECK-INST: luti4 z21.h, { z10.h, z11.h }, z21[1] + // CHECK-ENCODING: [0x55,0xb5,0x75,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 1, INS_OPTS_SCALABLE_H, + EA_UNKNOWN, INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); + // luti4 z31.h, {z31.h, z0.h}, z31[3] // 01000101-11111111-10110111-11111111 + // CHECK-INST: luti4 z31.h, { z31.h, z0.h }, z31[3] + // CHECK-ENCODING: [0xff,0xb7,0xff,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 3, INS_OPTS_SCALABLE_H, + EA_UNKNOWN, INS_SCALABLE_OPTS_WITH_VECTOR_PAIR); + + // IF_SVE_GH_3B_B + // LUTI4 .H, {.H }, [] + // luti4 z0.h, {z0.h}, z0[0] // 01000101-00100000-10111100-00000000 + // CHECK-INST: luti4 z0.h, { z0.h }, z0[0] + // CHECK-ENCODING: [0x00,0xbc,0x20,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); + // luti4 z21.h, {z10.h}, z21[1] // 01000101-01110101-10111101-01010101 + // CHECK-INST: luti4 z21.h, { z10.h }, z21[1] + // CHECK-ENCODING: [0x55,0xbd,0x75,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 1, INS_OPTS_SCALABLE_H); + // luti4 z31.h, {z31.h}, z31[3] // 01000101-11111111-10111111-11111111 + // CHECK-INST: luti4 z31.h, { z31.h }, z31[3] + // CHECK-ENCODING: [0xff,0xbf,0xff,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 3, INS_OPTS_SCALABLE_H); + + // IF_SVE_GG_3B + // LUTI2 .H, {.H }, [] + // luti2 z0.h, {z0.h}, z0[0] // 01000101-00100000-10101000-00000000 + // CHECK-INST: luti2 z0.h, { z0.h }, z0[0] + // CHECK-ENCODING: [0x00,0xa8,0x20,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); + // luti2 z21.h, {z10.h}, z21[3] // 01000101-01110101-10111001-01010101 + // CHECK-INST: luti2 z21.h, { z10.h }, z21[3] + // CHECK-ENCODING: [0x55,0xb9,0x75,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V21, REG_V10, REG_V21, 3, INS_OPTS_SCALABLE_H); + // luti2 z31.h, {z31.h}, z31[7] // 01000101-11111111-10111011-11111111 + // CHECK-INST: luti2 z31.h, { z31.h }, z31[7] + // CHECK-ENCODING: [0xff,0xbb,0xff,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti2, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_H); + + // IF_SVE_GH_3A + // LUTI4 .B, {.B }, [] + // luti4 z0.b, {z0.b}, z0[0] // 01000101-01100000-10100100-00000000 + // CHECK-INST: luti4 z0.b, { z0.b }, z0[0] + // CHECK-ENCODING: [0x00,0xa4,0x60,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V0, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); + // luti4 z31.b, {z31.b}, z31[1] // 01000101-11111111-10100111-11111111 + // CHECK-INST: luti4 z31.b, { z31.b }, z31[1] + // CHECK-ENCODING: [0xff,0xa7,0xff,0x45] + theEmitter->emitIns_R_R_R_I(INS_sve_luti4, EA_SCALABLE, REG_V31, REG_V31, REG_V31, 1, INS_OPTS_SCALABLE_B); +#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED + // IF_SVE_HY_3A theEmitter->emitIns_PRFOP_R_R_R(INS_sve_prfb, EA_SCALABLE, SVE_PRFOP_PLDL1KEEP, REG_P1, REG_R2, REG_V3, INS_OPTS_SCALABLE_S_UXTW); // PRFB , , [, .S, ] @@ -8192,6 +8562,215 @@ void CodeGen::genArm64EmitterUnitTestsSve() // IF_SVE_BI_2A // MOVPRFX , theEmitter->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, REG_V3, REG_V5); + + // IF_SVE_BF_2A + // ASR ., ., # + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_asr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + // LSL ., ., #emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 31, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V31, REG_V31, 63, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsl, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + // LSR ., ., #emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 5, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 9, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 15, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 33, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + theEmitter->emitIns_R_R_I(INS_sve_lsr, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D, + INS_SCALABLE_OPTS_UNPREDICATED); + + // IF_SVE_FT_2A + // SLI ., ., # + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 7, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 15, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 31, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V0, 0, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V31, REG_V31, 63, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_sli, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); + // SRI ., ., # + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_sri, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); + + // IF_SVE_FU_2A + // SRSRA ., ., # + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_srsra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); + // SSRA ., ., # + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_ssra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); + // URSRA ., ., # + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_ursra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); + // USRA ., ., # + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 8, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 3, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 4, INS_OPTS_SCALABLE_B); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 16, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 7, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 8, INS_OPTS_SCALABLE_H); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 32, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 17, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 16, INS_OPTS_SCALABLE_S); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V0, 1, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V31, REG_V31, 64, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 31, INS_OPTS_SCALABLE_D); + theEmitter->emitIns_R_R_I(INS_sve_usra, EA_SCALABLE, REG_V0, REG_V31, 32, INS_OPTS_SCALABLE_D); } #endif // defined(TARGET_ARM64) && defined(DEBUG) diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 8eb8d71b617a5..1535ed68d9ee4 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -441,6 +441,12 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) break; #endif // TARGET_ARM64 +#ifdef SWIFT_SUPPORT + case GT_SWIFT_ERROR: + genCodeForSwiftErrorReg(treeNode); + break; +#endif // SWIFT_SUPPORT + case GT_RELOAD: // do nothing - reload is just a marker. // The parent node will call genConsumeReg on this which will trigger the unspill of this node's child @@ -507,7 +513,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #endif break; - case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -3369,6 +3374,17 @@ void CodeGen::genCall(GenTreeCall* call) genDefineTempLabel(genCreateTempLabel()); } +#ifdef SWIFT_SUPPORT + // Clear the Swift error register before calling a Swift method, + // so we can check if it set the error register after returning. + // (Flag is only set if we know we need to check the error register) + if ((call->gtCallMoreFlags & GTF_CALL_M_SWIFT_ERROR_HANDLING) != 0) + { + assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); + instGen_Set_Reg_To_Zero(EA_PTRSIZE, REG_SWIFT_ERROR); + } +#endif // SWIFT_SUPPORT + genCallInstruction(call); genDefinePendingCallLabel(call); @@ -4551,14 +4567,14 @@ void CodeGen::inst_JMP(emitJumpKind jmp, BasicBlock* tgtBlock) } //------------------------------------------------------------------------ -// genCodeForStoreBlk: Produce code for a GT_STORE_DYN_BLK/GT_STORE_BLK node. +// genCodeForStoreBlk: Produce code for a GT_STORE_BLK node. // // Arguments: // tree - the node // void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) { - assert(blkOp->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); + assert(blkOp->OperIs(GT_STORE_BLK)); bool isCopyBlk = blkOp->OperIsCopyBlkOp(); diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index f5a32d241cc34..9d7992ea4efe2 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -6546,7 +6546,7 @@ void CodeGen::genDefinePendingCallLabel(GenTreeCall* call) // For certain indirect calls we may introduce helper calls before that we need to skip: // - CFG may introduce a call to the validator first // - Generic virtual methods may compute the target dynamically through a separate helper call - // - memset/memcpy helper calls emitted for GT_STORE_DYN_BLK/GT_STORE_BLK + // - memset/memcpy helper calls emitted for GT_STORE_BLK if (call->IsHelperCall()) { switch (compiler->eeGetHelperNum(call->gtCallMethHnd)) @@ -8569,3 +8569,31 @@ void CodeGen::genCodeForReuseVal(GenTree* treeNode) genDefineTempLabel(genCreateTempLabel()); } } + +#ifdef SWIFT_SUPPORT +//--------------------------------------------------------------------- +// genCodeForSwiftErrorReg - generate code for a GT_SWIFT_ERROR node +// +// Arguments +// tree - the GT_SWIFT_ERROR node +// +// Return value: +// None +// +void CodeGen::genCodeForSwiftErrorReg(GenTree* tree) +{ + assert(tree->OperIs(GT_SWIFT_ERROR)); + + var_types targetType = tree->TypeGet(); + regNumber targetReg = tree->GetRegNum(); + + // LSRA should have picked REG_SWIFT_ERROR as the destination register, too + // (see LinearScan::BuildNode for an explanation of why we want this) + assert(targetReg == REG_SWIFT_ERROR); + + inst_Mov(targetType, targetReg, REG_SWIFT_ERROR, /* canSkip */ true); + genTransferRegGCState(targetReg, REG_SWIFT_ERROR); + + genProduceReg(tree); +} +#endif // SWIFT_SUPPORT diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 913f3a47002a5..78df10811a4c2 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1935,18 +1935,9 @@ void CodeGen::genSetBlockSize(GenTreeBlk* blkNode, regNumber sizeReg) { if (sizeReg != REG_NA) { - unsigned blockSize = blkNode->Size(); - if (!blkNode->OperIs(GT_STORE_DYN_BLK)) - { - assert((blkNode->gtRsvdRegs & genRegMask(sizeReg)) != 0); - // This can go via helper which takes the size as a native uint. - instGen_Set_Reg_To_Imm(EA_PTRSIZE, sizeReg, blockSize); - } - else - { - GenTree* sizeNode = blkNode->AsStoreDynBlk()->gtDynamicSize; - inst_Mov(sizeNode->TypeGet(), sizeReg, sizeNode->GetRegNum(), /* canSkip */ true); - } + assert((blkNode->gtRsvdRegs & genRegMask(sizeReg)) != 0); + // This can go via helper which takes the size as a native uint. + instGen_Set_Reg_To_Imm(EA_PTRSIZE, sizeReg, blkNode->Size()); } } @@ -2052,12 +2043,6 @@ void CodeGen::genConsumeBlockOp(GenTreeBlk* blkNode, regNumber dstReg, regNumber genConsumeReg(dstAddr); // The source may be a local or in a register; 'genConsumeBlockSrc' will check that. genConsumeBlockSrc(blkNode); - // 'genSetBlockSize' (called below) will ensure that a register has been reserved as needed - // in the case where the size is a constant (i.e. it is not GT_STORE_DYN_BLK). - if (blkNode->OperGet() == GT_STORE_DYN_BLK) - { - genConsumeReg(blkNode->AsStoreDynBlk()->gtDynamicSize); - } // Next, perform any necessary moves. genCopyRegIfNeeded(dstAddr, dstReg); diff --git a/src/coreclr/jit/codegenloongarch64.cpp b/src/coreclr/jit/codegenloongarch64.cpp index f287785951964..e894d98763331 100644 --- a/src/coreclr/jit/codegenloongarch64.cpp +++ b/src/coreclr/jit/codegenloongarch64.cpp @@ -5022,7 +5022,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_ld_d, EA_PTRSIZE, genPendingCallLabel, targetReg); break; - case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -7249,14 +7248,14 @@ void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, } //------------------------------------------------------------------------ -// genCodeForStoreBlk: Produce code for a GT_STORE_DYN_BLK/GT_STORE_BLK node. +// genCodeForStoreBlk: Produce code for a GT_STORE_BLK node. // // Arguments: // tree - the node // void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) { - assert(blkOp->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); + assert(blkOp->OperIs(GT_STORE_BLK)); if (blkOp->gtBlkOpGcUnsafe) { diff --git a/src/coreclr/jit/codegenriscv64.cpp b/src/coreclr/jit/codegenriscv64.cpp index 6dc76478246be..90429b89584fd 100644 --- a/src/coreclr/jit/codegenriscv64.cpp +++ b/src/coreclr/jit/codegenriscv64.cpp @@ -2867,6 +2867,7 @@ void CodeGen::genJumpTable(GenTree* treeNode) for (unsigned i = 0; i < jumpCount; i++) { BasicBlock* target = (*jumpTable)->getDestinationBlock(); + jumpTable++; noway_assert(target->HasFlag(BBF_HAS_LABEL)); JITDUMP(" DD L_M%03u_" FMT_BB "\n", compiler->compMethodID, target->bbNum); @@ -5102,7 +5103,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_ld, EA_PTRSIZE, genPendingCallLabel, targetReg); break; - case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -7246,14 +7246,14 @@ void CodeGen::genCreateAndStoreGCInfo(unsigned codeSize, } //------------------------------------------------------------------------ -// genCodeForStoreBlk: Produce code for a GT_STORE_DYN_BLK/GT_STORE_BLK node. +// genCodeForStoreBlk: Produce code for a GT_STORE_BLK node. // // Arguments: // tree - the node // void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp) { - assert(blkOp->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); + assert(blkOp->OperIs(GT_STORE_BLK)); if (blkOp->gtBlkOpGcUnsafe) { diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 884ba901e5d83..584cb3aab19bb 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -332,7 +332,7 @@ BasicBlock* CodeGen::genCallFinally(BasicBlock* block) #endif // !FEATURE_EH_FUNCLETS // The BBJ_CALLFINALLYRET is used because the BBJ_CALLFINALLY can't point to the - // jump target using bbTarget - that is already used to point + // jump target using bbTargetEdge - that is already used to point // to the finally block. So just skip past the BBJ_CALLFINALLYRET unless the // block is RETLESS. if (!block->HasFlag(BBF_RETLESS_CALL)) @@ -2107,6 +2107,12 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_NOP: break; +#ifdef SWIFT_SUPPORT + case GT_SWIFT_ERROR: + genCodeForSwiftErrorReg(treeNode); + break; +#endif // SWIFT_SUPPORT + case GT_KEEPALIVE: genConsumeRegs(treeNode->AsOp()->gtOp1); break; @@ -2177,7 +2183,6 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) emit->emitIns_R_L(INS_lea, EA_PTR_DSP_RELOC, genPendingCallLabel, treeNode->GetRegNum()); break; - case GT_STORE_DYN_BLK: case GT_STORE_BLK: genCodeForStoreBlk(treeNode->AsBlk()); break; @@ -3051,7 +3056,7 @@ void CodeGen::genLclHeap(GenTree* tree) void CodeGen::genCodeForStoreBlk(GenTreeBlk* storeBlkNode) { - assert(storeBlkNode->OperIs(GT_STORE_DYN_BLK, GT_STORE_BLK)); + assert(storeBlkNode->OperIs(GT_STORE_BLK)); bool isCopyBlk = storeBlkNode->OperIsCopyBlkOp(); @@ -6035,6 +6040,17 @@ void CodeGen::genCall(GenTreeCall* call) instGen(INS_vzeroupper); } +#ifdef SWIFT_SUPPORT + // Clear the Swift error register before calling a Swift method, + // so we can check if it set the error register after returning. + // (Flag is only set if we know we need to check the error register) + if ((call->gtCallMoreFlags & GTF_CALL_M_SWIFT_ERROR_HANDLING) != 0) + { + assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); + instGen_Set_Reg_To_Zero(EA_PTRSIZE, REG_SWIFT_ERROR); + } +#endif // SWIFT_SUPPORT + genCallInstruction(call X86_ARG(stackArgBytes)); genDefinePendingCallLabel(call); @@ -10789,9 +10805,12 @@ void CodeGen::genFuncletProlog(BasicBlock* block) compiler->unwindEndProlog(); // TODO We may need EBP restore sequence here if we introduce PSPSym + CLANG_FORMAT_COMMENT_ANCHOR; +#ifdef UNIX_X86_ABI // Add a padding for 16-byte alignment inst_RV_IV(INS_sub, REG_SPBASE, 12, EA_PTRSIZE); +#endif } /***************************************************************************** @@ -10810,8 +10829,10 @@ void CodeGen::genFuncletEpilog() ScopedSetVariable _setGeneratingEpilog(&compiler->compGeneratingEpilog, true); +#ifdef UNIX_X86_ABI // Revert a padding that was added for 16-byte alignment inst_RV_IV(INS_add, REG_SPBASE, 12, EA_PTRSIZE); +#endif instGen_Return(0); } diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 9fded7a13ccb0..66ed4a63fda6b 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -4893,6 +4893,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl bool doValueNum = true; bool doLoopHoisting = true; bool doCopyProp = true; + bool doOptimizeIVs = true; bool doBranchOpt = true; bool doCse = true; bool doAssertionProp = true; @@ -4905,6 +4906,7 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl doSsa = (JitConfig.JitDoSsa() != 0); doEarlyProp = doSsa && (JitConfig.JitDoEarlyProp() != 0); doValueNum = doSsa && (JitConfig.JitDoValueNumber() != 0); + doOptimizeIVs = doSsa && (JitConfig.JitDoOptimizeIVs() != 0); doLoopHoisting = doValueNum && (JitConfig.JitDoLoopHoisting() != 0); doCopyProp = doValueNum && (JitConfig.JitDoCopyProp() != 0); doBranchOpt = doValueNum && (JitConfig.JitDoRedundantBranchOpts() != 0); @@ -5005,6 +5007,13 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl DoPhase(this, PHASE_OPTIMIZE_INDEX_CHECKS, &Compiler::rangeCheckPhase); } + if (doOptimizeIVs) + { + // Simplify and optimize induction variables used in natural loops + // + DoPhase(this, PHASE_OPTIMIZE_INDUCTION_VARIABLES, &Compiler::optInductionVariables); + } + if (doVNBasedDeadStoreRemoval) { // Note: this invalidates SSA and value numbers on tree nodes. @@ -9409,6 +9418,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:cLoops") #pragma comment(linker, "/include:cLoopsA") #pragma comment(linker, "/include:cLoop") +#pragma comment(linker, "/include:cScev") #pragma comment(linker, "/include:cTreeFlags") #pragma comment(linker, "/include:cVN") @@ -9434,6 +9444,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #pragma comment(linker, "/include:dCVarSet") #pragma comment(linker, "/include:dLoop") #pragma comment(linker, "/include:dLoops") +#pragma comment(linker, "/include:dScev") #pragma comment(linker, "/include:dTreeFlags") #pragma comment(linker, "/include:dVN") @@ -9677,24 +9688,38 @@ JITDBGAPI void __cdecl cCVarSet(Compiler* comp, VARSET_VALARG_TP vars) JITDBGAPI void __cdecl cLoops(Compiler* comp) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *NewLoops %u\n", sequenceNumber++); + printf("===================================================================== *Loops %u\n", sequenceNumber++); FlowGraphNaturalLoops::Dump(comp->m_loops); } JITDBGAPI void __cdecl cLoopsA(Compiler* comp, FlowGraphNaturalLoops* loops) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *NewLoopsA %u\n", sequenceNumber++); + printf("===================================================================== *LoopsA %u\n", sequenceNumber++); FlowGraphNaturalLoops::Dump(loops); } JITDBGAPI void __cdecl cLoop(Compiler* comp, FlowGraphNaturalLoop* loop) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called - printf("===================================================================== *NewLoop %u\n", sequenceNumber++); + printf("===================================================================== *Loop %u\n", sequenceNumber++); FlowGraphNaturalLoop::Dump(loop); } +JITDBGAPI void __cdecl cScev(Compiler* comp, Scev* scev) +{ + static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called + printf("===================================================================== *Scev %u\n", sequenceNumber++); + if (scev == nullptr) + { + printf(" NULL\n"); + } + else + { + scev->Dump(comp); + } +} + JITDBGAPI void __cdecl cTreeFlags(Compiler* comp, GenTree* tree) { static unsigned sequenceNumber = 0; // separate calls with a number to indicate this function has been called @@ -9920,7 +9945,6 @@ JITDBGAPI void __cdecl cTreeFlags(Compiler* comp, GenTree* tree) case GT_BLK: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: if (tree->gtFlags & GTF_IND_VOLATILE) { @@ -10285,6 +10309,11 @@ JITDBGAPI void __cdecl dLoop(FlowGraphNaturalLoop* loop) cLoop(JitTls::GetCompiler(), loop); } +JITDBGAPI void __cdecl dScev(Scev* scev) +{ + cScev(JitTls::GetCompiler(), scev); +} + JITDBGAPI void __cdecl dTreeFlags(GenTree* tree) { cTreeFlags(JitTls::GetCompiler(), tree); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index c8d7fe49b62a0..52038be8eb5fe 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -42,6 +42,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "jitexpandarray.h" #include "tinyarray.h" #include "valuenum.h" +#include "scev.h" #include "namedintrinsiclist.h" #ifdef LATE_DISASM #include "disasm.h" @@ -1583,9 +1584,10 @@ enum class ProfileChecks : unsigned int CHECK_NONE = 0, CHECK_CLASSIC = 1 << 0, // check "classic" jit weights CHECK_HASLIKELIHOOD = 1 << 1, // check all FlowEdges for hasLikelihood - CHECK_LIKELY = 1 << 2, // fully check likelihood based weights - RAISE_ASSERT = 1 << 3, // assert on check failure - CHECK_ALL_BLOCKS = 1 << 4, // check blocks even if bbHasProfileWeight is false + CHECK_LIKELIHOODSUM = 1 << 2, // check block succesor likelihoods sum to 1 + CHECK_LIKELY = 1 << 3, // fully check likelihood based weights + RAISE_ASSERT = 1 << 4, // assert on check failure + CHECK_ALL_BLOCKS = 1 << 5, // check blocks even if bbHasProfileWeight is false }; inline constexpr ProfileChecks operator ~(ProfileChecks a) @@ -3364,9 +3366,6 @@ class Compiler GenTreeBlk* gtNewStoreBlkNode( ClassLayout* layout, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); - GenTreeStoreDynBlk* gtNewStoreDynBlkNode( - GenTree* addr, GenTree* data, GenTree* dynamicSize, GenTreeFlags indirFlags = GTF_EMPTY); - GenTreeStoreInd* gtNewStoreIndNode( var_types type, GenTree* addr, GenTree* data, GenTreeFlags indirFlags = GTF_EMPTY); @@ -4368,7 +4367,11 @@ class Compiler void impCheckForPInvokeCall( GenTreeCall* call, CORINFO_METHOD_HANDLE methHnd, CORINFO_SIG_INFO* sig, unsigned mflags, BasicBlock* block); GenTreeCall* impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugInfo& di = DebugInfo()); - void impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig); + void impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig, /* OUT */ CallArg** swiftErrorArg, /* OUT */ CallArg** swiftSelfArg); + +#ifdef SWIFT_SUPPORT + void impAppendSwiftErrorStore(GenTreeCall* call, CallArg* const swiftErrorArg); +#endif // SWIFT_SUPPORT void impInsertHelperCall(CORINFO_HELPER_DESC* helperCall); void impHandleAccessAllowed(CorInfoIsAccessAllowedResult result, CORINFO_HELPER_DESC* helperCall); @@ -4973,7 +4976,7 @@ class Compiler #ifdef DEBUG jitstd::vector* fgBBOrder; // ordered vector of BBs #endif - // Used as a quick check for whether loop alignment should look for natural loops. + // Used as a quick check for whether phases downstream of loop finding should look for natural loops. // If true: there may or may not be any natural loops in the flow graph, so try to find them // If false: there's definitely not any natural loops in the flow graph bool fgMightHaveNaturalLoops; @@ -5079,34 +5082,31 @@ class Compiler void fgExtendEHRegionBefore(BasicBlock* block); void fgExtendEHRegionAfter(BasicBlock* block); - BasicBlock* fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool extendRegion, BasicBlock* jumpDest = nullptr); + BasicBlock* fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool extendRegion); - BasicBlock* fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool extendRegion, BasicBlock* jumpDest = nullptr); + BasicBlock* fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool extendRegion); - BasicBlock* fgNewBBFromTreeAfter(BBKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, BasicBlock* jumpDest = nullptr, bool updateSideEffects = false); + BasicBlock* fgNewBBFromTreeAfter(BBKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, bool updateSideEffects = false); BasicBlock* fgNewBBinRegion(BBKinds jumpKind, unsigned tryIndex, unsigned hndIndex, BasicBlock* nearBlk, - BasicBlock* jumpDest = nullptr, bool putInFilter = false, bool runRarely = false, bool insertAtEnd = false); BasicBlock* fgNewBBinRegion(BBKinds jumpKind, BasicBlock* srcBlk, - BasicBlock* jumpDest = nullptr, bool runRarely = false, bool insertAtEnd = false); - BasicBlock* fgNewBBinRegion(BBKinds jumpKind, BasicBlock* jumpDest = nullptr); + BasicBlock* fgNewBBinRegion(BBKinds jumpKind); BasicBlock* fgNewBBinRegionWorker(BBKinds jumpKind, BasicBlock* afterBlk, unsigned xcptnIndex, - bool putInTryRegion, - BasicBlock* jumpDest = nullptr); + bool putInTryRegion); void fgInsertBBbefore(BasicBlock* insertBeforeBlk, BasicBlock* newBlk); void fgInsertBBafter(BasicBlock* insertAfterBlk, BasicBlock* newBlk); @@ -5827,15 +5827,15 @@ class Compiler public: // For many purposes, it is desirable to be able to enumerate the *distinct* targets of a switch statement, // skipping duplicate targets. (E.g., in flow analyses that are only interested in the set of possible targets.) - // SwitchUniqueSuccSet contains the non-duplicated switch targets. + // SwitchUniqueSuccSet contains the non-duplicated switch successor edges. // Code that modifies the flowgraph (such as by renumbering blocks) must call Compiler::InvalidateUniqueSwitchSuccMap, // and code that modifies the targets of a switch block must call Compiler::fgInvalidateSwitchDescMapEntry. // If the unique targets of a switch block are needed later, they will be recomputed, ensuring they're up-to-date. struct SwitchUniqueSuccSet { - unsigned numDistinctSuccs; // Number of distinct targets of the switch. - BasicBlock** nonDuplicates; // Array of "numDistinctSuccs", containing all the distinct switch target - // successors. + unsigned numDistinctSuccs; // Number of distinct targets of the switch. + FlowEdge** nonDuplicates; // Array of "numDistinctSuccs", containing all the distinct switch target + // successor edges. }; typedef JitHashTable, SwitchUniqueSuccSet> BlockToSwitchDescMap; @@ -5898,8 +5898,6 @@ class Compiler void fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, BasicBlock* newTarget); - void fgReplacePred(BasicBlock* block, BasicBlock* oldPred, BasicBlock* newPred); - void fgReplacePred(FlowEdge* edge, BasicBlock* const newPred); // initializingPreds is only 'true' when we are computing preds in fgLinkBasicBlocks() @@ -6459,7 +6457,6 @@ class Compiler public: GenTree* fgMorphInitBlock(GenTree* tree); GenTree* fgMorphCopyBlock(GenTree* tree); - GenTree* fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree); private: GenTree* fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optAssertionPropDone = nullptr); void fgTryReplaceStructLocalWithField(GenTree* tree); @@ -7057,6 +7054,7 @@ class Compiler unsigned optCSEstart; // The first local variable number that is a CSE unsigned optCSEattempt; // The number of CSEs attempted so far. unsigned optCSEcount; // The total count of CSEs introduced. + unsigned optCSEunmarks; // Number of CSE trees unmarked weight_t optCSEweight; // The weight of the current block when we are doing PerformCSE CSE_HeuristicCommon* optCSEheuristic; // CSE Heuristic to use for this method @@ -7401,6 +7399,18 @@ class Compiler BasicBlock* basicBlock); #endif + PhaseStatus optInductionVariables(); + bool optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop); + bool optIsIVWideningProfitable(unsigned lclNum, + BasicBlock* initBlock, + bool initedToConstant, + FlowGraphNaturalLoop* loop, + ArrayStack& ivUses); + void optBestEffortReplaceNarrowIVUses( + unsigned lclNum, unsigned ssaNum, unsigned newLclNum, BasicBlock* block, Statement* firstStmt); + void optReplaceWidenedIV(unsigned lclNum, unsigned ssaNum, unsigned newLclNum, Statement* stmt); + void optSinkWidenedIV(unsigned lclNum, unsigned newLclNum, FlowGraphNaturalLoop* loop); + // Redundant branch opts // PhaseStatus optRedundantBranches(); @@ -11314,6 +11324,7 @@ class GenTreeVisitor case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: + case GT_SWIFT_ERROR: break; // Lclvar unary operators @@ -11441,28 +11452,6 @@ class GenTreeVisitor break; } - case GT_STORE_DYN_BLK: - { - GenTreeStoreDynBlk* const dynBlock = node->AsStoreDynBlk(); - - result = WalkTree(&dynBlock->gtOp1, dynBlock); - if (result == fgWalkResult::WALK_ABORT) - { - return result; - } - result = WalkTree(&dynBlock->gtOp2, dynBlock); - if (result == fgWalkResult::WALK_ABORT) - { - return result; - } - result = WalkTree(&dynBlock->gtDynamicSize, dynBlock); - if (result == fgWalkResult::WALK_ABORT) - { - return result; - } - break; - } - case GT_CALL: { GenTreeCall* const call = node->AsCall(); diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 67f5c59d93265..05842d837395f 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -664,27 +664,27 @@ BasicBlockVisit BasicBlock::VisitAllSuccs(Compiler* comp, TFunc func) return VisitEHSuccs(comp, func); case BBJ_CALLFINALLY: - RETURN_ON_ABORT(func(bbTarget)); + RETURN_ON_ABORT(func(GetTarget())); return ::VisitEHSuccs(comp, this, func); case BBJ_CALLFINALLYRET: // These are "pseudo-blocks" and control never actually flows into them // (codegen directly jumps to its successor after finally calls). - return func(bbTarget); + return func(GetTarget()); case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: case BBJ_LEAVE: case BBJ_ALWAYS: - RETURN_ON_ABORT(func(bbTarget)); + RETURN_ON_ABORT(func(GetTarget())); return VisitEHSuccs(comp, func); case BBJ_COND: - RETURN_ON_ABORT(func(bbFalseTarget)); + RETURN_ON_ABORT(func(GetFalseTarget())); - if (bbTrueTarget != bbFalseTarget) + if (!TrueEdgeIs(GetFalseEdge())) { - RETURN_ON_ABORT(func(bbTrueTarget)); + RETURN_ON_ABORT(func(GetTrueTarget())); } return VisitEHSuccs(comp, func); @@ -694,7 +694,7 @@ BasicBlockVisit BasicBlock::VisitAllSuccs(Compiler* comp, TFunc func) Compiler::SwitchUniqueSuccSet sd = comp->GetDescriptorForSwitch(this); for (unsigned i = 0; i < sd.numDistinctSuccs; i++) { - RETURN_ON_ABORT(func(sd.nonDuplicates[i])); + RETURN_ON_ABORT(func(sd.nonDuplicates[i]->getDestinationBlock())); } return VisitEHSuccs(comp, func); @@ -744,14 +744,14 @@ BasicBlockVisit BasicBlock::VisitRegularSuccs(Compiler* comp, TFunc func) case BBJ_EHFILTERRET: case BBJ_LEAVE: case BBJ_ALWAYS: - return func(bbTarget); + return func(GetTarget()); case BBJ_COND: - RETURN_ON_ABORT(func(bbFalseTarget)); + RETURN_ON_ABORT(func(GetFalseTarget())); - if (bbTrueTarget != bbFalseTarget) + if (!TrueEdgeIs(GetFalseEdge())) { - RETURN_ON_ABORT(func(bbTrueTarget)); + RETURN_ON_ABORT(func(GetTrueTarget())); } return BasicBlockVisit::Continue; @@ -761,7 +761,7 @@ BasicBlockVisit BasicBlock::VisitRegularSuccs(Compiler* comp, TFunc func) Compiler::SwitchUniqueSuccSet sd = comp->GetDescriptorForSwitch(this); for (unsigned i = 0; i < sd.numDistinctSuccs; i++) { - RETURN_ON_ABORT(func(sd.nonDuplicates[i])); + RETURN_ON_ABORT(func(sd.nonDuplicates[i]->getDestinationBlock())); } return BasicBlockVisit::Continue; @@ -4246,6 +4246,7 @@ void GenTree::VisitOperands(TVisitor visitor) case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: + case GT_SWIFT_ERROR: return; // Unary operators with an optional operand @@ -4361,21 +4362,6 @@ void GenTree::VisitOperands(TVisitor visitor) return; } - case GT_STORE_DYN_BLK: - { - GenTreeStoreDynBlk* const dynBlock = this->AsStoreDynBlk(); - if (visitor(dynBlock->gtOp1) == VisitResult::Abort) - { - return; - } - if (visitor(dynBlock->gtOp2) == VisitResult::Abort) - { - return; - } - visitor(dynBlock->gtDynamicSize); - return; - } - case GT_CALL: { GenTreeCall* const call = this->AsCall(); diff --git a/src/coreclr/jit/compmemkind.h b/src/coreclr/jit/compmemkind.h index 835d85f798d29..e986682894c3b 100644 --- a/src/coreclr/jit/compmemkind.h +++ b/src/coreclr/jit/compmemkind.h @@ -50,6 +50,7 @@ CompMemKindMacro(LoopOpt) CompMemKindMacro(LoopClone) CompMemKindMacro(LoopUnroll) CompMemKindMacro(LoopHoist) +CompMemKindMacro(LoopIVOpts) CompMemKindMacro(Unknown) CompMemKindMacro(RangeCheck) CompMemKindMacro(CopyProp) diff --git a/src/coreclr/jit/compphases.h b/src/coreclr/jit/compphases.h index 2393098531976..10b60167be422 100644 --- a/src/coreclr/jit/compphases.h +++ b/src/coreclr/jit/compphases.h @@ -84,6 +84,7 @@ CompPhaseNameMacro(PHASE_BUILD_SSA_DF, "SSA: DF", CompPhaseNameMacro(PHASE_BUILD_SSA_INSERT_PHIS, "SSA: insert phis", false, PHASE_BUILD_SSA, false) CompPhaseNameMacro(PHASE_BUILD_SSA_RENAME, "SSA: rename", false, PHASE_BUILD_SSA, false) CompPhaseNameMacro(PHASE_EARLY_PROP, "Early Value Propagation", false, -1, false) +CompPhaseNameMacro(PHASE_OPTIMIZE_INDUCTION_VARIABLES, "Optimize Induction Variables", false, -1, false) CompPhaseNameMacro(PHASE_VALUE_NUMBER, "Do value numbering", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_INDEX_CHECKS, "Optimize index checks", false, -1, false) CompPhaseNameMacro(PHASE_OPTIMIZE_VALNUM_CSES, "Optimize Valnum CSEs", false, -1, false) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index ec4627adc66d7..263aaf9791124 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1059,6 +1059,8 @@ void emitter::emitInsSanityCheck(instrDesc* id) // (predicated) case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand // (predicated) + case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) + case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend elemsize = id->idOpSize(); assert(insOptsScalableStandard(id->idInsOpt())); // xx assert(isVectorRegister(id->idReg1())); // ddddd @@ -1094,6 +1096,10 @@ void emitter::emitInsSanityCheck(instrDesc* id) case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long + case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part + case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) + case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) elemsize = id->idOpSize(); assert(insOptsScalableStandard(id->idInsOpt())); // xx assert(isVectorRegister(id->idReg1())); // ddddd @@ -1113,6 +1119,23 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isValidScalarDatasize(elemsize)); break; + case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation + assert(id->idInsOpt() == INS_OPTS_SCALABLE_S || id->idInsOpt() == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isValidUimm2(emitGetInsSC(id))); // hh + break; + + case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + assert(id->idInsOpt() == INS_OPTS_SCALABLE_D_SXTW || id->idInsOpt() == INS_OPTS_SCALABLE_D_UXTW); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isValidUimm2(emitGetInsSC(id))); // hh + break; + case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count elemsize = id->idOpSize(); assert(id->idInsOpt() == INS_OPTS_NONE); @@ -1121,6 +1144,52 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isValidUimm4From1(emitGetInsSC(id))); break; + case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector + assert(isPredicateRegister(id->idReg1())); // DDDD + assert(isVectorRegister(id->idReg2())); // nnnnn + break; + + case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector + assert(isPredicateRegister(id->idReg1())); // DDDD + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isValidUimm<3>(emitGetInsSC(id))); + break; + + case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector + assert(isPredicateRegister(id->idReg1())); // DDDD + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isValidUimm<1>(emitGetInsSC(id))); // i + break; + + case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector + assert(isPredicateRegister(id->idReg1())); // DDDD + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isValidUimm<3>(emitGetInsSC(id))); // ii + break; + + case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // NNNN + break; + + case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // NNNN + assert(isValidUimm<3>(emitGetInsSC(id))); + break; + + case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // NNNN + assert(isValidUimm<1>(emitGetInsSC(id))); // i + break; + + case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // NNNN + assert(isValidUimm<2>(emitGetInsSC(id))); // ii + break; + case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements elemsize = id->idOpSize(); assert(insOptsScalableStandard(id->idInsOpt())); @@ -1184,17 +1253,40 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isValidUimm7(emitGetInsSC(id))); // iiiii break; - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate + case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments + case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate + case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) + case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product + case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product + case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations + case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long + case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long + case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long + case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations assert(insOptsScalable(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); // ddddd assert(isVectorRegister(id->idReg2())); // nnnnn/mmmmm assert(isVectorRegister(id->idReg3())); // mmmmm/aaaaa break; + case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) + assert(insOptsNone(id->idInsOpt())); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn/aaaaa + assert(isVectorRegister(id->idReg3())); // mmmmm + break; + case IF_SVE_EG_3A: // ...........iimmm ......nnnnnddddd -- SVE two-way dot product (indexed) case IF_SVE_EY_3A: // ...........iimmm ......nnnnnddddd -- SVE integer dot product (indexed) case IF_SVE_EZ_3A: // ...........iimmm ......nnnnnddddd -- SVE mixed sign dot product (indexed) @@ -1417,6 +1509,16 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isScalableVectorSize(elemsize)); break; + case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + elemsize = id->idOpSize(); + assert(insOptsScalableStandard(id->idInsOpt())); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isLowPredicateRegister(id->idReg2())); // ggg + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isScalableVectorSize(elemsize)); + break; + // Scalable to Simd Vector. case IF_SVE_AG_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (quadwords) case IF_SVE_AJ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (quadwords) @@ -1513,6 +1615,19 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg3())); // nnnnn break; + case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) + elemsize = id->idOpSize(); + assert(isScalableVectorSize(elemsize)); // xx + assert(insOptsScalableStandard(id->idInsOpt())); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isPredicateRegister(id->idReg2())); // VVVV + assert(isVectorRegister(id->idReg3())); // nnnnn + if (id->idIns() == INS_sve_sel) + { + assert(isVectorRegister(id->idReg4())); // mmmmm + } + break; + // Scalable from general scalar (possibly SP) case IF_SVE_CQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy general register to vector (predicated) elemsize = id->idOpSize(); @@ -1618,6 +1733,21 @@ void emitter::emitInsSanityCheck(instrDesc* id) // x break; + case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + assert(insOptsNone(id->idInsOpt())); + assert(id->idOpSize() == EA_8BYTE); + assert(isGeneralRegisterOrZR(id->idReg1())); // ddddd + assert(isGeneralRegisterOrZR(id->idReg2())); // nnnnn + assert(isValidSimm6(emitGetInsSC(id))); // iiiiii + break; + + case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + assert(insOptsNone(id->idInsOpt())); + assert(id->idOpSize() == EA_8BYTE); + assert(isGeneralRegister(id->idReg1())); // ddddd + assert(isValidSimm6(emitGetInsSC(id))); // iiiiii + break; + case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long { assert(insOptsScalableWide(id->idInsOpt())); @@ -2377,6 +2507,46 @@ void emitter::emitInsSanityCheck(instrDesc* id) // iiiiii break; + case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isValidUimm2(emitGetInsSC(id))); // ii + assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); + break; + + case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isValidUimm2(emitGetInsSC(id))); // ii + assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); + break; + + case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isValidUimm3(emitGetInsSC(id))); // ii + // i + assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); + break; + + case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + assert(insOptsScalable(id->idInsOpt())); + assert(isVectorRegister(id->idReg1())); // ddddd + assert(isVectorRegister(id->idReg2())); // nnnnn + assert(isVectorRegister(id->idReg3())); // mmmmm + assert(isValidImm1(emitGetInsSC(id))); // i + assert(id->idInsOpt() == INS_OPTS_SCALABLE_B); + break; + case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled // offsets) case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit @@ -2539,6 +2709,19 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isVectorRegister(id->idReg2())); break; + case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) + case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert + case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(id->idInsOpt()), + emitInsIsVectorRightShift(id->idIns()))); + assert(insOptsScalableStandard(id->idInsOpt())); + assert(isVectorRegister(id->idReg1())); + assert(isVectorRegister(id->idReg2())); + assert(isScalableVectorSize(elemsize)); + break; + default: printf("unexpected format %s\n", emitIfName(id->idInsFmt())); assert(!"Unexpected format"); @@ -7593,6 +7776,15 @@ void emitter::emitIns_R_I(instruction ins, } break; + case INS_sve_rdvl: + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); + assert(isGeneralRegister(reg)); // ddddd + assert(isValidSimm6(imm)); // iiiiii + fmt = IF_SVE_BC_1A; + canEncode = true; + break; + case INS_sve_smax: case INS_sve_smin: signedImm = true; @@ -8629,6 +8821,30 @@ void emitter::emitIns_R_R(instruction ins, } break; + case INS_sve_pmov: + if (opt != INS_OPTS_SCALABLE_B) + { + assert(insOptsScalableStandard(opt)); + return emitIns_R_R_I(INS_sve_pmov, attr, reg1, reg2, 0, opt, sopt); + } + if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + { + assert(isPredicateRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_SVE_CE_2A; + } + else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + { + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + fmt = IF_SVE_CF_2A; + } + else + { + assert(!"invalid instruction"); + } + break; + case INS_sve_movs: { assert(opt == INS_OPTS_SCALABLE_B); @@ -9639,11 +9855,87 @@ void emitter::emitIns_R_R_I(instruction ins, case INS_sve_uqshl: case INS_sve_asrd: isRightShift = emitInsIsVectorRightShift(ins); - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isLowPredicateRegister(reg2)); // ggg assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); - fmt = IF_SVE_AM_2A; + assert(insOptsScalableStandard(opt)); + assert(isScalableVectorSize(size)); + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert((ins == INS_sve_asr) || (ins == INS_sve_lsl) || (ins == INS_sve_lsr)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + fmt = IF_SVE_BF_2A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_AM_2A; + } + break; + + case INS_sve_addvl: + case INS_sve_addpl: + assert(insOptsNone(opt)); + assert(size == EA_8BYTE); + assert(isGeneralRegisterOrSP(reg1)); // ddddd + assert(isGeneralRegisterOrSP(reg2)); // nnnnn + assert(isValidSimm6(imm)); // iiiiii + reg1 = encodingSPtoZR(reg1); + reg2 = encodingSPtoZR(reg2); + fmt = IF_SVE_BB_2A; + break; + + case INS_sve_pmov: + if (sopt == INS_SCALABLE_OPTS_TO_PREDICATE) + { + assert(isPredicateRegister(reg1)); + assert(isVectorRegister(reg2)); + switch (opt) + { + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<3>(imm)); + fmt = IF_SVE_CE_2B; + break; + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + fmt = IF_SVE_CE_2D; + break; + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<1>(imm)); + fmt = IF_SVE_CE_2C; + break; + default: + unreached(); + } + } + else if (sopt == INS_SCALABLE_OPTS_TO_VECTOR) + { + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + switch (opt) + { + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<3>(imm)); + fmt = IF_SVE_CF_2B; + break; + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + fmt = IF_SVE_CF_2D; + break; + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<1>(imm)); + fmt = IF_SVE_CF_2C; + break; + default: + unreached(); + } + } + else + { + unreached(); + } break; case INS_sve_sqrshrn: @@ -9813,6 +10105,32 @@ void emitter::emitIns_R_R_I(instruction ins, } break; + case INS_sve_sli: + case INS_sve_sri: + isRightShift = emitInsIsVectorRightShift(ins); + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_FT_2A; + break; + + case INS_sve_srsra: + case INS_sve_ssra: + case INS_sve_ursra: + case INS_sve_usra: + isRightShift = emitInsIsVectorRightShift(ins); + assert(isValidVectorShiftAmount(imm, optGetSveElemsize(opt), isRightShift)); + assert(insOptsScalableStandard(opt)); + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); + assert(isVectorRegister(reg2)); + assert(isScalableVectorSize(size)); + fmt = IF_SVE_FU_2A; + break; + default: unreached(); break; @@ -10735,12 +11053,22 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_sve_bic: case INS_sve_eor: case INS_sve_orr: - assert(isVectorRegister(reg1)); - assert(isLowPredicateRegister(reg2)); - assert(isVectorRegister(reg3)); assert(insOptsScalableStandard(opt)); - assert(insScalableOptsNone(sopt)); - fmt = IF_SVE_AA_3A; + assert(isVectorRegister(reg1)); // mmmmm + assert(isVectorRegister(reg3)); // ddddd + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_AU_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_AA_3A; + } break; case INS_sve_add: @@ -10763,6 +11091,26 @@ void emitter::emitIns_R_R_R(instruction ins, } break; + case INS_sve_addpt: + case INS_sve_subpt: + unreached(); // TODO-SVE: Not yet supported. + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg3)); // mmmmm + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_AT_3B; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_AB_3B; + } + break; + case INS_sve_sdiv: case INS_sve_sdivr: case INS_sve_udiv: @@ -10808,6 +11156,14 @@ void emitter::emitIns_R_R_R(instruction ins, } break; + case INS_sve_pmul: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_BD_3B; + break; + case INS_sve_andv: case INS_sve_eorv: case INS_sve_orv: @@ -10997,12 +11353,28 @@ void emitter::emitIns_R_R_R(instruction ins, case INS_sve_sdot: case INS_sve_udot: - assert(insOptsScalableWords(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm - assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - fmt = IF_SVE_EH_3A; + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + fmt = IF_SVE_EF_3A; + } + else + { + fmt = IF_SVE_EH_3A; + assert(insOptsScalableWords(opt)); + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + } + break; + + case INS_sve_usdot: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_EI_3A; break; case INS_sve_smlalb: @@ -11154,12 +11526,128 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_FO_3A; break; - case INS_sve_eorbt: - case INS_sve_eortb: - assert(insOptsScalableStandard(opt)); - assert(isVectorRegister(reg1)); // ddddd - assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmmmm + case INS_sve_rax1: + case INS_sve_sm4ekey: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (ins == INS_sve_rax1) + { + assert(opt == INS_OPTS_SCALABLE_D); + } + else + { + assert(opt == INS_OPTS_SCALABLE_S); + } + + fmt = IF_SVE_GJ_3A; + break; + + case INS_sve_fmlalb: + case INS_sve_fmlalt: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_B) + { + unreached(); // TODO-SVE: Not yet supported. + fmt = IF_SVE_GN_3A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_H); + fmt = IF_SVE_HB_3A; + } + break; + + case INS_sve_fmlslb: + case INS_sve_fmlslt: + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HB_3A; + break; + + case INS_sve_bfmmla: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HD_3A; + break; + + case INS_sve_fmmla: + unreached(); // TODO-SVE: Not yet supported. + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HD_3A_A; + break; + + case INS_sve_fmlallbb: + case INS_sve_fmlallbt: + case INS_sve_fmlalltb: + case INS_sve_fmlalltt: + unreached(); // TODO-SVE: Not yet supported. + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_GO_3A; + break; + + case INS_sve_bfclamp: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_GW_3B; + break; + + case INS_sve_bfdot: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_HA_3A; + break; + + case INS_sve_fdot: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + fmt = IF_SVE_HA_3A; + } + else if (opt == INS_OPTS_SCALABLE_B) + { + unreached(); // TODO-SVE: Not yet supported. + fmt = IF_SVE_HA_3A_E; + } + else + { + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsNone(opt)); + fmt = IF_SVE_HA_3A_F; + } + break; + + case INS_sve_eorbt: + case INS_sve_eortb: + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx fmt = IF_SVE_FP_3A; break; @@ -11208,6 +11696,41 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_FX_3A; break; + case INS_sve_addhnb: + case INS_sve_addhnt: + case INS_sve_raddhnb: + case INS_sve_raddhnt: + case INS_sve_subhnb: + case INS_sve_subhnt: + case INS_sve_rsubhnb: + case INS_sve_rsubhnt: + unreached(); // TODO-SVE: Not yet supported. + assert(insOptsScalableWide(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GC_3A; + break; + + case INS_sve_histseg: + assert(opt == INS_OPTS_SCALABLE_B); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GF_3A; + break; + + case INS_sve_fclamp: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GW_3A; + break; + case INS_sve_clz: case INS_sve_cls: case INS_sve_cnt: @@ -11354,14 +11877,35 @@ void emitter::emitIns_R_R_R(instruction ins, // TODO-SVE: Following checks can be simplified to check reg1 as predicate register only after adding // definitions for predicate registers. Currently, predicate registers P0 to P15 are aliased to simd // registers V0 to V15. - if (isPredicateRegister(reg3) && - (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(ins == INS_sve_mov); + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + fmt = IF_SVE_AU_3A; + // ORR is an alias for MOV, and is always the preferred disassembly. + ins = INS_sve_orr; + } + else if (isPredicateRegister(reg3) && + (sopt == INS_SCALABLE_OPTS_NONE || sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE)) { assert(opt == INS_OPTS_SCALABLE_B); assert(isPredicateRegister(reg1)); // DDDD assert(isPredicateRegister(reg2)); // gggg assert(isPredicateRegister(reg3)); // NNNN fmt = sopt == INS_SCALABLE_OPTS_NONE ? IF_SVE_CZ_4A : IF_SVE_CZ_4A_K; + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; + } + else if (sopt == INS_SCALABLE_OPTS_PREDICATE_MERGE) + { + assert(isVectorRegister(reg1)); + assert(isPredicateRegister(reg2)); + assert(isVectorRegister(reg3)); + assert(insOptsScalableStandard(opt)); + fmt = IF_SVE_CW_4A; } else { @@ -11379,10 +11923,10 @@ void emitter::emitIns_R_R_R(instruction ins, assert(isVectorRegister(reg3)); fmt = IF_SVE_CP_3A; } - } - // MOV is an alias for CPY, and is always the preferred disassembly. - ins = INS_sve_mov; + // MOV is an alias for CPY, and is always the preferred disassembly. + ins = INS_sve_mov; + } break; case INS_sve_lasta: @@ -11670,23 +12214,52 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_HJ_3A; break; - case INS_sve_fabd: + case INS_sve_frecps: + case INS_sve_frsqrts: + case INS_sve_ftsmul: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_HK_3A; + break; + case INS_sve_fadd: + case INS_sve_fsub: + case INS_sve_fmul: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_HK_3A; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_HL_3A; + } + break; + + case INS_sve_fabd: case INS_sve_fdiv: case INS_sve_fdivr: case INS_sve_fmax: case INS_sve_fmaxnm: case INS_sve_fmin: case INS_sve_fminnm: - case INS_sve_fmul: case INS_sve_fmulx: case INS_sve_fscale: - case INS_sve_fsub: case INS_sve_fsubr: assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); assert(isVectorRegister(reg3)); - assert(insOptsScalableFloat(opt)); + assert(insOptsScalableAtLeastHalf(opt)); assert(insScalableOptsNone(sopt)); fmt = IF_SVE_HL_3A; break; @@ -11702,6 +12275,43 @@ void emitter::emitIns_R_R_R(instruction ins, fmt = IF_SVE_HL_3A; break; + case INS_sve_bfmul: + case INS_sve_bfadd: + case INS_sve_bfsub: + case INS_sve_bfmaxnm: + case INS_sve_bfminnm: + case INS_sve_bfmax: + case INS_sve_bfmin: + assert(opt == INS_OPTS_SCALABLE_H); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg3)); // mmmmm + + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + assert(isVectorRegister(reg2)); // nnnnn + fmt = IF_SVE_HK_3B; + } + else + { + assert(insScalableOptsNone(sopt)); + assert(isLowPredicateRegister(reg2)); // ggg + fmt = IF_SVE_HL_3B; + } + break; + + case INS_sve_bsl: + case INS_sve_eor3: + case INS_sve_bcax: + case INS_sve_bsl1n: + case INS_sve_bsl2n: + case INS_sve_nbsl: + assert(opt == INS_OPTS_SCALABLE_D); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // mmmmm + assert(isVectorRegister(reg3)); // kkkkk + fmt = IF_SVE_AV_3A; + break; + case INS_sve_frintn: case INS_sve_frintm: case INS_sve_frintp: @@ -11801,9 +12411,6 @@ void emitter::emitIns_R_R_R(instruction ins, assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmmmm fmt = IF_SVE_EW_3A; - - // opt is set only for convenience in emitDispInsHelp - opt = INS_OPTS_SCALABLE_D; break; case INS_sve_madpt: @@ -11813,9 +12420,6 @@ void emitter::emitIns_R_R_R(instruction ins, assert(isVectorRegister(reg2)); // mmmmm assert(isVectorRegister(reg3)); // aaaaa fmt = IF_SVE_EW_3B; - - // opt is set only for convenience in emitDispInsHelp - opt = INS_OPTS_SCALABLE_D; break; case INS_sve_fcmeq: @@ -12009,14 +12613,15 @@ void emitter::emitIns_R_R_R_I_LdStPair(instruction ins, * Add an instruction referencing three registers and a constant. */ -void emitter::emitIns_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - emitAttr attrReg2 /* = EA_UNKNOWN */) +void emitter::emitIns_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + emitAttr attrReg2 /* = EA_UNKNOWN */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { emitAttr size = EA_SIZE(attr); emitAttr elemsize = EA_UNKNOWN; @@ -12306,10 +12911,12 @@ void emitter::emitIns_R_R_R_I(instruction ins, default: // fallback to emit SVE instructions. - return emitInsSve_R_R_R_I(ins, attr, reg1, reg2, reg3, imm, opt, attrReg2); + return emitInsSve_R_R_R_I(ins, attr, reg1, reg2, reg3, imm, opt, sopt); } // end switch (ins) + assert(insScalableOptsNone(sopt)); + if (isLdSt) { assert(!isAddSub); @@ -12468,14 +13075,14 @@ void emitter::emitIns_R_R_R_I(instruction ins, * Add a SVE instruction referencing three registers and a constant. */ -void emitter::emitInsSve_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt /* = INS_OPTS_NONE */, - emitAttr attrReg2 /* = EA_UNKNOWN */) +void emitter::emitInsSve_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt /* = INS_OPTS_NONE */, + insScalableOpts sopt /* = INS_SCALABLE_OPTS_NONE */) { emitAttr size = EA_SIZE(attr); emitAttr elemsize = EA_UNKNOWN; @@ -12484,12 +13091,37 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, /* Figure out the encoding format of the instruction */ switch (ins) { + case INS_sve_adr: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + assert(isValidUimm2(imm)); + switch (opt) + { + case INS_OPTS_SCALABLE_S: + case INS_OPTS_SCALABLE_D: + assert(sopt == INS_SCALABLE_OPTS_LSL_N); + fmt = IF_SVE_BH_3A; + break; + case INS_OPTS_SCALABLE_D_SXTW: + fmt = IF_SVE_BH_3B; + break; + case INS_OPTS_SCALABLE_D_UXTW: + fmt = IF_SVE_BH_3B_A; + break; + default: + assert(!"invalid instruction"); + break; + } + break; + case INS_sve_cmpeq: case INS_sve_cmpgt: case INS_sve_cmpge: case INS_sve_cmpne: case INS_sve_cmple: case INS_sve_cmplt: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isPredicateRegister(reg1)); // DDDD assert(isLowPredicateRegister(reg2)); // ggg @@ -12502,6 +13134,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_cmphs: case INS_sve_cmplo: case INS_sve_cmpls: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isPredicateRegister(reg1)); // DDDD assert(isLowPredicateRegister(reg2)); // ggg @@ -12512,6 +13145,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_sdot: case INS_sve_udot: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -12539,6 +13173,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_usdot: case INS_sve_sudot: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_B); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -12549,6 +13184,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_mul: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -12580,6 +13216,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_cdot: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -12594,6 +13231,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_cmla: case INS_sve_sqrdcmlah: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -12607,6 +13245,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1d: + assert(insScalableOptsNone(sopt)); assert(insOptsScalable(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12634,6 +13273,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ldff1d: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12643,6 +13283,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1w: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWordsOrQuadwords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12662,6 +13303,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1sw: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12680,6 +13322,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ldff1sw: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12689,6 +13332,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1sb: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12709,6 +13353,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1b: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12729,6 +13374,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldff1b: case INS_sve_ldff1sb: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12738,6 +13384,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1sh: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12756,6 +13403,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1h: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12775,6 +13423,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldff1h: case INS_sve_ldff1sh: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12784,6 +13433,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ldff1w: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12794,6 +13444,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldnf1sw: case INS_sve_ldnf1d: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12804,6 +13455,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldnf1sh: case INS_sve_ldnf1w: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12814,6 +13466,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldnf1h: case INS_sve_ldnf1sb: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12823,6 +13476,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ldnf1b: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isPredicateRegister(reg2)); @@ -12835,6 +13489,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ldnt1h: case INS_sve_ldnt1w: case INS_sve_ldnt1d: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12877,6 +13532,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld1row: case INS_sve_ld1rqd: case INS_sve_ld1rod: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12938,6 +13594,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld2q: case INS_sve_ld3q: case INS_sve_ld4q: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_Q); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -12979,6 +13636,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_ld2d: case INS_sve_ld3d: case INS_sve_ld4d: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13051,6 +13709,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st2q: case INS_sve_st3q: case INS_sve_st4q: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_Q); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13084,6 +13743,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_stnt1h: case INS_sve_stnt1w: case INS_sve_stnt1d: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13120,6 +13780,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st1w: case INS_sve_st1d: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13182,6 +13843,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st2d: case INS_sve_st3d: case INS_sve_st4d: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13253,6 +13915,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_st1b: case INS_sve_st1h: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13291,6 +13954,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_fmla: case INS_sve_fmls: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13311,6 +13975,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_bfmla: case INS_sve_bfmls: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -13321,6 +13986,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_fmul: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13340,6 +14006,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_bfmul: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -13350,6 +14017,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_fdot: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isVectorRegister(reg3)); // mmm @@ -13380,6 +14048,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_bfdot: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn @@ -13391,6 +14060,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_mla: case INS_sve_mls: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13419,6 +14089,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_smullt: case INS_sve_umullb: case INS_sve_umullt: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13445,6 +14116,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_smlslt: case INS_sve_umlslb: case INS_sve_umlslt: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13465,6 +14137,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_sqdmullb: case INS_sve_sqdmullt: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13485,6 +14158,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_sqdmulh: case INS_sve_sqrdmulh: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13513,6 +14187,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_sqdmlalt: case INS_sve_sqdmlslb: case INS_sve_sqdmlslt: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13533,6 +14208,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_sqrdmlah: case INS_sve_sqrdmlsh: + assert(insScalableOptsNone(sopt)); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn assert(isLowVectorRegister(reg3)); // mmmm @@ -13558,6 +14234,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_fcadd: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13568,6 +14245,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rd: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13577,6 +14255,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rsw: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_D); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13586,6 +14265,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rsh: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13595,6 +14275,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rw: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableWords(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13604,6 +14285,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rh: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13613,6 +14295,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rsb: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableAtLeastHalf(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13622,6 +14305,7 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, break; case INS_sve_ld1rb: + assert(insScalableOptsNone(sopt)); assert(insOptsScalableStandard(opt)); assert(isVectorRegister(reg1)); assert(isLowPredicateRegister(reg2)); @@ -13638,13 +14322,63 @@ void emitter::emitInsSve_R_R_R_I(instruction ins, case INS_sve_bfmlalt: case INS_sve_bfmlslb: case INS_sve_bfmlslt: + assert(insScalableOptsNone(sopt)); assert(opt == INS_OPTS_SCALABLE_H); assert(isVectorRegister(reg1)); // ddddd assert(isVectorRegister(reg2)); // nnnnn - assert(isVectorRegister(reg3)); // mmm - assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); - assert(isValidUimm3(imm)); // ii i - fmt = IF_SVE_GZ_3A; + assert(isVectorRegister(reg3)); // mmm + assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); + assert(isValidUimm3(imm)); // ii i + fmt = IF_SVE_GZ_3A; + break; + + case INS_sve_luti2: + assert(insScalableOptsNone(sopt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm3(imm)); // iii + fmt = IF_SVE_GG_3B; + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isValidUimm2(imm)); // ii i + fmt = IF_SVE_GG_3A; + } + unreached(); + break; + + case INS_sve_luti4: + assert(isVectorRegister(reg1)); // ddddd + assert(isVectorRegister(reg2)); // nnnnn + assert(isVectorRegister(reg3)); // mmmmm + + if (opt == INS_OPTS_SCALABLE_H) + { + assert(isValidUimm2(imm)); + + if (sopt == INS_SCALABLE_OPTS_WITH_VECTOR_PAIR) + { + fmt = IF_SVE_GH_3B; + } + else + { + assert(insScalableOptsNone(sopt)); + fmt = IF_SVE_GH_3B_B; + } + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(insScalableOptsNone(sopt)); + assert(isValidImm1(imm)); // i + fmt = IF_SVE_GH_3A; + } + unreached(); break; default: @@ -14119,6 +14853,33 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, /* Figure out the encoding format of the instruction */ switch (ins) { + case INS_sve_sel: + if (sopt == INS_SCALABLE_OPTS_UNPREDICATED) + { + if (reg1 == reg4) + { + // mov is a preferred alias for sel + return emitIns_R_R_R(INS_sve_mov, attr, reg1, reg2, reg3, opt, INS_SCALABLE_OPTS_PREDICATE_MERGE); + } + + assert(insOptsScalableStandard(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isPredicateRegister(reg2)); // VVVV + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + fmt = IF_SVE_CW_4A; + } + else + { + assert(opt == INS_OPTS_SCALABLE_B); + assert(isPredicateRegister(reg1)); // dddd + assert(isPredicateRegister(reg2)); // gggg + assert(isPredicateRegister(reg3)); // nnnn + assert(isPredicateRegister(reg4)); // mmmm + fmt = IF_SVE_CZ_4A; + } + break; + case INS_sve_cmpeq: case INS_sve_cmpgt: case INS_sve_cmpge: @@ -14154,7 +14915,6 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, case INS_sve_bic: case INS_sve_orn: case INS_sve_bics: - case INS_sve_sel: case INS_sve_eors: case INS_sve_nor: case INS_sve_nand: @@ -14224,6 +14984,29 @@ void emitter::emitInsSve_R_R_R_R(instruction ins, fmt = IF_SVE_AR_4A; break; + case INS_sve_histcnt: + assert(insOptsScalableWords(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_GI_4A; + break; + + case INS_sve_fmla: + case INS_sve_fmls: + case INS_sve_fnmla: + case INS_sve_fnmls: + assert(insOptsScalableAtLeastHalf(opt)); + assert(isVectorRegister(reg1)); // ddddd + assert(isLowPredicateRegister(reg2)); // ggg + assert(isVectorRegister(reg3)); // nnnnn + assert(isVectorRegister(reg4)); // mmmmm + assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx + fmt = IF_SVE_HU_4A; + break; + case INS_sve_mad: case INS_sve_msb: assert(insOptsScalableStandard(opt)); @@ -17300,7 +18083,7 @@ void emitter::emitIns_Call(EmitCallType callType, { assert(isLowPredicateRegister(reg)); emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); + assert((ureg >= 0) && (ureg <= 7)); return ureg << 10; } @@ -18577,6 +19360,51 @@ void emitter::emitIns_Call(EmitCallType callType, return encoding; } +/***************************************************************************** + * + * Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, + ssize_t imm, + bool isRightShift) +{ + code_t encoding = 0; + + imm = insEncodeShiftImmediate(optGetSveElemsize(opt), isRightShift, imm); + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + imm = imm & 0b111; // bits 18-16 + encoding |= (1 << 19); // bit 19 + break; + + case INS_OPTS_SCALABLE_H: + imm = imm & 0b1111; // bits 19-16 + encoding |= (1 << 20); // bit 20 + break; + + case INS_OPTS_SCALABLE_S: + imm = imm & 0b11111; // bits 20-16 + encoding |= (1 << 22); // bit 22 + break; + + case INS_OPTS_SCALABLE_D: + // this gets the last bit of 'imm' and tries to set bit 22 + encoding |= ((imm >> 5) << 22); + imm = imm & 0b11111; // bits 20-16 + encoding |= (1 << 23); // bit 23 + break; + + default: + assert(!"Invalid size for vector register"); + break; + } + + return (encoding | (code_t)(imm << 16)); +} + /***************************************************************************** * * Returns the encoding to select the elemsize for an Arm64 SVE vector instruction plus an immediate. @@ -18889,6 +19717,10 @@ void emitter::emitIns_Call(EmitCallType callType, case IF_SVE_CZ_4A_A: case IF_SVE_CZ_4A_L: + case IF_SVE_CE_2A: + case IF_SVE_CE_2B: + case IF_SVE_CE_2C: + case IF_SVE_CE_2D: case IF_SVE_CF_2A: case IF_SVE_CF_2B: case IF_SVE_CF_2C: @@ -20532,6 +21364,21 @@ void emitter::emitIns_Call(EmitCallType callType, return (code_t)imm << 16; } +/***************************************************************************** + * + * Returns the encoding for the immediate value as 6-bits at bit locations '10-5'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSimm6_10_to_5(ssize_t imm) +{ + assert(isValidSimm6(imm)); + if (imm < 0) + { + imm = (imm & 0x3F); + } + return (code_t)imm << 5; +} + /***************************************************************************** * * Returns the encoding for the immediate value as 6-bits at bit locations '20-16'. @@ -20580,6 +21427,43 @@ void emitter::emitIns_Call(EmitCallType callType, return (code_t)imm << 19; } +/***************************************************************************** + * + * Returns the encoding for the immediate value as 2-bits at bit locations '23-22'. + */ + +/*static*/ emitter::code_t emitter::insEncodeUimm2_23_to_22(ssize_t imm) +{ + assert(isValidUimm2(imm)); + return (code_t)imm << 22; +} + +/***************************************************************************** + * + * Returns the encoding for the immediate value as 1-bit at bit locations '23'. + */ + +/*static*/ emitter::code_t emitter::insEncodeUimm1_23(ssize_t imm) +{ + assert(isValidImm1(imm)); + return (code_t)imm << 23; +} + +/***************************************************************************** + * + * Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. + */ + +/*static*/ emitter::code_t emitter::insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm) +{ + assert(isValidUimm3(imm)); + + code_t h = (code_t)(imm & 0x6) << 21; // encode high 2-bits at locations '23-22' + code_t l = (code_t)(imm & 0x1) << 12; // encode low 1-bit at locations '12' + + return (h | l); +} + /***************************************************************************** * * Returns the encoding for the immediate value as 1 bit at bit location '10'. @@ -22943,6 +23827,15 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + // Scalable with Merge or Zero predicate case IF_SVE_AH_3A: // ........xx.....M ...gggnnnnnddddd -- SVE constructive prefix (predicated) code = emitInsCodeSve(ins, fmt); @@ -22971,6 +23864,8 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) // Scalable, 4 regs. Reg4 in mmmmm. case IF_SVE_AR_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE integer multiply-accumulate writing addend // (predicated) + case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) + case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg @@ -23018,6 +23913,10 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long + case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part + case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) + case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn @@ -23037,6 +23936,26 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm2_11_to_10(emitGetInsSC(id)); // hh + code |= insEncodeImm1_22(id->idInsOpt() == INS_OPTS_SCALABLE_D ? 1 : 0); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm2_11_to_10(emitGetInsSC(id)); // hh + dst += emitOutput_Instr(dst, code); + break; + // Immediate and patterm to general purpose. case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count imm = emitGetInsSC(id); @@ -23047,6 +23966,68 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; + case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD @@ -23128,6 +24109,19 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) + { + regNumber reg4 = (ins == INS_sve_mov ? id->idReg1() : id->idReg4()); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_13_to_10(id->idReg2()); // VVVV + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(reg4); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } + case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match @@ -23163,10 +24157,26 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments + case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate + case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) + case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product + case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product + case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations + case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long + case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long + case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long + case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn @@ -23174,6 +24184,29 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeReg_V_9_to_5(id->idReg3()); // kkkkk + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm6_10_to_5(emitGetInsSC(id)); // iiiiii + code |= insEncodeReg_R_20_to_16(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm6_10_to_5(emitGetInsSC(id)); // iiiiii + dst += emitOutput_Instr(dst, code); + break; + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) code = emitInsCodeSve(ins, fmt); code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd @@ -24108,6 +25141,44 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm2_23_to_22(imm); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm3h3l_23_to_22_and_12(imm); // ii + // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm1_23(imm); // i + dst += emitOutput_Instr(dst, code); + break; + case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled // offsets) case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit @@ -24350,6 +25421,18 @@ BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) dst += emitOutput_Instr(dst, code); break; + case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) + case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert + case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsizeWithShift_tszh_tszl_imm3(id->idInsOpt(), imm, + emitInsIsVectorRightShift(ins)); // xx xxiii + dst += emitOutput_Instr(dst, code); + break; + default: assert(!"Unexpected format"); break; @@ -24629,11 +25712,17 @@ void emitter::emitDispSveExtendOpts(insOpts opt) { switch (opt) { + case INS_OPTS_LSL: + printf("lsl"); + break; + + case INS_OPTS_UXTW: case INS_OPTS_SCALABLE_S_UXTW: case INS_OPTS_SCALABLE_D_UXTW: printf("uxtw"); break; + case INS_OPTS_SXTW: case INS_OPTS_SCALABLE_S_SXTW: case INS_OPTS_SCALABLE_D_SXTW: printf("sxtw"); @@ -24650,27 +25739,18 @@ void emitter::emitDispSveExtendOpts(insOpts opt) * Prints the encoding for the Extend Type encoding along with the N value */ -void emitter::emitDispSveExtendOptsModN(insOpts opt, int n) +void emitter::emitDispSveExtendOptsModN(insOpts opt, ssize_t imm) { - assert(n >= 0 && n <= 3); - - emitDispSveExtendOpts(opt); - switch (n) - { - case 3: - printf(" #3"); - break; + assert(imm >= 0 && imm <= 3); - case 2: - printf(" #2"); - break; - - case 1: - printf(" #1"); - break; - - default: - break; + if (imm == 0 && opt != INS_OPTS_LSL) + { + emitDispSveExtendOpts(opt); + } + else if (imm > 0) + { + emitDispSveExtendOpts(opt); + printf(" #%d", (int)imm); } } @@ -24840,6 +25920,18 @@ void emitter::emitDispReg(regNumber reg, emitAttr attr, bool addComma) emitDispComma(); } +//------------------------------------------------------------------------ +// emitDispSveReg: Display a scalable vector register name +// +void emitter::emitDispSveReg(regNumber reg, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + + if (addComma) + emitDispComma(); +} + //------------------------------------------------------------------------ // emitDispSveReg: Display a scalable vector register name with an arrangement suffix // @@ -24858,6 +25950,16 @@ void emitter::emitDispSveReg(regNumber reg, insOpts opt, bool addComma) emitDispComma(); } +//------------------------------------------------------------------------ +// emitDispSveRegIndex: Display a scalable vector register with indexed element +// +void emitter::emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + emitDispElementIndex(index, addComma); +} + //------------------------------------------------------------------------ // emitDispVectorReg: Display a SIMD vector register name with an arrangement suffix // @@ -26781,6 +27883,10 @@ void emitter::emitDispInsHelp( // (predicated) case IF_SVE_GR_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 floating-point pairwise operations case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + // .D, /M, .D, .D + case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + // .H, /M, .H, .H + case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispLowPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // ggg emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd @@ -26819,6 +27925,9 @@ void emitter::emitDispInsHelp( // (predicated) case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand // (predicated) + case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend + // ., /Z, ., . + case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispLowPredicateReg(id->idReg2(), insGetPredicateType(fmt), id->idInsOpt(), true); // ggg emitDispSveReg(id->idReg3(), id->idInsOpt(), true); @@ -26836,18 +27945,55 @@ void emitter::emitDispInsHelp( case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp + case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) // ., ., . case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate // .Q, .Q, .Q case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments + // .D, .D, .D + case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + // .D, .D, .D + case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) + // .B, .B, .B + case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) + case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + // .D, .D, .D + // .S, .S, .S + case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations + // .H, .H, .H + case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn/mmmmm + emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm/aaaaa + break; + // .D, .D, .D case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) // .D, .D, .D case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) + emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_D, true); // ddddd + emitDispSveReg(id->idReg2(), INS_OPTS_SCALABLE_D, true); // nnnnn + emitDispSveReg(id->idReg3(), INS_OPTS_SCALABLE_D, false); // mmmmm + break; + + // .D, .D, .D, .D + case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd - emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn/mmmmm - emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm/aaaaa + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // mmmmm + emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // kkkkk + break; + + // .H, .B, .B + case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long + case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_H, true); // ddddd + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn + emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm break; // ., ., . @@ -26914,6 +28060,36 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg3(), INS_OPTS_SCALABLE_D, false); // mmmmm break; + // ., [., .{, }] + case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + printf("["); + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); + emitDispSveReg(id->idReg3(), id->idInsOpt(), emitGetInsSC(id) > 0); + emitDispSveExtendOptsModN(INS_OPTS_LSL, emitGetInsSC(id)); + printf("]"); + break; + + // .D, [.D, .D, SXTW{ }] + case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + printf("["); + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); + emitDispSveReg(id->idReg3(), id->idInsOpt(), true); + emitDispSveExtendOptsModN(INS_OPTS_SXTW, emitGetInsSC(id)); + printf("]"); + break; + + // .D, [.D, .D, UXTW{ }] + case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + printf("["); + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); + emitDispSveReg(id->idReg3(), id->idInsOpt(), true); + emitDispSveExtendOptsModN(INS_OPTS_UXTW, emitGetInsSC(id)); + printf("]"); + break; + // .H, .B case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_H, true); // DDDD @@ -26980,6 +28156,39 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm break; + case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_B, true); // DDDD + emitDispSveReg(id->idReg2(), false); // nnnnn + break; + case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_D, true); // DDDD + emitDispSveRegIndex(id->idReg2(), emitGetInsSC(id), false); // nnnnn + break; + case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_H, true); // DDDD + emitDispSveRegIndex(id->idReg2(), emitGetInsSC(id), false); // nnnnn + break; + case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector + emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_S, true); // DDDD + emitDispSveRegIndex(id->idReg2(), emitGetInsSC(id), false); // nnnnn + break; + case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector + emitDispSveReg(id->idReg1(), true); // ddddd + emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_B, false); // NNNN + break; + case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector + emitDispSveRegIndex(id->idReg1(), emitGetInsSC(id), true); // ddddd + emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_D, false); // NNNN + break; + case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector + emitDispSveRegIndex(id->idReg1(), emitGetInsSC(id), true); // ddddd + emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_H, false); // NNNN + break; + case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector + emitDispSveRegIndex(id->idReg1(), emitGetInsSC(id), true); // ddddd + emitDispPredicateReg(id->idReg2(), insGetPredicateType(fmt), INS_OPTS_SCALABLE_S, false); // NNNN + break; + // ., ., . case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements emitDispPredicateReg(id->idReg1(), insGetPredicateType(fmt, 1), id->idInsOpt(), true); // DDDD @@ -27031,6 +28240,25 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm break; + // MOV ., /M, . or SEL ., , ., . + case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) + { + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + + if (id->idIns() == INS_sve_mov) + { + emitDispPredicateReg(id->idReg2(), PREDICATE_MERGE, id->idInsOpt(), true); // VVVV + emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // nnnnn + } + else + { + emitDispPredicateReg(id->idReg2(), PREDICATE_NONE, id->idInsOpt(), true); // VVVV + emitDispSveReg(id->idReg3(), id->idInsOpt(), true); // nnnnn + emitDispSveReg(id->idReg4(), id->idInsOpt(), false); // mmmmm + } + break; + } + // ., /Z, ., . case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match @@ -27082,6 +28310,25 @@ void emitter::emitDispInsHelp( emitDispElementIndex(emitGetInsSC(id), false); // ii/iii break; + // .S, .H, .H + case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product + case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long + case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product + case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long + emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_S, true); // ddddd + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); // nnnnn + emitDispSveReg(id->idReg3(), id->idInsOpt(), false); // mmmmm + break; + + // .S, .B, .B + case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + emitDispSveReg(id->idReg1(), INS_OPTS_SCALABLE_S, true); // ddddd + emitDispSveReg(id->idReg2(), INS_OPTS_SCALABLE_B, true); // nnnnn + emitDispSveReg(id->idReg3(), INS_OPTS_SCALABLE_B, false); // mmmmm + break; + // .D, .S, .S[] case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) @@ -27347,6 +28594,23 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg2(), optWidenSveElemsizeArrangement(id->idInsOpt()), false); // nnnnn break; + // , , # + case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + { + const regNumber reg1 = (id->idReg1() == REG_ZR) ? REG_SP : id->idReg1(); + const regNumber reg2 = (id->idReg2() == REG_ZR) ? REG_SP : id->idReg2(); + emitDispReg(reg1, id->idOpSize(), true); // ddddd + emitDispReg(reg2, id->idOpSize(), true); // nnnnn + emitDispImm(emitGetInsSC(id), false); // iiiiii + break; + } + + // , # + case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + emitDispReg(id->idReg1(), id->idOpSize(), true); // ddddd + emitDispImm(emitGetInsSC(id), false); // iiiiii + break; + // ., ., # case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long { @@ -27531,6 +28795,16 @@ void emitter::emitDispInsHelp( break; } + // ., ., . + case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part + { + const insOpts largeSizeSpecifier = (insOpts)(id->idInsOpt() + 1); + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd + emitDispSveReg(id->idReg2(), largeSizeSpecifier, true); // nnnnn + emitDispSveReg(id->idReg3(), largeSizeSpecifier, false); // mmmmm + break; + } + // ., ., . case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide { @@ -28079,6 +29353,28 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg4(), id->idInsOpt(), false); break; + // .B, { .B }, [] + case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + // .B, { .B }, [] + case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + // .H, { .H }, [] + case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + // .H, { .H, .H }, [] + case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + // .H, {.H }, [] + case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); + emitDispSveConsecutiveRegList(id->idReg1(), 1, id->idInsOpt(), true); + emitDispSveReg(id->idReg2(), id->idInsOpt(), false); + emitDispElementIndex(imm, false); + break; + // , , [, .S, ] // , , [, .S, #1] // , , [, .S, #2] @@ -28181,6 +29477,18 @@ void emitter::emitDispInsHelp( emitDispSveReg(id->idReg2(), id->idInsOpt(), false); break; + // ., ., # + case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) + // ., ., # + case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert + // ., ., # + case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate + imm = emitGetInsSC(id); + emitDispSveReg(id->idReg1(), id->idInsOpt(), true); + emitDispSveReg(id->idReg2(), id->idInsOpt(), true); + emitDispImm(imm, false); + break; + default: printf("unexpected format %s", emitIfName(id->idInsFmt())); assert(!"unexpectedFormat"); @@ -30504,6 +31812,7 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_GU_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) case IF_SVE_GU_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_4C; break; @@ -30532,13 +31841,87 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins } break; - case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) + case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + switch (ins) + { + case INS_sve_fdot: + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + case INS_sve_bfdot: + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_4C; + break; + default: + // all other instructions + perfScoreUnhandledInstruction(id, &result); + break; + } + break; + + case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long + switch (ins) + { + case INS_sve_fmlalb: + case INS_sve_fmlalt: + case INS_sve_fmlslb: + case INS_sve_fmlslt: + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_4C; + break; + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + default: + // all other instructions + perfScoreUnhandledInstruction(id, &result); + break; + } + break; + + case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations + switch (ins) + { + case INS_sve_eor3: + case INS_sve_bcax: + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_2C; + break; + case INS_sve_bsl: + case INS_sve_bsl1n: + case INS_sve_bsl2n: + case INS_sve_nbsl: + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_2C; + break; + default: + // all other instructions + perfScoreUnhandledInstruction(id, &result); + break; + } + break; + + case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads + case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp + case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) + case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long + case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix result.insLatency = PERFSCORE_LATENCY_1C; // need to fix break; @@ -30552,6 +31935,12 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long + case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part + case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) + case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) + case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) + case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_2C; break; @@ -30591,6 +31980,13 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_8C; break; + case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation + case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_2C; + break; + case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_2C; @@ -30604,10 +32000,23 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) result.insThroughput = PERFSCORE_THROUGHPUT_1C; result.insLatency = PERFSCORE_LATENCY_2C; break; + case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector + case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector + case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector + case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector + case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector + case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector + case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector + case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector + result.insThroughput = PERFSCORE_THROUGHPUT_140C; // @ToDo currently undocumented + result.insLatency = PERFSCORE_LATENCY_140C; + break; + case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements case IF_SVE_CJ_2A: // ........xx...... .......NNNN.DDDD -- SVE reverse predicate elements case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements @@ -30653,6 +32062,11 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insThroughput = PERFSCORE_THROUGHPUT_1C; break; + case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) + result.insLatency = PERFSCORE_LATENCY_2C; + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + break; + case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate @@ -30678,6 +32092,24 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insThroughput = PERFSCORE_THROUGHPUT_1C; break; + case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations + switch (ins) + { + case INS_sve_rax1: + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_2C; + break; + case INS_sve_sm4ekey: + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_4C; + break; + default: + // all other instructions + perfScoreUnhandledInstruction(id, &result); + break; + } + break; + case IF_SVE_GZ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE floating-point multiply-add long (indexed) switch (ins) { @@ -30926,12 +32358,9 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins // Arithmetic, shift complex case IF_SVE_EU_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating/rounding bitwise shift left // (predicated) - result.insLatency = PERFSCORE_LATENCY_4C; - result.insThroughput = PERFSCORE_THROUGHPUT_1C; - break; - // Arithmetic, pairwise add and accum long case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long + case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product result.insLatency = PERFSCORE_LATENCY_4C; result.insThroughput = PERFSCORE_THROUGHPUT_1C; break; @@ -30960,6 +32389,34 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insThroughput = PERFSCORE_THROUGHPUT_2X; break; + case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + switch (ins) + { + case INS_sve_frecps: + case INS_sve_frsqrts: + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_4C; + break; + + case INS_sve_fmul: + case INS_sve_ftsmul: + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_3C; + break; + + case INS_sve_fadd: + case INS_sve_fsub: + result.insThroughput = PERFSCORE_THROUGHPUT_2C; + result.insLatency = PERFSCORE_LATENCY_2C; + break; + + default: + // all other instructions + perfScoreUnhandledInstruction(id, &result); + break; + } + break; + case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) switch (ins) { @@ -31833,12 +33290,14 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins break; case IF_SVE_GP_3A: // ........xx.....r ...gggmmmmmddddd -- SVE floating-point complex add (predicated) + case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_3C; break; case IF_SVE_GV_3A: // ...........immmm ....rrnnnnnddddd -- SVE floating-point complex multiply-add (indexed) case IF_SVE_GT_4A: // ........xx.mmmmm .rrgggnnnnnddddd -- SVE floating-point complex multiply-add (predicated) + case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_5C; break; @@ -31896,6 +33355,7 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins case IF_SVE_HV_4A: // ........xx.aaaaa ...gggmmmmmddddd -- SVE floating-point multiply-accumulate writing // multiplicand + case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend result.insThroughput = PERFSCORE_THROUGHPUT_2C; result.insLatency = PERFSCORE_LATENCY_4C; break; @@ -31912,6 +33372,36 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_2C; break; + case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + + case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + + case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + + case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + + case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + result.insThroughput = PERFSCORE_THROUGHPUT_1C; // need to fix + result.insLatency = PERFSCORE_LATENCY_1C; // need to fix + break; + case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled // offsets) switch (ins) @@ -32186,6 +33676,17 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins result.insLatency = PERFSCORE_LATENCY_3C; break; + case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) + case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_2C; + break; + + case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate + result.insThroughput = PERFSCORE_THROUGHPUT_1C; + result.insLatency = PERFSCORE_LATENCY_4C; + break; + default: // all other instructions perfScoreUnhandledInstruction(id, &result); diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 3fa81419e9604..61647d6512cd0 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -51,14 +51,16 @@ void emitDispBarrier(insBarrier barrier); void emitDispShiftOpts(insOpts opt); void emitDispExtendOpts(insOpts opt); void emitDispSveExtendOpts(insOpts opt); -void emitDispSveExtendOptsModN(insOpts opt, int n); +void emitDispSveExtendOptsModN(insOpts opt, ssize_t imm); void emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts opt, insFormat fmt); void emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt); void emitDispSveImmMulVl(regNumber reg1, ssize_t imm); void emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm); void emitDispLSExtendOpts(insOpts opt); void emitDispReg(regNumber reg, emitAttr attr, bool addComma); +void emitDispSveReg(regNumber reg, bool addComma); void emitDispSveReg(regNumber reg, insOpts opt, bool addComma); +void emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma); void emitDispVectorReg(regNumber reg, insOpts opt, bool addComma); void emitDispVectorRegIndex(regNumber reg, emitAttr elemsize, ssize_t index, bool addComma); void emitDispVectorRegList(regNumber firstReg, unsigned listSize, insOpts opt, bool addComma); @@ -544,6 +546,9 @@ static code_t insEncodeSveElemsize_30_or_21(insFormat fmt, emitAttr size); // Returns the encoding for the field 'i1:tszh:tszl' at bit locations '23-22:20-18'. static code_t insEncodeSveElemsize_tszh_tszl_and_imm(const insOpts opt, const ssize_t imm); +// Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. +static code_t insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, ssize_t imm, bool isRightShift); + // Returns the encoding to select the constant values 90 or 270 for an Arm64 SVE vector instruction // This specifically encode the field 'rot' at bit location '16'. static code_t insEncodeSveImm90_or_270_rot(ssize_t imm); @@ -591,6 +596,49 @@ static code_t insEncodeSveElemsize_dtype_ld1w(instruction ins, insFormat fmt, em // for the 'dtypeh' and 'dtypel' fields. static code_t insEncodeSveElemsize_dtypeh_dtypel(instruction ins, insFormat fmt, emitAttr size, code_t code); +// Encodes an immediate value in consecutive bits from most significant position 'hi' to least significant +// position 'lo'. +template +static code_t insEncodeUimm(size_t imm) +{ + // lo <= hi < 32 + static_assert((hi >= lo) && (hi < sizeof(code_t) * BITS_PER_BYTE)); + + const size_t imm_bits = hi - lo + 1; + static_assert(imm_bits < sizeof(code_t) * BITS_PER_BYTE); + + const size_t imm_max = 1 << imm_bits; + assert(imm < imm_max); + + code_t result = static_cast(imm << lo); + assert((result >> lo) == imm); + return result; +} + +// Encodes an immediate value across two ranges of consecutive bits, splitting the bits of the immediate +// value between them. The bit ranges are from hi1-lo1, and hi2-lo2 where the second range is at a less +// significant position relative to the first. +template +static code_t insEncodeSplitUimm(size_t imm) +{ + static_assert((hi1 >= lo1) && (lo1 > hi2) && (hi2 >= lo2)); + static_assert(hi1 < sizeof(code_t) * BITS_PER_BYTE); + + const size_t hi_bits = hi1 - lo1 + 1; + const size_t lo_bits = hi2 - lo2 + 1; + + const size_t imm_max = 1 << (hi_bits + lo_bits); + assert(imm < imm_max); + + const size_t hi_max = 1 << hi_bits; + const size_t lo_max = 1 << lo_bits; + + size_t immhi = (imm >> lo_bits) & (hi_max - 1); + size_t immlo = imm & (lo_max - 1); + + return insEncodeUimm(immhi) | insEncodeUimm(immlo); +} + // Returns the encoding for the immediate value as 4-bits at bit locations '19-16'. static code_t insEncodeSimm4_19_to_16(ssize_t imm); @@ -621,6 +669,9 @@ static code_t insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm); // Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. static code_t insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm); +// Returns the encoding for the immediate value as 6-bits at bit locations '10-5'. +static code_t insEncodeSimm6_10_to_5(ssize_t imm); + // Returns the encoding for the immediate value as 6-bits at bit locations '21-16'. static code_t insEncodeSimm6_21_to_16(ssize_t imm); @@ -645,6 +696,15 @@ static code_t insEncodeUimm2_11_to_10(ssize_t imm); // Returns the encoding for the immediate value as 2-bits at bit locations '20-19'. static code_t insEncodeUimm2_20_to_19(ssize_t imm); +// Returns the encoding for the immediate value as 2-bits at bit locations '23-22'. +static code_t insEncodeUimm2_23_to_22(ssize_t imm); + +// Returns the encoding for the immediate value as 1-bit at bit locations '23'. +static code_t insEncodeUimm1_23(ssize_t imm); + +// Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. +static code_t insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm); + // Returns the encoding for the immediate value as 1 bit at bit location '10'. static code_t insEncodeImm1_10(ssize_t imm); @@ -776,6 +836,13 @@ static bool isValidUimm6_MultipleOf8(ssize_t value) return (0 <= value) && (value <= 504) && (value % 8 == 0); }; +template +static bool isValidUimm(ssize_t value) +{ + constexpr size_t max = 1 << bits; + return (0 <= value) && (value < max); +} + // Returns true if 'value' is a legal immediate 1 bit encoding (such as for PEXT). static bool isValidImm1(ssize_t value) { @@ -1438,23 +1505,24 @@ void emitIns_R_R_R(instruction ins, insOpts opt = INS_OPTS_NONE, insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); -void emitIns_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt = INS_OPTS_NONE, - emitAttr attrReg2 = EA_UNKNOWN); - -void emitInsSve_R_R_R_I(instruction ins, - emitAttr attr, - regNumber reg1, - regNumber reg2, - regNumber reg3, - ssize_t imm, - insOpts opt = INS_OPTS_NONE, - emitAttr attrReg2 = EA_UNKNOWN); +void emitIns_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt = INS_OPTS_NONE, + emitAttr attrReg2 = EA_UNKNOWN, + insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); + +void emitInsSve_R_R_R_I(instruction ins, + emitAttr attr, + regNumber reg1, + regNumber reg2, + regNumber reg3, + ssize_t imm, + insOpts opt = INS_OPTS_NONE, + insScalableOpts sopt = INS_SCALABLE_OPTS_NONE); void emitIns_R_R_R_I_I(instruction ins, emitAttr attr, diff --git a/src/coreclr/jit/emitloongarch64.cpp b/src/coreclr/jit/emitloongarch64.cpp index 0ea2546213a7a..86ba90fa36ede 100644 --- a/src/coreclr/jit/emitloongarch64.cpp +++ b/src/coreclr/jit/emitloongarch64.cpp @@ -4598,7 +4598,7 @@ void emitter::emitInsLoadStoreOp(instruction ins, emitAttr attr, regNumber dataR int offset = 0; DWORD lsl = 0; - if (addr->OperGet() == GT_LEA) + if (addr->OperIs(GT_LEA)) { offset = addr->AsAddrMode()->Offset(); if (addr->AsAddrMode()->gtScale > 0) @@ -4980,7 +4980,7 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, { emitIns_R_R_R(ins, attr, dst->GetRegNum(), src1->GetRegNum(), src2->GetRegNum()); } - else if (dst->OperGet() == GT_MUL) + else if (dst->OperIs(GT_MUL)) { if (!needCheckOv) { @@ -5048,10 +5048,14 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, // TODO-LOONGARCH64-CQ: here sign-extend dst when deal with 32bit data is too conservative. if (EA_SIZE(attr) == EA_4BYTE) + { emitIns_R_R_I(INS_slli_w, attr, dst->GetRegNum(), dst->GetRegNum(), 0); + } } else { + assert(dst->OperIs(GT_ADD, GT_SUB)); + regNumber regOp1 = src1->GetRegNum(); regNumber regOp2 = src2->GetRegNum(); regNumber saveOperReg1 = REG_NA; @@ -5064,26 +5068,38 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, assert(REG_R21 != dst->GetRegNum()); assert(REG_RA != dst->GetRegNum()); - if (dst->GetRegNum() == regOp1) + if (dst->OperIs(GT_ADD)) { - assert(REG_R21 != regOp1); - assert(REG_RA != regOp1); - saveOperReg1 = REG_R21; - saveOperReg2 = regOp2; - emitIns_R_R_R(INS_or, attr, REG_R21, regOp1, REG_R0); + saveOperReg1 = (dst->GetRegNum() == regOp1) ? regOp2 : regOp1; } - else if (dst->GetRegNum() == regOp2) + else { - assert(REG_R21 != regOp2); - assert(REG_RA != regOp2); - saveOperReg1 = regOp1; - saveOperReg2 = REG_R21; - emitIns_R_R_R(INS_or, attr, REG_R21, regOp2, REG_R0); + if (dst->GetRegNum() == regOp1) + { + assert(REG_R21 != regOp1); + assert(REG_RA != regOp1); + saveOperReg1 = REG_R21; + emitIns_R_R_R(INS_or, attr, REG_R21, regOp1, REG_R0); + } + else + { + saveOperReg1 = regOp1; + } } - else + + if ((dst->gtFlags & GTF_UNSIGNED) == 0) { - saveOperReg1 = regOp1; - saveOperReg2 = regOp2; + saveOperReg2 = dst->GetSingleTempReg(); + assert((saveOperReg2 != REG_RA) && (saveOperReg2 != REG_R21)); + assert(REG_RA != regOp1); + assert(saveOperReg2 != regOp2); + + ssize_t ui6 = (attr == EA_4BYTE) ? 31 : 63; + if (dst->OperIs(GT_ADD)) + { + emitIns_R_R_I(INS_srli_d, attr, REG_RA, regOp1, ui6); + } + emitIns_R_R_I(INS_srli_d, attr, saveOperReg2, regOp2, ui6); } } @@ -5091,86 +5107,56 @@ regNumber emitter::emitInsTernary(instruction ins, emitAttr attr, GenTree* dst, if (needCheckOv) { - if (dst->OperGet() == GT_ADD || dst->OperGet() == GT_SUB) + // ADD : A = B + C + // SUB : A = B - C <=> B = A + C + if ((dst->gtFlags & GTF_UNSIGNED) != 0) { - ssize_t imm; - regNumber tempReg1; - regNumber tempReg2; - // ADD : A = B + C - // SUB : C = A - B - if ((dst->gtFlags & GTF_UNSIGNED) != 0) + // ADD: if A < B, goto overflow + // SUB: if B < A, goto overflow + codeGen->genJumpToThrowHlpBlk_la(SCK_OVERFLOW, INS_bltu, + dst->OperIs(GT_ADD) ? dst->GetRegNum() : saveOperReg1, nullptr, + dst->OperIs(GT_ADD) ? saveOperReg1 : dst->GetRegNum()); + } + else + { + if (dst->OperIs(GT_SUB)) { - // if A < B, goto overflow - if (dst->OperGet() == GT_ADD) - { - tempReg1 = dst->GetRegNum(); - tempReg2 = saveOperReg1; - } - else - { - tempReg1 = saveOperReg1; - tempReg2 = saveOperReg2; - } - codeGen->genJumpToThrowHlpBlk_la(SCK_OVERFLOW, INS_bltu, tempReg1, nullptr, tempReg2); + emitIns_R_R_I(INS_srli_d, attr, REG_RA, dst->GetRegNum(), (attr == EA_4BYTE) ? 31 : 63); } - else - { - tempReg1 = REG_RA; - tempReg2 = dst->GetSingleTempReg(); - assert(tempReg1 != tempReg2); - assert(tempReg1 != saveOperReg1); - assert(tempReg2 != saveOperReg2); - - ssize_t ui6 = (attr == EA_4BYTE) ? 31 : 63; - if (dst->OperGet() == GT_ADD) - { - emitIns_R_R_I(INS_srli_d, attr, tempReg1, saveOperReg1, ui6); - } - else - { - emitIns_R_R_I(INS_srli_d, attr, tempReg1, dst->GetRegNum(), ui6); - } - emitIns_R_R_I(INS_srli_d, attr, tempReg2, saveOperReg2, ui6); - emitIns_R_R_R(INS_xor, attr, tempReg1, tempReg1, tempReg2); - if (attr == EA_4BYTE) - { - imm = 1; - emitIns_R_R_I(INS_andi, attr, tempReg1, tempReg1, imm); - emitIns_R_R_I(INS_andi, attr, tempReg2, tempReg2, imm); - } - // if (B > 0 && C < 0) || (B < 0 && C > 0), skip overflow - BasicBlock* tmpLabel = codeGen->genCreateTempLabel(); - BasicBlock* tmpLabel2 = codeGen->genCreateTempLabel(); - BasicBlock* tmpLabel3 = codeGen->genCreateTempLabel(); + emitIns_R_R_R(INS_xor, attr, REG_RA, REG_RA, saveOperReg2); + if (attr == EA_4BYTE) + { + emitIns_R_R_I(INS_andi, attr, REG_RA, REG_RA, 1); + emitIns_R_R_I(INS_andi, attr, saveOperReg2, saveOperReg2, 1); + } + // ADD: if (B > 0 && C < 0) || (B < 0 && C > 0), skip overflow + // SUB: if (A > 0 && C < 0) || (A < 0 && C > 0), skip overflow + BasicBlock* tmpLabel1 = codeGen->genCreateTempLabel(); + BasicBlock* tmpLabel2 = codeGen->genCreateTempLabel(); + BasicBlock* tmpLabel3 = codeGen->genCreateTempLabel(); - emitIns_J_cond_la(INS_bne, tmpLabel, tempReg1, REG_R0); + emitIns_J_cond_la(INS_bne, tmpLabel1, REG_RA, REG_R0); - emitIns_J_cond_la(INS_bne, tmpLabel3, tempReg2, REG_R0); + emitIns_J_cond_la(INS_bne, tmpLabel3, saveOperReg2, REG_R0); - // B > 0 and C > 0, if A < B, goto overflow - emitIns_J_cond_la(INS_bge, tmpLabel, dst->OperGet() == GT_ADD ? dst->GetRegNum() : saveOperReg1, - dst->OperGet() == GT_ADD ? saveOperReg1 : saveOperReg2); + // ADD: B > 0 and C > 0, if A < B, goto overflow + // SUB: A > 0 and C > 0, if B < A, goto overflow + emitIns_J_cond_la(INS_bge, tmpLabel1, dst->OperIs(GT_ADD) ? dst->GetRegNum() : saveOperReg1, + dst->OperIs(GT_ADD) ? saveOperReg1 : dst->GetRegNum()); - codeGen->genDefineTempLabel(tmpLabel2); + codeGen->genDefineTempLabel(tmpLabel2); - codeGen->genJumpToThrowHlpBlk(EJ_jmp, SCK_OVERFLOW); + codeGen->genJumpToThrowHlpBlk(EJ_jmp, SCK_OVERFLOW); - codeGen->genDefineTempLabel(tmpLabel3); + codeGen->genDefineTempLabel(tmpLabel3); - // B < 0 and C < 0, if A > B, goto overflow - emitIns_J_cond_la(INS_blt, tmpLabel2, dst->OperGet() == GT_ADD ? saveOperReg1 : saveOperReg2, - dst->OperGet() == GT_ADD ? dst->GetRegNum() : saveOperReg1); + // ADD: B < 0 and C < 0, if A > B, goto overflow + // SUB: A < 0 and C < 0, if B > A, goto overflow + emitIns_J_cond_la(INS_blt, tmpLabel2, dst->OperIs(GT_ADD) ? saveOperReg1 : dst->GetRegNum(), + dst->OperIs(GT_ADD) ? dst->GetRegNum() : saveOperReg1); - codeGen->genDefineTempLabel(tmpLabel); - } - } - else - { -#ifdef DEBUG - printf("---------[LOONGARCH64]-NOTE: UnsignedOverflow instruction %d\n", ins); -#endif - assert(!"unimplemented on LOONGARCH yet"); + codeGen->genDefineTempLabel(tmpLabel1); } } } diff --git a/src/coreclr/jit/emitriscv64.cpp b/src/coreclr/jit/emitriscv64.cpp index 66cb371aeeeca..9e9f900417096 100644 --- a/src/coreclr/jit/emitriscv64.cpp +++ b/src/coreclr/jit/emitriscv64.cpp @@ -625,7 +625,8 @@ void emitter::emitIns_R_R( assert(isGeneralRegisterOrR0(reg2)); code |= (reg1 & 0x1f) << 7; code |= reg2 << 15; - code |= 0x7 << 12; + if (INS_fcvt_d_w != ins && INS_fcvt_d_wu != ins) // fcvt.d.w[u] always produces an exact result + code |= 0x7 << 12; // round according to frm status register } else if (INS_fcvt_s_d == ins || INS_fcvt_d_s == ins) { @@ -633,7 +634,8 @@ void emitter::emitIns_R_R( assert(isFloatReg(reg2)); code |= (reg1 & 0x1f) << 7; code |= (reg2 & 0x1f) << 15; - code |= 0x7 << 12; + if (INS_fcvt_d_s != ins) // fcvt.d.s never rounds + code |= 0x7 << 12; // round according to frm status register } else { diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index d429f0d261f7a..3580b02db4dfc 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -15330,13 +15330,10 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) if (id->idIsCnsReloc()) { - if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI)) + if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && id->idAddr()->iiaSecRel) { - if (id->idAddr()->iiaSecRel) - { - // For section relative, the immediate offset is relocatable and hence need IMAGE_REL_SECREL - emitRecordRelocation((void*)(dst - (unsigned)EA_SIZE(size)), (void*)(size_t)val, IMAGE_REL_SECREL); - } + // For section relative, the immediate offset is relocatable and hence need IMAGE_REL_SECREL + emitRecordRelocation((void*)(dst - (unsigned)EA_SIZE(size)), (void*)(size_t)val, IMAGE_REL_SECREL); } else { diff --git a/src/coreclr/jit/fgbasic.cpp b/src/coreclr/jit/fgbasic.cpp index d1a7bc7fda298..cc83e856a7ee5 100644 --- a/src/coreclr/jit/fgbasic.cpp +++ b/src/coreclr/jit/fgbasic.cpp @@ -206,17 +206,10 @@ bool Compiler::fgEnsureFirstBBisScratch() assert(fgFirstBBScratch == nullptr); - BasicBlock* block = BasicBlock::New(this, BBJ_ALWAYS, fgFirstBB); - block->SetFlags(BBF_NONE_QUIRK); + BasicBlock* block; if (fgFirstBB != nullptr) { - // If we have profile data the new block will inherit fgFirstBlock's weight - if (fgFirstBB->hasProfileWeight()) - { - block->inheritWeight(fgFirstBB); - } - // The first block has an implicit ref count which we must // remove. Note the ref count could be greater than one, if // the first block is not scratch and is targeted by a @@ -224,14 +217,24 @@ bool Compiler::fgEnsureFirstBBisScratch() assert(fgFirstBB->bbRefs >= 1); fgFirstBB->bbRefs--; + block = BasicBlock::New(this); + + // If we have profile data the new block will inherit fgFirstBlock's weight + if (fgFirstBB->hasProfileWeight()) + { + block->inheritWeight(fgFirstBB); + } + // The new scratch bb will fall through to the old first bb FlowEdge* const edge = fgAddRefPred(fgFirstBB, block); edge->setLikelihood(1.0); + block->SetKindAndTargetEdge(BBJ_ALWAYS, edge); fgInsertBBbefore(fgFirstBB, block); } else { noway_assert(fgLastBB == nullptr); + block = BasicBlock::New(this, BBJ_ALWAYS); fgFirstBB = block; fgLastBB = block; } @@ -239,7 +242,7 @@ bool Compiler::fgEnsureFirstBBisScratch() noway_assert(fgLastBB != nullptr); // Set the expected flags - block->SetFlags(BBF_INTERNAL | BBF_IMPORTED); + block->SetFlags(BBF_INTERNAL | BBF_IMPORTED | BBF_NONE_QUIRK); // This new first BB has an implicit ref, and no others. // @@ -357,7 +360,7 @@ void Compiler::fgConvertBBToThrowBB(BasicBlock* block) fgRemoveBlockAsPred(block); // Update jump kind after the scrub. - block->SetKindAndTarget(BBJ_THROW); + block->SetKindAndTargetEdge(BBJ_THROW); block->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY // Any block with a throw is rare @@ -645,9 +648,9 @@ void Compiler::fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, Bas case BBJ_LEAVE: // This function can be called before import, so we still have BBJ_LEAVE { assert(block->TargetIs(oldTarget)); - block->SetTarget(newTarget); - FlowEdge* const oldEdge = fgRemoveRefPred(oldTarget, block); - fgAddRefPred(newTarget, block, oldEdge); + fgRemoveRefPred(block->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(newTarget, block, block->GetTargetEdge()); + block->SetTargetEdge(newEdge); break; } @@ -655,44 +658,50 @@ void Compiler::fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, Bas if (block->TrueTargetIs(oldTarget)) { - if (block->FalseTargetIs(oldTarget)) + FlowEdge* const oldEdge = block->GetTrueEdge(); + + if (block->FalseEdgeIs(oldEdge)) { // fgRemoveRefPred returns nullptr for BBJ_COND blocks with two flow edges to target fgRemoveConditionalJump(block); assert(block->KindIs(BBJ_ALWAYS)); assert(block->TargetIs(oldTarget)); - block->SetTarget(newTarget); - } - else - { - block->SetTrueTarget(newTarget); } // fgRemoveRefPred should have removed the flow edge - FlowEdge* oldEdge = fgRemoveRefPred(oldTarget, block); - assert(oldEdge != nullptr); + fgRemoveRefPred(oldEdge); + assert(oldEdge->getDupCount() == 0); // TODO-NoFallThrough: Proliferate weight from oldEdge // (as a quirk, we avoid doing so for the true target to reduce diffs for now) FlowEdge* const newEdge = fgAddRefPred(newTarget, block); + if (block->KindIs(BBJ_ALWAYS)) { newEdge->setLikelihood(1.0); + block->SetTargetEdge(newEdge); } - else if (oldEdge->hasLikelihood()) + else { - newEdge->setLikelihood(oldEdge->getLikelihood()); + assert(block->KindIs(BBJ_COND)); + block->SetTrueEdge(newEdge); + + if (oldEdge->hasLikelihood()) + { + newEdge->setLikelihood(oldEdge->getLikelihood()); + } } } else { assert(block->FalseTargetIs(oldTarget)); + FlowEdge* const oldEdge = block->GetFalseEdge(); // fgRemoveRefPred should have removed the flow edge - FlowEdge* oldEdge = fgRemoveRefPred(oldTarget, block); - assert(oldEdge != nullptr); - block->SetFalseTarget(newTarget); - fgAddRefPred(newTarget, block, oldEdge); + fgRemoveRefPred(oldEdge); + assert(oldEdge->getDupCount() == 0); + FlowEdge* const newEdge = fgAddRefPred(newTarget, block, oldEdge); + block->SetFalseEdge(newEdge); } break; @@ -730,54 +739,6 @@ void Compiler::fgReplaceJumpTarget(BasicBlock* block, BasicBlock* oldTarget, Bas } } -//------------------------------------------------------------------------ -// fgReplacePred: update the predecessor list, swapping one pred for another -// -// Arguments: -// block - block with the pred list we want to update -// oldPred - pred currently appearing in block's pred list -// newPred - pred that will take oldPred's place. -// -// Notes: -// -// A block can only appear once in the preds list. If a predecessor has multiple -// ways to get to this block, then the pred edge DupCount will be >1. -// -// This function assumes that all branches from the predecessor (practically, that all -// switch cases that target this block) are changed to branch from the new predecessor, -// with the same dup count. -// -// Note that the block bbRefs is not changed, since 'block' has the same number of -// references as before, just from a different predecessor block. -// -// Also note this may cause sorting of the pred list. -// -void Compiler::fgReplacePred(BasicBlock* block, BasicBlock* oldPred, BasicBlock* newPred) -{ - noway_assert(block != nullptr); - noway_assert(oldPred != nullptr); - noway_assert(newPred != nullptr); - - bool modified = false; - - for (FlowEdge* const pred : block->PredEdges()) - { - if (oldPred == pred->getSourceBlock()) - { - pred->setSourceBlock(newPred); - modified = true; - break; - } - } - - // We may now need to reorder the pred list. - // - if (modified) - { - block->ensurePredListOrder(this); - } -} - //------------------------------------------------------------------------ // fgReplacePred: redirects the given edge to a new predecessor block // @@ -2965,10 +2926,21 @@ void Compiler::fgLinkBasicBlocks() { BasicBlock* const trueTarget = fgLookupBB(curBBdesc->GetTargetOffs()); BasicBlock* const falseTarget = curBBdesc->Next(); - curBBdesc->SetTrueTarget(trueTarget); - curBBdesc->SetFalseTarget(falseTarget); - fgAddRefPred(trueTarget, curBBdesc); - fgAddRefPred(falseTarget, curBBdesc); + FlowEdge* const trueEdge = fgAddRefPred(trueTarget, curBBdesc); + FlowEdge* const falseEdge = fgAddRefPred(falseTarget, curBBdesc); + curBBdesc->SetTrueEdge(trueEdge); + curBBdesc->SetFalseEdge(falseEdge); + + if (trueEdge == falseEdge) + { + assert(trueEdge->getDupCount() == 2); + trueEdge->setLikelihood(1.0); + } + else + { + trueEdge->setLikelihood(0.5); + falseEdge->setLikelihood(0.5); + } if (trueTarget->bbNum <= curBBdesc->bbNum) { @@ -2991,10 +2963,11 @@ void Compiler::fgLinkBasicBlocks() assert(!(curBBdesc->IsLast() && jumpsToNext)); BasicBlock* const jumpDest = jumpsToNext ? curBBdesc->Next() : fgLookupBB(curBBdesc->GetTargetOffs()); - // Redundantly use SetKindAndTarget() instead of SetTarget() just this once, - // so we don't break the HasInitializedTarget() invariant of SetTarget(). - curBBdesc->SetKindAndTarget(curBBdesc->GetKind(), jumpDest); - fgAddRefPred(jumpDest, curBBdesc); + // Redundantly use SetKindAndTargetEdge() instead of SetTargetEdge() just this once, + // so we don't break the HasInitializedTarget() invariant of SetTargetEdge(). + FlowEdge* const newEdge = fgAddRefPred(jumpDest, curBBdesc); + newEdge->setLikelihood(1.0); + curBBdesc->SetKindAndTargetEdge(curBBdesc->GetKind(), newEdge); if (curBBdesc->GetTarget()->bbNum <= curBBdesc->bbNum) { @@ -3020,14 +2993,17 @@ void Compiler::fgLinkBasicBlocks() case BBJ_SWITCH: { - unsigned jumpCnt = curBBdesc->GetSwitchTargets()->bbsCount; - FlowEdge** jumpPtr = curBBdesc->GetSwitchTargets()->bbsDstTab; + const unsigned numSucc = curBBdesc->GetSwitchTargets()->bbsCount; + unsigned jumpCnt = numSucc; + FlowEdge** jumpPtr = curBBdesc->GetSwitchTargets()->bbsDstTab; do { BasicBlock* jumpDest = fgLookupBB((unsigned)*(size_t*)jumpPtr); FlowEdge* const newEdge = fgAddRefPred(jumpDest, curBBdesc); - *jumpPtr = newEdge; + + newEdge->setLikelihood((1.0 / numSucc) * newEdge->getDupCount()); + *jumpPtr = newEdge; if (jumpDest->bbNum <= curBBdesc->bbNum) { fgMarkBackwardJump(jumpDest, curBBdesc); @@ -3569,7 +3545,7 @@ unsigned Compiler::fgMakeBasicBlocks(const BYTE* codeAddr, IL_OFFSET codeSize, F noway_assert(codeAddr == codeEndp); - /* Finally link up the bbTarget of the blocks together */ + /* Finally link up the targets of the blocks together */ fgLinkBasicBlocks(); @@ -3887,10 +3863,10 @@ void Compiler::fgFindBasicBlocks() if (block->KindIs(BBJ_EHFILTERRET)) { // Mark catch handler as successor. - block->SetTarget(hndBegBB); FlowEdge* const newEdge = fgAddRefPred(hndBegBB, block); newEdge->setLikelihood(1.0); - assert(block->GetTarget()->bbCatchTyp == BBCT_FILTER_HANDLER); + block->SetTargetEdge(newEdge); + assert(hndBegBB->bbCatchTyp == BBCT_FILTER_HANDLER); break; } } @@ -4223,9 +4199,9 @@ void Compiler::fgFixEntryFlowForOSR() fgEnsureFirstBBisScratch(); assert(fgFirstBB->KindIs(BBJ_ALWAYS) && fgFirstBB->JumpsToNext()); fgRemoveRefPred(fgFirstBB->GetTarget(), fgFirstBB); - fgFirstBB->SetKindAndTarget(BBJ_ALWAYS, fgOSREntryBB); - FlowEdge* const edge = fgAddRefPred(fgOSREntryBB, fgFirstBB); - edge->setLikelihood(1.0); + FlowEdge* const newEdge = fgAddRefPred(fgOSREntryBB, fgFirstBB); + newEdge->setLikelihood(1.0); + fgFirstBB->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // We don't know the right weight for this block, since // execution of the method was interrupted within the @@ -4774,14 +4750,15 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr) { // For each successor of the original block, set the new block as their predecessor. - for (BasicBlock* const succ : curr->Succs(this)) + for (FlowEdge* const succEdge : curr->SuccEdges()) { - if (succ != newBlock) - { - JITDUMP(FMT_BB " previous predecessor was " FMT_BB ", now is " FMT_BB "\n", succ->bbNum, curr->bbNum, - newBlock->bbNum); - fgReplacePred(succ, curr, newBlock); - } + // For non-switch blocks, successor iterator should not iterate duplicates. + assert(succEdge->getSourceBlock() != newBlock); + + BasicBlock* const succBlock = succEdge->getDestinationBlock(); + JITDUMP(FMT_BB " previous predecessor was " FMT_BB ", now is " FMT_BB "\n", succBlock->bbNum, curr->bbNum, + newBlock->bbNum); + fgReplacePred(succEdge, newBlock); } } @@ -4813,19 +4790,18 @@ BasicBlock* Compiler::fgSplitBlockAtEnd(BasicBlock* curr) // Remove flags from the old block that are no longer possible. curr->RemoveFlags(BBF_HAS_JMP | BBF_RETLESS_CALL); + // Default to fallthrough, and add the arc for that. + FlowEdge* const newEdge = fgAddRefPred(newBlock, curr); + newEdge->setLikelihood(1.0); + // Transfer the kind and target. Do this after the code above, to avoid null-ing out the old targets used by the - // above code (and so newBlock->bbNext is valid, so SetCond() can initialize bbFalseTarget if newBlock is a - // BBJ_COND). + // above code. newBlock->TransferTarget(curr); - // Default to fallthrough, and add the arc for that. - curr->SetKindAndTarget(BBJ_ALWAYS, newBlock); + curr->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); curr->SetFlags(BBF_NONE_QUIRK); assert(curr->JumpsToNext()); - FlowEdge* const newEdge = fgAddRefPred(newBlock, curr); - newEdge->setLikelihood(1.0); - return newBlock; } @@ -5048,15 +5024,14 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) // an immediately following block of a BBJ_SWITCH (which has // no fall-through path). For this case, simply insert a new // fall-through block after 'curr'. - // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, this will be unnecessary for BBJ_COND - newBlock = fgNewBBafter(BBJ_ALWAYS, curr, true /* extendRegion */, /* jumpDest */ succ); + // TODO-NoFallThrough: Once false target can diverge from bbNext, this will be unnecessary for BBJ_COND + newBlock = fgNewBBafter(BBJ_ALWAYS, curr, true /* extendRegion */); newBlock->SetFlags(BBF_NONE_QUIRK); - assert(newBlock->JumpsToNext()); } else { // The new block always jumps to 'succ' - newBlock = fgNewBBinRegion(BBJ_ALWAYS, curr, /* jumpDest */ succ, /* isRunRarely */ curr->isRunRarely()); + newBlock = fgNewBBinRegion(BBJ_ALWAYS, curr, /* isRunRarely */ curr->isRunRarely()); } newBlock->CopyFlags(curr, succ->GetFlagsRaw() & BBF_BACKWARD_JUMP); @@ -5069,6 +5044,7 @@ BasicBlock* Compiler::fgSplitEdge(BasicBlock* curr, BasicBlock* succ) // And 'succ' has 'newBlock' as a new predecessor. FlowEdge* const newEdge = fgAddRefPred(succ, newBlock); newEdge->setLikelihood(1.0); + newBlock->SetTargetEdge(newEdge); // This isn't accurate, but it is complex to compute a reasonable number so just assume that we take the // branch 50% of the time. @@ -5371,7 +5347,7 @@ BasicBlock* Compiler::fgRemoveBlock(BasicBlock* block, bool unreachable) assert(!bPrev->FalseTargetIs(block)); /* Check if both sides of the BBJ_COND now jump to the same block */ - if (bPrev->TrueTargetIs(bPrev->GetFalseTarget())) + if (bPrev->TrueEdgeIs(bPrev->GetFalseEdge())) { fgRemoveConditionalJump(bPrev); } @@ -5447,16 +5423,19 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) if (bSrc->KindIs(BBJ_COND) && bSrc->FalseTargetIs(bDst) && !bSrc->NextIs(bDst)) { // Add a new block after bSrc which jumps to 'bDst' - jmpBlk = fgNewBBafter(BBJ_ALWAYS, bSrc, true, bDst); - bSrc->SetFalseTarget(jmpBlk); - fgAddRefPred(jmpBlk, bSrc, fgGetPredForBlock(bDst, bSrc)); + jmpBlk = fgNewBBafter(BBJ_ALWAYS, bSrc, true); + FlowEdge* oldEdge = bSrc->GetFalseEdge(); + fgReplacePred(oldEdge, jmpBlk); + jmpBlk->SetTargetEdge(oldEdge); + assert(jmpBlk->TargetIs(bDst)); + + FlowEdge* newEdge = fgAddRefPred(jmpBlk, bSrc, oldEdge); + bSrc->SetFalseEdge(newEdge); // When adding a new jmpBlk we will set the bbWeight and bbFlags // if (fgHaveValidEdgeWeights && fgHaveProfileWeights()) { - FlowEdge* const newEdge = fgGetPredForBlock(jmpBlk, bSrc); - jmpBlk->bbWeight = (newEdge->edgeWeightMin() + newEdge->edgeWeightMax()) / 2; if (bSrc->bbWeight == BB_ZERO_WEIGHT) { @@ -5494,8 +5473,6 @@ BasicBlock* Compiler::fgConnectFallThrough(BasicBlock* bSrc, BasicBlock* bDst) } } - fgReplacePred(bDst, bSrc, jmpBlk); - JITDUMP("Added an unconditional jump to " FMT_BB " after block " FMT_BB "\n", jmpBlk->GetTarget()->bbNum, bSrc->bbNum); } @@ -6052,14 +6029,11 @@ BasicBlock* Compiler::fgRelocateEHRange(unsigned regionIndex, FG_RELOCATE_TYPE r * Insert a BasicBlock before the given block. */ -BasicBlock* Compiler::fgNewBBbefore(BBKinds jumpKind, - BasicBlock* block, - bool extendRegion, - BasicBlock* jumpDest /* = nullptr */) +BasicBlock* Compiler::fgNewBBbefore(BBKinds jumpKind, BasicBlock* block, bool extendRegion) { // Create a new BasicBlock and chain it in - BasicBlock* newBlk = BasicBlock::New(this, jumpKind, jumpDest); + BasicBlock* newBlk = BasicBlock::New(this, jumpKind); newBlk->SetFlags(BBF_INTERNAL); fgInsertBBbefore(block, newBlk); @@ -6094,14 +6068,11 @@ BasicBlock* Compiler::fgNewBBbefore(BBKinds jumpKind, * Insert a BasicBlock after the given block. */ -BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, - BasicBlock* block, - bool extendRegion, - BasicBlock* jumpDest /* = nullptr */) +BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, BasicBlock* block, bool extendRegion) { // Create a new BasicBlock and chain it in - BasicBlock* newBlk = BasicBlock::New(this, jumpKind, jumpDest); + BasicBlock* newBlk = BasicBlock::New(this, jumpKind); newBlk->SetFlags(BBF_INTERNAL); fgInsertBBafter(block, newBlk); @@ -6141,7 +6112,6 @@ BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, // tree - tree that will be wrapped into a statement and // inserted in the new block. // debugInfo - debug info to propagate into the new statement. -// jumpDest - the jump target of the new block. Defaults to nullptr. // updateSideEffects - update side effects for the whole statement. // // Return Value: @@ -6150,14 +6120,10 @@ BasicBlock* Compiler::fgNewBBafter(BBKinds jumpKind, // Notes: // The new block will have BBF_INTERNAL flag and EH region will be extended // -BasicBlock* Compiler::fgNewBBFromTreeAfter(BBKinds jumpKind, - BasicBlock* block, - GenTree* tree, - DebugInfo& debugInfo, - BasicBlock* jumpDest /* = nullptr */, - bool updateSideEffects /* = false */) +BasicBlock* Compiler::fgNewBBFromTreeAfter( + BBKinds jumpKind, BasicBlock* block, GenTree* tree, DebugInfo& debugInfo, bool updateSideEffects /* = false */) { - BasicBlock* newBlock = fgNewBBafter(jumpKind, block, true, jumpDest); + BasicBlock* newBlock = fgNewBBafter(jumpKind, block, true); newBlock->SetFlags(BBF_INTERNAL); Statement* stmt = fgNewStmtFromTree(tree, debugInfo); fgInsertStmtAtEnd(newBlock, stmt); @@ -6579,7 +6545,6 @@ BasicBlock* Compiler::fgFindInsertPoint(unsigned regionIndex, // [0..compHndBBtabCount]. // nearBlk - insert the new block closely after this block, if possible. If nullptr, put the new block anywhere // in the requested region. -// jumpDest - the jump target of the new block. Defaults to nullptr. // putInFilter - put the new block in the filter region given by hndIndex, as described above. // runRarely - 'true' if the new block is run rarely. // insertAtEnd - 'true' if the block should be inserted at the end of the region. Note: this is currently only @@ -6592,7 +6557,6 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, unsigned tryIndex, unsigned hndIndex, BasicBlock* nearBlk, - BasicBlock* jumpDest /* = nullptr */, bool putInFilter /* = false */, bool runRarely /* = false */, bool insertAtEnd /* = false */) @@ -6718,7 +6682,7 @@ _FoundAfterBlk:; bbKindNames[jumpKind], tryIndex, hndIndex, dspBool(putInFilter), dspBool(runRarely), dspBool(insertAtEnd), afterBlk->bbNum); - return fgNewBBinRegionWorker(jumpKind, afterBlk, regionIndex, putInTryRegion, jumpDest); + return fgNewBBinRegionWorker(jumpKind, afterBlk, regionIndex, putInTryRegion); } //------------------------------------------------------------------------ @@ -6729,7 +6693,6 @@ _FoundAfterBlk:; // Arguments: // jumpKind - the jump kind of the new block to create. // srcBlk - insert the new block in the same EH region as this block, and closely after it if possible. -// jumpDest - the jump target of the new block. Defaults to nullptr. // runRarely - 'true' if the new block is run rarely. // insertAtEnd - 'true' if the block should be inserted at the end of the region. Note: this is currently only // implemented when inserting into the main function (not into any EH region). @@ -6739,7 +6702,6 @@ _FoundAfterBlk:; BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, BasicBlock* srcBlk, - BasicBlock* jumpDest /* = nullptr */, bool runRarely /* = false */, bool insertAtEnd /* = false */) { @@ -6758,7 +6720,7 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, putInFilter = ehGetDsc(hndIndex - 1)->InFilterRegionBBRange(srcBlk); } - return fgNewBBinRegion(jumpKind, tryIndex, hndIndex, srcBlk, jumpDest, putInFilter, runRarely, insertAtEnd); + return fgNewBBinRegion(jumpKind, tryIndex, hndIndex, srcBlk, putInFilter, runRarely, insertAtEnd); } //------------------------------------------------------------------------ @@ -6768,14 +6730,13 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, // // Arguments: // jumpKind - the jump kind of the new block to create. -// jumpDest - the jump target of the new block. Defaults to nullptr. // // Return Value: // The new block. -BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, BasicBlock* jumpDest /* = nullptr */) +BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind) { - return fgNewBBinRegion(jumpKind, 0, 0, nullptr, jumpDest, /* putInFilter */ false, /* runRarely */ false, + return fgNewBBinRegion(jumpKind, 0, 0, nullptr, /* putInFilter */ false, /* runRarely */ false, /* insertAtEnd */ true); } @@ -6794,7 +6755,6 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, BasicBlock* jumpDest /* // set its handler index to the most nested handler region enclosing that 'try' region. // Otherwise, put the block in the handler region specified by 'regionIndex', and set its 'try' // index to the most nested 'try' region enclosing that handler region. -// jumpDest - the jump target of the new block. Defaults to nullptr. // // Return Value: // The new block. @@ -6802,13 +6762,12 @@ BasicBlock* Compiler::fgNewBBinRegion(BBKinds jumpKind, BasicBlock* jumpDest /* BasicBlock* Compiler::fgNewBBinRegionWorker(BBKinds jumpKind, BasicBlock* afterBlk, unsigned regionIndex, - bool putInTryRegion, - BasicBlock* jumpDest /* = nullptr */) + bool putInTryRegion) { /* Insert the new block */ BasicBlock* afterBlkNext = afterBlk->Next(); (void)afterBlkNext; // prevent "unused variable" error from GCC - BasicBlock* newBlk = fgNewBBafter(jumpKind, afterBlk, false, jumpDest); + BasicBlock* newBlk = fgNewBBafter(jumpKind, afterBlk, false); if (putInTryRegion) { diff --git a/src/coreclr/jit/fgdiagnostic.cpp b/src/coreclr/jit/fgdiagnostic.cpp index 6a2068169e1a5..3818cbcd471df 100644 --- a/src/coreclr/jit/fgdiagnostic.cpp +++ b/src/coreclr/jit/fgdiagnostic.cpp @@ -134,7 +134,7 @@ void Compiler::fgDebugCheckUpdate() // Check for an unnecessary jumps to the next block. // A conditional branch should never jump to the next block as it can be folded into a BBJ_ALWAYS. - if (block->KindIs(BBJ_COND) && block->TrueTargetIs(block->GetFalseTarget())) + if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) { noway_assert(!"Unnecessary jump to the next block!"); } @@ -3003,7 +3003,7 @@ void Compiler::fgDebugCheckBBlist(bool checkBBNum /* = false */, bool checkBBRef { for (unsigned i = 0; i < sd.numDistinctSuccs; i++) { - const BasicBlock* const nonDuplicateSucc = sd.nonDuplicates[i]; + const BasicBlock* const nonDuplicateSucc = sd.nonDuplicates[i]->getDestinationBlock(); assert(nonDuplicateSucc != nullptr); assert(nonDuplicateSucc->bbTraversalStamp == curTraversalStamp); } @@ -3981,7 +3981,8 @@ void Compiler::fgDebugCheckBlockLinks() assert(uniqueSuccSet.numDistinctSuccs == count); for (unsigned i = 0; i < uniqueSuccSet.numDistinctSuccs; i++) { - assert(BitVecOps::IsMember(&bitVecTraits, succBlocks, uniqueSuccSet.nonDuplicates[i]->bbNum)); + assert(BitVecOps::IsMember(&bitVecTraits, succBlocks, + uniqueSuccSet.nonDuplicates[i]->getDestinationBlock()->bbNum)); } } } diff --git a/src/coreclr/jit/fgehopt.cpp b/src/coreclr/jit/fgehopt.cpp index 4c781cbc0c222..54a36e5c0d169 100644 --- a/src/coreclr/jit/fgehopt.cpp +++ b/src/coreclr/jit/fgehopt.cpp @@ -167,12 +167,12 @@ PhaseStatus Compiler::fgRemoveEmptyFinally() fgPrepareCallFinallyRetForRemoval(leaveBlock); fgRemoveBlock(leaveBlock, /* unreachable */ true); - currentBlock->SetKindAndTarget(BBJ_ALWAYS, postTryFinallyBlock); - currentBlock->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY - // Ref count updates. - fgAddRefPred(postTryFinallyBlock, currentBlock); - fgRemoveRefPred(firstBlock, currentBlock); + fgRemoveRefPred(currentBlock->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(postTryFinallyBlock, currentBlock); + + currentBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + currentBlock->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY // Cleanup the postTryFinallyBlock fgCleanupContinuation(postTryFinallyBlock); @@ -524,8 +524,8 @@ PhaseStatus Compiler::fgRemoveEmptyTry() GenTree* finallyRetExpr = finallyRet->GetRootNode(); assert(finallyRetExpr->gtOper == GT_RETFILT); fgRemoveStmt(block, finallyRet); - block->SetKindAndTarget(BBJ_ALWAYS, continuation); - fgAddRefPred(continuation, block); + FlowEdge* const newEdge = fgAddRefPred(continuation, block); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } } @@ -1093,9 +1093,9 @@ PhaseStatus Compiler::fgCloneFinally() GenTree* finallyRetExpr = finallyRet->GetRootNode(); assert(finallyRetExpr->gtOper == GT_RETFILT); fgRemoveStmt(newBlock, finallyRet); - newBlock->SetKindAndTarget(BBJ_ALWAYS, normalCallFinallyReturn); - fgAddRefPred(normalCallFinallyReturn, newBlock); + FlowEdge* const newEdge = fgAddRefPred(normalCallFinallyReturn, newBlock); + newBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } else { @@ -1135,13 +1135,13 @@ PhaseStatus Compiler::fgCloneFinally() fgPrepareCallFinallyRetForRemoval(leaveBlock); fgRemoveBlock(leaveBlock, /* unreachable */ true); + // Ref count updates. + fgRemoveRefPred(currentBlock->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(firstCloneBlock, currentBlock); + // This call returns to the expected spot, so retarget it to branch to the clone. - currentBlock->SetKindAndTarget(BBJ_ALWAYS, firstCloneBlock); currentBlock->RemoveFlags(BBF_RETLESS_CALL); // no longer a BBJ_CALLFINALLY - - // Ref count updates. - fgAddRefPred(firstCloneBlock, currentBlock); - fgRemoveRefPred(firstBlock, currentBlock); + currentBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // Make sure iteration isn't going off the deep end. assert(leaveBlock != endCallFinallyRangeBlock); @@ -1757,8 +1757,8 @@ bool Compiler::fgRetargetBranchesToCanonicalCallFinally(BasicBlock* block, JITDUMP("Redirecting branch in " FMT_BB " from " FMT_BB " to " FMT_BB ".\n", block->bbNum, callFinally->bbNum, canonicalCallFinally->bbNum); - block->SetTarget(canonicalCallFinally); - fgAddRefPred(canonicalCallFinally, block); + FlowEdge* const newEdge = fgAddRefPred(canonicalCallFinally, block); + block->SetTargetEdge(newEdge); assert(callFinally->bbRefs > 0); fgRemoveRefPred(callFinally, block); @@ -2103,19 +2103,20 @@ void Compiler::fgTailMergeThrowsFallThroughHelper(BasicBlock* predBlock, assert(predBlock->KindIs(BBJ_COND)); assert(predBlock->FalseTargetIs(nonCanonicalBlock)); - BasicBlock* const newBlock = fgNewBBafter(BBJ_ALWAYS, predBlock, true, canonicalBlock); - predBlock->SetFalseTarget(newBlock); + BasicBlock* const newBlock = fgNewBBafter(BBJ_ALWAYS, predBlock, true); JITDUMP("*** " FMT_BB " now falling through to empty " FMT_BB " and then to " FMT_BB "\n", predBlock->bbNum, newBlock->bbNum, canonicalBlock->bbNum); // Remove the old flow - fgRemoveRefPred(nonCanonicalBlock, predBlock); + fgRemoveRefPred(predEdge); // Wire up the new flow - fgAddRefPred(newBlock, predBlock, predEdge); + FlowEdge* const falseEdge = fgAddRefPred(newBlock, predBlock, predEdge); + predBlock->SetFalseEdge(falseEdge); - fgAddRefPred(canonicalBlock, newBlock, predEdge); + FlowEdge* const newEdge = fgAddRefPred(canonicalBlock, newBlock, predEdge); + newBlock->SetTargetEdge(newEdge); // If nonCanonicalBlock has only one pred, all its flow transfers. // If it has multiple preds, then we need edge counts or likelihoods @@ -2149,17 +2150,17 @@ void Compiler::fgTailMergeThrowsJumpToHelper(BasicBlock* predBlock, fgRemoveRefPred(nonCanonicalBlock, predBlock); // Wire up the new flow + FlowEdge* const newEdge = fgAddRefPred(canonicalBlock, predBlock, predEdge); + if (predBlock->KindIs(BBJ_ALWAYS)) { assert(predBlock->TargetIs(nonCanonicalBlock)); - predBlock->SetTarget(canonicalBlock); + predBlock->SetTargetEdge(newEdge); } else { assert(predBlock->KindIs(BBJ_COND)); assert(predBlock->TrueTargetIs(nonCanonicalBlock)); - predBlock->SetTrueTarget(canonicalBlock); + predBlock->SetTrueEdge(newEdge); } - - fgAddRefPred(canonicalBlock, predBlock, predEdge); } diff --git a/src/coreclr/jit/fgflow.cpp b/src/coreclr/jit/fgflow.cpp index 479e8b5fb1442..89da962024e54 100644 --- a/src/coreclr/jit/fgflow.cpp +++ b/src/coreclr/jit/fgflow.cpp @@ -136,32 +136,6 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (flowLast->getSourceBlock() == blockPred) { flow = flowLast; - - // This edge should have been given a likelihood when it was created. - // Since we're increasing its duplicate count, update the likelihood. - // - assert(flow->hasLikelihood()); - const unsigned numSucc = blockPred->NumSucc(); - assert(numSucc > 0); - - if (numSucc == 1) - { - // BasicBlock::NumSucc() returns 1 for BBJ_CONDs with the same true/false target. - // For blocks that only ever have one successor (BBJ_ALWAYS, BBJ_LEAVE, etc.), - // their successor edge should never have a duplicate count over 1. - // - assert(blockPred->KindIs(BBJ_COND)); - assert(blockPred->TrueTargetIs(blockPred->GetFalseTarget())); - flow->setLikelihood(1.0); - } - else - { - // Duplicate count isn't updated until later, so add 1 for now. - // - const unsigned dupCount = flow->getDupCount() + 1; - assert(dupCount > 1); - flow->setLikelihood((1.0 / numSucc) * dupCount); - } } } } @@ -211,14 +185,6 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE if (initializingPreds) { block->bbLastPred = flow; - - // When initializing preds, ensure edge likelihood is set, - // such that this edge is as likely as any other successor edge - // - const unsigned numSucc = blockPred->NumSucc(); - assert(numSucc > 0); - assert(flow->getDupCount() == 1); - flow->setLikelihood(1.0 / numSucc); } else if ((oldEdge != nullptr) && oldEdge->hasLikelihood()) { @@ -268,10 +234,6 @@ FlowEdge* Compiler::fgAddRefPred(BasicBlock* block, BasicBlock* blockPred, FlowE // assert(block->checkPredListOrder()); - // When initializing preds, edge likelihood should always be set. - // - assert(!initializingPreds || flow->hasLikelihood()); - return flow; } @@ -502,16 +464,20 @@ Compiler::SwitchUniqueSuccSet Compiler::GetDescriptorForSwitch(BasicBlock* switc // Now we have a set of unique successors. unsigned numNonDups = BitVecOps::Count(&blockVecTraits, uniqueSuccBlocks); - BasicBlock** nonDups = new (getAllocator()) BasicBlock*[numNonDups]; + FlowEdge** nonDups = new (getAllocator()) FlowEdge*[numNonDups]; unsigned nonDupInd = 0; + // At this point, all unique targets are in "uniqueSuccBlocks". As we encounter each, // add to nonDups, remove from "uniqueSuccBlocks". - for (BasicBlock* const targ : switchBlk->SwitchTargets()) + BBswtDesc* const swtDesc = switchBlk->GetSwitchTargets(); + for (unsigned i = 0; i < swtDesc->bbsCount; i++) { + FlowEdge* const succEdge = swtDesc->bbsDstTab[i]; + BasicBlock* const targ = succEdge->getDestinationBlock(); if (BitVecOps::IsMember(&blockVecTraits, uniqueSuccBlocks, targ->bbNum)) { - nonDups[nonDupInd] = targ; + nonDups[nonDupInd] = succEdge; nonDupInd++; BitVecOps::RemoveElemD(&blockVecTraits, uniqueSuccBlocks, targ->bbNum); } diff --git a/src/coreclr/jit/fginline.cpp b/src/coreclr/jit/fginline.cpp index 54003637e7e36..baf0b409841fb 100644 --- a/src/coreclr/jit/fginline.cpp +++ b/src/coreclr/jit/fginline.cpp @@ -675,15 +675,18 @@ class SubstitutePlaceholdersAndDevirtualizeWalker : public GenTreeVisitorIsIntegralConst(0)) { - m_compiler->fgRemoveRefPred(block->GetTrueTarget(), block); - block->SetKindAndTarget(BBJ_ALWAYS, block->Next()); + m_compiler->fgRemoveRefPred(block->GetTrueEdge()); + block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetFalseEdge()); block->SetFlags(BBF_NONE_QUIRK); } else { - m_compiler->fgRemoveRefPred(block->GetFalseTarget(), block); + m_compiler->fgRemoveRefPred(block->GetFalseEdge()); block->SetKind(BBJ_ALWAYS); } + + FlowEdge* const edge = m_compiler->fgGetPredForBlock(block->GetTarget(), block); + edge->setLikelihood(1.0); } } else @@ -1533,9 +1536,9 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo) JITDUMP("\nConvert bbKind of " FMT_BB " to BBJ_ALWAYS to bottomBlock " FMT_BB "\n", block->bbNum, bottomBlock->bbNum); - block->SetKindAndTarget(BBJ_ALWAYS, bottomBlock); FlowEdge* const newEdge = fgAddRefPred(bottomBlock, block); newEdge->setLikelihood(1.0); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); if (block == InlineeCompiler->fgLastBB) { @@ -1551,11 +1554,12 @@ void Compiler::fgInsertInlineeBlocks(InlineInfo* pInlineInfo) // Insert inlinee's blocks into inliner's block list. assert(topBlock->KindIs(BBJ_ALWAYS)); assert(topBlock->TargetIs(bottomBlock)); + FlowEdge* const oldEdge = fgRemoveRefPred(bottomBlock, topBlock); + FlowEdge* const newEdge = fgAddRefPred(InlineeCompiler->fgFirstBB, topBlock, oldEdge); + topBlock->SetNext(InlineeCompiler->fgFirstBB); - topBlock->SetTarget(topBlock->Next()); + topBlock->SetTargetEdge(newEdge); topBlock->SetFlags(BBF_NONE_QUIRK); - FlowEdge* const oldEdge = fgRemoveRefPred(bottomBlock, topBlock); - fgAddRefPred(InlineeCompiler->fgFirstBB, topBlock, oldEdge); InlineeCompiler->fgLastBB->SetNext(bottomBlock); // diff --git a/src/coreclr/jit/fgopt.cpp b/src/coreclr/jit/fgopt.cpp index 011562ac47309..a0065dd2c6af3 100644 --- a/src/coreclr/jit/fgopt.cpp +++ b/src/coreclr/jit/fgopt.cpp @@ -132,7 +132,7 @@ bool Compiler::fgRemoveUnreachableBlocks(CanRemoveBlockBody canRemoveBlock) block->RemoveFlags(BBF_REMOVED | BBF_INTERNAL); block->SetFlags(BBF_IMPORTED); - block->SetKindAndTarget(BBJ_THROW); + block->SetKindAndTargetEdge(BBJ_THROW); block->bbSetRunRarely(); } else @@ -623,8 +623,8 @@ PhaseStatus Compiler::fgPostImportationCleanup() // What follows is similar to fgNewBBInRegion, but we can't call that // here as the oldTryEntry is no longer in the main bb list. - newTryEntry = BasicBlock::New(this, BBJ_ALWAYS, tryEntryPrev->Next()); - newTryEntry->SetFlags(BBF_IMPORTED | BBF_INTERNAL | BBF_NONE_QUIRK); + newTryEntry = BasicBlock::New(this); + newTryEntry->SetFlags(BBF_IMPORTED | BBF_INTERNAL); newTryEntry->bbRefs = 0; // Set the right EH region indices on this new block. @@ -643,12 +643,14 @@ PhaseStatus Compiler::fgPostImportationCleanup() // plausible flow target. Simplest is to just mark it as a throw. if (bbIsHandlerBeg(newTryEntry->Next())) { - newTryEntry->SetKindAndTarget(BBJ_THROW); + newTryEntry->SetKindAndTargetEdge(BBJ_THROW); } else { FlowEdge* const newEdge = fgAddRefPred(newTryEntry->Next(), newTryEntry); newEdge->setLikelihood(1.0); + newTryEntry->SetFlags(BBF_NONE_QUIRK); + newTryEntry->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } JITDUMP("OSR: changing start of try region #%u from " FMT_BB " to new " FMT_BB "\n", @@ -774,7 +776,7 @@ PhaseStatus Compiler::fgPostImportationCleanup() fromBlock->SetFlags(BBF_INTERNAL); newBlock->RemoveFlags(BBF_DONT_REMOVE); addedBlocks++; - FlowEdge* const normalTryEntryEdge = fgGetPredForBlock(newBlock, fromBlock); + FlowEdge* const normalTryEntryEdge = fromBlock->GetTargetEdge(); GenTree* const entryStateLcl = gtNewLclvNode(entryStateVar, TYP_INT); GenTree* const compareEntryStateToZero = @@ -782,9 +784,9 @@ PhaseStatus Compiler::fgPostImportationCleanup() GenTree* const jumpIfEntryStateZero = gtNewOperNode(GT_JTRUE, TYP_VOID, compareEntryStateToZero); fgNewStmtAtBeg(fromBlock, jumpIfEntryStateZero); - fromBlock->SetCond(toBlock, newBlock); FlowEdge* const osrTryEntryEdge = fgAddRefPred(toBlock, fromBlock); newBlock->inheritWeight(fromBlock); + fromBlock->SetCond(osrTryEntryEdge, normalTryEntryEdge); // Not sure what the correct edge likelihoods are just yet; // for now we'll say the OSR path is the likely one. @@ -833,9 +835,9 @@ PhaseStatus Compiler::fgPostImportationCleanup() if (entryJumpTarget != osrEntry) { - fgFirstBB->SetTarget(entryJumpTarget); FlowEdge* const oldEdge = fgRemoveRefPred(osrEntry, fgFirstBB); - fgAddRefPred(entryJumpTarget, fgFirstBB, oldEdge); + FlowEdge* const newEdge = fgAddRefPred(entryJumpTarget, fgFirstBB, oldEdge); + fgFirstBB->SetTargetEdge(newEdge); JITDUMP("OSR: redirecting flow from method entry " FMT_BB " to OSR entry " FMT_BB " via step blocks.\n", @@ -1286,24 +1288,31 @@ void Compiler::fgCompactBlocks(BasicBlock* block, BasicBlock* bNext) case BBJ_ALWAYS: case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: - block->SetKindAndTarget(bNext->GetKind(), bNext->GetTarget()); + { + /* Update the predecessor list for bNext's target */ + FlowEdge* const targetEdge = bNext->GetTargetEdge(); + fgReplacePred(targetEdge, block); - /* Update the predecessor list for 'bNext->bbTarget' */ - fgReplacePred(bNext->GetTarget(), bNext, block); + block->SetKindAndTargetEdge(bNext->GetKind(), targetEdge); break; + } case BBJ_COND: - block->SetCond(bNext->GetTrueTarget(), bNext->GetFalseTarget()); - - /* Update the predecessor list for 'bNext->bbTrueTarget' */ - fgReplacePred(bNext->GetTrueTarget(), bNext, block); + { + /* Update the predecessor list for bNext's true target */ + FlowEdge* const trueEdge = bNext->GetTrueEdge(); + FlowEdge* const falseEdge = bNext->GetFalseEdge(); + fgReplacePred(trueEdge, block); - /* Update the predecessor list for 'bNext->bbFalseTarget' if it is different than 'bNext->bbTrueTarget' */ - if (!bNext->TrueTargetIs(bNext->GetFalseTarget())) + /* Update the predecessor list for bNext's false target if it is different from the true target */ + if (trueEdge != falseEdge) { - fgReplacePred(bNext->GetFalseTarget(), bNext, block); + fgReplacePred(falseEdge, block); } + + block->SetCond(trueEdge, falseEdge); break; + } case BBJ_EHFINALLYRET: block->SetEhf(bNext->GetEhfTargets()); @@ -1557,23 +1566,25 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc } // Optimize the JUMP to empty unconditional JUMP to go to the new target + FlowEdge* const newEdge = fgAddRefPred(bDest->GetTarget(), block, fgRemoveRefPred(bDest, block)); + switch (block->GetKind()) { case BBJ_ALWAYS: case BBJ_CALLFINALLYRET: - block->SetTarget(bDest->GetTarget()); + block->SetTargetEdge(newEdge); break; case BBJ_COND: if (block->TrueTargetIs(bDest)) { assert(!block->FalseTargetIs(bDest)); - block->SetTrueTarget(bDest->GetTarget()); + block->SetTrueEdge(newEdge); } else { assert(block->FalseTargetIs(bDest)); - block->SetFalseTarget(bDest->GetTarget()); + block->SetFalseEdge(newEdge); } break; @@ -1581,8 +1592,6 @@ bool Compiler::fgOptimizeBranchToEmptyUnconditional(BasicBlock* block, BasicBloc unreached(); } - fgAddRefPred(bDest->GetTarget(), block, fgRemoveRefPred(bDest, block)); - return true; } return false; @@ -1642,7 +1651,7 @@ bool Compiler::fgOptimizeEmptyBlock(BasicBlock* block) else { // TODO-NoFallThrough: Once BBJ_COND blocks have pointers to their false branches, - // allow removing empty BBJ_ALWAYS and pointing bPrev's false branch to block->bbTarget. + // allow removing empty BBJ_ALWAYS and pointing bPrev's false branch to block's target. if (bPrev->bbFallsThrough() && !block->JumpsToNext()) { break; @@ -1998,10 +2007,10 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) } // Change the switch jump into a BBJ_ALWAYS - block->SetKindAndTarget(BBJ_ALWAYS, block->GetSwitchTargets()->bbsDstTab[0]->getDestinationBlock()); + block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetSwitchTargets()->bbsDstTab[0]); for (unsigned i = 1; i < jmpCnt; ++i) { - fgRemoveRefPred(jmpTab[i]->getDestinationBlock(), block); + fgRemoveRefPred(jmpTab[i]); } return true; @@ -2060,9 +2069,9 @@ bool Compiler::fgOptimizeSwitchBranches(BasicBlock* block) fgSetStmtSeq(switchStmt); } - BasicBlock* const trueTarget = block->GetSwitchTargets()->bbsDstTab[0]->getDestinationBlock(); - BasicBlock* const falseTarget = block->GetSwitchTargets()->bbsDstTab[1]->getDestinationBlock(); - block->SetCond(trueTarget, falseTarget); + FlowEdge* const trueEdge = block->GetSwitchTargets()->bbsDstTab[0]; + FlowEdge* const falseEdge = block->GetSwitchTargets()->bbsDstTab[1]; + block->SetCond(trueEdge, falseEdge); JITDUMP("After:\n"); DISPNODE(switchTree); @@ -2470,27 +2479,30 @@ bool Compiler::fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* fgInsertStmtAtEnd(block, cloneStmt); } - // add an unconditional block after this block to jump to the target block's fallthrough block - // - assert(!target->IsLast()); - BasicBlock* next = fgNewBBafter(BBJ_ALWAYS, block, true, target->GetFalseTarget()); - - // Fix up block's flow + // Fix up block's flow. + // Assume edge likelihoods transfer over. // - block->SetCond(target->GetTrueTarget(), next); - fgAddRefPred(block->GetTrueTarget(), block); fgRemoveRefPred(target, block); - // The new block 'next' will inherit its weight from 'block' - // - next->inheritWeight(block); - fgAddRefPred(next, block); - fgAddRefPred(next->GetTarget(), next); + FlowEdge* const trueEdge = fgAddRefPred(target->GetTrueTarget(), block, target->GetTrueEdge()); + FlowEdge* const falseEdge = fgAddRefPred(target->GetFalseTarget(), block, target->GetFalseEdge()); + block->SetCond(trueEdge, falseEdge); - JITDUMP("fgOptimizeUncondBranchToSimpleCond(from " FMT_BB " to cond " FMT_BB "), created new uncond " FMT_BB "\n", - block->bbNum, target->bbNum, next->bbNum); + JITDUMP("fgOptimizeUncondBranchToSimpleCond(from " FMT_BB " to cond " FMT_BB "), modified " FMT_BB "\n", + block->bbNum, target->bbNum, block->bbNum); JITDUMP(" expecting opts to key off V%02u in " FMT_BB "\n", lclNum, block->bbNum); + if (target->hasProfileWeight() && block->hasProfileWeight()) + { + // Remove weight from target since block now bypasses it... + // + weight_t targetWeight = target->bbWeight; + weight_t blockWeight = block->bbWeight; + target->setBBProfileWeight(max(0, targetWeight - blockWeight)); + JITDUMP("Decreased " FMT_BB " profile weight from " FMT_WT " to " FMT_WT "\n", target->bbNum, targetWeight, + target->bbWeight); + } + return true; } @@ -2504,7 +2516,7 @@ bool Compiler::fgOptimizeUncondBranchToSimpleCond(BasicBlock* block, BasicBlock* void Compiler::fgRemoveConditionalJump(BasicBlock* block) { assert(block->KindIs(BBJ_COND)); - assert(block->TrueTargetIs(block->GetFalseTarget())); + assert(block->TrueEdgeIs(block->GetFalseEdge())); BasicBlock* target = block->GetTrueTarget(); @@ -2624,7 +2636,7 @@ void Compiler::fgRemoveConditionalJump(BasicBlock* block) * block are counted twice so we have to remove one of them */ noway_assert(target->countOfInEdges() > 1); - fgRemoveRefPred(target, block); + fgRemoveRefPred(block->GetTargetEdge()); } //------------------------------------------------------------- @@ -2884,13 +2896,11 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) // We need to update the following flags of the bJump block if they were set in the bDest block bJump->CopyFlags(bDest, BBF_COPY_PROPAGATE); - bJump->SetCond(bDestNormalTarget, bJump->Next()); - /* Update bbRefs and bbPreds */ // bJump now falls through into the next block // - fgAddRefPred(bJump->GetFalseTarget(), bJump); + FlowEdge* const falseEdge = fgAddRefPred(bJump->Next(), bJump); // bJump no longer jumps to bDest // @@ -2898,7 +2908,9 @@ bool Compiler::fgOptimizeBranch(BasicBlock* bJump) // bJump now jumps to bDest's normal jump target // - fgAddRefPred(bDestNormalTarget, bJump); + FlowEdge* const trueEdge = fgAddRefPred(bDestNormalTarget, bJump); + + bJump->SetCond(trueEdge, falseEdge); if (weightJump > 0) { @@ -3044,11 +3056,9 @@ bool Compiler::fgOptimizeSwitchJumps() // Wire up the new control flow. // - block->SetCond(dominantTarget, newBlock); FlowEdge* const blockToTargetEdge = fgAddRefPred(dominantTarget, block); FlowEdge* const blockToNewBlockEdge = newBlock->bbPreds; - assert(blockToNewBlockEdge->getSourceBlock() == block); - assert(blockToTargetEdge->getSourceBlock() == block); + block->SetCond(blockToTargetEdge, blockToNewBlockEdge); // Update profile data // @@ -3517,11 +3527,11 @@ bool Compiler::fgReorderBlocks(bool useProfile) assert(test->OperIsConditionalJump()); test->AsOp()->gtOp1 = gtReverseCond(test->AsOp()->gtOp1); - BasicBlock* newFalseTarget = block->GetTrueTarget(); - BasicBlock* newTrueTarget = block->GetFalseTarget(); - block->SetTrueTarget(newTrueTarget); - block->SetFalseTarget(newFalseTarget); - assert(block->CanRemoveJumpToTarget(newFalseTarget, this)); + FlowEdge* const newFalseEdge = block->GetTrueEdge(); + FlowEdge* const newTrueEdge = block->GetFalseEdge(); + block->SetTrueEdge(newTrueEdge); + block->SetFalseEdge(newFalseEdge); + assert(block->CanRemoveJumpToTarget(block->GetFalseTarget(), this)); } else { @@ -4578,10 +4588,10 @@ bool Compiler::fgReorderBlocks(bool useProfile) noway_assert(condTest->gtOper == GT_JTRUE); condTest->AsOp()->gtOp1 = gtReverseCond(condTest->AsOp()->gtOp1); - BasicBlock* trueTarget = bPrev->GetTrueTarget(); - BasicBlock* falseTarget = bPrev->GetFalseTarget(); - bPrev->SetTrueTarget(falseTarget); - bPrev->SetFalseTarget(trueTarget); + FlowEdge* const trueEdge = bPrev->GetTrueEdge(); + FlowEdge* const falseEdge = bPrev->GetFalseEdge(); + bPrev->SetTrueEdge(falseEdge); + bPrev->SetFalseEdge(trueEdge); // may need to rethread // @@ -4817,13 +4827,11 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh if (doTailDuplication && fgOptimizeUncondBranchToSimpleCond(block, bDest)) { assert(block->KindIs(BBJ_COND)); - change = true; - modified = true; - bDest = block->GetTrueTarget(); - bNext = block->GetFalseTarget(); - - // TODO-NoFallThrough: Adjust the above logic once bbFalseTarget can diverge from bbNext - assert(block->NextIs(bNext)); + assert(bNext == block->Next()); + change = true; + modified = true; + bDest = block->GetTrueTarget(); + bFalseDest = block->GetFalseTarget(); } } @@ -4984,13 +4992,15 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh if (bDest->KindIs(BBJ_COND) && !bDest->NextIs(bDest->GetFalseTarget())) { BasicBlock* const bDestFalseTarget = bDest->GetFalseTarget(); - BasicBlock* const bFixup = fgNewBBafter(BBJ_ALWAYS, bDest, true, bDestFalseTarget); - bDest->SetFalseTarget(bFixup); + BasicBlock* const bFixup = fgNewBBafter(BBJ_ALWAYS, bDest, true); bFixup->inheritWeight(bDestFalseTarget); fgRemoveRefPred(bDestFalseTarget, bDest); - fgAddRefPred(bFixup, bDest); - fgAddRefPred(bDestFalseTarget, bFixup); + FlowEdge* const falseEdge = fgAddRefPred(bFixup, bDest); + bDest->SetFalseEdge(falseEdge); + + FlowEdge* const newEdge = fgAddRefPred(bDestFalseTarget, bFixup); + bFixup->SetTargetEdge(newEdge); } } } @@ -5018,10 +5028,11 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh } // Optimize the Conditional JUMP to go to the new target - block->SetTrueTarget(bNext->GetTarget()); - block->SetFalseTarget(bNext->Next()); - - fgAddRefPred(bNext->GetTarget(), block, fgRemoveRefPred(bNext->GetTarget(), bNext)); + fgRemoveRefPred(block->GetFalseEdge()); + fgRemoveRefPred(bNext->GetTargetEdge()); + block->SetFalseEdge(block->GetTrueEdge()); + FlowEdge* const newEdge = fgAddRefPred(bNext->GetTarget(), block, bNext->GetTargetEdge()); + block->SetTrueEdge(newEdge); /* Unlink bNext from the BasicBlock list; note that we can @@ -5033,7 +5044,6 @@ bool Compiler::fgUpdateFlowGraph(bool doTailDuplication /* = false */, bool isPh to the final target by the time we're done here. */ - fgRemoveRefPred(bNext, block); fgUnlinkBlockForRemoval(bNext); /* Mark the block as removed */ @@ -5666,13 +5676,13 @@ PhaseStatus Compiler::fgHeadTailMerge(bool early) // Fix up the flow. // - predBlock->SetKindAndTarget(BBJ_ALWAYS, crossJumpTarget); - if (commSucc != nullptr) { fgRemoveRefPred(commSucc, predBlock); } - fgAddRefPred(crossJumpTarget, predBlock); + + FlowEdge* const newEdge = fgAddRefPred(crossJumpTarget, predBlock); + predBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } // We changed things @@ -5841,7 +5851,7 @@ bool Compiler::fgTryOneHeadMerge(BasicBlock* block, bool early) // ternaries in C#). // The logic below could be generalized to BBJ_SWITCH, but this currently // has almost no CQ benefit but does have a TP impact. - if (!block->KindIs(BBJ_COND) || block->TrueTargetIs(block->GetFalseTarget())) + if (!block->KindIs(BBJ_COND) || block->TrueEdgeIs(block->GetFalseEdge())) { return false; } diff --git a/src/coreclr/jit/fgprofile.cpp b/src/coreclr/jit/fgprofile.cpp index a1c9da833fd3f..656f2dec8c188 100644 --- a/src/coreclr/jit/fgprofile.cpp +++ b/src/coreclr/jit/fgprofile.cpp @@ -507,12 +507,12 @@ void BlockCountInstrumentor::RelocateProbes() // if (criticalPreds.Height() > 0) { - BasicBlock* const intermediary = - m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true, /* jumpDest */ block); + BasicBlock* const intermediary = m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true); intermediary->SetFlags(BBF_IMPORTED | BBF_MARKED | BBF_NONE_QUIRK); intermediary->inheritWeight(block); FlowEdge* const newEdge = m_comp->fgAddRefPred(block, intermediary); newEdge->setLikelihood(1.0); + intermediary->SetTargetEdge(newEdge); SetModifiedFlow(); while (criticalPreds.Height() > 0) @@ -1679,12 +1679,12 @@ void EfficientEdgeCountInstrumentor::RelocateProbes() // if (criticalPreds.Height() > 0) { - BasicBlock* intermediary = - m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true, /* jumpDest */ block); + BasicBlock* intermediary = m_comp->fgNewBBbefore(BBJ_ALWAYS, block, /* extendRegion */ true); intermediary->SetFlags(BBF_IMPORTED | BBF_NONE_QUIRK); intermediary->inheritWeight(block); FlowEdge* const newEdge = m_comp->fgAddRefPred(block, intermediary); newEdge->setLikelihood(1.0); + intermediary->SetTargetEdge(newEdge); NewRelocatedProbe(intermediary, probe->source, probe->target, &leader); SetModifiedFlow(); @@ -3862,18 +3862,22 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, { // We expect one pseudo-edge and at least one normal edge. // - Edge* pseudoEdge = nullptr; - unsigned nEdges = 0; + Edge* pseudoEdge = nullptr; + weight_t pseudoEdgeWeight = 0; + unsigned nEdges = 0; + weight_t successorWeight = BB_ZERO_WEIGHT; for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { if (edge->m_isPseudoEdge) { assert(pseudoEdge == nullptr); - pseudoEdge = edge; + pseudoEdge = edge; + pseudoEdgeWeight = edge->m_weight; continue; } + successorWeight += edge->m_weight; nEdges++; } @@ -3890,28 +3894,25 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, assert(nEdges == nSucc); - if (info->m_weight == BB_ZERO_WEIGHT) + if ((info->m_weight == BB_ZERO_WEIGHT) || (successorWeight == BB_ZERO_WEIGHT)) { - JITDUMP("\nPropagate: OSR entry block weight is zero\n"); + JITDUMP("\nPropagate: OSR entry block or successor weight is zero\n"); EntryWeightZero(); return; } // Transfer model edge weight onto the FlowEdges as likelihoods. // - assert(nEdges == nSucc); - weight_t totalLikelihood = 0; + JITDUMP("Normalizing OSR successor likelihoods with factor 1/" FMT_WT "\n", successorWeight); for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { assert(block == edge->m_sourceBlock); - // The pseudo edge doesn't correspond to a flow edge, - // but it carries away some flow. + // The pseudo edge doesn't correspond to a flow edge. // if (edge == pseudoEdge) { - totalLikelihood += edge->m_weight / info->m_weight; continue; } @@ -3927,51 +3928,19 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, if (nEdges == 1) { - // Conceptually we could assert(edge->m_weight == info->m_weight); - // but we can have inconsistencies. - // // Go with what we know for sure, edge should be 100% likely. // likelihood = 1.0; JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (uniq)\n", block->bbNum, edge->m_targetBlock->bbNum, likelihood); flowEdge->setLikelihood(likelihood); - totalLikelihood += likelihood; break; } - assert(info->m_weight != BB_ZERO_WEIGHT); - - // We may see nonsensical weights here, cap likelihood. - // - bool capped = false; - if (edge->m_weight > info->m_weight) - { - capped = true; - likelihood = 1.0; - } - else - { - likelihood = edge->m_weight / info->m_weight; - } - JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (%s)\n", block->bbNum, - edge->m_targetBlock->bbNum, likelihood, capped ? "pgo -- capped" : "pgo"); + likelihood = edge->m_weight / successorWeight; + JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (pgo)\n", block->bbNum, + edge->m_targetBlock->bbNum, likelihood); flowEdge->setLikelihood(likelihood); - totalLikelihood += likelihood; - } - - // Note we expect real flow imbalances here as it's likely there - // was no observed flow from the OSR entry to some of its successors. - // Since we added in the pseudo edge likelihood above, the check below - // probably won't flag this. - // - // Seems like for OSR we will always want to run synthesis/repair. - // - if (totalLikelihood != 1.0) - { - // Consider what to do here... flag this method as needing immediate profile repairs? - // - JITDUMP(FMT_BB " total outgoing likelihood inaccurate: " FMT_WT "\n", block->bbNum, totalLikelihood); } } @@ -3984,10 +3953,6 @@ void EfficientEdgeCountReconstructor::PropagateOSREntryEdges(BasicBlock* block, // info - model info for the block // nSucc - number of successors of the block in the flow graph // -// Notes: -// This block requires special handling because original method flow -// was interrupted here. -// void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInfo* info, unsigned nSucc) { // There is at least one FlowEdge. @@ -3995,8 +3960,9 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // Check the reconstruction graph edges. For normal blocks, if we have // any pseudo-edges there should be only one pseudo-edge, and no regular edges. // - Edge* pseudoEdge = nullptr; - unsigned nEdges = 0; + Edge* pseudoEdge = nullptr; + unsigned nEdges = 0; + weight_t successorWeight = BB_ZERO_WEIGHT; for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { @@ -4007,6 +3973,7 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf continue; } + successorWeight += edge->m_weight; nEdges++; } @@ -4033,7 +4000,7 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // // This can happen because bome BBJ_LEAVE blocks may have been missed during // our spanning tree walk since we don't know where all the finallies can return - // to just yet (specially, in WalkSpanningTree, we may not add the bbTarget of + // to just yet (specially, in WalkSpanningTree, we may not add the target of // a BBJ_LEAVE to the worklist). // // Worst case those missed blocks dominate other blocks so we can't limit @@ -4046,19 +4013,19 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // // (TODO: use synthesis here) // - if ((nEdges != nSucc) || (info->m_weight == BB_ZERO_WEIGHT)) + if ((nEdges != nSucc) || (info->m_weight == BB_ZERO_WEIGHT) || (successorWeight == BB_ZERO_WEIGHT)) { JITDUMP(FMT_BB " %s , setting outgoing likelihoods heuristically\n", block->bbNum, (nEdges != nSucc) ? "has inaccurate flow model" : "has zero weight"); weight_t equalLikelihood = 1.0 / nSucc; - for (BasicBlock* succ : block->Succs(m_comp)) + for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) { - FlowEdge* const flowEdge = m_comp->fgGetPredForBlock(succ, block); - JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (heur)\n", block->bbNum, succ->bbNum, - equalLikelihood); - flowEdge->setLikelihood(equalLikelihood); + BasicBlock* const succBlock = succEdge->getDestinationBlock(); + JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (heur)\n", block->bbNum, + succBlock->bbNum, equalLikelihood); + succEdge->setLikelihood(equalLikelihood); } return; @@ -4067,7 +4034,7 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf // Transfer model edge weight onto the FlowEdges as likelihoods. // assert(nEdges == nSucc); - weight_t totalLikelihood = 0; + JITDUMP("Normalizing successor likelihoods with factor 1/" FMT_WT "\n", successorWeight); for (Edge* edge = info->m_outgoingEdges; edge != nullptr; edge = edge->m_nextOutgoingEdge) { @@ -4079,45 +4046,17 @@ void EfficientEdgeCountReconstructor::PropagateEdges(BasicBlock* block, BlockInf if (nEdges == 1) { assert(nSucc == 1); - - // Conceptually we could assert(edge->m_weight == info->m_weight); - // but we can have inconsistencies. - // - // Go with what we know for sure, edge should be 100% likely. - // likelihood = 1.0; JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (uniq)\n", block->bbNum, edge->m_targetBlock->bbNum, likelihood); flowEdge->setLikelihood(likelihood); - totalLikelihood += likelihood; break; } - assert(info->m_weight != BB_ZERO_WEIGHT); - - // We may see nonsensical weights here, cap likelihood. - // - bool capped = false; - if (edge->m_weight > info->m_weight) - { - capped = true; - likelihood = 1.0; - } - else - { - likelihood = edge->m_weight / info->m_weight; - } - JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (%s)\n", block->bbNum, - edge->m_targetBlock->bbNum, likelihood, capped ? "pgo -- capped" : "pgo"); + likelihood = edge->m_weight / successorWeight; + JITDUMP("Setting likelihood of " FMT_BB " -> " FMT_BB " to " FMT_WT " (pgo)\n", block->bbNum, + edge->m_targetBlock->bbNum, likelihood); flowEdge->setLikelihood(likelihood); - totalLikelihood += likelihood; - } - - if (totalLikelihood != 1.0) - { - // Consider what to do here... flag this method as needing immediate profile repairs? - // - JITDUMP(FMT_BB " total outgoing likelihood inaccurate: " FMT_WT "\n", block->bbNum, totalLikelihood); } } @@ -5296,7 +5235,8 @@ void Compiler::fgDebugCheckProfileWeights() } else { - ProfileChecks checks = ProfileChecks::CHECK_HASLIKELIHOOD | ProfileChecks::RAISE_ASSERT; + ProfileChecks checks = + ProfileChecks::CHECK_HASLIKELIHOOD | ProfileChecks::CHECK_LIKELIHOODSUM | ProfileChecks::RAISE_ASSERT; fgDebugCheckProfileWeights(checks); } } @@ -5328,6 +5268,7 @@ void Compiler::fgDebugCheckProfileWeights(ProfileChecks checks) const bool verifyClassicWeights = fgEdgeWeightsComputed && hasFlag(checks, ProfileChecks::CHECK_CLASSIC); const bool verifyLikelyWeights = hasFlag(checks, ProfileChecks::CHECK_LIKELY); const bool verifyHasLikelihood = hasFlag(checks, ProfileChecks::CHECK_HASLIKELIHOOD); + const bool verifyLikelihoodSum = hasFlag(checks, ProfileChecks::CHECK_LIKELIHOODSUM); const bool assertOnFailure = hasFlag(checks, ProfileChecks::RAISE_ASSERT); const bool checkAllBlocks = hasFlag(checks, ProfileChecks::CHECK_ALL_BLOCKS); @@ -5478,6 +5419,10 @@ void Compiler::fgDebugCheckProfileWeights(ProfileChecks checks) JITDUMP("Profile is self-consistent (%d profiled blocks, %d unprofiled)\n", profiledBlocks, unprofiledBlocks); } + else if (verifyLikelihoodSum) + { + JITDUMP("All block successor flow edge likelihoods sum to 1.0\n"); + } else if (verifyHasLikelihood) { JITDUMP("All flow edges have likelihoods\n"); @@ -5618,10 +5563,10 @@ bool Compiler::fgDebugCheckIncomingProfileData(BasicBlock* block, ProfileChecks bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks checks) { const bool verifyClassicWeights = fgEdgeWeightsComputed && hasFlag(checks, ProfileChecks::CHECK_CLASSIC); - const bool verifyLikelyWeights = hasFlag(checks, ProfileChecks::CHECK_LIKELY); const bool verifyHasLikelihood = hasFlag(checks, ProfileChecks::CHECK_HASLIKELIHOOD); + const bool verifyLikelihoodSum = hasFlag(checks, ProfileChecks::CHECK_LIKELIHOODSUM); - if (!(verifyClassicWeights || verifyLikelyWeights || verifyHasLikelihood)) + if (!(verifyClassicWeights || verifyHasLikelihood || verifyLikelihoodSum)) { return true; } @@ -5645,17 +5590,10 @@ bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks unsigned missingEdges = 0; unsigned missingLikelihood = 0; - for (unsigned i = 0; i < numSuccs; i++) + for (FlowEdge* succEdge : block->SuccEdges(this)) { - BasicBlock* succBlock = block->GetSucc(i, this); - FlowEdge* succEdge = fgGetPredForBlock(succBlock, block); - - if (succEdge == nullptr) - { - missingEdges++; - JITDUMP(" " FMT_BB " can't find successor edge to " FMT_BB "\n", block->bbNum, succBlock->bbNum); - continue; - } + assert(succEdge != nullptr); + BasicBlock* succBlock = succEdge->getDestinationBlock(); outgoingWeightMin += succEdge->edgeWeightMin(); outgoingWeightMax += succEdge->edgeWeightMax(); @@ -5711,7 +5649,7 @@ bool Compiler::fgDebugCheckOutgoingProfileData(BasicBlock* block, ProfileChecks } } - if (verifyLikelyWeights) + if (verifyLikelihoodSum) { if (!fgProfileWeightsConsistent(outgoingLikelihood, 1.0)) { diff --git a/src/coreclr/jit/fgprofilesynthesis.cpp b/src/coreclr/jit/fgprofilesynthesis.cpp index e315e33015e13..d4f1c2ffe3f40 100644 --- a/src/coreclr/jit/fgprofilesynthesis.cpp +++ b/src/coreclr/jit/fgprofilesynthesis.cpp @@ -190,7 +190,7 @@ void ProfileSynthesis::AssignLikelihoodNext(BasicBlock* block) //------------------------------------------------------------------------ // AssignLikelihoodJump: update edge likelihood for a block that always -// transfers control to bbTarget +// transfers control to its target block // // Arguments; // block -- block in question @@ -342,10 +342,9 @@ void ProfileSynthesis::AssignLikelihoodSwitch(BasicBlock* block) // Each unique edge gets some multiple of that basic probability // - for (BasicBlock* const succ : block->Succs(m_comp)) + for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) { - FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); - edge->setLikelihood(p * edge->getDupCount()); + succEdge->setLikelihood(p * succEdge->getDupCount()); } } @@ -368,10 +367,9 @@ weight_t ProfileSynthesis::SumOutgoingLikelihoods(BasicBlock* block, WeightVecto likelihoods->clear(); } - for (BasicBlock* const succ : block->Succs(m_comp)) + for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) { - FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); - weight_t likelihood = edge->getLikelihood(); + weight_t likelihood = succEdge->getLikelihood(); if (likelihoods != nullptr) { likelihoods->push_back(likelihood); @@ -560,15 +558,15 @@ void ProfileSynthesis::BlendLikelihoods() JITDUMP("Blending likelihoods in " FMT_BB " with blend factor " FMT_WT " \n", block->bbNum, blendFactor); iter = likelihoods.begin(); - for (BasicBlock* const succ : block->Succs(m_comp)) + for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) { - FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); - weight_t newLikelihood = edge->getLikelihood(); - weight_t oldLikelihood = *iter; + weight_t newLikelihood = succEdge->getLikelihood(); + weight_t oldLikelihood = *iter; - edge->setLikelihood((blendFactor * oldLikelihood) + ((1.0 - blendFactor) * newLikelihood)); - JITDUMP(FMT_BB " -> " FMT_BB " was " FMT_WT " now " FMT_WT "\n", block->bbNum, succ->bbNum, - oldLikelihood, edge->getLikelihood()); + succEdge->setLikelihood((blendFactor * oldLikelihood) + ((1.0 - blendFactor) * newLikelihood)); + BasicBlock* const succBlock = succEdge->getDestinationBlock(); + JITDUMP(FMT_BB " -> " FMT_BB " was " FMT_WT " now " FMT_WT "\n", block->bbNum, succBlock->bbNum, + oldLikelihood, succEdge->getLikelihood()); iter++; } @@ -588,10 +586,9 @@ void ProfileSynthesis::ClearLikelihoods() { for (BasicBlock* const block : m_comp->Blocks()) { - for (BasicBlock* const succ : block->Succs(m_comp)) + for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) { - FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); - edge->clearLikelihood(); + succEdge->clearLikelihood(); } } } @@ -664,10 +661,9 @@ void ProfileSynthesis::RandomizeLikelihoods() } i = 0; - for (BasicBlock* const succ : block->Succs(m_comp)) + for (FlowEdge* const succEdge : block->SuccEdges(m_comp)) { - FlowEdge* const edge = m_comp->fgGetPredForBlock(succ, block); - edge->setLikelihood(likelihoods[i++] / sum); + succEdge->setLikelihood(likelihoods[i++] / sum); } } #endif // DEBUG diff --git a/src/coreclr/jit/flowgraph.cpp b/src/coreclr/jit/flowgraph.cpp index c393648c843ef..b621739a41cdd 100644 --- a/src/coreclr/jit/flowgraph.cpp +++ b/src/coreclr/jit/flowgraph.cpp @@ -266,17 +266,11 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) // I want to create: // top -> poll -> bottom (lexically) // so that we jump over poll to get to bottom. - BasicBlock* top = block; - BBKinds oldJumpKind = top->GetKind(); + BasicBlock* top = block; BasicBlock* poll = fgNewBBafter(BBJ_ALWAYS, top, true); bottom = fgNewBBafter(top->GetKind(), poll, true); - poll->SetTarget(bottom); - assert(poll->JumpsToNext()); - - bottom->TransferTarget(top); - // Update block flags const BasicBlockFlags originalFlags = top->GetFlagsRaw() | BBF_GC_SAFE_POINT; @@ -300,7 +294,7 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) } // Remove the last statement from Top and add it to Bottom if necessary. - if ((oldJumpKind == BBJ_COND) || (oldJumpKind == BBJ_RETURN) || (oldJumpKind == BBJ_THROW)) + if (top->KindIs(BBJ_COND, BBJ_RETURN, BBJ_THROW)) { Statement* stmt = top->firstStmt(); while (stmt->GetNextStmt() != nullptr) @@ -364,38 +358,47 @@ BasicBlock* Compiler::fgCreateGCPoll(GCPollType pollType, BasicBlock* block) } #endif - top->SetCond(bottom, poll); // Bottom has Top and Poll as its predecessors. Poll has just Top as a predecessor. - fgAddRefPred(bottom, poll); - fgAddRefPred(bottom, top); - fgAddRefPred(poll, top); + FlowEdge* const trueEdge = fgAddRefPred(bottom, top); + FlowEdge* const falseEdge = fgAddRefPred(poll, top); + + FlowEdge* const newEdge = fgAddRefPred(bottom, poll); + poll->SetTargetEdge(newEdge); + assert(poll->JumpsToNext()); // Replace Top with Bottom in the predecessor list of all outgoing edges from Bottom // (1 for unconditional branches, 2 for conditional branches, N for switches). - switch (oldJumpKind) + switch (top->GetKind()) { case BBJ_RETURN: case BBJ_THROW: // no successors break; + case BBJ_COND: // replace predecessor in true/false successors. noway_assert(!bottom->IsLast()); - fgReplacePred(bottom->GetFalseTarget(), top, bottom); - fgReplacePred(bottom->GetTrueTarget(), top, bottom); + fgReplacePred(top->GetFalseEdge(), bottom); + fgReplacePred(top->GetTrueEdge(), bottom); break; case BBJ_ALWAYS: case BBJ_CALLFINALLY: - fgReplacePred(bottom->GetTarget(), top, bottom); + fgReplacePred(top->GetTargetEdge(), bottom); break; + case BBJ_SWITCH: NO_WAY("SWITCH should be a call rather than an inlined poll."); break; + default: NO_WAY("Unknown block type for updating predecessor lists."); + break; } + bottom->TransferTarget(top); + top->SetCond(trueEdge, falseEdge); + if (compCurBB == top) { compCurBB = bottom; @@ -1625,9 +1628,9 @@ void Compiler::fgConvertSyncReturnToLeave(BasicBlock* block) assert(ehDsc->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX); // Convert the BBJ_RETURN to BBJ_ALWAYS, jumping to genReturnBB. - block->SetKindAndTarget(BBJ_ALWAYS, genReturnBB); FlowEdge* const newEdge = fgAddRefPred(genReturnBB, block); newEdge->setLikelihood(1.0); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); #ifdef DEBUG if (verbose) @@ -2097,9 +2100,9 @@ class MergedReturns // Change BBJ_RETURN to BBJ_ALWAYS targeting const return block. assert((comp->info.compFlags & CORINFO_FLG_SYNCH) == 0); - returnBlock->SetKindAndTarget(BBJ_ALWAYS, constReturnBlock); FlowEdge* const newEdge = comp->fgAddRefPred(constReturnBlock, returnBlock); newEdge->setLikelihood(1.0); + returnBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // Remove GT_RETURN since constReturnBlock returns the constant. assert(returnBlock->lastStmt()->GetRootNode()->OperIs(GT_RETURN)); @@ -2758,15 +2761,14 @@ void Compiler::fgInsertFuncletPrologBlock(BasicBlock* block) /* Allocate a new basic block */ - BasicBlock* newHead = BasicBlock::New(this, BBJ_ALWAYS, block); + BasicBlock* newHead = BasicBlock::New(this); newHead->SetFlags(BBF_INTERNAL | BBF_NONE_QUIRK); newHead->inheritWeight(block); newHead->bbRefs = 0; fgInsertBBbefore(block, newHead); // insert the new block in the block list - assert(newHead->JumpsToNext()); - fgExtendEHRegionBefore(block); // Update the EH table to make the prolog block the first block in the block's EH - // block. + fgExtendEHRegionBefore(block); // Update the EH table to make the prolog block the first block in the block's EH + // block. // Distribute the pred list between newHead and block. Incoming edges coming from outside // the handler go to the prolog. Edges coming from with the handler are back-edges, and @@ -2782,11 +2784,13 @@ void Compiler::fgInsertFuncletPrologBlock(BasicBlock* block) switch (predBlock->GetKind()) { case BBJ_CALLFINALLY: + { noway_assert(predBlock->TargetIs(block)); - predBlock->SetTarget(newHead); - fgRemoveRefPred(block, predBlock); - fgAddRefPred(newHead, predBlock); + fgRemoveRefPred(predBlock->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(newHead, predBlock); + predBlock->SetTargetEdge(newEdge); break; + } default: // The only way into the handler is via a BBJ_CALLFINALLY (to a finally handler), or @@ -2797,10 +2801,10 @@ void Compiler::fgInsertFuncletPrologBlock(BasicBlock* block) } } - assert(nullptr == fgGetPredForBlock(block, newHead)); - fgAddRefPred(block, newHead); - - assert(newHead->HasFlag(BBF_INTERNAL)); + assert(fgGetPredForBlock(block, newHead) == nullptr); + FlowEdge* const newEdge = fgAddRefPred(block, newHead); + newHead->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); + assert(newHead->JumpsToNext()); } //------------------------------------------------------------------------ @@ -3374,7 +3378,7 @@ PhaseStatus Compiler::fgCreateThrowHelperBlocks() assert((add->acdKind == SCK_FAIL_FAST) || (bbThrowIndex(srcBlk) == add->acdData)); assert(add->acdKind != SCK_NONE); - BasicBlock* const newBlk = fgNewBBinRegion(jumpKinds[add->acdKind], srcBlk, /* jumpDest */ nullptr, + BasicBlock* const newBlk = fgNewBBinRegion(jumpKinds[add->acdKind], srcBlk, /* runRarely */ true, /* insertAtEnd */ true); // Update the descriptor @@ -3438,7 +3442,7 @@ PhaseStatus Compiler::fgCreateThrowHelperBlocks() #endif // DEBUG // Mark the block as added by the compiler and not removable by future flow - // graph optimizations. Note that no bbTarget points to these blocks. + // graph optimizations. Note that no target block points to these blocks. // newBlk->SetFlags(BBF_IMPORTED | BBF_DONT_REMOVE); diff --git a/src/coreclr/jit/gcencode.cpp b/src/coreclr/jit/gcencode.cpp index 96005e6057662..a942cc24a9af2 100644 --- a/src/coreclr/jit/gcencode.cpp +++ b/src/coreclr/jit/gcencode.cpp @@ -1564,7 +1564,7 @@ size_t GCInfo::gcInfoBlockHdrSave( header->syncStartOffset = INVALID_SYNC_OFFSET; header->syncEndOffset = INVALID_SYNC_OFFSET; -#ifndef UNIX_X86_ABI +#if !defined(FEATURE_EH_FUNCLETS) // JIT is responsible for synchronization on funclet-based EH model that x86/Linux uses. if (compiler->info.compFlags & CORINFO_FLG_SYNCH) { diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 7194fbeab6325..7b96d73cef95e 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -252,7 +252,6 @@ void GenTree::InitNodeSize() GenTree::s_gtNodeSizes[GT_FIELD_ADDR] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_CMPXCHG] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_QMARK] = TREE_NODE_SZ_LARGE; - GenTree::s_gtNodeSizes[GT_STORE_DYN_BLK] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_INTRINSIC] = TREE_NODE_SZ_LARGE; GenTree::s_gtNodeSizes[GT_ALLOCOBJ] = TREE_NODE_SZ_LARGE; #if USE_HELPERS_FOR_INT_DIV @@ -318,7 +317,6 @@ void GenTree::InitNodeSize() static_assert_no_msg(sizeof(GenTreeStoreInd) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeAddrMode) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreeBlk) <= TREE_NODE_SZ_SMALL); - static_assert_no_msg(sizeof(GenTreeStoreDynBlk) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeRetExpr) <= TREE_NODE_SZ_LARGE); // *** large node static_assert_no_msg(sizeof(GenTreeILOffset) <= TREE_NODE_SZ_SMALL); static_assert_no_msg(sizeof(GenTreePhiArg) <= TREE_NODE_SZ_SMALL); @@ -3151,11 +3149,6 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK) Compare(op1->AsCmpXchg()->Data(), op2->AsCmpXchg()->Data()) && Compare(op1->AsCmpXchg()->Comparand(), op2->AsCmpXchg()->Comparand()); - case GT_STORE_DYN_BLK: - return Compare(op1->AsStoreDynBlk()->Addr(), op2->AsStoreDynBlk()->Addr()) && - Compare(op1->AsStoreDynBlk()->Data(), op2->AsStoreDynBlk()->Data()) && - Compare(op1->AsStoreDynBlk()->gtDynamicSize, op2->AsStoreDynBlk()->gtDynamicSize); - default: assert(!"unexpected operator"); } @@ -3704,12 +3697,6 @@ unsigned Compiler::gtHashValue(GenTree* tree) hash = genTreeHashAdd(hash, gtHashValue(tree->AsCmpXchg()->Comparand())); break; - case GT_STORE_DYN_BLK: - hash = genTreeHashAdd(hash, gtHashValue(tree->AsStoreDynBlk()->Data())); - hash = genTreeHashAdd(hash, gtHashValue(tree->AsStoreDynBlk()->Addr())); - hash = genTreeHashAdd(hash, gtHashValue(tree->AsStoreDynBlk()->gtDynamicSize)); - break; - default: #ifdef DEBUG gtDispTree(tree); @@ -4520,12 +4507,6 @@ bool Compiler::gtGetIndNodeCost(GenTreeIndir* node, int* pCostEx, int* pCostSz) { // See if we can form a complex addressing mode. bool doAddrMode = true; - - // TODO-1stClassStructs: delete once IND nodes are no more. - if (node->TypeGet() == TYP_STRUCT) - { - doAddrMode = false; - } #ifdef TARGET_ARM64 if (node->IsVolatile()) { @@ -6385,22 +6366,6 @@ unsigned Compiler::gtSetEvalOrder(GenTree* tree) } break; - case GT_STORE_DYN_BLK: - level = gtSetEvalOrder(tree->AsStoreDynBlk()->Addr()); - costEx = tree->AsStoreDynBlk()->Addr()->GetCostEx(); - costSz = tree->AsStoreDynBlk()->Addr()->GetCostSz(); - - lvl2 = gtSetEvalOrder(tree->AsStoreDynBlk()->Data()); - level = max(level, lvl2); - costEx += tree->AsStoreDynBlk()->Data()->GetCostEx(); - costSz += tree->AsStoreDynBlk()->Data()->GetCostSz(); - - lvl2 = gtSetEvalOrder(tree->AsStoreDynBlk()->gtDynamicSize); - level = max(level, lvl2); - costEx += tree->AsStoreDynBlk()->gtDynamicSize->GetCostEx(); - costSz += tree->AsStoreDynBlk()->gtDynamicSize->GetCostSz(); - break; - case GT_SELECT: level = gtSetEvalOrder(tree->AsConditional()->gtCond); costEx = tree->AsConditional()->gtCond->GetCostEx(); @@ -6847,27 +6812,6 @@ bool GenTree::TryGetUse(GenTree* operand, GenTree*** pUse) return false; } - case GT_STORE_DYN_BLK: - { - GenTreeStoreDynBlk* const dynBlock = this->AsStoreDynBlk(); - if (operand == dynBlock->gtOp1) - { - *pUse = &dynBlock->gtOp1; - return true; - } - if (operand == dynBlock->gtOp2) - { - *pUse = &dynBlock->gtOp2; - return true; - } - if (operand == dynBlock->gtDynamicSize) - { - *pUse = &dynBlock->gtDynamicSize; - return true; - } - return false; - } - case GT_CALL: { GenTreeCall* const call = this->AsCall(); @@ -7027,7 +6971,6 @@ bool GenTree::OperRequiresAsgFlag() const case GT_STORE_LCL_FLD: case GT_STOREIND: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_XADD: case GT_XORR: case GT_XAND: @@ -7067,6 +7010,9 @@ bool GenTree::OperRequiresCallFlag(Compiler* comp) const case GT_KEEPALIVE: return true; + case GT_SWIFT_ERROR: + return true; + case GT_INTRINSIC: return comp->IsIntrinsicImplementedByUserCall(this->AsIntrinsic()->gtIntrinsicName); @@ -7137,7 +7083,6 @@ bool GenTree::OperIsImplicitIndir() const case GT_CMPXCHG: case GT_BLK: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_BOX: case GT_ARR_ELEM: case GT_ARR_LENGTH: @@ -7234,7 +7179,6 @@ ExceptionSetFlags GenTree::OperExceptions(Compiler* comp) case GT_BLK: case GT_NULLCHECK: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_ARR_LENGTH: case GT_MDARR_LENGTH: case GT_MDARR_LOWER_BOUND: @@ -7354,7 +7298,6 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const case GT_STOREIND: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_XADD: case GT_XORR: case GT_XAND: @@ -7363,6 +7306,7 @@ bool GenTree::OperRequiresGlobRefFlag(Compiler* comp) const case GT_CMPXCHG: case GT_MEMORYBARRIER: case GT_KEEPALIVE: + case GT_SWIFT_ERROR: return true; case GT_CALL: @@ -7412,7 +7356,6 @@ bool GenTree::OperSupportsOrderingSideEffect() const case GT_STOREIND: case GT_NULLCHECK: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_XADD: case GT_XORR: case GT_XAND: @@ -7421,6 +7364,7 @@ bool GenTree::OperSupportsOrderingSideEffect() const case GT_CMPXCHG: case GT_MEMORYBARRIER: case GT_CATCH_ARG: + case GT_SWIFT_ERROR: return true; default: return false; @@ -8746,40 +8690,12 @@ GenTreeBlk* Compiler::gtNewStoreBlkNode(ClassLayout* layout, GenTree* addr, GenT return store; } -//------------------------------------------------------------------------------ -// gtNewStoreDynBlkNode : Create a dynamic block store node. -// -// Arguments: -// addr - Destination address -// data - Value to store (init val or indirection representing a location) -// dynamicSize - Node that computes number of bytes to store -// indirFlags - Indirection flags -// -// Return Value: -// The created GT_STORE_DYN_BLK node. -// -GenTreeStoreDynBlk* Compiler::gtNewStoreDynBlkNode(GenTree* addr, - GenTree* data, - GenTree* dynamicSize, - GenTreeFlags indirFlags) -{ - assert((indirFlags & GTF_IND_INVARIANT) == 0); - assert(data->IsInitVal() || data->OperIs(GT_IND)); - - GenTreeStoreDynBlk* store = new (this, GT_STORE_DYN_BLK) GenTreeStoreDynBlk(addr, data, dynamicSize); - store->gtFlags |= GTF_ASG; - gtInitializeIndirNode(store, indirFlags); - gtInitializeStoreNode(store, data); - - return store; -} - //------------------------------------------------------------------------------ // gtNewStoreIndNode : Create an indirect store node. // // Arguments: // type - Type of the store -// addr - Destionation address +// addr - Destination address // data - Value to store // indirFlags - Indirection flags // @@ -9811,12 +9727,6 @@ GenTree* Compiler::gtCloneExpr(GenTree* tree) gtCloneExpr(tree->AsCmpXchg()->Data()), gtCloneExpr(tree->AsCmpXchg()->Comparand())); break; - case GT_STORE_DYN_BLK: - copy = new (this, oper) GenTreeStoreDynBlk(gtCloneExpr(tree->AsStoreDynBlk()->Addr()), - gtCloneExpr(tree->AsStoreDynBlk()->Data()), - gtCloneExpr(tree->AsStoreDynBlk()->gtDynamicSize)); - break; - case GT_SELECT: copy = new (this, oper) GenTreeConditional(oper, tree->TypeGet(), gtCloneExpr(tree->AsConditional()->gtCond), @@ -10325,6 +10235,7 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) case GT_PINVOKE_EPILOG: case GT_IL_OFFSET: case GT_NOP: + case GT_SWIFT_ERROR: m_state = -1; return; @@ -10428,12 +10339,6 @@ GenTreeUseEdgeIterator::GenTreeUseEdgeIterator(GenTree* node) m_advance = &GenTreeUseEdgeIterator::AdvanceArrElem; return; - case GT_STORE_DYN_BLK: - m_edge = &m_node->AsStoreDynBlk()->Addr(); - assert(*m_edge != nullptr); - m_advance = &GenTreeUseEdgeIterator::AdvanceStoreDynBlk; - return; - case GT_CALL: m_statePtr = m_node->AsCall()->gtArgs.Args().begin().GetArg(); m_advance = &GenTreeUseEdgeIterator::AdvanceCall; @@ -10499,29 +10404,6 @@ void GenTreeUseEdgeIterator::AdvanceArrElem() } } -//------------------------------------------------------------------------ -// GenTreeUseEdgeIterator::AdvanceStoreDynBlk: produces the next operand of a StoreDynBlk node and advances the state. -// -void GenTreeUseEdgeIterator::AdvanceStoreDynBlk() -{ - GenTreeStoreDynBlk* const dynBlock = m_node->AsStoreDynBlk(); - switch (m_state) - { - case 0: - m_edge = &dynBlock->Data(); - m_state = 1; - break; - case 1: - m_edge = &dynBlock->gtDynamicSize; - m_advance = &GenTreeUseEdgeIterator::Terminate; - break; - default: - unreached(); - } - - assert(*m_edge != nullptr); -} - //------------------------------------------------------------------------ // GenTreeUseEdgeIterator::AdvanceFieldList: produces the next operand of a FieldList node and advances the state. // @@ -10877,7 +10759,7 @@ bool GenTree::Precedes(GenTree* other) // void GenTree::SetIndirExceptionFlags(Compiler* comp) { - assert(OperIsIndirOrArrMetaData() && (OperIsSimple() || OperIs(GT_CMPXCHG, GT_STORE_DYN_BLK))); + assert(OperIsIndirOrArrMetaData() && (OperIsSimple() || OperIs(GT_CMPXCHG))); if (IndirMayFault(comp)) { @@ -10899,11 +10781,6 @@ void GenTree::SetIndirExceptionFlags(Compiler* comp) gtFlags |= AsCmpXchg()->Data()->gtFlags & GTF_EXCEPT; gtFlags |= AsCmpXchg()->Comparand()->gtFlags & GTF_EXCEPT; } - else if (OperIs(GT_STORE_DYN_BLK)) - { - gtFlags |= AsStoreDynBlk()->Data()->gtFlags & GTF_EXCEPT; - gtFlags |= AsStoreDynBlk()->gtDynamicSize->gtFlags & GTF_EXCEPT; - } } #ifdef DEBUG @@ -11326,7 +11203,6 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ case GT_IND: case GT_STOREIND: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: // We prefer printing V or U if ((tree->gtFlags & (GTF_IND_VOLATILE | GTF_IND_UNALIGNED)) == 0) { @@ -12452,6 +12328,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) case GT_MEMORYBARRIER: case GT_PINVOKE_PROLOG: case GT_JMPTABLE: + case GT_SWIFT_ERROR: break; case GT_RET_EXPR: @@ -13153,28 +13030,6 @@ void Compiler::gtDispTree(GenTree* tree, } break; - case GT_STORE_DYN_BLK: - if (tree->OperIsCopyBlkOp()) - { - printf(" (copy)"); - } - else if (tree->OperIsInitBlkOp()) - { - printf(" (init)"); - } - gtDispCommonEndLine(tree); - - if (!topOnly) - { - gtDispChild(tree->AsStoreDynBlk()->Addr(), indentStack, IIArc, nullptr, topOnly); - if (tree->AsStoreDynBlk()->Data() != nullptr) - { - gtDispChild(tree->AsStoreDynBlk()->Data(), indentStack, IIArc, nullptr, topOnly); - } - gtDispChild(tree->AsStoreDynBlk()->gtDynamicSize, indentStack, IIArcBottom, nullptr, topOnly); - } - break; - case GT_SELECT: gtDispCommonEndLine(tree); @@ -13606,22 +13461,6 @@ void Compiler::gtDispLIRNode(GenTree* node, const char* prefixMsg /* = nullptr * displayOperand(operand, buf, operandArc, indentStack, prefixIndent); } } - else if (node->OperIs(GT_STORE_DYN_BLK)) - { - if (operand == node->AsBlk()->Addr()) - { - displayOperand(operand, "lhs", operandArc, indentStack, prefixIndent); - } - else if (operand == node->AsBlk()->Data()) - { - displayOperand(operand, "rhs", operandArc, indentStack, prefixIndent); - } - else - { - assert(operand == node->AsStoreDynBlk()->gtDynamicSize); - displayOperand(operand, "size", operandArc, indentStack, prefixIndent); - } - } else { displayOperand(operand, "", operandArc, indentStack, prefixIndent); @@ -16978,22 +16817,6 @@ bool Compiler::gtSplitTree( } private: - bool IsLocation(const UseInfo& useInf) - { - if (useInf.User == nullptr) - { - return false; - } - - if (useInf.User->OperIs(GT_STORE_DYN_BLK) && !(*useInf.Use)->OperIs(GT_CNS_INT, GT_INIT_VAL) && - (useInf.Use == &useInf.User->AsStoreDynBlk()->Data())) - { - return true; - } - - return false; - } - bool IsReturned(const UseInfo& useInf, bool userIsReturned) { if (useInf.User != nullptr) @@ -17087,18 +16910,6 @@ bool Compiler::gtSplitTree( return; } - if (IsLocation(useInf)) - { - // Only a handful of nodes can be location, and they are all unary or nullary. - assert((*use)->OperIs(GT_IND, GT_BLK, GT_LCL_VAR, GT_LCL_FLD)); - if ((*use)->OperIsUnary()) - { - SplitOutUse(UseInfo{&(*use)->AsUnOp()->gtOp1, user}, false); - } - - return; - } - #ifndef TARGET_64BIT // GT_MUL with GTF_MUL_64RSLT is required to stay with casts on the // operands. Note that one operand may also be a constant, but we diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 26ab5b2c705a3..e1f225a9bd8bf 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -194,13 +194,13 @@ inline AssertionIndex GetAssertionIndex(unsigned index) class AssertionInfo { - // true if the assertion holds on the bbNext edge instead of the bbTarget edge (for GT_JTRUE nodes) - unsigned short m_isNextEdgeAssertion : 1; + // true if the assertion holds on the false edge instead of the true edge (for GT_JTRUE nodes) + unsigned short m_assertionHoldsOnFalseEdge : 1; // 1-based index of the assertion unsigned short m_assertionIndex : 15; - AssertionInfo(bool isNextEdgeAssertion, AssertionIndex assertionIndex) - : m_isNextEdgeAssertion(isNextEdgeAssertion), m_assertionIndex(assertionIndex) + AssertionInfo(bool assertionHoldsOnFalseEdge, AssertionIndex assertionIndex) + : m_assertionHoldsOnFalseEdge(assertionHoldsOnFalseEdge), m_assertionIndex(assertionIndex) { assert(m_assertionIndex == assertionIndex); } @@ -223,8 +223,8 @@ class AssertionInfo void Clear() { - m_isNextEdgeAssertion = 0; - m_assertionIndex = NO_ASSERTION_INDEX; + m_assertionHoldsOnFalseEdge = 0; + m_assertionIndex = NO_ASSERTION_INDEX; } bool HasAssertion() const @@ -237,9 +237,9 @@ class AssertionInfo return m_assertionIndex; } - bool IsNextEdgeAssertion() const + bool AssertionHoldsOnFalseEdge() const { - return m_isNextEdgeAssertion; + return m_assertionHoldsOnFalseEdge; } }; @@ -1216,7 +1216,7 @@ struct GenTree static bool OperIsStoreBlk(genTreeOps gtOper) { - return StaticOperIs(gtOper, GT_STORE_BLK, GT_STORE_DYN_BLK); + return StaticOperIs(gtOper, GT_STORE_BLK); } bool OperIsStoreBlk() const @@ -1545,7 +1545,7 @@ struct GenTree static bool OperIsIndir(genTreeOps gtOper) { static_assert_no_msg(AreContiguous(GT_LOCKADD, GT_XAND, GT_XORR, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_IND, - GT_STOREIND, GT_BLK, GT_STORE_BLK, GT_STORE_DYN_BLK, GT_NULLCHECK)); + GT_STOREIND, GT_BLK, GT_STORE_BLK, GT_NULLCHECK)); return (GT_LOCKADD <= gtOper) && (gtOper <= GT_NULLCHECK); } @@ -2862,7 +2862,6 @@ class GenTreeUseEdgeIterator final // Advance functions for special nodes void AdvanceCmpXchg(); void AdvanceArrElem(); - void AdvanceStoreDynBlk(); void AdvanceFieldList(); void AdvancePhi(); void AdvanceConditional(); @@ -4113,6 +4112,10 @@ enum GenTreeCallFlags : unsigned int GTF_CALL_M_CAST_CAN_BE_EXPANDED = 0x04000000, // this cast (helper call) can be expanded if it's profitable. To be removed. GTF_CALL_M_CAST_OBJ_NONNULL = 0x08000000, // if we expand this specific cast we don't need to check the input object for null // NOTE: if needed, this flag can be removed, and we can introduce new _NONNUL cast helpers + +#ifdef SWIFT_SUPPORT + GTF_CALL_M_SWIFT_ERROR_HANDLING = 0x10000000, // call uses the Swift calling convention, and error register will be checked after it returns. +#endif // SWIFT_SUPPORT }; inline constexpr GenTreeCallFlags operator ~(GenTreeCallFlags a) @@ -7354,15 +7357,10 @@ struct GenTreeBlk : public GenTreeIndir public: ClassLayout* GetLayout() const { + assert(m_layout != nullptr); return m_layout; } - void SetLayout(ClassLayout* layout) - { - assert((layout != nullptr) || OperIs(GT_STORE_DYN_BLK)); - m_layout = layout; - } - // The data to be stored (null for GT_BLK) GenTree*& Data() { @@ -7376,8 +7374,7 @@ struct GenTreeBlk : public GenTreeIndir // The size of the buffer to be copied. unsigned Size() const { - assert((m_layout != nullptr) || OperIs(GT_STORE_DYN_BLK)); - return (m_layout != nullptr) ? m_layout->GetSize() : 0; + return m_layout->GetSize(); } // Instruction selection: during codegen time, what code sequence we will be using @@ -7403,7 +7400,7 @@ struct GenTreeBlk : public GenTreeIndir bool ContainsReferences() { - return (m_layout != nullptr) && m_layout->HasGCPtr(); + return m_layout->HasGCPtr(); } bool IsOnHeapAndContainsReferences() @@ -7434,8 +7431,8 @@ struct GenTreeBlk : public GenTreeIndir void Initialize(ClassLayout* layout) { - assert(OperIsBlk(OperGet()) && ((layout != nullptr) || OperIs(GT_STORE_DYN_BLK))); - assert((layout == nullptr) || (layout->GetSize() != 0)); + assert(layout != nullptr); + assert(layout->GetSize() != 0); m_layout = layout; gtBlkOpKind = BlkOpKindInvalid; @@ -7453,35 +7450,6 @@ struct GenTreeBlk : public GenTreeIndir #endif // DEBUGGABLE_GENTREE }; -// GenTreeStoreDynBlk -- 'dynamic block store' (GT_STORE_DYN_BLK). -// -// This node is used to represent stores that have a dynamic size - the "cpblk" and "initblk" -// IL instructions are implemented with it. Note that such stores assume the input has no GC -// pointers in it, and as such do not ever use write barriers. -// -// The "Data()" member of this node will either be a "dummy" IND(struct) node, for "cpblk", or -// the zero constant/INIT_VAL for "initblk". -// -struct GenTreeStoreDynBlk : public GenTreeBlk -{ -public: - GenTree* gtDynamicSize; - - GenTreeStoreDynBlk(GenTree* dstAddr, GenTree* data, GenTree* dynamicSize) - : GenTreeBlk(GT_STORE_DYN_BLK, TYP_VOID, dstAddr, data, nullptr), gtDynamicSize(dynamicSize) - { - gtFlags |= dynamicSize->gtFlags & GTF_ALL_EFFECT; - } - -#if DEBUGGABLE_GENTREE -protected: - friend GenTree; - GenTreeStoreDynBlk() : GenTreeBlk() - { - } -#endif // DEBUGGABLE_GENTREE -}; - // Read-modify-write status of a RMW memory op rooted at a storeInd enum RMWStatus { @@ -8896,10 +8864,6 @@ struct GenTreeCCMP final : public GenTreeOpCC inline bool GenTree::OperIsBlkOp() { - if (OperIs(GT_STORE_DYN_BLK)) - { - return true; - } if (OperIsStore()) { return varTypeIsStruct(this); @@ -9307,7 +9271,7 @@ inline GenTree* GenTree::gtGetOp2IfPresent() const inline GenTree*& GenTree::Data() { - assert(OperIsStore() || OperIs(GT_STORE_DYN_BLK)); + assert(OperIsStore()); return OperIsLocalStore() ? AsLclVarCommon()->Data() : AsIndir()->Data(); } diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 00696e6398fcd..26513f0f3441b 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -37,6 +37,7 @@ GTNODE(LABEL , GenTree ,0,0,GTK_LEAF) // Jump- GTNODE(JMP , GenTreeVal ,0,0,GTK_LEAF|GTK_NOVALUE) // Jump to another function GTNODE(FTN_ADDR , GenTreeFptrVal ,0,0,GTK_LEAF) // Address of a function GTNODE(RET_EXPR , GenTreeRetExpr ,0,0,GTK_LEAF|DBK_NOTLIR) // Place holder for the return expression from an inline candidate +GTNODE(SWIFT_ERROR , GenTree ,0,0,GTK_LEAF) // Error register value post-Swift call //----------------------------------------------------------------------------- // Constant nodes: @@ -82,7 +83,6 @@ GTNODE(IND , GenTreeIndir ,0,1,GTK_UNOP) GTNODE(STOREIND , GenTreeStoreInd ,0,1,GTK_BINOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // Store indirection GTNODE(BLK , GenTreeBlk ,0,1,GTK_UNOP|GTK_EXOP) // Struct load GTNODE(STORE_BLK , GenTreeBlk ,0,1,GTK_BINOP|GTK_EXOP|GTK_NOVALUE|GTK_STORE) // Struct store -GTNODE(STORE_DYN_BLK , GenTreeStoreDynBlk ,0,1,GTK_SPECIAL|GTK_NOVALUE) // Dynamically sized block store, with native uint size GTNODE(NULLCHECK , GenTreeIndir ,0,1,GTK_UNOP|GTK_NOVALUE) // Null checks the source GTNODE(ARR_LENGTH , GenTreeArrLen ,0,0,GTK_UNOP|GTK_EXOP) // single-dimension (SZ) array length diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index 989b6e554eae7..e6823478a3c9a 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -88,9 +88,8 @@ GTSTRUCT_1(AddrMode , GT_LEA) GTSTRUCT_1(Qmark , GT_QMARK) GTSTRUCT_1(PhiArg , GT_PHI_ARG) GTSTRUCT_1(Phi , GT_PHI) -GTSTRUCT_N(Indir , GT_IND, GT_NULLCHECK, GT_BLK, GT_STORE_BLK, GT_STORE_DYN_BLK, GT_LOCKADD, GT_XAND, GT_XORR, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_STOREIND) -GTSTRUCT_N(Blk , GT_BLK, GT_STORE_BLK, GT_STORE_DYN_BLK) -GTSTRUCT_1(StoreDynBlk , GT_STORE_DYN_BLK) +GTSTRUCT_N(Indir , GT_IND, GT_NULLCHECK, GT_BLK, GT_STORE_BLK, GT_LOCKADD, GT_XAND, GT_XORR, GT_XADD, GT_XCHG, GT_CMPXCHG, GT_STOREIND) +GTSTRUCT_N(Blk , GT_BLK, GT_STORE_BLK) GTSTRUCT_1(StoreInd , GT_STOREIND) GTSTRUCT_1(CmpXchg , GT_CMPXCHG) #ifdef TARGET_ARM64 diff --git a/src/coreclr/jit/helperexpansion.cpp b/src/coreclr/jit/helperexpansion.cpp index 39f7b8f09d276..d6a9ac01df969 100644 --- a/src/coreclr/jit/helperexpansion.cpp +++ b/src/coreclr/jit/helperexpansion.cpp @@ -319,21 +319,11 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm // Fallback basic block GenTree* fallbackValueDef = gtNewStoreLclVarNode(rtLookupLcl->GetLclNum(), call); - BasicBlock* fallbackBb = - fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fallbackValueDef, debugInfo, nullcheckBb->Next(), true); - - assert(fallbackBb->JumpsToNext()); - fallbackBb->SetFlags(BBF_NONE_QUIRK); - - // Set nullcheckBb's true jump target - nullcheckBb->SetTrueTarget(fallbackBb); + BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fallbackValueDef, debugInfo, true); // Fast-path basic block GenTree* fastpathValueDef = gtNewStoreLclVarNode(rtLookupLcl->GetLclNum(), fastPathValueClone); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fastpathValueDef, debugInfo, block); - - // Set nullcheckBb's false jump target - nullcheckBb->SetFalseTarget(fastPathBb); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, nullcheckBb, fastpathValueDef, debugInfo); BasicBlock* sizeCheckBb = nullptr; if (needsSizeCheck) @@ -375,42 +365,60 @@ bool Compiler::fgExpandRuntimeLookupsForCall(BasicBlock** pBlock, Statement* stm GenTree* jtrue = gtNewOperNode(GT_JTRUE, TYP_VOID, sizeCheck); // sizeCheckBb fails - jump to fallbackBb - sizeCheckBb = fgNewBBFromTreeAfter(BBJ_COND, prevBb, jtrue, debugInfo, fallbackBb); - sizeCheckBb->SetFalseTarget(nullcheckBb); + sizeCheckBb = fgNewBBFromTreeAfter(BBJ_COND, prevBb, jtrue, debugInfo); } // // Update preds in all new blocks // fgRemoveRefPred(block, prevBb); - fgAddRefPred(block, fastPathBb); - fgAddRefPred(block, fallbackBb); + + { + FlowEdge* const newEdge = fgAddRefPred(block, fastPathBb); + fastPathBb->SetTargetEdge(newEdge); + } + + { + FlowEdge* const newEdge = fgAddRefPred(block, fallbackBb); + fallbackBb->SetTargetEdge(newEdge); + assert(fallbackBb->JumpsToNext()); + fallbackBb->SetFlags(BBF_NONE_QUIRK); + } + assert(prevBb->KindIs(BBJ_ALWAYS)); if (needsSizeCheck) { // sizeCheckBb is the first block after prevBb - prevBb->SetTarget(sizeCheckBb); - fgAddRefPred(sizeCheckBb, prevBb); + FlowEdge* const newEdge = fgAddRefPred(sizeCheckBb, prevBb); + prevBb->SetTargetEdge(newEdge); + // sizeCheckBb flows into nullcheckBb in case if the size check passes - fgAddRefPred(nullcheckBb, sizeCheckBb); + { + FlowEdge* const trueEdge = fgAddRefPred(fallbackBb, sizeCheckBb); + FlowEdge* const falseEdge = fgAddRefPred(nullcheckBb, sizeCheckBb); + sizeCheckBb->SetTrueEdge(trueEdge); + sizeCheckBb->SetFalseEdge(falseEdge); + } + // fallbackBb is reachable from both nullcheckBb and sizeCheckBb - fgAddRefPred(fallbackBb, nullcheckBb); - fgAddRefPred(fallbackBb, sizeCheckBb); // fastPathBb is only reachable from successful nullcheckBb - fgAddRefPred(fastPathBb, nullcheckBb); } else { // nullcheckBb is the first block after prevBb - prevBb->SetTarget(nullcheckBb); - fgAddRefPred(nullcheckBb, prevBb); + FlowEdge* const newEdge = fgAddRefPred(nullcheckBb, prevBb); + prevBb->SetTargetEdge(newEdge); + // No size check, nullcheckBb jumps to fast path - fgAddRefPred(fastPathBb, nullcheckBb); // fallbackBb is only reachable from nullcheckBb (jump destination) - fgAddRefPred(fallbackBb, nullcheckBb); } + FlowEdge* const trueEdge = fgAddRefPred(fallbackBb, nullcheckBb); + FlowEdge* const falseEdge = fgAddRefPred(fastPathBb, nullcheckBb); + nullcheckBb->SetTrueEdge(trueEdge); + nullcheckBb->SetFalseEdge(falseEdge); + // // Re-distribute weights (see '[weight: X]' on the diagrams above) // TODO: consider marking fallbackBb as rarely-taken @@ -699,11 +707,10 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St // fallbackBb GenTree* fallbackValueDef = gtNewStoreLclVarNode(finalLclNum, slowHelper); - BasicBlock* fallbackBb = - fgNewBBFromTreeAfter(BBJ_ALWAYS, tlsRootNullCondBB, fallbackValueDef, debugInfo, block, true); + BasicBlock* fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, tlsRootNullCondBB, fallbackValueDef, debugInfo, true); GenTree* fastPathValueDef = gtNewStoreLclVarNode(finalLclNum, gtCloneExpr(finalLcl)); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, block, true); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, true); *callUse = finalLcl; @@ -713,14 +720,20 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St // // Update preds in all new blocks // - fgAddRefPred(fallbackBb, tlsRootNullCondBB); - fgAddRefPred(fastPathBb, tlsRootNullCondBB); + FlowEdge* const trueEdge = fgAddRefPred(fastPathBb, tlsRootNullCondBB); + FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, tlsRootNullCondBB); + tlsRootNullCondBB->SetTrueEdge(trueEdge); + tlsRootNullCondBB->SetFalseEdge(falseEdge); - fgAddRefPred(block, fallbackBb); - fgAddRefPred(block, fastPathBb); + { + FlowEdge* const newEdge = fgAddRefPred(block, fallbackBb); + fallbackBb->SetTargetEdge(newEdge); + } - tlsRootNullCondBB->SetTrueTarget(fastPathBb); - tlsRootNullCondBB->SetFalseTarget(fallbackBb); + { + FlowEdge* const newEdge = fgAddRefPred(block, fastPathBb); + fastPathBb->SetTargetEdge(newEdge); + } // Inherit the weights block->inheritWeight(prevBb); @@ -730,9 +743,9 @@ bool Compiler::fgExpandThreadLocalAccessForCallNativeAOT(BasicBlock** pBlock, St // fallback will just execute first time fallbackBb->bbSetRunRarely(); - fgRemoveRefPred(block, prevBb); - fgAddRefPred(tlsRootNullCondBB, prevBb); - prevBb->SetTarget(tlsRootNullCondBB); + fgRemoveRefPred(prevBb->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(tlsRootNullCondBB, prevBb); + prevBb->SetTargetEdge(newEdge); // All blocks are expected to be in the same EH region assert(BasicBlock::sameEHRegion(prevBb, block)); @@ -1056,7 +1069,7 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* // fallbackBb GenTree* fallbackValueDef = gtNewStoreLclVarNode(threadStaticBlockLclNum, call); BasicBlock* fallbackBb = - fgNewBBFromTreeAfter(BBJ_ALWAYS, threadStaticBlockNullCondBB, fallbackValueDef, debugInfo, block, true); + fgNewBBFromTreeAfter(BBJ_ALWAYS, threadStaticBlockNullCondBB, fallbackValueDef, debugInfo, true); // fastPathBb if (isGCThreadStatic) @@ -1071,32 +1084,42 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement* GenTree* fastPathValueDef = gtNewStoreLclVarNode(threadStaticBlockLclNum, gtCloneExpr(threadStaticBlockBaseLclValueUse)); - BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, block, true); - - // Set maxThreadStaticBlocksCondBB's jump targets - maxThreadStaticBlocksCondBB->SetTrueTarget(fallbackBb); - maxThreadStaticBlocksCondBB->SetFalseTarget(threadStaticBlockNullCondBB); - - // Set threadStaticBlockNullCondBB's jump targets - threadStaticBlockNullCondBB->SetTrueTarget(fastPathBb); - threadStaticBlockNullCondBB->SetFalseTarget(fallbackBb); + BasicBlock* fastPathBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, fastPathValueDef, debugInfo, true); // // Update preds in all new blocks // assert(prevBb->KindIs(BBJ_ALWAYS)); - prevBb->SetTarget(maxThreadStaticBlocksCondBB); - fgRemoveRefPred(block, prevBb); - fgAddRefPred(maxThreadStaticBlocksCondBB, prevBb); + fgRemoveRefPred(prevBb->GetTargetEdge()); - fgAddRefPred(threadStaticBlockNullCondBB, maxThreadStaticBlocksCondBB); - fgAddRefPred(fallbackBb, maxThreadStaticBlocksCondBB); + { + FlowEdge* const newEdge = fgAddRefPred(maxThreadStaticBlocksCondBB, prevBb); + prevBb->SetTargetEdge(newEdge); + } + + { + FlowEdge* const trueEdge = fgAddRefPred(fallbackBb, maxThreadStaticBlocksCondBB); + FlowEdge* const falseEdge = fgAddRefPred(threadStaticBlockNullCondBB, maxThreadStaticBlocksCondBB); + maxThreadStaticBlocksCondBB->SetTrueEdge(trueEdge); + maxThreadStaticBlocksCondBB->SetFalseEdge(falseEdge); + } - fgAddRefPred(fastPathBb, threadStaticBlockNullCondBB); - fgAddRefPred(fallbackBb, threadStaticBlockNullCondBB); + { + FlowEdge* const trueEdge = fgAddRefPred(fastPathBb, threadStaticBlockNullCondBB); + FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, threadStaticBlockNullCondBB); + threadStaticBlockNullCondBB->SetTrueEdge(trueEdge); + threadStaticBlockNullCondBB->SetFalseEdge(falseEdge); + } + + { + FlowEdge* const newEdge = fgAddRefPred(block, fastPathBb); + fastPathBb->SetTargetEdge(newEdge); + } - fgAddRefPred(block, fastPathBb); - fgAddRefPred(block, fallbackBb); + { + FlowEdge* const newEdge = fgAddRefPred(block, fallbackBb); + fallbackBb->SetTargetEdge(newEdge); + } // Inherit the weights block->inheritWeight(prevBb); @@ -1376,14 +1399,12 @@ bool Compiler::fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, G GenTree* isInitedCmp = gtNewOperNode(GT_EQ, TYP_INT, isInitedActualValueNode, isInitedExpectedValue); isInitedCmp->gtFlags |= GTF_RELOP_JMP_USED; BasicBlock* isInitedBb = - fgNewBBFromTreeAfter(BBJ_COND, prevBb, gtNewOperNode(GT_JTRUE, TYP_VOID, isInitedCmp), debugInfo, block); + fgNewBBFromTreeAfter(BBJ_COND, prevBb, gtNewOperNode(GT_JTRUE, TYP_VOID, isInitedCmp), debugInfo); // Fallback basic block // TODO-CQ: for JIT we can replace the original call with CORINFO_HELP_INITCLASS // that only accepts a single argument - BasicBlock* helperCallBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, isInitedBb, call, debugInfo, isInitedBb->Next(), true); - assert(helperCallBb->JumpsToNext()); - helperCallBb->SetFlags(BBF_NONE_QUIRK); + BasicBlock* helperCallBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, isInitedBb, call, debugInfo, true); GenTree* replacementNode = nullptr; if (retValKind == SHRV_STATIC_BASE_PTR) @@ -1443,22 +1464,32 @@ bool Compiler::fgExpandStaticInitForCall(BasicBlock** pBlock, Statement* stmt, G // // Unlink block and prevBb - fgRemoveRefPred(block, prevBb); + fgRemoveRefPred(prevBb->GetTargetEdge()); - // Block has two preds now: either isInitedBb or helperCallBb - fgAddRefPred(block, isInitedBb); - fgAddRefPred(block, helperCallBb); + { + // Block has two preds now: either isInitedBb or helperCallBb + FlowEdge* const newEdge = fgAddRefPred(block, helperCallBb); + helperCallBb->SetTargetEdge(newEdge); + assert(helperCallBb->JumpsToNext()); + helperCallBb->SetFlags(BBF_NONE_QUIRK); + } - // prevBb always flows into isInitedBb - assert(prevBb->KindIs(BBJ_ALWAYS)); - prevBb->SetTarget(isInitedBb); - prevBb->SetFlags(BBF_NONE_QUIRK); - assert(prevBb->JumpsToNext()); - fgAddRefPred(isInitedBb, prevBb); + { + // prevBb always flows into isInitedBb + assert(prevBb->KindIs(BBJ_ALWAYS)); + FlowEdge* const newEdge = fgAddRefPred(isInitedBb, prevBb); + prevBb->SetTargetEdge(newEdge); + prevBb->SetFlags(BBF_NONE_QUIRK); + assert(prevBb->JumpsToNext()); + } - // Both fastPathBb and helperCallBb have a single common pred - isInitedBb - isInitedBb->SetFalseTarget(helperCallBb); - fgAddRefPred(helperCallBb, isInitedBb); + { + // Both fastPathBb and helperCallBb have a single common pred - isInitedBb + FlowEdge* const trueEdge = fgAddRefPred(block, isInitedBb); + FlowEdge* const falseEdge = fgAddRefPred(helperCallBb, isInitedBb); + isInitedBb->SetTrueEdge(trueEdge); + isInitedBb->SetFalseEdge(falseEdge); + } // // Re-distribute weights @@ -1687,7 +1718,7 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, // // Block 1: lengthCheckBb (we check that dstLen < srcLen) // - BasicBlock* lengthCheckBb = fgNewBBafter(BBJ_COND, prevBb, true, block); + BasicBlock* lengthCheckBb = fgNewBBafter(BBJ_COND, prevBb, true); lengthCheckBb->SetFlags(BBF_INTERNAL); // Set bytesWritten -1 by default, if the fast path is not taken we'll return it as the result. @@ -1709,9 +1740,8 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, // In theory, we could just emit the const U8 data to the data section and use GT_BLK here // but that would be a bit less efficient since we would have to load the data from memory. // - BasicBlock* fastpathBb = fgNewBBafter(BBJ_ALWAYS, lengthCheckBb, true, lengthCheckBb->Next()); - assert(fastpathBb->JumpsToNext()); - fastpathBb->SetFlags(BBF_INTERNAL | BBF_NONE_QUIRK); + BasicBlock* fastpathBb = fgNewBBafter(BBJ_ALWAYS, lengthCheckBb, true); + fastpathBb->SetFlags(BBF_INTERNAL); // The widest type we can use for loads const var_types maxLoadType = roundDownMaxType(srcLenU8); @@ -1764,19 +1794,32 @@ bool Compiler::fgVNBasedIntrinsicExpansionForCall_ReadUtf8(BasicBlock** pBlock, // Update preds in all new blocks // // block is no longer a predecessor of prevBb - fgRemoveRefPred(block, prevBb); - // prevBb flows into lengthCheckBb - assert(prevBb->KindIs(BBJ_ALWAYS)); - prevBb->SetTarget(lengthCheckBb); - prevBb->SetFlags(BBF_NONE_QUIRK); - assert(prevBb->JumpsToNext()); - fgAddRefPred(lengthCheckBb, prevBb); - // lengthCheckBb has two successors: block and fastpathBb - lengthCheckBb->SetFalseTarget(fastpathBb); - fgAddRefPred(fastpathBb, lengthCheckBb); - fgAddRefPred(block, lengthCheckBb); - // fastpathBb flows into block - fgAddRefPred(block, fastpathBb); + fgRemoveRefPred(prevBb->GetTargetEdge()); + + { + // prevBb flows into lengthCheckBb + assert(prevBb->KindIs(BBJ_ALWAYS)); + FlowEdge* const newEdge = fgAddRefPred(lengthCheckBb, prevBb); + prevBb->SetTargetEdge(newEdge); + prevBb->SetFlags(BBF_NONE_QUIRK); + assert(prevBb->JumpsToNext()); + } + + { + // lengthCheckBb has two successors: block and fastpathBb + FlowEdge* const trueEdge = fgAddRefPred(block, lengthCheckBb); + FlowEdge* const falseEdge = fgAddRefPred(fastpathBb, lengthCheckBb); + lengthCheckBb->SetTrueEdge(trueEdge); + lengthCheckBb->SetFalseEdge(falseEdge); + } + + { + // fastpathBb flows into block + FlowEdge* const newEdge = fgAddRefPred(block, fastpathBb); + fastpathBb->SetTargetEdge(newEdge); + assert(fastpathBb->JumpsToNext()); + fastpathBb->SetFlags(BBF_NONE_QUIRK); + } // // Re-distribute weights @@ -2344,8 +2387,8 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, // it's too late to rely on upstream phases to do this for us (unless we do optRepeat). GenTree* nullcheckOp = gtNewOperNode(GT_EQ, TYP_INT, tmpNode, gtNewNull()); nullcheckOp->gtFlags |= GTF_RELOP_JMP_USED; - BasicBlock* nullcheckBb = fgNewBBFromTreeAfter(BBJ_COND, firstBb, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), - debugInfo, lastBb, true); + BasicBlock* nullcheckBb = + fgNewBBFromTreeAfter(BBJ_COND, firstBb, gtNewOperNode(GT_JTRUE, TYP_VOID, nullcheckOp), debugInfo, true); // The very first statement in the whole expansion is to assign obj to tmp. // We assume it's the value we're going to return in most cases. @@ -2385,7 +2428,7 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, GenTree* mtCheck = gtNewOperNode(GT_EQ, TYP_INT, gtNewMethodTableLookup(gtCloneExpr(tmpNode)), expectedClsNode); mtCheck->gtFlags |= GTF_RELOP_JMP_USED; GenTree* jtrue = gtNewOperNode(GT_JTRUE, TYP_VOID, mtCheck); - typeChecksBbs[candidateId] = fgNewBBFromTreeAfter(BBJ_COND, lastTypeCheckBb, jtrue, debugInfo, lastBb, true); + typeChecksBbs[candidateId] = fgNewBBFromTreeAfter(BBJ_COND, lastTypeCheckBb, jtrue, debugInfo, true); lastTypeCheckBb = typeChecksBbs[candidateId]; // Insert the CSE node as the first statement in the block @@ -2407,13 +2450,13 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, { // fallback call is used only to throw InvalidCastException call->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; - fallbackBb = fgNewBBFromTreeAfter(BBJ_THROW, lastTypeCheckBb, call, debugInfo, nullptr, true); + fallbackBb = fgNewBBFromTreeAfter(BBJ_THROW, lastTypeCheckBb, call, debugInfo, true); } else if (typeCheckFailedAction == TypeCheckFailedAction::ReturnNull) { // if fallback call is not needed, we just assign null to tmp GenTree* fallbackTree = gtNewTempStore(tmpNum, gtNewNull()); - fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, lastBb, true); + fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, true); } else { @@ -2424,7 +2467,7 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, call->gtCallMethHnd = eeFindHelper(CORINFO_HELP_CHKCASTCLASS_SPECIAL); } GenTree* fallbackTree = gtNewTempStore(tmpNum, call); - fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, lastBb, true); + fallbackBb = fgNewBBFromTreeAfter(BBJ_ALWAYS, lastTypeCheckBb, fallbackTree, debugInfo, true); } // Block 4: typeCheckSucceedBb @@ -2439,15 +2482,11 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, typeCheckSucceedTree = gtNewNothingNode(); } BasicBlock* typeCheckSucceedBb = - typeCheckNotNeeded ? nullptr - : fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, typeCheckSucceedTree, debugInfo, lastBb); + typeCheckNotNeeded ? nullptr : fgNewBBFromTreeAfter(BBJ_ALWAYS, fallbackBb, typeCheckSucceedTree, debugInfo); // // Wire up the blocks // - firstBb->SetTarget(nullcheckBb); - nullcheckBb->SetTrueTarget(lastBb); - nullcheckBb->SetFalseTarget(typeCheckNotNeeded ? fallbackBb : typeChecksBbs[0]); // Tricky case - wire up multiple type check blocks (in most cases there is only one) for (int candidateId = 0; candidateId < numOfCandidates; candidateId++) @@ -2455,41 +2494,48 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, BasicBlock* curTypeCheckBb = typeChecksBbs[candidateId]; // All type checks jump straight to the typeCheckSucceedBb on success - curTypeCheckBb->SetTrueTarget(typeCheckSucceedBb); - fgAddRefPred(typeCheckSucceedBb, curTypeCheckBb); + FlowEdge* const trueEdge = fgAddRefPred(typeCheckSucceedBb, curTypeCheckBb); + curTypeCheckBb->SetTrueEdge(trueEdge); // or ... if (candidateId == numOfCandidates - 1) { // ... jump to the fallbackBb on last type check's failure - curTypeCheckBb->SetFalseTarget(fallbackBb); - fgAddRefPred(fallbackBb, curTypeCheckBb); + FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, curTypeCheckBb); + curTypeCheckBb->SetFalseEdge(falseEdge); } else { // ... jump to the next type check on failure - curTypeCheckBb->SetFalseTarget(typeChecksBbs[candidateId + 1]); - fgAddRefPred(typeChecksBbs[candidateId + 1], curTypeCheckBb); + FlowEdge* const falseEdge = fgAddRefPred(typeChecksBbs[candidateId + 1], curTypeCheckBb); + curTypeCheckBb->SetFalseEdge(falseEdge); } } - fgRemoveRefPred(lastBb, firstBb); - fgAddRefPred(nullcheckBb, firstBb); - fgAddRefPred(lastBb, nullcheckBb); - if (typeCheckNotNeeded) + fgRemoveRefPred(firstBb->GetTargetEdge()); + { - fgAddRefPred(fallbackBb, nullcheckBb); + FlowEdge* const newEdge = fgAddRefPred(nullcheckBb, firstBb); + firstBb->SetTargetEdge(newEdge); } - else + { - fgAddRefPred(typeChecksBbs[0], nullcheckBb); - fgAddRefPred(lastBb, typeCheckSucceedBb); + FlowEdge* const trueEdge = fgAddRefPred(lastBb, nullcheckBb); + nullcheckBb->SetTrueEdge(trueEdge); } - if (!fallbackBb->KindIs(BBJ_THROW)) + if (typeCheckNotNeeded) + { + FlowEdge* const falseEdge = fgAddRefPred(fallbackBb, nullcheckBb); + nullcheckBb->SetFalseEdge(falseEdge); + } + else { - // if fallbackBb is BBJ_THROW then it has no successors - fgAddRefPred(lastBb, fallbackBb); + FlowEdge* const falseEdge = fgAddRefPred(typeChecksBbs[0], nullcheckBb); + nullcheckBb->SetFalseEdge(falseEdge); + + FlowEdge* const newEdge = fgAddRefPred(lastBb, typeCheckSucceedBb); + typeCheckSucceedBb->SetTargetEdge(newEdge); } // @@ -2521,12 +2567,18 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, } else { + assert(fallbackBb->KindIs(BBJ_ALWAYS)); + FlowEdge* const newEdge = fgAddRefPred(lastBb, fallbackBb); + fallbackBb->SetTargetEdge(newEdge); + fallbackBb->inheritWeightPercentage(lastTypeCheckBb, 100 - totalLikelihood); } + if (!typeCheckNotNeeded) { typeCheckSucceedBb->inheritWeightPercentage(typeChecksBbs[0], totalLikelihood); } + lastBb->inheritWeight(firstBb); // @@ -2537,12 +2589,12 @@ bool Compiler::fgLateCastExpansionForCall(BasicBlock** pBlock, Statement* stmt, assert(BasicBlock::sameEHRegion(firstBb, fallbackBb)); // call guarantees that obj is never null, we can drop the nullcheck - // by converting it to a BBJ_ALWAYS to typeCheckBb. + // by converting it to a BBJ_ALWAYS to its false target. if ((call->gtCallMoreFlags & GTF_CALL_M_CAST_OBJ_NONNULL) != 0) { fgRemoveStmt(nullcheckBb, nullcheckBb->lastStmt()); - nullcheckBb->SetKindAndTarget(BBJ_ALWAYS, typeCheckNotNeeded ? fallbackBb : typeChecksBbs[0]); - fgRemoveRefPred(lastBb, nullcheckBb); + fgRemoveRefPred(nullcheckBb->GetTrueEdge()); + nullcheckBb->SetKindAndTargetEdge(BBJ_ALWAYS, nullcheckBb->GetFalseEdge()); } // Bonus step: merge prevBb with nullcheckBb as they are likely to be mergeable diff --git a/src/coreclr/jit/ifconversion.cpp b/src/coreclr/jit/ifconversion.cpp index f9cb5af17925e..1e6a573aa7b0d 100644 --- a/src/coreclr/jit/ifconversion.cpp +++ b/src/coreclr/jit/ifconversion.cpp @@ -739,7 +739,7 @@ bool OptIfConversionDsc::optIfConvert() // Update the flow from the original block. m_comp->fgRemoveAllRefPreds(m_startBlock->GetFalseTarget(), m_startBlock); - m_startBlock->SetKindAndTarget(BBJ_ALWAYS, m_startBlock->GetTrueTarget()); + m_startBlock->SetKindAndTargetEdge(BBJ_ALWAYS, m_startBlock->GetTrueEdge()); #ifdef DEBUG if (m_comp->verbose) diff --git a/src/coreclr/jit/importer.cpp b/src/coreclr/jit/importer.cpp index e0d472b2538e1..8e429196c873c 100644 --- a/src/coreclr/jit/importer.cpp +++ b/src/coreclr/jit/importer.cpp @@ -2020,13 +2020,14 @@ BasicBlock* Compiler::impPushCatchArgOnStack(BasicBlock* hndBlk, CORINFO_CLASS_H { // Create extra basic block for the spill // - BasicBlock* newBlk = fgNewBBbefore(BBJ_ALWAYS, hndBlk, /* extendRegion */ true, /* jumpDest */ hndBlk); + BasicBlock* newBlk = fgNewBBbefore(BBJ_ALWAYS, hndBlk, /* extendRegion */ true); newBlk->SetFlags(BBF_IMPORTED | BBF_DONT_REMOVE | BBF_NONE_QUIRK); newBlk->inheritWeight(hndBlk); newBlk->bbCodeOffs = hndBlk->bbCodeOffs; FlowEdge* const newEdge = fgAddRefPred(hndBlk, newBlk); newEdge->setLikelihood(1.0); + newBlk->SetTargetEdge(newEdge); // Spill into a temp. unsigned tempNum = lvaGrabTemp(false DEBUGARG("SpillCatchArg")); @@ -2493,7 +2494,7 @@ GenTree* Compiler::impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom) void Compiler::verConvertBBToThrowVerificationException(BasicBlock* block DEBUGARG(bool logMsg)) { - block->SetKindAndTarget(BBJ_THROW); + block->SetKindAndTargetEdge(BBJ_THROW); block->SetFlags(BBF_FAILED_VERIFICATION); block->RemoveFlags(BBF_IMPORTED); @@ -4404,10 +4405,9 @@ void Compiler::impImportLeave(BasicBlock* block) callBlock = block; assert(callBlock->HasInitializedTarget()); - fgRemoveRefPred(callBlock->GetTarget(), callBlock); + fgRemoveRefPred(callBlock->GetTargetEdge()); - // callBlock will call the finally handler. Convert the BBJ_LEAVE to BBJ_CALLFINALLY. - callBlock->SetKindAndTarget(BBJ_CALLFINALLY, HBtab->ebdHndBeg); + // callBlock will call the finally handler. This will be set up later. if (endCatches) { @@ -4429,16 +4429,16 @@ void Compiler::impImportLeave(BasicBlock* block) // Calling the finally block. - // callBlock will call the finally handler - callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step, HBtab->ebdHndBeg); + // callBlock will call the finally handler. This will be set up later. + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, XTnum + 1, 0, step); // step's jump target shouldn't be set yet assert(!step->HasInitializedTarget()); // the previous call to a finally returns to this call (to the next finally in the chain) - step->SetTarget(callBlock); FlowEdge* const newEdge = fgAddRefPred(callBlock, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); // The new block will inherit this block's weight. callBlock->inheritWeight(block); @@ -4486,10 +4486,9 @@ void Compiler::impImportLeave(BasicBlock* block) unsigned finallyNesting = compHndBBtab[XTnum].ebdHandlerNestingLevel; assert(finallyNesting <= compHndBBtabCount); - assert(callBlock->KindIs(BBJ_CALLFINALLY)); - assert(callBlock->TargetIs(HBtab->ebdHndBeg)); - FlowEdge* const newEdge = fgAddRefPred(callBlock->GetTarget(), callBlock); + FlowEdge* const newEdge = fgAddRefPred(HBtab->ebdHndBeg, callBlock); newEdge->setLikelihood(1.0); + callBlock->SetKindAndTargetEdge(BBJ_CALLFINALLY, newEdge); GenTree* endLFin = new (this, GT_END_LFIN) GenTreeVal(GT_END_LFIN, TYP_VOID, finallyNesting); endLFinStmt = gtNewStmt(endLFin); @@ -4532,16 +4531,16 @@ void Compiler::impImportLeave(BasicBlock* block) // Insert a new BB either in the try region indicated by tryIndex or // the handler region indicated by leaveTarget->bbHndIndex, // depending on which is the inner region. - BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step, leaveTarget); + BasicBlock* finalStep = fgNewBBinRegion(BBJ_ALWAYS, tryIndex, leaveTarget->bbHndIndex, step); finalStep->SetFlags(BBF_KEEP_BBJ_ALWAYS); // step's jump target shouldn't be set yet assert(!step->HasInitializedTarget()); - step->SetTarget(finalStep); { FlowEdge* const newEdge = fgAddRefPred(finalStep, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); } // The new block will inherit this block's weight. @@ -4574,6 +4573,7 @@ void Compiler::impImportLeave(BasicBlock* block) { FlowEdge* const newEdge = fgAddRefPred(leaveTarget, finalStep); newEdge->setLikelihood(1.0); + finalStep->SetTargetEdge(newEdge); } // Queue up the jump target for importing @@ -4690,10 +4690,11 @@ void Compiler::impImportLeave(BasicBlock* block) { fgRemoveRefPred(step->GetTarget(), step); } - step->SetTarget(exitBlock); // the previous step (maybe a call to a nested finally, or a nested catch - // exit) returns to this block + FlowEdge* const newEdge = fgAddRefPred(exitBlock, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); // the previous step (maybe a call to a nested finally, or a nested catch + // exit) returns to this block // The new block will inherit this block's weight. exitBlock->inheritWeight(block); @@ -4728,17 +4729,16 @@ void Compiler::impImportLeave(BasicBlock* block) (HBtab->ebdEnclosingTryIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingTryIndex + 1; unsigned callFinallyHndIndex = (HBtab->ebdEnclosingHndIndex == EHblkDsc::NO_ENCLOSING_INDEX) ? 0 : HBtab->ebdEnclosingHndIndex + 1; - callBlock = - fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block, HBtab->ebdHndBeg); + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, block); // Convert the BBJ_LEAVE to BBJ_ALWAYS, jumping to the new BBJ_CALLFINALLY. This is because // the new BBJ_CALLFINALLY is in a different EH region, thus it can't just replace the BBJ_LEAVE, // which might be in the middle of the "try". In most cases, the BBJ_ALWAYS will jump to the // next block, and flow optimizations will remove it. - fgRemoveRefPred(block->GetTarget(), block); - block->SetKindAndTarget(BBJ_ALWAYS, callBlock); + fgRemoveRefPred(block->GetTargetEdge()); FlowEdge* const newEdge = fgAddRefPred(callBlock, block); newEdge->setLikelihood(1.0); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // The new block will inherit this block's weight. callBlock->inheritWeight(block); @@ -4758,10 +4758,9 @@ void Compiler::impImportLeave(BasicBlock* block) callBlock = block; assert(callBlock->HasInitializedTarget()); - fgRemoveRefPred(callBlock->GetTarget(), callBlock); + fgRemoveRefPred(callBlock->GetTargetEdge()); - // callBlock will call the finally handler. Convert the BBJ_LEAVE to BBJ_CALLFINALLY - callBlock->SetKindAndTarget(BBJ_CALLFINALLY, HBtab->ebdHndBeg); +// callBlock will call the finally handler. This will be set up later. #ifdef DEBUG if (verbose) @@ -4804,11 +4803,12 @@ void Compiler::impImportLeave(BasicBlock* block) BasicBlock* step2 = fgNewBBinRegion(BBJ_ALWAYS, XTnum + 1, 0, step); if (step == block) { - fgRemoveRefPred(step->GetTarget(), step); + fgRemoveRefPred(step->GetTargetEdge()); } - step->SetTarget(step2); + FlowEdge* const newEdge = fgAddRefPred(step2, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); step2->inheritWeight(block); step2->CopyFlags(block, BBF_RUN_RARELY); step2->SetFlags(BBF_IMPORTED); @@ -4841,16 +4841,16 @@ void Compiler::impImportLeave(BasicBlock* block) assert((step == block) || !step->HasInitializedTarget()); // callBlock will call the finally handler - callBlock = - fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step, HBtab->ebdHndBeg); + callBlock = fgNewBBinRegion(BBJ_CALLFINALLY, callFinallyTryIndex, callFinallyHndIndex, step); if (step == block) { - fgRemoveRefPred(step->GetTarget(), step); + fgRemoveRefPred(step->GetTargetEdge()); } - step->SetTarget(callBlock); // the previous call to a finally returns to this call (to the next - // finally in the chain) + FlowEdge* const newEdge = fgAddRefPred(callBlock, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); // the previous call to a finally returns to this call (to the next + // finally in the chain) // The new block will inherit this block's weight. callBlock->inheritWeight(block); @@ -4884,10 +4884,9 @@ void Compiler::impImportLeave(BasicBlock* block) } #endif - assert(callBlock->KindIs(BBJ_CALLFINALLY)); - assert(callBlock->TargetIs(HBtab->ebdHndBeg)); - FlowEdge* const newEdge = fgAddRefPred(callBlock->GetTarget(), callBlock); + FlowEdge* const newEdge = fgAddRefPred(HBtab->ebdHndBeg, callBlock); newEdge->setLikelihood(1.0); + callBlock->SetKindAndTargetEdge(BBJ_CALLFINALLY, newEdge); } else if (HBtab->HasCatchHandler() && jitIsBetween(blkAddr, tryBeg, tryEnd) && !jitIsBetween(jmpAddr, tryBeg, tryEnd)) @@ -4951,11 +4950,12 @@ void Compiler::impImportLeave(BasicBlock* block) if (step == block) { - fgRemoveRefPred(step->GetTarget(), step); + fgRemoveRefPred(step->GetTargetEdge()); } - step->SetTarget(catchStep); + FlowEdge* const newEdge = fgAddRefPred(catchStep, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); // The new block will inherit this block's weight. catchStep->inheritWeight(block); @@ -5008,9 +5008,9 @@ void Compiler::impImportLeave(BasicBlock* block) { fgRemoveRefPred(step->GetTarget(), step); } - step->SetTarget(leaveTarget); // this is the ultimate destination of the LEAVE FlowEdge* const newEdge = fgAddRefPred(leaveTarget, step); newEdge->setLikelihood(1.0); + step->SetTargetEdge(newEdge); // this is the ultimate destination of the LEAVE #ifdef DEBUG if (verbose) @@ -5069,10 +5069,11 @@ void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) // will be treated as pair and handled correctly. if (block->KindIs(BBJ_CALLFINALLY)) { - BasicBlock* dupBlock = BasicBlock::New(this, BBJ_CALLFINALLY, block->GetTarget()); + BasicBlock* dupBlock = BasicBlock::New(this); dupBlock->CopyFlags(block); - FlowEdge* const newEdge = fgAddRefPred(dupBlock->GetTarget(), dupBlock); + FlowEdge* const newEdge = fgAddRefPred(block->GetTarget(), dupBlock); newEdge->setLikelihood(1.0); + dupBlock->SetKindAndTargetEdge(BBJ_CALLFINALLY, newEdge); dupBlock->copyEHRegion(block); dupBlock->bbCatchTyp = block->bbCatchTyp; @@ -5101,10 +5102,10 @@ void Compiler::impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr) fgInitBBLookup(); - fgRemoveRefPred(block->GetTarget(), block); - block->SetKindAndTarget(BBJ_LEAVE, fgLookupBB(jmpAddr)); - FlowEdge* const newEdge = fgAddRefPred(block->GetTarget(), block); + fgRemoveRefPred(block->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(fgLookupBB(jmpAddr), block); newEdge->setLikelihood(1.0); + block->SetKindAndTargetEdge(BBJ_LEAVE, newEdge); // We will leave the BBJ_ALWAYS block we introduced. When it's reimported // the BBJ_ALWAYS block will be unreachable, and will be removed after. The @@ -5427,8 +5428,7 @@ GenTree* Compiler::impCastClassOrIsInstToTree( assert(op1->TypeGet() == TYP_REF); // Optimistically assume the jit should expand this as an inline test - bool shouldExpandInline = true; - bool isClassExact = info.compCompHnd->isExactType(pResolvedToken->hClass); + bool isClassExact = info.compCompHnd->isExactType(pResolvedToken->hClass); // ECMA-335 III.4.3: If typeTok is a nullable type, Nullable, it is interpreted as "boxed" T // We can convert constant-ish tokens of nullable to its underlying type. @@ -5437,7 +5437,6 @@ GenTree* Compiler::impCastClassOrIsInstToTree( if (isClassExact && !(info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & CORINFO_FLG_SHAREDINST)) { CORINFO_CLASS_HANDLE hClass = info.compCompHnd->getTypeForBox(pResolvedToken->hClass); - if (hClass != pResolvedToken->hClass) { bool runtimeLookup; @@ -5447,101 +5446,34 @@ GenTree* Compiler::impCastClassOrIsInstToTree( } } - // Profitability check. - // - // Don't bother with inline expansion when jit is trying to generate code quickly - if (opts.OptimizationDisabled()) - { - // not worth the code expansion if jitting fast or in a rarely run block - shouldExpandInline = false; - } - else if ((op1->gtFlags & GTF_GLOB_EFFECT) && lvaHaveManyLocals()) - { - // not worth creating an untracked local variable - shouldExpandInline = false; - } - else if (opts.jitFlags->IsSet(JitFlags::JIT_FLAG_BBINSTR) && (JitConfig.JitProfileCasts() == 1)) - { - // Optimizations are enabled but we're still instrumenting (including casts) - if (isCastClass && !isClassExact) - { - // Usually, we make a speculative assumption that it makes sense to expand castclass - // even for non-sealed classes, but let's rely on PGO in this specific case - shouldExpandInline = false; - } - } - - if (shouldExpandInline && compCurBB->isRunRarely()) - { - // For cold blocks we only expand castclass against exact classes because it's cheap - shouldExpandInline = isCastClass && isClassExact; - } - - // Pessimistically assume the jit cannot expand this as an inline test - bool canExpandInline = false; - bool reversedMTCheck = false; - const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); + const CorInfoHelpFunc helper = info.compCompHnd->getCastingHelper(pResolvedToken, isCastClass); - CORINFO_CLASS_HANDLE exactCls = NO_CLASS_HANDLE; - - // By default, we assume it's 50/50 with the slow path. - unsigned fastPathLikelihood = 50; - - // Legality check. - // - // Not all classclass/isinst operations can be inline expanded. - // Check legality only if an inline expansion is desirable. - if (shouldExpandInline) + bool shouldExpandEarly = false; + const bool tooManyLocals = (((op1->gtFlags & GTF_GLOB_EFFECT) != 0) && lvaHaveManyLocals()); + if (isClassExact && opts.OptimizationEnabled() && !compCurBB->isRunRarely() && !tooManyLocals) { - if (isCastClass) + // TODO-InlineCast: Fix size regressions for these two cases if they're moved to the + // late cast expansion path and remove this early expansion entirely. + if (helper == CORINFO_HELP_ISINSTANCEOFCLASS) { - // Jit can only inline expand CHKCASTCLASS and CHKCASTARRAY helpers. - canExpandInline = (helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY); - - // For ChkCastAny we ignore cases where the class is known to be abstract or is an interface. - if (helper == CORINFO_HELP_CHKCASTANY) - { - const bool isAbstract = (info.compCompHnd->getClassAttribs(pResolvedToken->hClass) & - (CORINFO_FLG_INTERFACE | CORINFO_FLG_ABSTRACT)) != 0; - canExpandInline = !isAbstract; - } + shouldExpandEarly = true; } - else if ((helper == CORINFO_HELP_ISINSTANCEOFCLASS) || (helper == CORINFO_HELP_ISINSTANCEOFARRAY)) + else if (helper == CORINFO_HELP_ISINSTANCEOFARRAY && !op2->IsIconHandle(GTF_ICON_CLASS_HDL)) { - // If the class is exact, the jit can expand the IsInst check inline. - canExpandInline = isClassExact; + shouldExpandEarly = true; } } - bool expandInline = canExpandInline && shouldExpandInline; - - if ((helper == CORINFO_HELP_ISINSTANCEOFCLASS) && isClassExact) - { - // TODO-InlineCast: isinst against exact class - // It's already supported by the late cast expansion phase, but - // produces unexpected size regressions in some cases. - } - else if (!isCastClass && !op2->IsIconHandle(GTF_ICON_CLASS_HDL)) - { - // TODO-InlineCast: isinst against Class<_Canon> - } - else - { - // Expand later - expandInline = false; - } - - if (!expandInline) + if (!shouldExpandEarly) { - JITDUMP("\nExpanding %s as call because %s\n", isCastClass ? "castclass" : "isinst", - canExpandInline ? "want smaller code or faster jitting" : "inline expansion not legal"); + JITDUMP("\nImporting %s as call because %s\n", isCastClass ? "castclass" : "isinst"); // If we CSE this class handle we prevent assertionProp from making SubType assertions // so instead we force the CSE logic to not consider CSE-ing this class handle. // op2->gtFlags |= GTF_DONT_CSE; - - GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); + GenTreeCall* call = gtNewHelperCallNode(helper, TYP_REF, op2, op1); + call->gtCastHelperILOffset = ilOffset; // Instrument this castclass/isinst if ((JitConfig.JitClassProfiling() > 0) && impIsCastHelperEligibleForClassProbe(call) && !isClassExact && @@ -5567,128 +5499,41 @@ GenTree* Compiler::impCastClassOrIsInstToTree( return call; } - JITDUMP("\nExpanding %s inline\n", isCastClass ? "castclass" : "isinst"); - - impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling QMark2")); - - GenTree* temp; - GenTree* condMT; - // - // expand the methodtable match: - // - // condMT ==> GT_NE - // / \. - // GT_IND op2 (typically CNS_INT) - // | - // op1Copy - // - - // This can replace op1 with a GT_COMMA that evaluates op1 into a local - // - op1 = impCloneExpr(op1, &temp, CHECK_SPILL_ALL, nullptr DEBUGARG("CASTCLASS eval op1")); - // - // op1 is now known to be a non-complex tree - // thus we can use gtClone(op1) from now on - // - - GenTree* op2Var = op2; - if (isCastClass && (exactCls == NO_CLASS_HANDLE)) - { - // if exactCls is not null we won't have to clone op2 (it will be used only for the fallback) - op2Var = fgInsertCommaFormTemp(&op2); - lvaTable[op2Var->AsLclVarCommon()->GetLclNum()].lvIsCSE = true; - } - temp = gtNewMethodTableLookup(temp); - condMT = - gtNewOperNode(GT_NE, TYP_INT, temp, (exactCls != NO_CLASS_HANDLE) ? gtNewIconEmbClsHndNode(exactCls) : op2); - - GenTree* condNull; - // - // expand the null check: - // - // condNull ==> GT_EQ - // / \. - // op1Copy CNS_INT - // null - // - condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewNull()); - - // - // expand the true and false trees for the condMT - // - GenTree* condFalse = reversedMTCheck ? gtNewNull() : gtClone(op1); - GenTree* condTrue; - if (isCastClass) - { - assert((helper == CORINFO_HELP_CHKCASTCLASS) || (helper == CORINFO_HELP_CHKCASTARRAY) || - (helper == CORINFO_HELP_CHKCASTANY) || (helper == CORINFO_HELP_CHKCASTINTERFACE)); + JITDUMP("\nExpanding isinst inline\n"); - CorInfoHelpFunc specialHelper = helper; - if ((helper == CORINFO_HELP_CHKCASTCLASS) && - ((exactCls == nullptr) || (exactCls == gtGetHelperArgClassHandle(op2)))) - { - // use the special helper that skips the cases checked by our inlined cast - specialHelper = CORINFO_HELP_CHKCASTCLASS_SPECIAL; - } - condTrue = gtNewHelperCallNode(specialHelper, TYP_REF, op2Var, gtClone(op1)); - } - else - { - condTrue = gtNewIconNode(0, TYP_REF); - } + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("bubbling ")); - GenTreeQmark* qmarkMT; + // Now we import it as two QMark nodes representing this: // - // Generate first QMARK - COLON tree + // tmp = op1; + // if (tmp != null) // qmarkNull + // { + // if (tmp->pMT == op2) // qmarkMT + // result = tmp; + // else + // result = null; + // } + // else + // result = null; // - // qmarkMT ==> GT_QMARK - // / \. - // condMT GT_COLON - // / \. - // condFalse condTrue - // - temp = new (this, GT_COLON) GenTreeColon(TYP_REF, condTrue, condFalse); - qmarkMT = gtNewQmarkNode(TYP_REF, condMT, temp->AsColon()); - qmarkMT->SetThenNodeLikelihood(fastPathLikelihood); - - if (isCastClass && isClassExact && condTrue->OperIs(GT_CALL)) - { - if (helper == CORINFO_HELP_CHKCASTCLASS) - { - // condTrue is used only for throwing InvalidCastException in case of casting to an exact class. - condTrue->AsCall()->gtCallMoreFlags |= GTF_CALL_M_DOES_NOT_RETURN; - // defer calling setMethodHasNoReturnCalls until qmark expasion - } - } + // Spill op1 if it's a complex expression + GenTree* op1Clone; + op1 = impCloneExpr(op1, &op1Clone, CHECK_SPILL_ALL, nullptr DEBUGARG("ISINST eval op1")); - GenTree* qmarkNull; - // - // Generate second QMARK - COLON tree - // - // qmarkNull ==> GT_QMARK - // / \. - // condNull GT_COLON - // / \. - // qmarkMT op1Copy - // - temp = new (this, GT_COLON) GenTreeColon(TYP_REF, reversedMTCheck ? gtNewNull() : gtClone(op1), qmarkMT); - qmarkNull = gtNewQmarkNode(TYP_REF, condNull, temp->AsColon()); + GenTreeOp* condMT = gtNewOperNode(GT_NE, TYP_INT, gtNewMethodTableLookup(op1Clone), op2); + GenTreeOp* condNull = gtNewOperNode(GT_EQ, TYP_INT, gtClone(op1), gtNewNull()); + GenTreeQmark* qmarkMT = gtNewQmarkNode(TYP_REF, condMT, gtNewColonNode(TYP_REF, gtNewNull(), gtClone(op1))); + GenTreeQmark* qmarkNull = gtNewQmarkNode(TYP_REF, condNull, gtNewColonNode(TYP_REF, gtClone(op1), qmarkMT)); // Make QMark node a top level node by spilling it. - unsigned tmp = lvaGrabTemp(true DEBUGARG("spilling QMark2")); - impStoreTemp(tmp, qmarkNull, CHECK_SPILL_NONE); + const unsigned result = lvaGrabTemp(true DEBUGARG("spilling qmarkNull")); + impStoreTemp(result, qmarkNull, CHECK_SPILL_NONE); - // TODO-CQ: Is it possible op1 has a better type? - // // See also gtGetHelperCallClassHandle where we make the same // determination for the helper call variants. - LclVarDsc* lclDsc = lvaGetDesc(tmp); - assert(lclDsc->lvSingleDef == 0); - lclDsc->lvSingleDef = 1; - JITDUMP("Marked V%02u as a single def temp\n", tmp); - lvaSetClass(tmp, pResolvedToken->hClass); - return gtNewLclvNode(tmp, TYP_REF); + lvaSetClass(result, pResolvedToken->hClass); + return gtNewLclvNode(result, TYP_REF); } #ifndef DEBUG @@ -6021,7 +5866,7 @@ void Compiler::impImportBlockCode(BasicBlock* block) // Change block to BBJ_THROW so we won't trigger importation of successors. // - block->SetKindAndTarget(BBJ_THROW); + block->SetKindAndTargetEdge(BBJ_THROW); // If this method has a explicit generic context, the only uses of it may be in // the IL for this block. So assume it's used. @@ -7337,14 +7182,14 @@ void Compiler::impImportBlockCode(BasicBlock* block) // We may have already modified `block`'s jump kind, if this is a re-importation. // bool jumpToNextOptimization = false; - if (block->KindIs(BBJ_COND) && block->TrueTargetIs(block->GetFalseTarget())) + if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) { JITDUMP(FMT_BB " always branches to " FMT_BB ", changing to BBJ_ALWAYS\n", block->bbNum, block->GetFalseTarget()->bbNum); fgRemoveRefPred(block->GetFalseTarget(), block); block->SetKind(BBJ_ALWAYS); - // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, it may not make sense to + // TODO-NoFallThrough: Once false target can diverge from bbNext, it may not make sense to // set BBF_NONE_QUIRK block->SetFlags(BBF_NONE_QUIRK); @@ -7416,21 +7261,24 @@ void Compiler::impImportBlockCode(BasicBlock* block) { JITDUMP("\nThe conditional jump becomes an unconditional jump to " FMT_BB "\n", block->GetTrueTarget()->bbNum); - fgRemoveRefPred(block->GetFalseTarget(), block); + fgRemoveRefPred(block->GetFalseEdge()); block->SetKind(BBJ_ALWAYS); } else { - // TODO-NoFallThrough: Update once bbFalseTarget can diverge from bbNext + // TODO-NoFallThrough: Update once false target can diverge from bbNext assert(block->NextIs(block->GetFalseTarget())); JITDUMP("\nThe block jumps to the next " FMT_BB "\n", block->Next()->bbNum); - fgRemoveRefPred(block->GetTrueTarget(), block); - block->SetKindAndTarget(BBJ_ALWAYS, block->Next()); + fgRemoveRefPred(block->GetTrueEdge()); + block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetFalseEdge()); - // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, it may not make sense + // TODO-NoFallThrough: Once false target can diverge from bbNext, it may not make sense // to set BBF_NONE_QUIRK block->SetFlags(BBF_NONE_QUIRK); } + + FlowEdge* const edge = fgGetPredForBlock(block->GetTarget(), block); + edge->setLikelihood(1.0); } break; @@ -7599,14 +7447,14 @@ void Compiler::impImportBlockCode(BasicBlock* block) // We may have already modified `block`'s jump kind, if this is a re-importation. // bool jumpToNextOptimization = false; - if (block->KindIs(BBJ_COND) && block->TrueTargetIs(block->GetFalseTarget())) + if (block->KindIs(BBJ_COND) && block->TrueEdgeIs(block->GetFalseEdge())) { JITDUMP(FMT_BB " always branches to " FMT_BB ", changing to BBJ_ALWAYS\n", block->bbNum, block->GetFalseTarget()->bbNum); fgRemoveRefPred(block->GetFalseTarget(), block); block->SetKind(BBJ_ALWAYS); - // TODO-NoFallThrough: Once bbFalseTarget can diverge from bbNext, it may not make sense to + // TODO-NoFallThrough: Once false target can diverge from bbNext, it may not make sense to // set BBF_NONE_QUIRK block->SetFlags(BBF_NONE_QUIRK); @@ -7691,7 +7539,8 @@ void Compiler::impImportBlockCode(BasicBlock* block) if ((val == switchVal) || (!foundVal && (val == jumpCnt - 1))) { // transform the basic block into a BBJ_ALWAYS - block->SetKindAndTarget(BBJ_ALWAYS, curEdge->getDestinationBlock()); + block->SetKindAndTargetEdge(BBJ_ALWAYS, curEdge); + curEdge->setLikelihood(1.0); foundVal = true; } else @@ -10340,9 +10189,6 @@ void Compiler::impImportBlockCode(BasicBlock* block) op3 = gtNewCastNode(TYP_LONG, op3, /* fromUnsigned */ true, TYP_LONG); } -// TODO: enable for X86 as well, it currently doesn't support memset/memcpy helpers -// Then, get rid of GT_STORE_DYN_BLK entirely. -#ifndef TARGET_X86 GenTreeCall* call; if (opcode == CEE_INITBLK) { @@ -10372,20 +10218,6 @@ void Compiler::impImportBlockCode(BasicBlock* block) { op1 = call; } -#else - if (opcode == CEE_INITBLK) - { - if (!op2->IsIntegralConst(0)) - { - op2 = gtNewOperNode(GT_INIT_VAL, TYP_INT, op2); - } - } - else - { - op2 = gtNewIndir(TYP_STRUCT, op2); - } - op1 = gtNewStoreDynBlkNode(op1, op2, op3, indirFlags); -#endif } goto SPILL_APPEND; } diff --git a/src/coreclr/jit/importercalls.cpp b/src/coreclr/jit/importercalls.cpp index a1f5768ee6832..f0ae5cf97bbaa 100644 --- a/src/coreclr/jit/importercalls.cpp +++ b/src/coreclr/jit/importercalls.cpp @@ -98,6 +98,11 @@ var_types Compiler::impImportCall(OPCODE opcode, CORINFO_SIG_INFO calliSig; NewCallArg extraArg; + // Swift calls may use special register types that require additional IR to handle, + // so if we're importing a Swift call, look for these types in the signature + CallArg* swiftErrorArg = nullptr; + CallArg* swiftSelfArg = nullptr; + /*------------------------------------------------------------------------- * First create the call node */ @@ -651,6 +656,8 @@ var_types Compiler::impImportCall(OPCODE opcode, if (call->gtFlags & GTF_CALL_UNMANAGED) { + assert(call->IsCall()); + // We set up the unmanaged call by linking the frame, disabling GC, etc // This needs to be cleaned up on return. // In addition, native calls have different normalization rules than managed code @@ -663,7 +670,7 @@ var_types Compiler::impImportCall(OPCODE opcode, checkForSmallType = true; - impPopArgsForUnmanagedCall(call->AsCall(), sig); + impPopArgsForUnmanagedCall(call->AsCall(), sig, &swiftErrorArg, &swiftSelfArg); goto DONE; } @@ -1485,6 +1492,15 @@ var_types Compiler::impImportCall(OPCODE opcode, impPushOnStack(call, tiRetVal); } +#ifdef SWIFT_SUPPORT + // If call is a Swift call with error handling, append additional IR + // to handle storing the error register's value post-call. + if (swiftErrorArg != nullptr) + { + impAppendSwiftErrorStore(call->AsCall(), swiftErrorArg); + } +#endif // SWIFT_SUPPORT + return callRetTyp; } #ifdef _PREFAST_ @@ -1822,7 +1838,10 @@ GenTreeCall* Compiler::impImportIndirectCall(CORINFO_SIG_INFO* sig, const DebugI /*****************************************************************************/ -void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* sig) +void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, + CORINFO_SIG_INFO* sig, + /* OUT */ CallArg** swiftErrorArg, + /* OUT */ CallArg** swiftSelfArg) { assert(call->gtFlags & GTF_CALL_UNMANAGED); @@ -1842,10 +1861,74 @@ void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* s if (call->unmgdCallConv == CorInfoCallConvExtension::Thiscall) { - assert(argsToReverse); + assert(argsToReverse != 0); argsToReverse--; } +#ifdef SWIFT_SUPPORT + unsigned short swiftErrorIndex = sig->numArgs; + + // We are importing an unmanaged Swift call, which might require special parameter handling + if (call->unmgdCallConv == CorInfoCallConvExtension::Swift) + { + bool checkEntireStack = false; + + // Check the signature of the Swift call for the special types + CORINFO_ARG_LIST_HANDLE sigArg = sig->args; + + for (unsigned short argIndex = 0; argIndex < sig->numArgs; + sigArg = info.compCompHnd->getArgNext(sigArg), argIndex++) + { + CORINFO_CLASS_HANDLE argClass; + CorInfoType argType = strip(info.compCompHnd->getArgType(sig, sigArg, &argClass)); + bool argIsByrefOrPtr = false; + + if ((argType == CORINFO_TYPE_BYREF) || (argType == CORINFO_TYPE_PTR)) + { + argClass = info.compCompHnd->getArgClass(sig, sigArg); + argType = info.compCompHnd->getChildType(argClass, &argClass); + argIsByrefOrPtr = true; + } + + if ((argType != CORINFO_TYPE_VALUECLASS) || !info.compCompHnd->isIntrinsicType(argClass)) + { + continue; + } + + const char* namespaceName; + const char* className = info.compCompHnd->getClassNameFromMetadata(argClass, &namespaceName); + + if ((strcmp(className, "SwiftError") == 0) && + (strcmp(namespaceName, "System.Runtime.InteropServices.Swift") == 0)) + { + // For error handling purposes, we expect a pointer/reference to a SwiftError to be passed + if (!argIsByrefOrPtr) + { + BADCODE("Expected SwiftError pointer/reference, got struct"); + } + + if (swiftErrorIndex != sig->numArgs) + { + BADCODE("Duplicate SwiftError* parameter"); + } + + swiftErrorIndex = argIndex; + checkEntireStack = true; + } + // TODO: Handle SwiftSelf, SwiftAsync + } + + // Don't need to reverse args for Swift calls + argsToReverse = 0; + + // If using one of the Swift register types, check entire stack for side effects + if (checkEntireStack) + { + impSpillSideEffects(true, CHECK_SPILL_ALL DEBUGARG("Spill for swift calls")); + } + } +#endif // SWIFT_SUPPORT + #ifndef TARGET_X86 // Don't reverse args on ARM or x64 - first four args always placed in regs in order argsToReverse = 0; @@ -1892,6 +1975,7 @@ void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* s assert(thisPtr->TypeGet() == TYP_I_IMPL || thisPtr->TypeGet() == TYP_BYREF); } + unsigned short argIndex = 0; for (CallArg& arg : call->gtArgs.Args()) { GenTree* argNode = arg.GetEarlyNode(); @@ -1914,9 +1998,55 @@ void Compiler::impPopArgsForUnmanagedCall(GenTreeCall* call, CORINFO_SIG_INFO* s assert(!"*** invalid IL: gc ref passed to unmanaged call"); } } + +#ifdef SWIFT_SUPPORT + if (argIndex == swiftErrorIndex) + { + // Found the SwiftError* arg + assert(swiftErrorArg != nullptr); + *swiftErrorArg = &arg; + } +// TODO: SwiftSelf, SwiftAsync +#endif // SWIFT_SUPPORT + + argIndex++; } } +#ifdef SWIFT_SUPPORT +//------------------------------------------------------------------------ +// impAppendSwiftErrorStore: Append IR to store the Swift error register value +// to the SwiftError* argument specified by swiftErrorArg, post-Swift call +// +// Arguments: +// call - the Swift call +// swiftErrorArg - the SwiftError* argument passed to call +// +void Compiler::impAppendSwiftErrorStore(GenTreeCall* call, CallArg* const swiftErrorArg) +{ + assert(call != nullptr); + assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); + assert(swiftErrorArg != nullptr); + + GenTree* const argNode = swiftErrorArg->GetNode(); + assert(argNode != nullptr); + + // Store the error register value to where the SwiftError* points to + GenTree* errorRegNode = new (this, GT_SWIFT_ERROR) GenTree(GT_SWIFT_ERROR, TYP_I_IMPL); + errorRegNode->SetHasOrderingSideEffect(); + errorRegNode->gtFlags |= (GTF_CALL | GTF_GLOB_REF); + + GenTreeStoreInd* swiftErrorStore = gtNewStoreIndNode(argNode->TypeGet(), argNode, errorRegNode); + impAppendTree(swiftErrorStore, CHECK_SPILL_ALL, impCurStmtDI, false); + + // Indicate the error register will be checked after this call returns + call->gtCallMoreFlags |= GTF_CALL_M_SWIFT_ERROR_HANDLING; + + // Swift call isn't going to use the SwiftError* arg, so don't bother emitting it + call->gtArgs.Remove(swiftErrorArg); +} +#endif // SWIFT_SUPPORT + //------------------------------------------------------------------------ // impInitializeArrayIntrinsic: Attempts to replace a call to InitializeArray // with a GT_COPYBLK node. @@ -5629,8 +5759,7 @@ void Compiler::impCheckForPInvokeCall( // return here without inlining the native call. if (unmanagedCallConv == CorInfoCallConvExtension::Managed || unmanagedCallConv == CorInfoCallConvExtension::Fastcall || - unmanagedCallConv == CorInfoCallConvExtension::FastcallMemberFunction || - unmanagedCallConv == CorInfoCallConvExtension::Swift) + unmanagedCallConv == CorInfoCallConvExtension::FastcallMemberFunction) { return; } diff --git a/src/coreclr/jit/indirectcalltransformer.cpp b/src/coreclr/jit/indirectcalltransformer.cpp index fbbe08d66fc3c..1a295384f464c 100644 --- a/src/coreclr/jit/indirectcalltransformer.cpp +++ b/src/coreclr/jit/indirectcalltransformer.cpp @@ -218,13 +218,12 @@ class IndirectCallTransformer // Arguments: // jumpKind - jump kind for the new basic block // insertAfter - basic block, after which compiler has to insert the new one. - // jumpDest - jump target for the new basic block. Defaults to nullptr. // // Return Value: // new basic block. - BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter, BasicBlock* jumpDest = nullptr) + BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter) { - BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true, jumpDest); + BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true); block->SetFlags(BBF_IMPORTED); return block; } @@ -272,32 +271,35 @@ class IndirectCallTransformer if (checkBlock != currBlock) { assert(currBlock->KindIs(BBJ_ALWAYS)); - currBlock->SetTarget(checkBlock); FlowEdge* const newEdge = compiler->fgAddRefPred(checkBlock, currBlock); newEdge->setLikelihood(1.0); + currBlock->SetTargetEdge(newEdge); } // checkBlock // Todo: get likelihoods right // assert(checkBlock->KindIs(BBJ_ALWAYS)); - checkBlock->SetCond(elseBlock, thenBlock); FlowEdge* const thenEdge = compiler->fgAddRefPred(thenBlock, checkBlock); thenEdge->setLikelihood(0.5); FlowEdge* const elseEdge = compiler->fgAddRefPred(elseBlock, checkBlock); elseEdge->setLikelihood(0.5); + checkBlock->SetCond(elseEdge, thenEdge); // thenBlock - assert(thenBlock->TargetIs(remainderBlock)); { + assert(thenBlock->KindIs(BBJ_ALWAYS)); FlowEdge* const newEdge = compiler->fgAddRefPred(remainderBlock, thenBlock); newEdge->setLikelihood(1.0); + thenBlock->SetTargetEdge(newEdge); } // elseBlock { + assert(elseBlock->KindIs(BBJ_ALWAYS)); FlowEdge* const newEdge = compiler->fgAddRefPred(remainderBlock, elseBlock); newEdge->setLikelihood(1.0); + elseBlock->SetTargetEdge(newEdge); } } @@ -376,7 +378,7 @@ class IndirectCallTransformer { assert(checkIdx == 0); - checkBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, currBlock, currBlock->Next()); + checkBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, currBlock); checkBlock->SetFlags(BBF_NONE_QUIRK); GenTree* fatPointerMask = new (compiler, GT_CNS_INT) GenTreeIntCon(TYP_I_IMPL, FAT_POINTER_MASK); GenTree* fptrAddressCopy = compiler->gtCloneExpr(fptrAddress); @@ -395,7 +397,7 @@ class IndirectCallTransformer virtual void CreateThen(uint8_t checkIdx) { assert(remainderBlock != nullptr); - thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock, remainderBlock); + thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); Statement* copyOfOriginalStmt = compiler->gtCloneStmt(stmt); compiler->fgInsertStmtAtEnd(thenBlock, copyOfOriginalStmt); } @@ -405,7 +407,7 @@ class IndirectCallTransformer // virtual void CreateElse() { - elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock, thenBlock->Next()); + elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock); elseBlock->SetFlags(BBF_NONE_QUIRK); GenTree* fixedFptrAddress = GetFixedFptrAddress(); @@ -601,23 +603,25 @@ class IndirectCallTransformer checkBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock); checkFallsThrough = false; - // Calculate the total likelihood for this check as a sum of likelihoods - // of all previous candidates (thenBlocks) - unsigned checkLikelihood = 100; - for (uint8_t previousCandidate = 0; previousCandidate < checkIdx; previousCandidate++) - { - checkLikelihood -= origCall->GetGDVCandidateInfo(previousCandidate)->likelihood; - } + // We computed the "then" likelihood in CreateThen, so we + // just use that to figure out the "else" likelihood. + // + assert(prevCheckBlock->KindIs(BBJ_ALWAYS)); + assert(prevCheckBlock->JumpsToNext()); + FlowEdge* const prevCheckThenEdge = prevCheckBlock->GetTargetEdge(); + assert(prevCheckThenEdge->hasLikelihood()); + weight_t checkLikelihood = max(0.0, 1.0 - prevCheckThenEdge->getLikelihood()); - // Make sure we didn't overflow - assert(checkLikelihood <= 100); - weight_t checkLikelihoodWt = ((weight_t)checkLikelihood) / 100.0; + JITDUMP("Level %u Check block " FMT_BB " success likelihood " FMT_WT "\n", checkIdx, checkBlock->bbNum, + checkLikelihood); // prevCheckBlock is expected to jump to this new check (if its type check doesn't succeed) - prevCheckBlock->SetCond(checkBlock, prevCheckBlock->Next()); - FlowEdge* const checkEdge = compiler->fgAddRefPred(checkBlock, prevCheckBlock); - checkEdge->setLikelihood(checkLikelihoodWt); - checkBlock->inheritWeightPercentage(currBlock, checkLikelihood); + // + FlowEdge* const prevCheckCheckEdge = compiler->fgAddRefPred(checkBlock, prevCheckBlock); + prevCheckCheckEdge->setLikelihood(checkLikelihood); + checkBlock->inheritWeight(prevCheckBlock); + checkBlock->scaleBBWeight(checkLikelihood); + prevCheckBlock->SetCond(prevCheckCheckEdge, prevCheckThenEdge); } // Find last arg with a side effect. All args with any effect @@ -1016,25 +1020,59 @@ class IndirectCallTransformer { // Compute likelihoods // - unsigned const thenLikelihood = origCall->GetGDVCandidateInfo(checkIdx)->likelihood; - weight_t thenLikelihoodWt = min(((weight_t)thenLikelihood) / 100.0, 100.0); - weight_t elseLikelihoodWt = max(1.0 - thenLikelihoodWt, 0.0); + // If this is the first check the likelihood is just the candidate likelihood. + // If there are multiple checks things are a bit more complicated. + // + // Say we had three candidates with likelihoods 0.5, 0.3, and 0.1. + // + // The first one's likelihood is 0.5. + // + // The second one (given that we've already checked the first and failed) + // is (0.3) / (1.0 - 0.5) = 0.6. + // + // The third one is (0.1) / (1.0 - (0.5 + 0.3)) = (0.1)/(0.2) = 0.5 + // + // So to figure out the proper divisor, we start with 1.0 and subtract off each + // preceeding test's likelihood of success. + // + unsigned const thenLikelihood = origCall->GetGDVCandidateInfo(checkIdx)->likelihood; + unsigned baseLikelihood = 0; + + for (uint8_t i = 0; i < checkIdx; i++) + { + baseLikelihood += origCall->GetGDVCandidateInfo(i)->likelihood; + } + assert(baseLikelihood < 100); + baseLikelihood = 100 - baseLikelihood; + + weight_t adjustedThenLikelihood = min(((weight_t)thenLikelihood) / baseLikelihood, 100.0); + JITDUMP("For check in " FMT_BB ": orig likelihood " FMT_WT ", base likelihood " FMT_WT + ", adjusted likelihood " FMT_WT "\n", + checkBlock->bbNum, (weight_t)thenLikelihood / 100.0, (weight_t)baseLikelihood / 100.0, + adjustedThenLikelihood); // thenBlock always jumps to remainderBlock - thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock, remainderBlock); + // + thenBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, checkBlock); thenBlock->CopyFlags(currBlock, BBF_SPLIT_GAINED); - thenBlock->inheritWeightPercentage(currBlock, thenLikelihood); + thenBlock->inheritWeight(currBlock); + thenBlock->scaleBBWeight(adjustedThenLikelihood); + FlowEdge* const thenRemainderEdge = compiler->fgAddRefPred(remainderBlock, thenBlock); + thenBlock->SetTargetEdge(thenRemainderEdge); + thenRemainderEdge->setLikelihood(1.0); - // Also, thenBlock has a single pred - last checkBlock + // thenBlock has a single pred - last checkBlock. + // assert(checkBlock->KindIs(BBJ_ALWAYS)); - checkBlock->SetTarget(thenBlock); + FlowEdge* const checkThenEdge = compiler->fgAddRefPred(thenBlock, checkBlock); + checkThenEdge->setLikelihood(adjustedThenLikelihood); + checkBlock->SetTargetEdge(checkThenEdge); checkBlock->SetFlags(BBF_NONE_QUIRK); assert(checkBlock->JumpsToNext()); - FlowEdge* const thenEdge = compiler->fgAddRefPred(thenBlock, checkBlock); - thenEdge->setLikelihood(thenLikelihoodWt); - FlowEdge* const elseEdge = compiler->fgAddRefPred(remainderBlock, thenBlock); - elseEdge->setLikelihood(elseLikelihoodWt); + // We will set the "else edge" likelihood in CreateElse later, + // based on the thenEdge likelihood. + // DevirtualizeCall(thenBlock, checkIdx); } @@ -1043,28 +1081,28 @@ class IndirectCallTransformer // virtual void CreateElse() { - // Calculate the likelihood of the else block as a remainder of the sum - // of all the other likelihoods. - unsigned elseLikelihood = 100; - for (uint8_t i = 0; i < origCall->GetInlineCandidatesCount(); i++) - { - elseLikelihood -= origCall->GetGDVCandidateInfo(i)->likelihood; - } - // Make sure it didn't overflow - assert(elseLikelihood <= 100); - weight_t elseLikelihoodDbl = ((weight_t)elseLikelihood) / 100.0; - - elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock, thenBlock->Next()); + elseBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, thenBlock); elseBlock->CopyFlags(currBlock, BBF_SPLIT_GAINED); elseBlock->SetFlags(BBF_NONE_QUIRK); + // We computed the "then" likelihood in CreateThen, so we + // just use that to figure out the "else" likelihood. + // + assert(checkBlock->KindIs(BBJ_ALWAYS)); + FlowEdge* const checkThenEdge = checkBlock->GetTargetEdge(); + assert(checkThenEdge->hasLikelihood()); + weight_t elseLikelihood = max(0.0, 1.0 - checkThenEdge->getLikelihood()); + // CheckBlock flows into elseBlock unless we deal with the case // where we know the last check is always true (in case of "exact" GDV) + // if (!checkFallsThrough) { - checkBlock->SetCond(elseBlock, checkBlock->Next()); - FlowEdge* const checkEdge = compiler->fgAddRefPred(elseBlock, checkBlock); - checkEdge->setLikelihood(elseLikelihoodDbl); + assert(checkBlock->KindIs(BBJ_ALWAYS)); + assert(checkBlock->JumpsToNext()); + FlowEdge* const checkElseEdge = compiler->fgAddRefPred(elseBlock, checkBlock); + checkElseEdge->setLikelihood(elseLikelihood); + checkBlock->SetCond(checkElseEdge, checkThenEdge); } else { @@ -1072,16 +1110,23 @@ class IndirectCallTransformer // and is NativeAOT-only, we just assume the unreached block will be removed // by other phases. assert(origCall->gtCallMoreFlags & GTF_CALL_M_GUARDED_DEVIRT_EXACT); + + // Since we're not modifying the check block a BBJ_COND, update the + // then edge likelihood (it should already have the right value, so perhaps assert instead?) + // + checkThenEdge->setLikelihood(1.0); } // elseBlock always flows into remainderBlock - FlowEdge* const elseEdge = compiler->fgAddRefPred(remainderBlock, elseBlock); - elseEdge->setLikelihood(1.0); + FlowEdge* const elseRemainderEdge = compiler->fgAddRefPred(remainderBlock, elseBlock); + elseRemainderEdge->setLikelihood(1.0); + elseBlock->SetTargetEdge(elseRemainderEdge); // Remove everything related to inlining from the original call origCall->ClearInlineInfo(); - elseBlock->inheritWeightPercentage(currBlock, elseLikelihood); + elseBlock->inheritWeight(checkBlock); + elseBlock->scaleBBWeight(elseLikelihood); GenTreeCall* call = origCall; Statement* newStmt = compiler->gtNewStmt(call, stmt->GetDebugInfo()); @@ -1176,12 +1221,12 @@ class IndirectCallTransformer // Finally, rewire the cold block to jump to the else block, // not fall through to the check block. // - FlowEdge* const oldEdge = compiler->fgRemoveRefPred(checkBlock, coldBlock); - coldBlock->SetKindAndTarget(BBJ_ALWAYS, elseBlock); - compiler->fgAddRefPred(elseBlock, coldBlock, oldEdge); + compiler->fgRemoveRefPred(coldBlock->GetTargetEdge()); + FlowEdge* const newEdge = compiler->fgAddRefPred(elseBlock, coldBlock, coldBlock->GetTargetEdge()); + coldBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } - // When the current candidate hads sufficiently high likelihood, scan + // When the current candidate has sufficiently high likelihood, scan // the remainer block looking for another GDV candidate. // // (also consider: if currBlock has sufficiently high execution frequency) diff --git a/src/coreclr/jit/inductionvariableopts.cpp b/src/coreclr/jit/inductionvariableopts.cpp new file mode 100644 index 0000000000000..b76d9e0be7438 --- /dev/null +++ b/src/coreclr/jit/inductionvariableopts.cpp @@ -0,0 +1,686 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file contains code to optimize induction variables in loops based on +// scalar evolution analysis (see scev.h and scev.cpp for more information +// about the scalar evolution analysis). +// +// Currently the only optimization done is widening of primary induction +// variables from 32 bits into 64 bits. This is generally only profitable on +// x64 that does not allow zero extension of 32-bit values in addressing modes +// (in contrast, arm64 does have the capability of including zero extensions in +// addressing modes). For x64 this saves a zero extension for every array +// access inside the loop, in exchange for some widening or narrowing stores +// outside the loop: +// - To make sure the new widened IV starts at the right value it is +// initialized to the value of the narrow IV outside the loop (either in the +// preheader or at the def location of the narrow IV). Usually the start +// value is a constant, in which case the widened IV is just initialized to +// the constant value. +// - If the narrow IV is used after the loop we need to store it back from +// the widened IV in the exits. We depend on liveness sets to figure out +// which exits to insert IR into. +// +// These steps ensure that the wide IV has the right value to begin with and +// the old narrow IV still has the right value after the loop. Additionally, +// we must replace every use of the narrow IV inside the loop with the widened +// IV. This is done by a traversal of the IR inside the loop. We do not +// actually widen the uses of the IV; rather, we keep all uses and defs as +// 32-bit, which the backend is able to handle efficiently on x64. Because of +// this we do not need to worry about overflow. +// + +#include "jitpch.h" +#include "scev.h" + +//------------------------------------------------------------------------ +// optCanSinkWidenedIV: Check to see if we are able to sink a store to the old +// local into the exits of a loop if we decide to widen. +// +// Parameters: +// lclNum - The primary induction variable +// loop - The loop +// +// Returns: +// True if we can sink a store to the old local after widening. +// +// Remarks: +// This handles the situation where the primary induction variable is used +// after the loop. In those cases we need to store the widened local back +// into the old one in the exits where the IV variable is live. +// +// We are able to sink when none of the exits are critical blocks, in the +// sense that all their predecessors must come from inside the loop. Loop +// exit canonicalization guarantees this for regular exit blocks. It is not +// guaranteed for exceptional exits, but we do not expect to widen IVs that +// are live into exceptional exits since those are marked DNER which makes it +// unprofitable anyway. +// +// Note that there may be natural loops that have not had their regular exits +// canonicalized at the time when IV opts run, in particular if RBO/assertion +// prop makes a previously unnatural loop natural. This function accounts for +// and rejects these cases. +// +bool Compiler::optCanSinkWidenedIV(unsigned lclNum, FlowGraphNaturalLoop* loop) +{ + LclVarDsc* dsc = lvaGetDesc(lclNum); + + BasicBlockVisit result = loop->VisitRegularExitBlocks([=](BasicBlock* exit) { + + if (!VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) + { + JITDUMP(" Exit " FMT_BB " does not need a sink; V%02u is not live-in\n", exit->bbNum, lclNum); + return BasicBlockVisit::Continue; + } + + for (BasicBlock* pred : exit->PredBlocks()) + { + if (!loop->ContainsBlock(pred)) + { + JITDUMP(" Cannot safely sink widened version of V%02u into exit " FMT_BB " of " FMT_LP + "; it has a non-loop pred " FMT_BB "\n", + lclNum, exit->bbNum, loop->GetIndex(), pred->bbNum); + return BasicBlockVisit::Abort; + } + } + + return BasicBlockVisit::Continue; + }); + +#ifdef DEBUG + // We currently do not expect to ever widen IVs that are live into + // exceptional exits. Such IVs are expected to have been marked DNER + // previously (EH write-thru is only for single def locals) which makes it + // unprofitable. If this ever changes we need some more expansive handling + // here. + loop->VisitLoopBlocks([=](BasicBlock* block) { + + block->VisitAllSuccs(this, [=](BasicBlock* succ) { + if (!loop->ContainsBlock(succ) && bbIsHandlerBeg(succ)) + { + assert(!VarSetOps::IsMember(this, succ->bbLiveIn, dsc->lvVarIndex) && + "Candidate IV for widening is live into exceptional exit"); + } + + return BasicBlockVisit::Continue; + }); + + return BasicBlockVisit::Continue; + }); +#endif + + return result != BasicBlockVisit::Abort; +} + +//------------------------------------------------------------------------ +// optIsIVWideningProfitable: Check to see if IV widening is profitable. +// +// Parameters: +// lclNum - The primary induction variable +// initBlock - The block in where the new IV would be initialized +// initedToConstant - Whether or not the new IV will be initialized to a constant +// loop - The loop +// ivUses - Statements in which "lclNum" appears will be added to this list +// +// +// Returns: +// True if IV widening is profitable. +// +// Remarks: +// IV widening is generally profitable when it allows us to remove casts +// inside the loop. However, it may also introduce other reg-reg moves: +// 1. We may need to store the narrow IV into the wide one in the +// preheader. This is necessary when the start value is not constant. If +// the start value _is_ constant then we assume that the constant store to +// the narrow local will be a DCE'd. +// 2. We need to store the wide IV back into the narrow one in each of +// the exits where the narrow IV is live-in. +// +bool Compiler::optIsIVWideningProfitable(unsigned lclNum, + BasicBlock* initBlock, + bool initedToConstant, + FlowGraphNaturalLoop* loop, + ArrayStack& ivUses) +{ + for (FlowGraphNaturalLoop* otherLoop : m_loops->InReversePostOrder()) + { + if (otherLoop == loop) + continue; + + for (Statement* stmt : otherLoop->GetHeader()->Statements()) + { + if (!stmt->IsPhiDefnStmt()) + break; + + if (stmt->GetRootNode()->AsLclVarCommon()->GetLclNum() == lclNum) + { + JITDUMP(" V%02u has a phi [%06u] in " FMT_LP "'s header " FMT_BB "\n", lclNum, + dspTreeID(stmt->GetRootNode()), otherLoop->GetIndex(), otherLoop->GetHeader()->bbNum); + // TODO-CQ: We can legally widen these cases, but LSRA is + // unhappy about some of the lifetimes we create when we do + // this. This particularly affects cloned loops. + return false; + } + } + } + + const weight_t ExtensionCost = 2; + const int ExtensionSize = 3; + + weight_t savedCost = 0; + int savedSize = 0; + + loop->VisitLoopBlocks([&](BasicBlock* block) { + for (Statement* stmt : block->NonPhiStatements()) + { + bool hasUse = false; + int numExtensions = 0; + for (GenTree* node : stmt->TreeList()) + { + if (!node->OperIs(GT_CAST)) + { + hasUse |= node->OperIsLocal() && (node->AsLclVarCommon()->GetLclNum() == lclNum); + continue; + } + + GenTreeCast* cast = node->AsCast(); + if ((cast->gtCastType != TYP_LONG) || !cast->IsUnsigned() || cast->gtOverflow()) + { + continue; + } + + GenTree* op = cast->CastOp(); + if (!op->OperIs(GT_LCL_VAR) || (op->AsLclVarCommon()->GetLclNum() != lclNum)) + { + continue; + } + + // If this is already the source of a store then it is going to be + // free in our backends regardless. + GenTree* parent = node->gtGetParent(nullptr); + if ((parent != nullptr) && parent->OperIs(GT_STORE_LCL_VAR)) + { + continue; + } + + numExtensions++; + } + + if (hasUse) + { + ivUses.Push(stmt); + } + + if (numExtensions > 0) + { + JITDUMP(" Found %d zero extensions in " FMT_STMT "\n", numExtensions, stmt->GetID()); + + savedSize += numExtensions * ExtensionSize; + savedCost += numExtensions * block->getBBWeight(this) * ExtensionCost; + } + } + + return BasicBlockVisit::Continue; + }); + + if (!initedToConstant) + { + // We will need to store the narrow IV into the wide one in the init + // block. We only cost this when init value is not a constant since + // otherwise we assume that constant initialization of the narrow local + // will be DCE'd. + savedSize -= ExtensionSize; + savedCost -= initBlock->getBBWeight(this) * ExtensionCost; + } + + // Now account for the cost of sinks. + LclVarDsc* dsc = lvaGetDesc(lclNum); + loop->VisitRegularExitBlocks([&](BasicBlock* exit) { + if (VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) + { + savedSize -= ExtensionSize; + savedCost -= exit->getBBWeight(this) * ExtensionCost; + } + return BasicBlockVisit::Continue; + }); + + const weight_t ALLOWED_SIZE_REGRESSION_PER_CYCLE_IMPROVEMENT = 2; + weight_t cycleImprovementPerInvoc = savedCost / fgFirstBB->getBBWeight(this); + + JITDUMP(" Estimated cycle improvement: " FMT_WT " cycles per invocation\n", cycleImprovementPerInvoc); + JITDUMP(" Estimated size improvement: %d bytes\n", savedSize); + + if ((cycleImprovementPerInvoc > 0) && + ((cycleImprovementPerInvoc * ALLOWED_SIZE_REGRESSION_PER_CYCLE_IMPROVEMENT) >= -savedSize)) + { + JITDUMP(" Widening is profitable (cycle improvement)\n"); + return true; + } + + const weight_t ALLOWED_CYCLE_REGRESSION_PER_SIZE_IMPROVEMENT = 0.01; + + if ((savedSize > 0) && ((savedSize * ALLOWED_CYCLE_REGRESSION_PER_SIZE_IMPROVEMENT) >= -cycleImprovementPerInvoc)) + { + JITDUMP(" Widening is profitable (size improvement)\n"); + return true; + } + + JITDUMP(" Widening is not profitable\n"); + return false; +} + +//------------------------------------------------------------------------ +// optSinkWidenedIV: Create stores back to the narrow IV in the exits where +// that is necessary. +// +// Parameters: +// lclNum - Narrow version of primary induction variable +// newLclNum - Wide version of primary induction variable +// loop - The loop +// +// Returns: +// True if any store was created in any exit block. +// +void Compiler::optSinkWidenedIV(unsigned lclNum, unsigned newLclNum, FlowGraphNaturalLoop* loop) +{ + LclVarDsc* dsc = lvaGetDesc(lclNum); + loop->VisitRegularExitBlocks([=](BasicBlock* exit) { + if (!VarSetOps::IsMember(this, exit->bbLiveIn, dsc->lvVarIndex)) + { + return BasicBlockVisit::Continue; + } + + GenTree* narrowing = gtNewCastNode(TYP_INT, gtNewLclvNode(newLclNum, TYP_LONG), false, TYP_INT); + GenTree* store = gtNewStoreLclVarNode(lclNum, narrowing); + Statement* newStmt = fgNewStmtFromTree(store); + JITDUMP("Narrow IV local V%02u live into exit block " FMT_BB "; sinking a narrowing\n", lclNum, exit->bbNum); + DISPSTMT(newStmt); + fgInsertStmtAtBeg(exit, newStmt); + + return BasicBlockVisit::Continue; + }); +} + +//------------------------------------------------------------------------ +// optReplaceWidenedIV: Replace uses of the narrow IV with the wide IV in the +// specified statement. +// +// Parameters: +// lclNum - Narrow version of primary induction variable +// newLclNum - Wide version of primary induction variable +// stmt - The statement to replace uses in. +// +void Compiler::optReplaceWidenedIV(unsigned lclNum, unsigned ssaNum, unsigned newLclNum, Statement* stmt) +{ + struct ReplaceVisitor : GenTreeVisitor + { + private: + unsigned m_lclNum; + unsigned m_ssaNum; + unsigned m_newLclNum; + + bool IsLocal(GenTreeLclVarCommon* tree) + { + return (tree->GetLclNum() == m_lclNum) && + ((m_ssaNum == SsaConfig::RESERVED_SSA_NUM) || (tree->GetSsaNum() == m_ssaNum)); + } + + public: + bool MadeChanges = false; + + enum + { + DoPreOrder = true, + }; + + ReplaceVisitor(Compiler* comp, unsigned lclNum, unsigned ssaNum, unsigned newLclNum) + : GenTreeVisitor(comp), m_lclNum(lclNum), m_ssaNum(ssaNum), m_newLclNum(newLclNum) + { + } + + fgWalkResult PreOrderVisit(GenTree** use, GenTree* user) + { + GenTree* node = *use; + if (node->OperIs(GT_CAST)) + { + GenTreeCast* cast = node->AsCast(); + if ((cast->gtCastType == TYP_LONG) && cast->IsUnsigned() && !cast->gtOverflow()) + { + GenTree* op = cast->CastOp(); + if (op->OperIs(GT_LCL_VAR) && IsLocal(op->AsLclVarCommon())) + { + *use = m_compiler->gtNewLclvNode(m_newLclNum, TYP_LONG); + MadeChanges = true; + return fgWalkResult::WALK_SKIP_SUBTREES; + } + } + } + else if (node->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_LCL_FLD, GT_STORE_LCL_FLD) && + IsLocal(node->AsLclVarCommon())) + { + switch (node->OperGet()) + { + case GT_LCL_VAR: + node->AsLclVarCommon()->SetLclNum(m_newLclNum); + // No cast needed -- the backend allows TYP_INT uses of TYP_LONG locals. + break; + case GT_STORE_LCL_VAR: + { + node->AsLclVarCommon()->SetLclNum(m_newLclNum); + node->gtType = TYP_LONG; + node->AsLclVarCommon()->Data() = + m_compiler->gtNewCastNode(TYP_LONG, node->AsLclVarCommon()->Data(), true, TYP_LONG); + break; + } + case GT_LCL_FLD: + case GT_STORE_LCL_FLD: + assert(!"Unexpected field use for local not marked as DNER"); + break; + default: + break; + } + + MadeChanges = true; + } + + return fgWalkResult::WALK_CONTINUE; + } + }; + + ReplaceVisitor visitor(this, lclNum, ssaNum, newLclNum); + visitor.WalkTree(stmt->GetRootNodePointer(), nullptr); + if (visitor.MadeChanges) + { + gtSetStmtInfo(stmt); + fgSetStmtSeq(stmt); + JITDUMP("New tree:\n", dspTreeID(stmt->GetRootNode())); + DISPTREE(stmt->GetRootNode()); + JITDUMP("\n"); + } + else + { + JITDUMP("No replacements made\n"); + } +} + +//------------------------------------------------------------------------ +// optBestEffortReplaceNarrowIVUses: Try to find and replace uses of the specified +// SSA def with a new local. +// +// Parameters: +// lclNum - Previous local +// ssaNum - Previous local SSA num +// newLclNum - New local to replace with +// block - Block to replace in +// firstStmt - First statement in "block" to start replacing in +// +// Remarks: +// This function is best effort; it might not find all uses of the provided +// SSA num, particularly because it does not follow into joins. Note that we +// only use this to replace uses of the narrow IV outside the loop; inside +// the loop we do ensure that all uses/defs are replaced. +// Keeping it best-effort outside the loop is ok; there is no correctness +// issue since we do not invalidate the value of the old narrow IV in any +// way, but it may mean we end up leaving the narrow IV live concurrently +// with the new widened IV, increasing register pressure. +// +void Compiler::optBestEffortReplaceNarrowIVUses( + unsigned lclNum, unsigned ssaNum, unsigned newLclNum, BasicBlock* block, Statement* firstStmt) +{ + JITDUMP("Replacing V%02u -> V%02u in " FMT_BB " starting at " FMT_STMT "\n", lclNum, newLclNum, block->bbNum, + firstStmt == nullptr ? 0 : firstStmt->GetID()); + + for (Statement* stmt = firstStmt; stmt != nullptr; stmt = stmt->GetNextStmt()) + { + JITDUMP("Replacing V%02u -> V%02u in [%06u]\n", lclNum, newLclNum, dspTreeID(stmt->GetRootNode())); + DISPSTMT(stmt); + JITDUMP("\n"); + + optReplaceWidenedIV(lclNum, ssaNum, newLclNum, stmt); + } + + block->VisitRegularSuccs(this, [=](BasicBlock* succ) { + if (succ->GetUniquePred(this) == block) + { + optBestEffortReplaceNarrowIVUses(lclNum, ssaNum, newLclNum, succ, succ->firstStmt()); + } + + return BasicBlockVisit::Continue; + }); +} + +//------------------------------------------------------------------------ +// optInductionVariables: Try and optimize induction variables in the method. +// +// Returns: +// PhaseStatus indicating if anything changed. +// +PhaseStatus Compiler::optInductionVariables() +{ + JITDUMP("*************** In optInductionVariables()\n"); + +#ifdef DEBUG + static ConfigMethodRange s_range; + s_range.EnsureInit(JitConfig.JitEnableInductionVariableOptsRange()); + + if (!s_range.Contains(info.compMethodHash())) + { + return PhaseStatus::MODIFIED_NOTHING; + } +#endif + + if (!fgMightHaveNaturalLoops) + { + JITDUMP(" Skipping since this method has no natural loops\n"); + return PhaseStatus::MODIFIED_NOTHING; + } + + bool changed = false; + + // Currently we only do IV widening which generally is only profitable for + // x64 because arm64 addressing modes can include the zero/sign-extension + // of the index for free. + CLANG_FORMAT_COMMENT_ANCHOR; +#if defined(TARGET_XARCH) && defined(TARGET_64BIT) + m_dfsTree = fgComputeDfs(); + m_loops = FlowGraphNaturalLoops::Find(m_dfsTree); + + ScalarEvolutionContext scevContext(this); + JITDUMP("Widening primary induction variables:\n"); + ArrayStack ivUses(getAllocator(CMK_LoopIVOpts)); + for (FlowGraphNaturalLoop* loop : m_loops->InReversePostOrder()) + { + JITDUMP("Processing "); + DBEXEC(verbose, FlowGraphNaturalLoop::Dump(loop)); + scevContext.ResetForLoop(loop); + + int numWidened = 0; + + for (Statement* stmt : loop->GetHeader()->Statements()) + { + if (!stmt->IsPhiDefnStmt()) + { + break; + } + + JITDUMP("\n"); + + DISPSTMT(stmt); + + GenTreeLclVarCommon* lcl = stmt->GetRootNode()->AsLclVarCommon(); + LclVarDsc* lclDsc = lvaGetDesc(lcl); + if (lclDsc->TypeGet() != TYP_INT) + { + JITDUMP(" Type is %s, no widening to be done\n", varTypeName(lclDsc->TypeGet())); + continue; + } + + // If the IV is not enregisterable then uses/defs are going to go + // to stack regardless. This check also filters out IVs that may be + // live into exceptional exits since those are always marked DNER. + if (lclDsc->lvDoNotEnregister) + { + JITDUMP(" V%02u is marked DNER\n", lcl->GetLclNum()); + continue; + } + + Scev* scev = scevContext.Analyze(loop->GetHeader(), stmt->GetRootNode()); + if (scev == nullptr) + { + JITDUMP(" Could not analyze header PHI\n"); + continue; + } + + scev = scevContext.Simplify(scev); + JITDUMP(" => "); + DBEXEC(verbose, scev->Dump(this)); + JITDUMP("\n"); + if (!scev->OperIs(ScevOper::AddRec)) + { + JITDUMP(" Not an addrec\n"); + continue; + } + + ScevAddRec* addRec = (ScevAddRec*)scev; + + JITDUMP(" V%02u is a primary induction variable in " FMT_LP "\n", lcl->GetLclNum(), loop->GetIndex()); + + if (!optCanSinkWidenedIV(lcl->GetLclNum(), loop)) + { + continue; + } + + // Start value should always be an SSA use from outside the loop + // since we only widen primary IVs. + assert(addRec->Start->OperIs(ScevOper::Local)); + ScevLocal* startLocal = (ScevLocal*)addRec->Start; + int64_t startConstant = 0; + bool initToConstant = startLocal->GetConstantValue(this, &startConstant); + LclSsaVarDsc* startSsaDsc = lclDsc->GetPerSsaData(startLocal->SsaNum); + + BasicBlock* preheader = loop->EntryEdge(0)->getSourceBlock(); + BasicBlock* initBlock = preheader; + if ((startSsaDsc->GetBlock() != nullptr) && (startSsaDsc->GetDefNode() != nullptr)) + { + initBlock = startSsaDsc->GetBlock(); + } + + ivUses.Reset(); + if (!optIsIVWideningProfitable(lcl->GetLclNum(), initBlock, initToConstant, loop, ivUses)) + { + continue; + } + + changed = true; + + Statement* insertInitAfter = nullptr; + if (initBlock != preheader) + { + GenTree* narrowInitRoot = startSsaDsc->GetDefNode(); + while (true) + { + GenTree* parent = narrowInitRoot->gtGetParent(nullptr); + if (parent == nullptr) + break; + + narrowInitRoot = parent; + } + + for (Statement* stmt : initBlock->Statements()) + { + if (stmt->GetRootNode() == narrowInitRoot) + { + insertInitAfter = stmt; + break; + } + } + + assert(insertInitAfter != nullptr); + + if (insertInitAfter->IsPhiDefnStmt()) + { + while ((insertInitAfter->GetNextStmt() != nullptr) && + insertInitAfter->GetNextStmt()->IsPhiDefnStmt()) + { + insertInitAfter = insertInitAfter->GetNextStmt(); + } + } + } + + Statement* initStmt = nullptr; + unsigned newLclNum = lvaGrabTemp(false DEBUGARG(printfAlloc("Widened IV V%02u", lcl->GetLclNum()))); + INDEBUG(lclDsc = nullptr); + assert(startLocal->LclNum == lcl->GetLclNum()); + + if (initBlock != preheader) + { + JITDUMP("Adding initialization of new widened local to same block as reaching def outside loop, " FMT_BB + "\n", + initBlock->bbNum); + } + else + { + JITDUMP("Adding initialization of new widened local to preheader " FMT_BB "\n", initBlock->bbNum); + } + + GenTree* initVal; + if (initToConstant) + { + initVal = gtNewIconNode((int64_t)(uint32_t)startConstant, TYP_LONG); + } + else + { + initVal = gtNewCastNode(TYP_LONG, gtNewLclvNode(lcl->GetLclNum(), TYP_INT), true, TYP_LONG); + } + + GenTree* widenStore = gtNewTempStore(newLclNum, initVal); + initStmt = fgNewStmtFromTree(widenStore); + if (insertInitAfter != nullptr) + { + fgInsertStmtAfter(initBlock, insertInitAfter, initStmt); + } + else + { + fgInsertStmtNearEnd(initBlock, initStmt); + } + + DISPSTMT(initStmt); + JITDUMP("\n"); + + JITDUMP(" Replacing uses of V%02u with widened version V%02u\n", lcl->GetLclNum(), newLclNum); + + if (initStmt != nullptr) + { + JITDUMP(" Replacing on the way to the loop\n"); + optBestEffortReplaceNarrowIVUses(lcl->GetLclNum(), startLocal->SsaNum, newLclNum, initBlock, + initStmt->GetNextStmt()); + } + + JITDUMP(" Replacing in the loop; %d statements with appearences\n", ivUses.Height()); + for (int i = 0; i < ivUses.Height(); i++) + { + Statement* stmt = ivUses.Bottom(i); + JITDUMP("Replacing V%02u -> V%02u in [%06u]\n", lcl->GetLclNum(), newLclNum, + dspTreeID(stmt->GetRootNode())); + DISPSTMT(stmt); + JITDUMP("\n"); + optReplaceWidenedIV(lcl->GetLclNum(), SsaConfig::RESERVED_SSA_NUM, newLclNum, stmt); + } + + optSinkWidenedIV(lcl->GetLclNum(), newLclNum, loop); + + numWidened++; + } + + Metrics.WidenedIVs += numWidened; + if (numWidened > 0) + { + Metrics.LoopsIVWidened++; + } + } + + fgInvalidateDfsTree(); +#endif + + return changed ? PhaseStatus::MODIFIED_EVERYTHING : PhaseStatus::MODIFIED_NOTHING; +} diff --git a/src/coreclr/jit/instr.h b/src/coreclr/jit/instr.h index 3120b2ac87fc6..1ff07e4301719 100644 --- a/src/coreclr/jit/instr.h +++ b/src/coreclr/jit/instr.h @@ -380,6 +380,8 @@ enum insScalableOpts : unsigned // Removable once REG_V0 and REG_P0 are distinct INS_SCALABLE_OPTS_UNPREDICATED, // Variants without a predicate (eg add) INS_SCALABLE_OPTS_UNPREDICATED_WIDE, // Variants without a predicate and wide elements (eg asr) + INS_SCALABLE_OPTS_TO_PREDICATE, // Variants moving to a predicate from a vector (e.g. pmov) + INS_SCALABLE_OPTS_TO_VECTOR // Variants moving to a vector from a predicate (e.g. pmov) }; // Maps directly to the pattern used in SVE instructions such as cntb. diff --git a/src/coreclr/jit/jitconfigvalues.h b/src/coreclr/jit/jitconfigvalues.h index d84c2f79b7dda..ee8320344ce42 100644 --- a/src/coreclr/jit/jitconfigvalues.h +++ b/src/coreclr/jit/jitconfigvalues.h @@ -481,8 +481,9 @@ CONFIG_INTEGER(JitNoRngChks, W("JitNoRngChks"), 0) // If 1, don't generate range #if defined(OPT_CONFIG) CONFIG_INTEGER(JitDoAssertionProp, W("JitDoAssertionProp"), 1) // Perform assertion propagation optimization -CONFIG_INTEGER(JitDoCopyProp, W("JitDoCopyProp"), 1) // Perform copy propagation on variables that appear redundant -CONFIG_INTEGER(JitDoEarlyProp, W("JitDoEarlyProp"), 1) // Perform Early Value Propagation +CONFIG_INTEGER(JitDoCopyProp, W("JitDoCopyProp"), 1) // Perform copy propagation on variables that appear redundant +CONFIG_INTEGER(JitDoOptimizeIVs, W("JitDoOptimizeIVs"), 1) // Perform optimization of induction variables +CONFIG_INTEGER(JitDoEarlyProp, W("JitDoEarlyProp"), 1) // Perform Early Value Propagation CONFIG_INTEGER(JitDoLoopHoisting, W("JitDoLoopHoisting"), 1) // Perform loop hoisting on loop invariant values CONFIG_INTEGER(JitDoLoopInversion, W("JitDoLoopInversion"), 1) // Perform loop inversion on "for/while" loops CONFIG_INTEGER(JitDoRangeAnalysis, W("JitDoRangeAnalysis"), 1) // Perform range check analysis @@ -497,6 +498,7 @@ CONFIG_STRING(JitOnlyOptimizeRange, W("JitOnlyOptimizeRange")) // If set, all methods that do _not_ match are forced into MinOpts CONFIG_STRING(JitEnablePhysicalPromotionRange, W("JitEnablePhysicalPromotionRange")) CONFIG_STRING(JitEnableCrossBlockLocalAssertionPropRange, W("JitEnableCrossBlockLocalAssertionPropRange")) +CONFIG_STRING(JitEnableInductionVariableOptsRange, W("JitEnableInductionVariableOptsRange")) CONFIG_INTEGER(JitDoSsa, W("JitDoSsa"), 1) // Perform Static Single Assignment (SSA) numbering on the variables CONFIG_INTEGER(JitDoValueNumber, W("JitDoValueNumber"), 1) // Perform value numbering on method expressions diff --git a/src/coreclr/jit/jiteh.cpp b/src/coreclr/jit/jiteh.cpp index a4eacd9069db4..e7195b5c98cb2 100644 --- a/src/coreclr/jit/jiteh.cpp +++ b/src/coreclr/jit/jiteh.cpp @@ -1984,10 +1984,11 @@ bool Compiler::fgNormalizeEHCase1() { // ...then we want to insert an empty, non-removable block outside the try to be the new first block of the // handler. - BasicBlock* newHndStart = BasicBlock::New(this, BBJ_ALWAYS, handlerStart); + BasicBlock* newHndStart = BasicBlock::New(this); fgInsertBBbefore(handlerStart, newHndStart); FlowEdge* newEdge = fgAddRefPred(handlerStart, newHndStart); newEdge->setLikelihood(1.0); + newHndStart->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // Handler begins have an extra implicit ref count. // BasicBlock::New has already handled this for newHndStart. @@ -2154,11 +2155,12 @@ bool Compiler::fgNormalizeEHCase2() // We've got multiple 'try' blocks starting at the same place! // Add a new first 'try' block for 'ehOuter' that will be outside 'eh'. - BasicBlock* newTryStart = BasicBlock::New(this, BBJ_ALWAYS, insertBeforeBlk); + BasicBlock* newTryStart = BasicBlock::New(this); newTryStart->bbRefs = 0; fgInsertBBbefore(insertBeforeBlk, newTryStart); FlowEdge* const newEdge = fgAddRefPred(insertBeforeBlk, newTryStart); newEdge->setLikelihood(1.0); + newTryStart->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // It's possible for a try to start at the beginning of a method. If so, we need // to adjust the implicit ref counts as we've just created a new first bb @@ -2346,7 +2348,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions() // Create a new bb for the fake filter BasicBlock* handlerBb = eh->ebdHndBeg; - BasicBlock* filterBb = BasicBlock::New(this, BBJ_EHFILTERRET, handlerBb); + BasicBlock* filterBb = BasicBlock::New(this); // Now we need to spill CATCH_ARG (it should be the first thing evaluated) GenTree* arg = new (this, GT_CATCH_ARG) GenTree(GT_CATCH_ARG, TYP_REF); @@ -2376,6 +2378,7 @@ bool Compiler::fgCreateFiltersForGenericExceptions() fgInsertBBbefore(handlerBb, filterBb); FlowEdge* const newEdge = fgAddRefPred(handlerBb, filterBb); newEdge->setLikelihood(1.0); + filterBb->SetKindAndTargetEdge(BBJ_EHFILTERRET, newEdge); fgNewStmtAtEnd(filterBb, retFilt, handlerBb->firstStmt()->GetDebugInfo()); filterBb->bbCatchTyp = BBCT_FILTER; @@ -2632,7 +2635,7 @@ bool Compiler::fgNormalizeEHCase3() // Add a new last block for 'ehOuter' that will be outside the EH region with which it encloses and // shares a 'last' pointer - BasicBlock* newLast = BasicBlock::New(this, BBJ_ALWAYS, insertAfterBlk->Next()); + BasicBlock* newLast = BasicBlock::New(this); newLast->bbRefs = 0; assert(insertAfterBlk != nullptr); fgInsertBBafter(insertAfterBlk, newLast); @@ -2683,6 +2686,7 @@ bool Compiler::fgNormalizeEHCase3() newLast->SetFlags(BBF_INTERNAL | BBF_NONE_QUIRK); FlowEdge* const newEdge = fgAddRefPred(newLast, insertAfterBlk); newEdge->setLikelihood(1.0); + insertAfterBlk->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); // Move the insert pointer. More enclosing equivalent 'last' blocks will be inserted after this. insertAfterBlk = newLast; @@ -4325,8 +4329,8 @@ void Compiler::fgExtendEHRegionBefore(BasicBlock* block) #endif // FEATURE_EH_FUNCLETS // If this is a handler for a filter, the last block of the filter will end with - // a BBJ_EHFILTERRET block that has a bbTarget that jumps to the first block of - // its handler. So we need to update it to keep things in sync. + // a BBJ_EHFILTERRET block that jumps to the first block of its handler. + // So we need to update it to keep things in sync. // if (HBtab->HasFilter()) { @@ -4337,15 +4341,15 @@ void Compiler::fgExtendEHRegionBefore(BasicBlock* block) #ifdef DEBUG if (verbose) { - printf("EH#%u: Updating bbTarget for filter ret block: " FMT_BB " => " FMT_BB "\n", - ehGetIndex(HBtab), bFilterLast->bbNum, bPrev->bbNum); + printf("EH#%u: Updating target for filter ret block: " FMT_BB " => " FMT_BB "\n", ehGetIndex(HBtab), + bFilterLast->bbNum, bPrev->bbNum); } #endif // DEBUG - // Change the bbTarget for bFilterLast from the old first 'block' to the new first 'bPrev' - fgRemoveRefPred(bFilterLast->GetTarget(), bFilterLast); - bFilterLast->SetTarget(bPrev); + // Change the target for bFilterLast from the old first 'block' to the new first 'bPrev' + fgRemoveRefPred(bFilterLast->GetTargetEdge()); FlowEdge* const newEdge = fgAddRefPred(bPrev, bFilterLast); newEdge->setLikelihood(1.0); + bFilterLast->SetTargetEdge(newEdge); } } diff --git a/src/coreclr/jit/jitmetadatalist.h b/src/coreclr/jit/jitmetadatalist.h index 8b69644d1a8fb..f36c15ab9991d 100644 --- a/src/coreclr/jit/jitmetadatalist.h +++ b/src/coreclr/jit/jitmetadatalist.h @@ -33,6 +33,8 @@ JITMETADATAMETRIC(LoopsCloned, int, 0) JITMETADATAMETRIC(LoopsUnrolled, int, 0) JITMETADATAMETRIC(LoopAlignmentCandidates, int, 0) JITMETADATAMETRIC(LoopsAligned, int, 0) +JITMETADATAMETRIC(LoopsIVWidened, int, 0) +JITMETADATAMETRIC(WidenedIVs, int, 0) JITMETADATAMETRIC(VarsInSsa, int, 0) JITMETADATAMETRIC(HoistedExpressions, int, 0) JITMETADATAMETRIC(RedundantBranchesEliminated, int, JIT_METADATA_HIGHER_IS_BETTER) diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index c4a4d44489f0d..78fb96fe3d77d 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -245,7 +245,6 @@ void Compiler::fgPerNodeLocalVarLiveness(GenTree* tree) case GT_STOREIND: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_MEMORYBARRIER: // Similar to Volatile indirections, we must handle this as a memory def. fgCurMemoryDef |= memoryKindSet(GcHeap, ByrefExposed); break; @@ -1937,7 +1936,6 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR case GT_STOREIND: case GT_BOUNDS_CHECK: case GT_STORE_BLK: - case GT_STORE_DYN_BLK: case GT_JCMP: case GT_JTEST: case GT_JCC: diff --git a/src/coreclr/jit/loopcloning.cpp b/src/coreclr/jit/loopcloning.cpp index 96cd8bff01593..c6d37dc507b95 100644 --- a/src/coreclr/jit/loopcloning.cpp +++ b/src/coreclr/jit/loopcloning.cpp @@ -860,17 +860,18 @@ BasicBlock* LoopCloneContext::CondToStmtInBlock(Compiler* { for (unsigned i = 0; i < conds.Size(); ++i) { - BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true, slowPreheader); + BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true); newBlk->inheritWeight(insertAfter); - JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, newBlk->GetTrueTarget()->bbNum); - comp->fgAddRefPred(newBlk->GetTrueTarget(), newBlk); + JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, slowPreheader->bbNum); + FlowEdge* const trueEdge = comp->fgAddRefPred(slowPreheader, newBlk); + newBlk->SetTrueEdge(trueEdge); if (insertAfter->KindIs(BBJ_COND)) { JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", insertAfter->bbNum, newBlk->bbNum); - insertAfter->SetFalseTarget(newBlk); - comp->fgAddRefPred(newBlk, insertAfter); + FlowEdge* const falseEdge = comp->fgAddRefPred(newBlk, insertAfter); + insertAfter->SetFalseEdge(falseEdge); } JITDUMP("Adding conditions %u to " FMT_BB "\n", i, newBlk->bbNum); @@ -894,16 +895,18 @@ BasicBlock* LoopCloneContext::CondToStmtInBlock(Compiler* } else { - BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true, slowPreheader); + BasicBlock* newBlk = comp->fgNewBBafter(BBJ_COND, insertAfter, /*extendRegion*/ true); newBlk->inheritWeight(insertAfter); - JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, newBlk->GetTrueTarget()->bbNum); - comp->fgAddRefPred(newBlk->GetTrueTarget(), newBlk); + JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", newBlk->bbNum, slowPreheader->bbNum); + FlowEdge* const trueEdge = comp->fgAddRefPred(slowPreheader, newBlk); + newBlk->SetTrueEdge(trueEdge); - if (insertAfter->bbFallsThrough()) + if (insertAfter->KindIs(BBJ_COND)) { JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", insertAfter->bbNum, newBlk->bbNum); - comp->fgAddRefPred(newBlk, insertAfter); + FlowEdge* const falseEdge = comp->fgAddRefPred(newBlk, insertAfter); + insertAfter->SetFalseEdge(falseEdge); } JITDUMP("Adding conditions to " FMT_BB "\n", newBlk->bbNum); @@ -1959,12 +1962,11 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // Make a new pre-header block for the fast loop. JITDUMP("Create new preheader block for fast loop\n"); - BasicBlock* fastPreheader = - fgNewBBafter(BBJ_ALWAYS, preheader, /*extendRegion*/ true, /*jumpDest*/ loop->GetHeader()); + BasicBlock* fastPreheader = fgNewBBafter(BBJ_ALWAYS, preheader, /*extendRegion*/ true); JITDUMP("Adding " FMT_BB " after " FMT_BB "\n", fastPreheader->bbNum, preheader->bbNum); fastPreheader->bbWeight = fastPreheader->isRunRarely() ? BB_ZERO_WEIGHT : ambientWeight; - if (fastPreheader->JumpsToNext()) + if (fastPreheader->NextIs(loop->GetHeader())) { fastPreheader->SetFlags(BBF_NONE_QUIRK); } @@ -1972,7 +1974,10 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex assert(preheader->KindIs(BBJ_ALWAYS)); assert(preheader->TargetIs(loop->GetHeader())); - fgReplacePred(loop->GetHeader(), preheader, fastPreheader); + FlowEdge* const oldEdge = preheader->GetTargetEdge(); + fgReplacePred(oldEdge, fastPreheader); + fastPreheader->SetTargetEdge(oldEdge); + JITDUMP("Replace " FMT_BB " -> " FMT_BB " with " FMT_BB " -> " FMT_BB "\n", preheader->bbNum, loop->GetHeader()->bbNum, fastPreheader->bbNum, loop->GetHeader()->bbNum); @@ -2039,9 +2044,12 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // We haven't set the jump target yet assert(slowPreheader->KindIs(BBJ_ALWAYS)); assert(!slowPreheader->HasInitializedTarget()); - slowPreheader->SetTarget(slowHeader); - fgAddRefPred(slowHeader, slowPreheader); + { + FlowEdge* const newEdge = fgAddRefPred(slowHeader, slowPreheader); + slowPreheader->SetTargetEdge(newEdge); + } + JITDUMP("Adding " FMT_BB " -> " FMT_BB "\n", slowPreheader->bbNum, slowHeader->bbNum); BasicBlock* condLast = optInsertLoopChoiceConditions(context, loop, slowPreheader, preheader); @@ -2049,14 +2057,18 @@ void Compiler::optCloneLoop(FlowGraphNaturalLoop* loop, LoopCloneContext* contex // Now redirect the old preheader to jump to the first new condition that // was inserted by the above function. assert(preheader->KindIs(BBJ_ALWAYS)); - preheader->SetTarget(preheader->Next()); - fgAddRefPred(preheader->Next(), preheader); + + { + FlowEdge* const newEdge = fgAddRefPred(preheader->Next(), preheader); + preheader->SetTargetEdge(newEdge); + } + preheader->SetFlags(BBF_NONE_QUIRK); // And make sure we insert a pred link for the final fallthrough into the fast preheader. assert(condLast->NextIs(fastPreheader)); - condLast->SetFalseTarget(fastPreheader); - fgAddRefPred(fastPreheader, condLast); + FlowEdge* const falseEdge = fgAddRefPred(fastPreheader, condLast); + condLast->SetFalseEdge(falseEdge); } //------------------------------------------------------------------------- diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 79935bffcae23..29ca8148dbe90 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -570,8 +570,6 @@ GenTree* Lowering::LowerNode(GenTree* node) LowerStoreSingleRegCallStruct(node->AsBlk()); break; } - FALLTHROUGH; - case GT_STORE_DYN_BLK: LowerBlockStoreCommon(node->AsBlk()); break; @@ -861,7 +859,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) { JITDUMP("Lowering switch " FMT_BB ": single target; converting to BBJ_ALWAYS\n", originalSwitchBB->bbNum); noway_assert(comp->opts.OptimizationDisabled()); - originalSwitchBB->SetKindAndTarget(BBJ_ALWAYS, jumpTab[0]->getDestinationBlock()); + originalSwitchBB->SetKindAndTargetEdge(BBJ_ALWAYS, jumpTab[0]); if (originalSwitchBB->JumpsToNext()) { @@ -951,7 +949,8 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // originalSwitchBB is now a BBJ_ALWAYS, and there is a predecessor edge in afterDefaultCondBlock // representing the fall-through flow from originalSwitchBB. assert(originalSwitchBB->KindIs(BBJ_ALWAYS)); - assert(originalSwitchBB->NextIs(afterDefaultCondBlock)); + assert(originalSwitchBB->TargetIs(afterDefaultCondBlock)); + assert(originalSwitchBB->JumpsToNext()); assert(afterDefaultCondBlock->KindIs(BBJ_SWITCH)); assert(afterDefaultCondBlock->GetSwitchTargets()->bbsHasDefault); assert(afterDefaultCondBlock->isEmpty()); // Nothing here yet. @@ -962,10 +961,10 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // as a predecessor, but the fgSplitBlockAfterStatement() moved all predecessors to point // to afterDefaultCondBlock. comp->fgRemoveRefPred(jumpTab[jumpCnt - 1]); - comp->fgAddRefPred(defaultBB, originalSwitchBB, jumpTab[jumpCnt - 1]); + FlowEdge* const trueEdge = comp->fgAddRefPred(defaultBB, originalSwitchBB, jumpTab[jumpCnt - 1]); // Turn originalSwitchBB into a BBJ_COND. - originalSwitchBB->SetCond(defaultBB, afterDefaultCondBlock); + originalSwitchBB->SetCond(trueEdge, originalSwitchBB->GetTargetEdge()); bool useJumpSequence = jumpCnt < minSwitchTabJumpCnt; @@ -1014,7 +1013,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) comp->fgRemoveRefPred(uniqueSucc); } - afterDefaultCondBlock->SetKindAndTarget(BBJ_ALWAYS, uniqueSucc->getDestinationBlock()); + afterDefaultCondBlock->SetKindAndTargetEdge(BBJ_ALWAYS, uniqueSucc); if (afterDefaultCondBlock->JumpsToNext()) { @@ -1067,10 +1066,10 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // If we haven't used the afterDefaultCondBlock yet, then use that. if (fUsedAfterDefaultCondBlock) { - BasicBlock* newBlock = comp->fgNewBBafter(BBJ_ALWAYS, currentBlock, true, currentBlock->Next()); + BasicBlock* newBlock = comp->fgNewBBafter(BBJ_ALWAYS, currentBlock, true); newBlock->SetFlags(BBF_NONE_QUIRK); - currentBlock->SetFalseTarget(newBlock); - comp->fgAddRefPred(newBlock, currentBlock); // The fall-through predecessor. + FlowEdge* const falseEdge = comp->fgAddRefPred(newBlock, currentBlock); // The fall-through predecessor. + currentBlock->SetFalseEdge(falseEdge); currentBlock = newBlock; currentBBRange = &LIR::AsRange(currentBlock); } @@ -1081,7 +1080,7 @@ GenTree* Lowering::LowerSwitch(GenTree* node) } // Wire up the predecessor list for the "branch" case. - comp->fgAddRefPred(targetBlock, currentBlock, jumpTab[i]); + FlowEdge* const newEdge = comp->fgAddRefPred(targetBlock, currentBlock, jumpTab[i]); if (!fAnyTargetFollows && (i == jumpCnt - 2)) { @@ -1090,13 +1089,14 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // case: there is no need to compare against the case index, since it's // guaranteed to be taken (since the default case was handled first, above). - currentBlock->SetKindAndTarget(BBJ_ALWAYS, targetBlock); + currentBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } else { // Otherwise, it's a conditional branch. Set the branch kind, then add the // condition statement. - currentBlock->SetCond(targetBlock, currentBlock->Next()); + // We will set the false edge in a later iteration of the loop, or after. + currentBlock->SetCond(newEdge); // Now, build the conditional statement for the current case that is // being evaluated: @@ -1118,7 +1118,8 @@ GenTree* Lowering::LowerSwitch(GenTree* node) // There is a fall-through to the following block. In the loop // above, we deleted all the predecessor edges from the switch. // In this case, we need to add one back. - comp->fgAddRefPred(currentBlock->Next(), currentBlock); + FlowEdge* const falseEdge = comp->fgAddRefPred(currentBlock->Next(), currentBlock); + currentBlock->SetFalseEdge(falseEdge); } if (!fUsedAfterDefaultCondBlock) @@ -1129,7 +1130,8 @@ GenTree* Lowering::LowerSwitch(GenTree* node) JITDUMP("Lowering switch " FMT_BB ": all switch cases were fall-through\n", originalSwitchBB->bbNum); assert(currentBlock == afterDefaultCondBlock); assert(currentBlock->KindIs(BBJ_SWITCH)); - currentBlock->SetKindAndTarget(BBJ_ALWAYS, currentBlock->Next()); + FlowEdge* const newEdge = comp->fgAddRefPred(currentBlock->Next(), currentBlock); + currentBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); currentBlock->RemoveFlags(BBF_DONT_REMOVE); comp->fgRemoveBlock(currentBlock, /* unreachable */ false); // It's an empty block. } @@ -1307,11 +1309,15 @@ bool Lowering::TryLowerSwitchToBitTest( comp->fgRemoveAllRefPreds(bbCase1, bbSwitch); comp->fgRemoveAllRefPreds(bbCase0, bbSwitch); + // TODO: Use old edges to influence new edge likelihoods? + case0Edge = comp->fgAddRefPred(bbCase0, bbSwitch); + case1Edge = comp->fgAddRefPred(bbCase1, bbSwitch); + if (bbSwitch->NextIs(bbCase0)) { // GenCondition::C generates JC so we jump to bbCase1 when the bit is set bbSwitchCondition = GenCondition::C; - bbSwitch->SetCond(bbCase1, bbCase0); + bbSwitch->SetCond(case1Edge, case0Edge); } else { @@ -1319,13 +1325,9 @@ bool Lowering::TryLowerSwitchToBitTest( // GenCondition::NC generates JNC so we jump to bbCase0 when the bit is not set bbSwitchCondition = GenCondition::NC; - bbSwitch->SetCond(bbCase0, bbCase1); + bbSwitch->SetCond(case0Edge, case1Edge); } - // TODO: Use old edges to influence new edge likelihoods? - comp->fgAddRefPred(bbCase0, bbSwitch); - comp->fgAddRefPred(bbCase1, bbSwitch); - var_types bitTableType = (bitCount <= (genTypeSize(TYP_INT) * 8)) ? TYP_INT : TYP_LONG; GenTree* bitTableIcon = comp->gtNewIconNode(bitTable, bitTableType); @@ -5997,6 +5999,26 @@ GenTree* Lowering::LowerNonvirtPinvokeCall(GenTreeCall* call) InsertPInvokeCallEpilog(call); } +#ifdef SWIFT_SUPPORT + // For Swift calls that require error handling, ensure the GT_SWIFT_ERROR node + // that consumes the error register is the call node's successor. + // This is to simplify logic for marking the error register as busy in LSRA. + if ((call->gtCallMoreFlags & GTF_CALL_M_SWIFT_ERROR_HANDLING) != 0) + { + GenTree* swiftErrorNode = call->gtNext; + assert(swiftErrorNode != nullptr); + + while (!swiftErrorNode->OperIs(GT_SWIFT_ERROR)) + { + swiftErrorNode = swiftErrorNode->gtNext; + assert(swiftErrorNode != nullptr); + } + + BlockRange().Remove(swiftErrorNode); + BlockRange().InsertAfter(call, swiftErrorNode); + } +#endif // SWIFT_SUPPORT + return result; } @@ -8108,17 +8130,9 @@ void Lowering::LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode) } } - if (blkNode->OperIs(GT_STORE_DYN_BLK)) - { - // Size is not a constant - size = blkNode->AsStoreDynBlk()->gtDynamicSize; - } - else - { - // Size is a constant - size = comp->gtNewIconNode(blkNode->Size(), TYP_I_IMPL); - BlockRange().InsertBefore(data, size); - } + // Size is a constant + size = comp->gtNewIconNode(blkNode->Size(), TYP_I_IMPL); + BlockRange().InsertBefore(data, size); // A hacky way to safely call fgMorphTree in Lower GenTree* destPlaceholder = comp->gtNewZeroConNode(dest->TypeGet()); @@ -9207,13 +9221,10 @@ void Lowering::LowerLclHeap(GenTree* node) // void Lowering::LowerBlockStoreCommon(GenTreeBlk* blkNode) { - assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK)); if (blkNode->ContainsReferences() && !blkNode->OperIsCopyBlkOp()) { - // Make sure we don't use GT_STORE_DYN_BLK - assert(blkNode->OperIs(GT_STORE_BLK)); - // and we only zero it (and that zero is better to be not hoisted/CSE'd) assert(blkNode->Data()->IsIntegralConst(0)); } @@ -9249,17 +9260,12 @@ void Lowering::LowerBlockStoreCommon(GenTreeBlk* blkNode) // bool Lowering::TryTransformStoreObjAsStoreInd(GenTreeBlk* blkNode) { - assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK)); if (!comp->opts.OptimizationEnabled()) { return false; } - if (blkNode->OperIs(GT_STORE_DYN_BLK)) - { - return false; - } - var_types regType = blkNode->GetLayout()->GetRegisterType(); if (regType == TYP_UNDEF) { diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 4cc7144ad15d3..df987c1aeab55 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -585,8 +585,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && - src->OperIs(GT_CNS_INT)) + if ((size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -651,7 +650,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); + bool doCpObj = layout->HasGCPtr(); unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy); if (doCpObj && (size <= copyBlockUnrollLimit)) @@ -686,7 +685,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK)); LowerBlockStoreAsHelperCall(blkNode); } } diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index 507ef59256ca4..7b3fb43dfa5dc 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -296,8 +296,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && - src->OperIs(GT_CNS_INT)) + if ((size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -351,7 +350,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); + bool doCpObj = layout->HasGCPtr(); unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy); if (doCpObj && (size <= copyBlockUnrollLimit)) @@ -387,7 +386,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK)); LowerBlockStoreAsHelperCall(blkNode); } } diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index 405b9707c4114..d172ded699174 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -245,7 +245,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= INITBLK_UNROLL_LIMIT) && src->OperIs(GT_CNS_INT)) + if ((size <= INITBLK_UNROLL_LIMIT) && src->OperIs(GT_CNS_INT)) { blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll; @@ -299,7 +299,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); + bool doCpObj = layout->HasGCPtr(); if (doCpObj && (size <= CPBLK_UNROLL_LIMIT)) { @@ -334,7 +334,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK)); LowerBlockStoreAsHelperCall(blkNode); } } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index ea4db9fedbac4..e0ca67574e1c9 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -349,7 +349,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) src = src->AsUnOp()->gtGetOp1(); } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset))) + if (size <= comp->getUnrollThreshold(Compiler::UnrollKind::Memset)) { if (!src->OperIs(GT_CNS_INT)) { @@ -436,7 +436,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } ClassLayout* layout = blkNode->GetLayout(); - bool doCpObj = !blkNode->OperIs(GT_STORE_DYN_BLK) && layout->HasGCPtr(); + bool doCpObj = layout->HasGCPtr(); unsigned copyBlockUnrollLimit = comp->getUnrollThreshold(Compiler::UnrollKind::Memcpy, false); #ifndef JIT32_GCENCODER @@ -516,7 +516,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) } else { - assert(blkNode->OperIs(GT_STORE_BLK, GT_STORE_DYN_BLK)); + assert(blkNode->OperIs(GT_STORE_BLK)); #ifdef TARGET_AMD64 LowerBlockStoreAsHelperCall(blkNode); diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 9efda15face0a..e411de81ed80e 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -5109,6 +5109,13 @@ void LinearScan::allocateRegistersMinimal() } regsInUseThisLocation |= currentRefPosition.registerAssignment; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); + +#ifdef SWIFT_SUPPORT + if (currentRefPosition.delayRegFree) + { + regsInUseNextLocation |= currentRefPosition.registerAssignment; + } +#endif // SWIFT_SUPPORT } else { @@ -5818,6 +5825,13 @@ void LinearScan::allocateRegisters() } regsInUseThisLocation |= currentRefPosition.registerAssignment; INDEBUG(dumpLsraAllocationEvent(LSRA_EVENT_FIXED_REG, nullptr, currentRefPosition.assignedReg())); + +#ifdef SWIFT_SUPPORT + if (currentRefPosition.delayRegFree) + { + regsInUseNextLocation |= currentRefPosition.registerAssignment; + } +#endif // SWIFT_SUPPORT } else { diff --git a/src/coreclr/jit/lsraarm.cpp b/src/coreclr/jit/lsraarm.cpp index 30991778868d6..2192265984d68 100644 --- a/src/coreclr/jit/lsraarm.cpp +++ b/src/coreclr/jit/lsraarm.cpp @@ -579,7 +579,6 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: - case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 1dddc7463501a..52db29fd95abe 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1076,7 +1076,6 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: - case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -1282,6 +1281,20 @@ int LinearScan::BuildNode(GenTree* tree) srcCount = BuildSelect(tree->AsOp()); break; +#ifdef SWIFT_SUPPORT + case GT_SWIFT_ERROR: + srcCount = 0; + assert(dstCount == 1); + + // Any register should do here, but the error register value should immediately + // be moved from GT_SWIFT_ERROR's destination register to the SwiftError struct, + // and we know REG_SWIFT_ERROR should be busy up to this point, anyway. + // By forcing LSRA to use REG_SWIFT_ERROR as both the source and destination register, + // we can ensure the redundant move is elided. + BuildDef(tree, RBM_SWIFT_ERROR); + break; +#endif // SWIFT_SUPPORT + } // end switch (tree->OperGet()) if (tree->IsUnusedValue() && (dstCount != 0)) diff --git a/src/coreclr/jit/lsraarmarch.cpp b/src/coreclr/jit/lsraarmarch.cpp index ad112a817220f..1df68f5f3f574 100644 --- a/src/coreclr/jit/lsraarmarch.cpp +++ b/src/coreclr/jit/lsraarmarch.cpp @@ -393,6 +393,30 @@ int LinearScan::BuildCall(GenTreeCall* call) regMaskTP killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); +#ifdef SWIFT_SUPPORT + if ((call->gtCallMoreFlags & GTF_CALL_M_SWIFT_ERROR_HANDLING) != 0) + { + // Tree is a Swift call with error handling; error register should have been killed + assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); + assert((killMask & RBM_SWIFT_ERROR) != 0); + + // After a Swift call that might throw returns, we expect the error register to be consumed + // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed + // before GT_SWIFT_ERROR can consume it. + // (For example, the PInvoke epilog comes before the error register store.) + // To do so, delay the freeing of the error register until the next node. + // This only works if the next node after the call is the GT_SWIFT_ERROR node. + // (InsertPInvokeCallEpilog should have moved the GT_SWIFT_ERROR node during lowering.) + assert(call->gtNext != nullptr); + assert(call->gtNext->OperIs(GT_SWIFT_ERROR)); + + // We could use RefTypeKill, but RefTypeFixedReg is used less commonly, so the check for delayRegFree + // during register allocation should be cheaper in terms of TP. + RefPosition* pos = newRefPosition(REG_SWIFT_ERROR, currentLoc, RefTypeFixedReg, call, RBM_SWIFT_ERROR); + setDelayFree(pos); + } +#endif // SWIFT_SUPPORT + // No args are placed in registers anymore. placedArgRegs = RBM_NONE; numPlacedArgLocals = 0; @@ -781,7 +805,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) + if (sizeRegMask != RBM_NONE) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -812,12 +836,6 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (blkNode->OperIs(GT_STORE_DYN_BLK)) - { - useCount++; - BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); - } - buildInternalRegisterUses(); regMaskTP killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index df1af6a419a3e..6e746d35c93ad 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -880,6 +880,19 @@ regMaskTP LinearScan::getKillSetForCall(GenTreeCall* call) assert(!call->IsVirtualStub() || ((killMask & compiler->virtualStubParamInfo->GetRegMask()) == compiler->virtualStubParamInfo->GetRegMask())); #endif // !TARGET_ARM + +#ifdef SWIFT_SUPPORT + // Swift calls that throw may trash the callee-saved error register, + // so don't use the register post-call until it is consumed by SwiftError. + // GTF_CALL_M_SWIFT_ERROR_HANDLING indicates the call has a SwiftError* argument, + // so the error register value will eventually be consumed post-call. + if ((call->gtCallMoreFlags & GTF_CALL_M_SWIFT_ERROR_HANDLING) != 0) + { + assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); + killMask |= RBM_SWIFT_ERROR; + } +#endif // SWIFT_SUPPORT + return killMask; } @@ -1043,7 +1056,6 @@ regMaskTP LinearScan::getKillSetForNode(GenTree* tree) break; case GT_STORE_BLK: - case GT_STORE_DYN_BLK: killMask = getKillSetForBlockStore(tree->AsBlk()); break; diff --git a/src/coreclr/jit/lsraloongarch64.cpp b/src/coreclr/jit/lsraloongarch64.cpp index b6e3b53d63ee5..51b1871b40bdc 100644 --- a/src/coreclr/jit/lsraloongarch64.cpp +++ b/src/coreclr/jit/lsraloongarch64.cpp @@ -394,7 +394,6 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: - case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -1157,7 +1156,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) + if (sizeRegMask != RBM_NONE) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -1188,12 +1187,6 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (blkNode->OperIs(GT_STORE_DYN_BLK)) - { - useCount++; - BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); - } - buildInternalRegisterUses(); regMaskTP killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); diff --git a/src/coreclr/jit/lsrariscv64.cpp b/src/coreclr/jit/lsrariscv64.cpp index c6e148d53d031..0a38d27574a13 100644 --- a/src/coreclr/jit/lsrariscv64.cpp +++ b/src/coreclr/jit/lsrariscv64.cpp @@ -512,7 +512,6 @@ int LinearScan::BuildNode(GenTree* tree) break; case GT_STORE_BLK: - case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -1313,7 +1312,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) + if (sizeRegMask != RBM_NONE) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -1344,12 +1343,6 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (blkNode->OperIs(GT_STORE_DYN_BLK)) - { - useCount++; - BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); - } - buildInternalRegisterUses(); regMaskTP killMask = getKillSetForBlockStore(blkNode); BuildDefsWithKills(blkNode, 0, RBM_NONE, killMask); diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 41121ee9bed28..a01c6ce5df057 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -507,7 +507,6 @@ int LinearScan::BuildNode(GenTree* tree) #endif // FEATURE_PUT_STRUCT_ARG_STK case GT_STORE_BLK: - case GT_STORE_DYN_BLK: srcCount = BuildBlockStore(tree->AsBlk()); break; @@ -633,6 +632,20 @@ int LinearScan::BuildNode(GenTree* tree) } break; +#ifdef SWIFT_SUPPORT + case GT_SWIFT_ERROR: + srcCount = 0; + assert(dstCount == 1); + + // Any register should do here, but the error register value should immediately + // be moved from GT_SWIFT_ERROR's destination register to the SwiftError struct, + // and we know REG_SWIFT_ERROR should be busy up to this point, anyway. + // By forcing LSRA to use REG_SWIFT_ERROR as both the source and destination register, + // we can ensure the redundant move is elided. + BuildDef(tree, RBM_SWIFT_ERROR); + break; +#endif // SWIFT_SUPPORT + } // end switch (tree->OperGet()) // We need to be sure that we've set srcCount and dstCount appropriately. @@ -1357,6 +1370,30 @@ int LinearScan::BuildCall(GenTreeCall* call) regMaskTP killMask = getKillSetForCall(call); BuildDefsWithKills(call, dstCount, dstCandidates, killMask); +#ifdef SWIFT_SUPPORT + if ((call->gtCallMoreFlags & GTF_CALL_M_SWIFT_ERROR_HANDLING) != 0) + { + // Tree is a Swift call with error handling; error register should have been killed + assert(call->unmgdCallConv == CorInfoCallConvExtension::Swift); + assert((killMask & RBM_SWIFT_ERROR) != 0); + + // After a Swift call that might throw returns, we expect the error register to be consumed + // by a GT_SWIFT_ERROR node. However, we want to ensure the error register won't be trashed + // before GT_SWIFT_ERROR can consume it. + // (For example, the PInvoke epilog comes before the error register store.) + // To do so, delay the freeing of the error register until the next node. + // This only works if the next node after the call is the GT_SWIFT_ERROR node. + // (InsertPInvokeCallEpilog should have moved the GT_SWIFT_ERROR node during lowering.) + assert(call->gtNext != nullptr); + assert(call->gtNext->OperIs(GT_SWIFT_ERROR)); + + // We could use RefTypeKill, but RefTypeFixedReg is used less commonly, so the check for delayRegFree + // during register allocation should be cheaper in terms of TP. + RefPosition* pos = newRefPosition(REG_SWIFT_ERROR, currentLoc, RefTypeFixedReg, call, RBM_SWIFT_ERROR); + setDelayFree(pos); + } +#endif // SWIFT_SUPPORT + // No args are placed in registers anymore. placedArgRegs = RBM_NONE; numPlacedArgLocals = 0; @@ -1566,7 +1603,7 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (!blkNode->OperIs(GT_STORE_DYN_BLK) && (sizeRegMask != RBM_NONE)) + if (sizeRegMask != RBM_NONE) { // Reserve a temp register for the block size argument. buildInternalIntRegisterDefForNode(blkNode, sizeRegMask); @@ -1597,12 +1634,6 @@ int LinearScan::BuildBlockStore(GenTreeBlk* blkNode) } } - if (blkNode->OperIs(GT_STORE_DYN_BLK)) - { - useCount++; - BuildUse(blkNode->AsStoreDynBlk()->gtDynamicSize, sizeRegMask); - } - #ifdef TARGET_X86 // If we require a byte register on x86, we may run into an over-constrained situation // if we have BYTE_REG_COUNT or more uses (currently, it can be at most 4, if both the diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 25024f9330d99..3c27bc4e74d3c 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -2045,17 +2045,23 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call PushBack(comp, NewCallArg::Primitive(newArg).WellKnown(WellKnownArg::WrapperDelegateCell)); } #endif // defined(TARGET_ARM) -#ifndef TARGET_X86 + + bool addStubCellArg = true; + +#ifdef TARGET_X86 // TODO-X86-CQ: Currently RyuJIT/x86 passes args on the stack, so this is not needed. // If/when we change that, the following code needs to be changed to correctly support the (TBD) managed calling // convention for x86/SSE. + addStubCellArg = call->gtCallType != CT_INDIRECT && comp->IsTargetAbi(CORINFO_NATIVEAOT_ABI); +#endif + // We are allowed to have a ret buffer argument combined // with any of the remaining non-standard arguments // CLANG_FORMAT_COMMENT_ANCHOR; - if (call->IsVirtualStub()) + if (call->IsVirtualStub() && addStubCellArg) { if (!call->IsTailCallViaJitHelper()) { @@ -2072,9 +2078,7 @@ void CallArgs::AddFinalArgsAndDetermineABIInfo(Compiler* comp, GenTreeCall* call // add as a non-standard arg. } } - else -#endif // !TARGET_X86 - if (call->gtCallType == CT_INDIRECT && (call->gtCallCookie != nullptr)) + else if (call->gtCallType == CT_INDIRECT && (call->gtCallCookie != nullptr)) { assert(!call->IsUnmanaged()); @@ -6318,7 +6322,7 @@ GenTree* Compiler::fgMorphPotentialTailCall(GenTreeCall* call) { // We call CORINFO_HELP_TAILCALL which does not return, so we will // not need epilogue. - compCurBB->SetKindAndTarget(BBJ_THROW); + compCurBB->SetKindAndTargetEdge(BBJ_THROW); } if (isRootReplaced) @@ -7465,7 +7469,8 @@ void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCa { // Todo: this may not look like a viable loop header. // Might need the moral equivalent of a scratch BB. - block->SetKindAndTarget(BBJ_ALWAYS, fgEntryBB); + FlowEdge* const newEdge = fgAddRefPred(fgEntryBB, block); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } else { @@ -7480,11 +7485,11 @@ void Compiler::fgMorphRecursiveFastTailCallIntoLoop(BasicBlock* block, GenTreeCa // block removal on it. // fgFirstBB->SetFlags(BBF_DONT_REMOVE); - block->SetKindAndTarget(BBJ_ALWAYS, fgFirstBB->Next()); + FlowEdge* const newEdge = fgAddRefPred(fgFirstBB->Next(), block); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); } // Finish hooking things up. - fgAddRefPred(block->GetTarget(), block); block->RemoveFlags(BBF_HAS_JMP); } @@ -9217,11 +9222,11 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA // TODO #4104: there are a lot of other places where // this condition is not checked before transformations. - if (fgGlobalMorph) + noway_assert(op2); + if (fgGlobalMorph && !op2->TypeIs(TYP_BYREF)) { /* Check for "op1 - cns2" , we change it to "op1 + (-cns2)" */ - noway_assert(op2); if (op2->IsCnsIntOrI() && !op2->IsIconHandle()) { // Negate the constant and change the node to be "+", @@ -9239,7 +9244,7 @@ GenTree* Compiler::fgMorphSmpOp(GenTree* tree, MorphAddrContext* mac, bool* optA noway_assert(op1); if (op1->IsCnsIntOrI()) { - noway_assert(varTypeIsIntOrI(tree)); + noway_assert(varTypeIsIntegralOrI(tree)); // The type of the new GT_NEG node cannot just be op2->TypeGet(). // Otherwise we may sign-extend incorrectly in cases where the GT_NEG @@ -12787,10 +12792,6 @@ GenTree* Compiler::fgMorphTree(GenTree* tree, MorphAddrContext* mac) gtUpdateNodeSideEffects(tree); break; - case GT_STORE_DYN_BLK: - tree = fgMorphStoreDynBlock(tree->AsStoreDynBlk()); - break; - case GT_SELECT: tree->AsConditional()->gtCond = fgMorphTree(tree->AsConditional()->gtCond); tree->AsConditional()->gtOp1 = fgMorphTree(tree->AsConditional()->gtOp1); @@ -12994,7 +12995,7 @@ void Compiler::fgAssertionGen(GenTree* tree) AssertionIndex ifFalseAssertionIndex; AssertionIndex ifTrueAssertionIndex; - if (info.IsNextEdgeAssertion()) + if (info.AssertionHoldsOnFalseEdge()) { ifFalseAssertionIndex = info.GetAssertionIndex(); ifTrueAssertionIndex = optFindComplementary(ifFalseAssertionIndex); @@ -13207,7 +13208,7 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) // JTRUE 0 - transform the basic block into a BBJ_ALWAYS bTaken = block->GetFalseTarget(); bNotTaken = block->GetTrueTarget(); - block->SetKindAndTarget(BBJ_ALWAYS, bTaken); + block->SetKindAndTargetEdge(BBJ_ALWAYS, block->GetFalseEdge()); block->SetFlags(BBF_NONE_QUIRK); } @@ -13375,13 +13376,13 @@ Compiler::FoldResult Compiler::fgFoldConditional(BasicBlock* block) if ((val == switchVal) || (!foundVal && (val == jumpCnt - 1))) { - block->SetKindAndTarget(BBJ_ALWAYS, curEdge->getDestinationBlock()); + block->SetKindAndTargetEdge(BBJ_ALWAYS, curEdge); foundVal = true; } else { // Remove 'curEdge' - fgRemoveRefPred(curEdge->getDestinationBlock(), block); + fgRemoveRefPred(curEdge); } } @@ -13563,11 +13564,7 @@ void Compiler::fgMorphStmtBlockOps(BasicBlock* block, Statement* stmt) { if ((*use)->OperIsBlkOp()) { - if ((*use)->OperIs(GT_STORE_DYN_BLK)) - { - *use = m_compiler->fgMorphStoreDynBlock((*use)->AsStoreDynBlk()); - } - else if ((*use)->OperIsInitBlkOp()) + if ((*use)->OperIsInitBlkOp()) { *use = m_compiler->fgMorphInitBlock(*use); } @@ -14131,8 +14128,8 @@ void Compiler::fgMergeBlockReturn(BasicBlock* block) else #endif // !TARGET_X86 { - block->SetKindAndTarget(BBJ_ALWAYS, genReturnBB); - fgAddRefPred(genReturnBB, block); + FlowEdge* const newEdge = fgAddRefPred(genReturnBB, block); + block->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); fgReturnCount--; } @@ -14634,16 +14631,24 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) assert(condBlock->bbWeight == remainderBlock->bbWeight); assert(block->KindIs(BBJ_ALWAYS)); - block->SetTarget(condBlock); - condBlock->SetTarget(elseBlock); - elseBlock->SetTarget(remainderBlock); + { + FlowEdge* const newEdge = fgAddRefPred(condBlock, block); + block->SetTargetEdge(newEdge); + } + + { + FlowEdge* const newEdge = fgAddRefPred(elseBlock, condBlock); + condBlock->SetTargetEdge(newEdge); + } + + { + FlowEdge* const newEdge = fgAddRefPred(remainderBlock, elseBlock); + elseBlock->SetTargetEdge(newEdge); + } + assert(condBlock->JumpsToNext()); assert(elseBlock->JumpsToNext()); - fgAddRefPred(condBlock, block); - fgAddRefPred(elseBlock, condBlock); - fgAddRefPred(remainderBlock, elseBlock); - condBlock->SetFlags(propagateFlagsToAll | BBF_NONE_QUIRK); elseBlock->SetFlags(propagateFlagsToAll | BBF_NONE_QUIRK); @@ -14660,17 +14665,20 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // gtReverseCond(condExpr); - thenBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true, remainderBlock); + thenBlock = fgNewBBafter(BBJ_ALWAYS, condBlock, true); thenBlock->SetFlags(propagateFlagsToAll); - condBlock->SetCond(elseBlock, thenBlock); if (!block->HasFlag(BBF_INTERNAL)) { thenBlock->RemoveFlags(BBF_INTERNAL); thenBlock->SetFlags(BBF_IMPORTED); } - fgAddRefPred(thenBlock, condBlock); - fgAddRefPred(remainderBlock, thenBlock); + FlowEdge* const newEdge = fgAddRefPred(remainderBlock, thenBlock); + thenBlock->SetTargetEdge(newEdge); + + assert(condBlock->TargetIs(elseBlock)); + FlowEdge* const falseEdge = fgAddRefPred(thenBlock, condBlock); + condBlock->SetCond(condBlock->GetTargetEdge(), falseEdge); thenBlock->inheritWeightPercentage(condBlock, qmark->ThenNodeLikelihood()); elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood()); @@ -14684,8 +14692,11 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // bbj_cond(true) // gtReverseCond(condExpr); - condBlock->SetCond(remainderBlock, elseBlock); - fgAddRefPred(remainderBlock, condBlock); + + assert(condBlock->TargetIs(elseBlock)); + FlowEdge* const trueEdge = fgAddRefPred(remainderBlock, condBlock); + condBlock->SetCond(trueEdge, condBlock->GetTargetEdge()); + // Since we have no false expr, use the one we'd already created. thenBlock = elseBlock; elseBlock = nullptr; @@ -14700,8 +14711,9 @@ bool Compiler::fgExpandQmarkStmt(BasicBlock* block, Statement* stmt) // +-->------------+ // bbj_cond(true) // - condBlock->SetCond(remainderBlock, elseBlock); - fgAddRefPred(remainderBlock, condBlock); + assert(condBlock->TargetIs(elseBlock)); + FlowEdge* const trueEdge = fgAddRefPred(remainderBlock, condBlock); + condBlock->SetCond(trueEdge, condBlock->GetTargetEdge()); elseBlock->inheritWeightPercentage(condBlock, qmark->ElseNodeLikelihood()); } diff --git a/src/coreclr/jit/morphblock.cpp b/src/coreclr/jit/morphblock.cpp index 94d10dd5887f5..3f4a20215ead8 100644 --- a/src/coreclr/jit/morphblock.cpp +++ b/src/coreclr/jit/morphblock.cpp @@ -1512,70 +1512,3 @@ GenTree* Compiler::fgMorphInitBlock(GenTree* tree) { return MorphInitBlockHelper::MorphInitBlock(this, tree); } - -//------------------------------------------------------------------------ -// fgMorphStoreDynBlock: Morph a dynamic block store (GT_STORE_DYN_BLK). -// -// Performs full (pre-order and post-order) morphing for a STORE_DYN_BLK. -// -// Arguments: -// tree - The GT_STORE_DYN_BLK tree to morph. -// -// Return Value: -// In case the size turns into a constant - the store, transformed -// into an "ordinary" STORE_BLK one, and further morphed by -// "fgMorphInitBlock"/"fgMorphCopyBlock". Otherwise, the original -// tree (fully morphed). -// -GenTree* Compiler::fgMorphStoreDynBlock(GenTreeStoreDynBlk* tree) -{ - if (!tree->Data()->OperIs(GT_CNS_INT, GT_INIT_VAL)) - { - // Data is a location and required to have GTF_DONT_CSE. - tree->Data()->gtFlags |= GTF_DONT_CSE; - } - - tree->Addr() = fgMorphTree(tree->Addr()); - tree->Data() = fgMorphTree(tree->Data()); - tree->gtDynamicSize = fgMorphTree(tree->gtDynamicSize); - - if (tree->gtDynamicSize->IsIntegralConst()) - { - int64_t size = tree->gtDynamicSize->AsIntConCommon()->IntegralValue(); - - if ((size != 0) && FitsIn(size)) - { - ClassLayout* layout = typGetBlkLayout(static_cast(size)); - GenTree* src = tree->Data(); - if (src->OperIs(GT_IND)) - { - assert(src->TypeIs(TYP_STRUCT)); - src->SetOper(GT_BLK); - src->AsBlk()->Initialize(layout); - } - - GenTree* store = gtNewStoreValueNode(layout, tree->Addr(), src, tree->gtFlags & GTF_IND_FLAGS); - store->AddAllEffectsFlags(tree); - INDEBUG(store->gtDebugFlags |= GTF_DEBUG_NODE_MORPHED); - - JITDUMP("MorphStoreDynBlock: transformed STORE_DYN_BLK into STORE_BLK\n"); - - return tree->OperIsCopyBlkOp() ? fgMorphCopyBlock(store) : fgMorphInitBlock(store); - } - } - - tree->SetAllEffectsFlags(tree->Addr(), tree->Data(), tree->gtDynamicSize); - - if (tree->OperMayThrow(this)) - { - tree->gtFlags |= GTF_EXCEPT; - } - else - { - tree->gtFlags |= GTF_IND_NONFAULTING; - } - - tree->gtFlags |= GTF_ASG; - - return tree; -} diff --git a/src/coreclr/jit/optcse.cpp b/src/coreclr/jit/optcse.cpp index 0166d4cd47431..cd554f02e5637 100644 --- a/src/coreclr/jit/optcse.cpp +++ b/src/coreclr/jit/optcse.cpp @@ -164,6 +164,11 @@ bool Compiler::optUnmarkCSE(GenTree* tree) // 2. Unmark the CSE information in the node tree->gtCSEnum = NO_CSE; + + // 3. Leave breadcrumbs so we know some dsc was altered + + optCSEunmarks++; + return true; } else @@ -2436,10 +2441,12 @@ void CSE_HeuristicParameterized::GreedyPolicy() // const int numCandidates = m_pCompiler->optCSECandidateCount; ArrayStack choices(m_pCompiler->getAllocator(CMK_CSE), numCandidates + 1); + unsigned numUnmarked = m_pCompiler->optCSEunmarks; + bool recomputeFeatures = true; while (true) { - Choice& choice = ChooseGreedy(choices); + Choice& choice = ChooseGreedy(choices, recomputeFeatures); CSEdsc* const dsc = choice.m_dsc; #ifdef DEBUG @@ -2472,7 +2479,16 @@ void CSE_HeuristicParameterized::GreedyPolicy() JITDUMP("\n"); PerformCSE(&candidate); - madeChanges = true; + madeChanges = true; + choice.m_performed = true; + + // If performing this CSE impacted other CSEs, we need to + // recompute all cse features. + // + unsigned newNumUnmarked = m_pCompiler->optCSEunmarks; + assert(newNumUnmarked >= numUnmarked); + recomputeFeatures = (numUnmarked != newNumUnmarked); + numUnmarked = newNumUnmarked; } return; @@ -2575,7 +2591,7 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) unsigned maxPostorderNum = 0; BasicBlock* minPostorderBlock = nullptr; BasicBlock* maxPostorderBlock = nullptr; - for (treeStmtLst* treeList = cse->csdTreeList; treeList != nullptr && !isMakeCse; treeList = treeList->tslNext) + for (treeStmtLst* treeList = cse->csdTreeList; treeList != nullptr; treeList = treeList->tslNext) { BasicBlock* const treeBlock = treeList->tslBlock; unsigned postorderNum = treeBlock->bbPostorderNum; @@ -2616,7 +2632,6 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) // LSRA "is live across call" // bool isLiveAcrossCallLSRA = isLiveAcrossCall; - if (!isLiveAcrossCallLSRA) { unsigned count = 0; @@ -2630,7 +2645,6 @@ void CSE_HeuristicParameterized::GetFeatures(CSEdsc* cse, double* features) } } } - features[23] = booleanScale * isLiveAcrossCallLSRA; } @@ -2748,6 +2762,10 @@ double CSE_HeuristicParameterized::StoppingPreference() // ChooseGreedy: examine candidates and choose the next CSE to perform // via greedy policy // +// Arguments: +// choices -- array of choices, possibly already filled in +// recompute -- if true, rebuild the choice array from scratch +// // Returns: // Choice of CSE to perform // @@ -2755,10 +2773,25 @@ double CSE_HeuristicParameterized::StoppingPreference() // Picks the most-preferred candidate. // If there is a tie, picks stop, or the lowest cse index. // -CSE_HeuristicParameterized::Choice& CSE_HeuristicParameterized::ChooseGreedy(ArrayStack& choices) +CSE_HeuristicParameterized::Choice& CSE_HeuristicParameterized::ChooseGreedy(ArrayStack& choices, + bool recompute) { - choices.Reset(); - BuildChoices(choices); + if (recompute) + { + choices.Reset(); + BuildChoices(choices); + } + else + { + // Always recompute the stopping preference as this + // reflects ambient state after each CSE. + // + // By convention, this is at TopRef(0). + // + Choice& stopping = choices.TopRef(0); + assert(stopping.m_dsc == nullptr); + stopping.m_preference = StoppingPreference(); + } // Find the maximally preferred case. // @@ -2766,8 +2799,14 @@ CSE_HeuristicParameterized::Choice& CSE_HeuristicParameterized::ChooseGreedy(Arr for (int i = 1; i < choices.Height(); i++) { - Choice& choice = choices.TopRef(i); - Choice& bestChoice = choices.TopRef(choiceNum); + const Choice& choice = choices.TopRef(i); + + if (choice.m_performed == true) + { + continue; + } + + const Choice& bestChoice = choices.TopRef(choiceNum); const double delta = choice.m_preference - bestChoice.m_preference; @@ -2811,6 +2850,8 @@ CSE_HeuristicParameterized::Choice& CSE_HeuristicParameterized::ChooseGreedy(Arr // void CSE_HeuristicParameterized::BuildChoices(ArrayStack& choices) { + JITDUMP("Building choice array...\n"); + for (unsigned i = 0; i < m_pCompiler->optCSECandidateCount; i++) { CSEdsc* const dsc = sortTab[i]; @@ -2893,9 +2934,15 @@ void CSE_HeuristicParameterized::DumpChoices(ArrayStack& choices, int hi { for (int i = 0; i < choices.Height(); i++) { - Choice& choice = choices.TopRef(i); - CSEdsc* const cse = choice.m_dsc; - const char* msg = i == highlight ? "=>" : " "; + const Choice& choice = choices.TopRef(i); + + if (choice.m_performed == true) + { + continue; + } + + CSEdsc* const cse = choice.m_dsc; + const char* msg = (i == highlight) ? "=>" : " "; if (cse != nullptr) { printf("%s%2d: " FMT_CSE " preference %10.7f likelihood %10.7f\n", msg, i, cse->csdIndex, @@ -2920,9 +2967,15 @@ void CSE_HeuristicParameterized::DumpChoices(ArrayStack& choices, CSEdsc { for (int i = 0; i < choices.Height(); i++) { - Choice& choice = choices.TopRef(i); - CSEdsc* const cse = choice.m_dsc; - const char* msg = cse == highlight ? "=>" : " "; + const Choice& choice = choices.TopRef(i); + + if (choice.m_performed == true) + { + continue; + } + + CSEdsc* const cse = choice.m_dsc; + const char* msg = (cse == highlight) ? "=>" : " "; if (cse != nullptr) { printf("%s%2d: " FMT_CSE " preference %10.7f likelihood %10.7f\n", msg, i, cse->csdIndex, @@ -4422,50 +4475,62 @@ bool CSE_HeuristicCommon::IsCompatibleType(var_types cseLclVarTyp, var_types exp return false; } -// PerformCSE() takes a successful candidate and performs the appropriate replacements: +//------------------------------------------------------------------------ +// PerformCSE: takes a successful candidate and performs the appropriate replacements +// +// Arguments: +// successfulCandidate - cse candidate to perform // // It will replace all of the CSE defs with assignments to a new "cse0" LclVar // and will replace all of the CSE uses with reads of the "cse0" LclVar // // It will also put cse0 into SSA if there is just one def. +// void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) { AdjustHeuristic(successfulCandidate); + CSEdsc* const dsc = successfulCandidate->CseDsc(); #ifdef DEBUG // Setup the message arg for lvaGrabTemp() // - const char* grabTempMessage = "CSE - unknown"; + const char* heuristicTempMessage = ""; if (successfulCandidate->IsAggressive()) { - grabTempMessage = "CSE - aggressive"; + heuristicTempMessage = ": aggressive"; } else if (successfulCandidate->IsModerate()) { - grabTempMessage = "CSE - moderate"; + heuristicTempMessage = ": moderate"; } else if (successfulCandidate->IsConservative()) { - grabTempMessage = "CSE - conservative"; + heuristicTempMessage = ": conservative"; } else if (successfulCandidate->IsStressCSE()) { - grabTempMessage = "CSE - stress mode"; + heuristicTempMessage = ": stress"; } else if (successfulCandidate->IsRandom()) { - grabTempMessage = "CSE - random"; + heuristicTempMessage = ": random"; } -#endif // DEBUG - /* Introduce a new temp for the CSE */ + const char* const grabTempMessage = m_pCompiler->printfAlloc(FMT_CSE "%s", dsc->csdIndex, heuristicTempMessage); + + // Add this candidate to the CSE sequence + // + m_sequence->push_back(dsc->csdIndex); + +#endif // DEBUG - // we will create a long lifetime temp for the new CSE LclVar + // Allocate a CSE temp + // unsigned cseLclVarNum = m_pCompiler->lvaGrabTemp(false DEBUGARG(grabTempMessage)); var_types cseLclVarTyp = genActualType(successfulCandidate->Expr()->TypeGet()); - LclVarDsc* lclDsc = m_pCompiler->lvaGetDesc(cseLclVarNum); + LclVarDsc* const lclDsc = m_pCompiler->lvaGetDesc(cseLclVarNum); if (cseLclVarTyp == TYP_STRUCT) { m_pCompiler->lvaSetStruct(cseLclVarNum, successfulCandidate->Expr()->GetLayout(m_pCompiler), false); @@ -4474,6 +4539,7 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) lclDsc->lvIsCSE = true; // Record that we created a new LclVar for use as a CSE temp + // m_addCSEcount++; m_pCompiler->optCSEcount++; m_pCompiler->Metrics.CseCount++; @@ -4484,11 +4550,9 @@ void CSE_HeuristicCommon::PerformCSE(CSE_Candidate* successfulCandidate) // // Later we will unmark any nested CSE's for the CSE uses. // - CSEdsc* dsc = successfulCandidate->CseDsc(); - INDEBUG(m_sequence->push_back(dsc->csdIndex)); - // If there's just a single def for the CSE, we'll put this // CSE into SSA form on the fly. We won't need any PHIs. + // unsigned cseSsaNum = SsaConfig::RESERVED_SSA_NUM; LclSsaVarDsc* ssaVarDsc = nullptr; diff --git a/src/coreclr/jit/optcse.h b/src/coreclr/jit/optcse.h index 3d1c7f0702ba9..550f754f6a8b6 100644 --- a/src/coreclr/jit/optcse.h +++ b/src/coreclr/jit/optcse.h @@ -151,12 +151,14 @@ class CSE_HeuristicParameterized : public CSE_HeuristicCommon protected: struct Choice { - Choice(CSEdsc* dsc, double preference) : m_dsc(dsc), m_preference(preference), m_softmax(0) + Choice(CSEdsc* dsc, double preference) : m_dsc(dsc), m_preference(preference), m_softmax(0), m_performed(false) { } + CSEdsc* m_dsc; double m_preference; double m_softmax; + bool m_performed; }; enum @@ -185,7 +187,7 @@ class CSE_HeuristicParameterized : public CSE_HeuristicCommon double StoppingPreference(); void BuildChoices(ArrayStack& choices); - Choice& ChooseGreedy(ArrayStack& choices); + Choice& ChooseGreedy(ArrayStack& choices, bool recompute); virtual const char* Name() const { diff --git a/src/coreclr/jit/optimizebools.cpp b/src/coreclr/jit/optimizebools.cpp index f9616636681b5..713d9f17c5834 100644 --- a/src/coreclr/jit/optimizebools.cpp +++ b/src/coreclr/jit/optimizebools.cpp @@ -46,7 +46,7 @@ class OptBoolsDsc private: BasicBlock* m_b1; // The first basic block with the BBJ_COND conditional jump type BasicBlock* m_b2; // The next basic block of m_b1. Either BBJ_COND or BBJ_RETURN type - BasicBlock* m_b3; // m_b1->bbTarget. Null if m_b2 is not a return block. + BasicBlock* m_b3; // m_b1's target block. Null if m_b2 is not a return block. Compiler* m_comp; // The pointer to the Compiler instance @@ -89,7 +89,7 @@ class OptBoolsDsc // Notes: // m_b1 and m_b2 are set on entry. // -// Case 1: if b1.bbTarget == b2.bbTarget, it transforms +// Case 1: if b1->TargetIs(b2->GetTarget()), it transforms // B1 : brtrue(t1, Bx) // B2 : brtrue(t2, Bx) // B3 : @@ -107,7 +107,7 @@ class OptBoolsDsc // B3: GT_RETURN (BBJ_RETURN) // B4: GT_RETURN (BBJ_RETURN) // -// Case 2: if B2->FalseTargetIs(B1.bbTarget), it transforms +// Case 2: if B2->FalseTargetIs(B1->GetTarget()), it transforms // B1 : brtrue(t1, B3) // B2 : brtrue(t2, Bx) // B3 : @@ -123,7 +123,7 @@ bool OptBoolsDsc::optOptimizeBoolsCondBlock() m_t3 = nullptr; - // Check if m_b1 and m_b2 have the same bbTarget + // Check if m_b1 and m_b2 have the same target if (m_b1->TrueTargetIs(m_b2->GetTrueTarget())) { @@ -808,15 +808,20 @@ bool OptBoolsDsc::optOptimizeRangeTests() } // Re-direct firstBlock to jump to inRangeBb - m_comp->fgAddRefPred(inRangeBb, m_b1); + FlowEdge* const newEdge = m_comp->fgAddRefPred(inRangeBb, m_b1); + if (!cmp2IsReversed) { - m_b1->SetTrueTarget(inRangeBb); - m_b1->SetFalseTarget(notInRangeBb); + m_b1->SetFalseEdge(m_b1->GetTrueEdge()); + m_b1->SetTrueEdge(newEdge); + assert(m_b1->TrueTargetIs(inRangeBb)); + assert(m_b1->FalseTargetIs(notInRangeBb)); } else { - m_b1->SetFalseTarget(inRangeBb); + m_b1->SetFalseEdge(newEdge); + assert(m_b1->TrueTargetIs(notInRangeBb)); + assert(m_b1->FalseTargetIs(inRangeBb)); } // Remove the 2nd condition block as we no longer need it @@ -1012,8 +1017,8 @@ bool OptBoolsDsc::optOptimizeCompareChainCondBlock() m_comp->fgSetStmtSeq(s2); // Update the flow. - m_comp->fgRemoveRefPred(m_b1->GetTrueTarget(), m_b1); - m_b1->SetKindAndTarget(BBJ_ALWAYS, m_b1->GetFalseTarget()); + m_comp->fgRemoveRefPred(m_b1->GetTrueEdge()); + m_b1->SetKindAndTargetEdge(BBJ_ALWAYS, m_b1->GetFalseEdge()); m_b1->SetFlags(BBF_NONE_QUIRK); // Fixup flags. @@ -1266,22 +1271,19 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() { // Update edges if m_b1: BBJ_COND and m_b2: BBJ_COND - FlowEdge* edge1 = m_comp->fgGetPredForBlock(m_b1->GetTrueTarget(), m_b1); + FlowEdge* edge1 = m_b1->GetTrueEdge(); FlowEdge* edge2; if (m_sameTarget) { - edge2 = m_comp->fgGetPredForBlock(m_b2->GetTrueTarget(), m_b2); + edge2 = m_b2->GetTrueEdge(); } else { - edge2 = m_comp->fgGetPredForBlock(m_b2->GetFalseTarget(), m_b2); - - m_comp->fgRemoveRefPred(m_b1->GetTrueTarget(), m_b1); - - m_b1->SetTrueTarget(m_b2->GetTrueTarget()); - - m_comp->fgAddRefPred(m_b2->GetTrueTarget(), m_b1); + edge2 = m_b2->GetFalseEdge(); + m_comp->fgRemoveRefPred(m_b1->GetTrueEdge()); + FlowEdge* const newEdge = m_comp->fgAddRefPred(m_b2->GetTrueTarget(), m_b1); + m_b1->SetTrueEdge(newEdge); } assert(edge1 != nullptr); @@ -1307,7 +1309,7 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() assert(m_b2->KindIs(BBJ_RETURN)); assert(m_b1->FalseTargetIs(m_b2)); assert(m_b3 != nullptr); - m_b1->SetKindAndTarget(BBJ_RETURN); + m_b1->SetKindAndTargetEdge(BBJ_RETURN); } else { @@ -1322,11 +1324,12 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() { // Update bbRefs and bbPreds // - // Replace pred 'm_b2' for 'm_b2->bbFalseTarget' with 'm_b1' - // Remove pred 'm_b2' for 'm_b2->bbTrueTarget' - m_comp->fgReplacePred(m_b2->GetFalseTarget(), m_b2, m_b1); - m_comp->fgRemoveRefPred(m_b2->GetTrueTarget(), m_b2); - m_b1->SetFalseTarget(m_b2->GetFalseTarget()); + // Replace pred 'm_b2' for m_b2's false target with 'm_b1' + // Remove pred 'm_b2' for m_b2's true target + FlowEdge* falseEdge = m_b2->GetFalseEdge(); + m_comp->fgReplacePred(falseEdge, m_b1); + m_comp->fgRemoveRefPred(m_b2->GetTrueEdge()); + m_b1->SetFalseEdge(falseEdge); } // Get rid of the second block @@ -1361,7 +1364,7 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees() // Notes: // m_b1, m_b2 and m_b3 of OptBoolsDsc are set on entry. // -// if B1.bbTarget == b3, it transforms +// if B1->TargetIs(b3), it transforms // B1 : brtrue(t1, B3) // B2 : ret(t2) // B3 : ret(0) diff --git a/src/coreclr/jit/optimizer.cpp b/src/coreclr/jit/optimizer.cpp index 169d5f2677493..6975bcd27bf99 100644 --- a/src/coreclr/jit/optimizer.cpp +++ b/src/coreclr/jit/optimizer.cpp @@ -34,6 +34,7 @@ void Compiler::optInit() optCSECandidateCount = 0; optCSEattempt = 0; optCSEheuristic = nullptr; + optCSEunmarks = 0; } DataFlow::DataFlow(Compiler* pCompiler) : m_pCompiler(pCompiler) @@ -584,20 +585,24 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo case BBJ_CALLFINALLY: case BBJ_CALLFINALLYRET: case BBJ_LEAVE: + { + FlowEdge* newEdge; + // Determine if newBlk should be redirected to a different target from blk's target if (redirectMap->Lookup(blk->GetTarget(), &newTarget)) { // newBlk needs to be redirected to a new target - newBlk->SetKindAndTarget(blk->GetKind(), newTarget); + newEdge = fgAddRefPred(newTarget, newBlk); } else { // newBlk uses the same target as blk - newBlk->SetKindAndTarget(blk->GetKind(), blk->GetTarget()); + newEdge = fgAddRefPred(blk->GetTarget(), newBlk); } - fgAddRefPred(newBlk->GetTarget(), newBlk); + newBlk->SetKindAndTargetEdge(blk->GetKind(), newEdge); break; + } case BBJ_COND: { @@ -626,9 +631,9 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo falseTarget = blk->GetFalseTarget(); } - fgAddRefPred(trueTarget, newBlk); - fgAddRefPred(falseTarget, newBlk); - newBlk->SetCond(trueTarget, falseTarget); + FlowEdge* const trueEdge = fgAddRefPred(trueTarget, newBlk); + FlowEdge* const falseEdge = fgAddRefPred(falseTarget, newBlk); + newBlk->SetCond(trueEdge, falseEdge); break; } @@ -695,16 +700,18 @@ void Compiler::optSetMappedBlockTargets(BasicBlock* blk, BasicBlock* newBlk, Blo case BBJ_EHCATCHRET: case BBJ_EHFILTERRET: + { // newBlk's jump target should not need to be redirected assert(!redirectMap->Lookup(blk->GetTarget(), &newTarget)); - newBlk->SetKindAndTarget(blk->GetKind(), blk->GetTarget()); - fgAddRefPred(newBlk->GetTarget(), newBlk); + FlowEdge* newEdge = fgAddRefPred(newBlk->GetTarget(), newBlk); + newBlk->SetKindAndTargetEdge(blk->GetKind(), newEdge); break; + } default: // blk doesn't have a jump destination assert(blk->NumSucc() == 0); - newBlk->SetKindAndTarget(blk->GetKind()); + newBlk->SetKindAndTargetEdge(blk->GetKind()); break; } @@ -1709,12 +1716,12 @@ void Compiler::optRedirectPrevUnrollIteration(FlowGraphNaturalLoop* loop, BasicB testCopyStmt->SetRootNode(sideEffList); } - fgRemoveRefPred(prevTestBlock->GetTrueTarget(), prevTestBlock); - fgRemoveRefPred(prevTestBlock->GetFalseTarget(), prevTestBlock); + fgRemoveRefPred(prevTestBlock->GetTrueEdge()); + fgRemoveRefPred(prevTestBlock->GetFalseEdge()); // Redirect exit edge from previous iteration to new entry. - prevTestBlock->SetKindAndTarget(BBJ_ALWAYS, target); - fgAddRefPred(target, prevTestBlock); + FlowEdge* const newEdge = fgAddRefPred(target, prevTestBlock); + prevTestBlock->SetKindAndTargetEdge(BBJ_ALWAYS, newEdge); JITDUMP("Redirecting previously created exiting " FMT_BB " -> " FMT_BB "\n", prevTestBlock->bbNum, target->bbNum); @@ -1923,7 +1930,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) return false; } - // Since bTest is a BBJ_COND it will have a bbFalseTarget + // Since bTest is a BBJ_COND it will have a false target // BasicBlock* const bJoin = bTest->GetFalseTarget(); noway_assert(bJoin != nullptr); @@ -1945,7 +1952,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) } // It has to be a forward jump. Defer this check until after all the cheap checks - // are done, since it iterates forward in the block list looking for bbTarget. + // are done, since it iterates forward in the block list looking for block's target. // TODO-CQ: Check if we can also optimize the backwards jump as well. // if (!fgIsForwardBranch(block, block->GetTarget())) @@ -2135,10 +2142,7 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) bool foundCondTree = false; // Create a new block after `block` to put the copied condition code. - BasicBlock* bNewCond = fgNewBBafter(BBJ_COND, block, /*extendRegion*/ true, bJoin); - block->SetKindAndTarget(BBJ_ALWAYS, bNewCond); - block->SetFlags(BBF_NONE_QUIRK); - assert(block->JumpsToNext()); + BasicBlock* bNewCond = fgNewBBafter(BBJ_COND, block, /*extendRegion*/ true); // Clone each statement in bTest and append to bNewCond. for (Statement* const stmt : bTest->Statements()) @@ -2197,12 +2201,17 @@ bool Compiler::optInvertWhileLoop(BasicBlock* block) // Update pred info // - bNewCond->SetFalseTarget(bTop); - fgAddRefPred(bJoin, bNewCond); - fgAddRefPred(bTop, bNewCond); + FlowEdge* const trueEdge = fgAddRefPred(bJoin, bNewCond); + FlowEdge* const falseEdge = fgAddRefPred(bTop, bNewCond); + bNewCond->SetTrueEdge(trueEdge); + bNewCond->SetFalseEdge(falseEdge); + + fgRemoveRefPred(block->GetTargetEdge()); + FlowEdge* const newEdge = fgAddRefPred(bNewCond, block); - fgAddRefPred(bNewCond, block); - fgRemoveRefPred(bTest, block); + block->SetTargetEdge(newEdge); + block->SetFlags(BBF_NONE_QUIRK); + assert(block->JumpsToNext()); // Move all predecessor edges that look like loop entry edges to point to the new cloned condition // block, not the existing condition block. The idea is that if we only move `block` to point to @@ -2976,7 +2985,7 @@ bool Compiler::optCreatePreheader(FlowGraphNaturalLoop* loop) insertBefore = header; } - BasicBlock* preheader = fgNewBBbefore(BBJ_ALWAYS, insertBefore, false, header); + BasicBlock* preheader = fgNewBBbefore(BBJ_ALWAYS, insertBefore, false); preheader->SetFlags(BBF_INTERNAL); fgSetEHRegionForNewPreheaderOrExit(preheader); @@ -2989,7 +2998,8 @@ bool Compiler::optCreatePreheader(FlowGraphNaturalLoop* loop) JITDUMP("Created new preheader " FMT_BB " for " FMT_LP "\n", preheader->bbNum, loop->GetIndex()); - fgAddRefPred(header, preheader); + FlowEdge* const newEdge = fgAddRefPred(header, preheader); + preheader->SetTargetEdge(newEdge); for (FlowEdge* enterEdge : loop->EntryEdges()) { @@ -3092,26 +3102,27 @@ bool Compiler::optCanonicalizeExit(FlowGraphNaturalLoop* loop, BasicBlock* exit) BasicBlock* bottom = loop->GetLexicallyBottomMostBlock(); if (bottom->hasTryIndex() && (bottom->getTryIndex() == finallyBlock->getHndIndex()) && !bottom->hasHndIndex()) { - newExit = fgNewBBafter(BBJ_ALWAYS, bottom, true, exit); + newExit = fgNewBBafter(BBJ_ALWAYS, bottom, true); } else { // Otherwise just do the heavy-handed thing and insert it anywhere in the right region. - newExit = fgNewBBinRegion(BBJ_ALWAYS, finallyBlock->bbHndIndex, 0, nullptr, exit, /* putInFilter */ false, + newExit = fgNewBBinRegion(BBJ_ALWAYS, finallyBlock->bbHndIndex, 0, nullptr, /* putInFilter */ false, /* runRarely */ false, /* insertAtEnd */ true); } } else #endif { - newExit = fgNewBBbefore(BBJ_ALWAYS, exit, false, exit); + newExit = fgNewBBbefore(BBJ_ALWAYS, exit, false); newExit->SetFlags(BBF_NONE_QUIRK); fgSetEHRegionForNewPreheaderOrExit(newExit); } newExit->SetFlags(BBF_INTERNAL); - fgAddRefPred(exit, newExit); + FlowEdge* const newEdge = fgAddRefPred(exit, newExit); + newExit->SetTargetEdge(newEdge); newExit->bbCodeOffs = exit->bbCodeOffs; @@ -5615,7 +5626,6 @@ void Compiler::optComputeLoopSideEffectsOfBlock(BasicBlock* blk, FlowGraphNatura case GT_XCHG: case GT_CMPXCHG: case GT_MEMORYBARRIER: - case GT_STORE_DYN_BLK: { memoryHavoc |= memoryKindSet(GcHeap, ByrefExposed); } diff --git a/src/coreclr/jit/patchpoint.cpp b/src/coreclr/jit/patchpoint.cpp index 27b94470962ef..a22534c9fe294 100644 --- a/src/coreclr/jit/patchpoint.cpp +++ b/src/coreclr/jit/patchpoint.cpp @@ -101,13 +101,12 @@ class PatchpointTransformer // Arguments: // jumpKind - jump kind for the new basic block // insertAfter - basic block, after which compiler has to insert the new one. - // jumpDest - jump target for the new basic block. Defaults to nullptr. // // Return Value: // new basic block. - BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter, BasicBlock* jumpDest = nullptr) + BasicBlock* CreateAndInsertBasicBlock(BBKinds jumpKind, BasicBlock* insertAfter) { - BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true, jumpDest); + BasicBlock* block = compiler->fgNewBBafter(jumpKind, insertAfter, true); block->SetFlags(BBF_IMPORTED); return block; } @@ -143,21 +142,21 @@ class PatchpointTransformer // Current block now becomes the test block BasicBlock* remainderBlock = compiler->fgSplitBlockAtBeginning(block); - BasicBlock* helperBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, block, block->Next()); + BasicBlock* helperBlock = CreateAndInsertBasicBlock(BBJ_ALWAYS, block); // Update flow and flags - block->SetCond(remainderBlock, helperBlock); block->SetFlags(BBF_INTERNAL); - helperBlock->SetFlags(BBF_BACKWARD_JUMP | BBF_NONE_QUIRK); FlowEdge* const falseEdge = compiler->fgAddRefPred(helperBlock, block); FlowEdge* const trueEdge = compiler->fgGetPredForBlock(remainderBlock, block); trueEdge->setLikelihood(HIGH_PROBABILITY / 100.0); falseEdge->setLikelihood((100 - HIGH_PROBABILITY) / 100.0); + block->SetCond(trueEdge, falseEdge); FlowEdge* const newEdge = compiler->fgAddRefPred(remainderBlock, helperBlock); newEdge->setLikelihood(1.0); + helperBlock->SetTargetEdge(newEdge); // Update weights remainderBlock->inheritWeight(block); @@ -238,7 +237,7 @@ class PatchpointTransformer } // Update flow - block->SetKindAndTarget(BBJ_THROW); + block->SetKindAndTargetEdge(BBJ_THROW); // Add helper call // diff --git a/src/coreclr/jit/scev.cpp b/src/coreclr/jit/scev.cpp new file mode 100644 index 0000000000000..81760593a8aba --- /dev/null +++ b/src/coreclr/jit/scev.cpp @@ -0,0 +1,821 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +// This file contains code to analyze how the value of induction variables +// evolve (scalar evolution analysis), and to turn them into the SCEV IR +// defined in scev.h. The analysis is inspired by "Michael Wolfe. 1992. Beyond +// induction variables." and also by LLVM's scalar evolution analysis. +// +// The main idea of scalar evolution nalysis is to give a closed form +// describing the value of tree nodes inside loops even when taking into +// account that they are changing on each loop iteration. This is useful for +// optimizations that want to reason about values of IR nodes inside loops, +// such as IV widening or strength reduction. +// +// To represent the possibility of evolution the SCEV IR includes the concept +// of an add recurrence , which describes a value that +// starts at "start" and changes by adding "step" at each iteration. The IR +// nodes that change in this way (or depend on something that changes in this +// way) are generally called induction variables. +// +// An add recurrence arises only when a local exists in the loop that is +// mutated in each iteration. Such a local will naturally end up with a phi +// node in the loop header. These locals are called primary (or basic) +// induction variables. The non-primary IVs (which always must depend on the +// primary IVs) are sometimes called secondary IVs. +// +// The job of the analysis is to go from a tree node to a SCEV node that +// describes its value (possibly taking its evolution into account). Note that +// SCEV nodes are immutable and the values they represent are _not_ +// flow-dependent; that is, they don't exist at a specific location inside the +// loop, even though some particular tree node gave rise to that SCEV node. The +// analysis itself _is_ flow-dependent and guarantees that the Scev* returned +// describes the value that corresponds to what the tree node computes at its +// specific location. However, it would be perfectly legal for two trees at +// different locations in the loop to analyze to the same SCEV node (even +// potentially returning the same pointer). For example, in theory "i" and "j" +// in the following loop would both be represented by the same add recurrence +// , and the analysis could even return the same Scev* for both of +// them, even if it does not today: +// +// int i = 0; +// while (true) +// { +// i++; +// ... +// int j = i - 1; +// } +// +// Actually materializing the value of a SCEV node back into tree IR is not +// implemented yet, but generally would depend on the availability of tree +// nodes that compute the dependent values at the point where the IR is to be +// materialized. +// +// Besides the add recurrences the analysis itself is generally a +// straightforward translation from JIT IR into the SCEV IR. Creating the add +// recurrences requires paying attention to the structure of PHIs, and +// disambiguating the values coming from outside the loop and the values coming +// from the backedges. Currently only simplistic add recurrences that do not +// require recursive analysis are supported. These simplistic add recurrences +// are always on the form i = i + k. +// + +#include "jitpch.h" + +//------------------------------------------------------------------------ +// GetConstantValue: If this SSA use refers to a constant, then fetch that +// constant. +// +// Parameters: +// comp - Compiler instance +// cns - [out] Constant value; only valid if this function returns true. +// +// Returns: +// True if this SSA use refers to a constant; otherwise false, +// +bool ScevLocal::GetConstantValue(Compiler* comp, int64_t* cns) +{ + LclVarDsc* dsc = comp->lvaGetDesc(LclNum); + LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(SsaNum); + GenTreeLclVarCommon* defNode = ssaDsc->GetDefNode(); + if ((defNode != nullptr) && defNode->Data()->OperIs(GT_CNS_INT, GT_CNS_LNG)) + { + *cns = defNode->Data()->AsIntConCommon()->IntegralValue(); + return true; + } + + return false; +} + +//------------------------------------------------------------------------ +// Scev::GetConstantValue: If this SCEV is always a constant (i.e. either an +// inline constant or an SSA use referring to a constant) then obtain that +// constant. +// +// Parameters: +// comp - Compiler instance +// cns - [out] Constant value; only valid if this function returns true. +// +// Returns: +// True if a constant could be extracted. +// +bool Scev::GetConstantValue(Compiler* comp, int64_t* cns) +{ + if (OperIs(ScevOper::Constant)) + { + *cns = ((ScevConstant*)this)->Value; + return true; + } + + if (OperIs(ScevOper::Local)) + { + return ((ScevLocal*)this)->GetConstantValue(comp, cns); + } + + return false; +} + +#ifdef DEBUG +//------------------------------------------------------------------------ +// Dump: Print this scev node to stdout. +// +// Parameters: +// comp - Compiler instance +// +void Scev::Dump(Compiler* comp) +{ + switch (Oper) + { + case ScevOper::Constant: + { + ScevConstant* cns = (ScevConstant*)this; + printf("%zd", (ssize_t)cns->Value); + break; + } + case ScevOper::Local: + { + ScevLocal* invariantLocal = (ScevLocal*)this; + printf("V%02u.%u", invariantLocal->LclNum, invariantLocal->SsaNum); + + int64_t cns; + if (invariantLocal->GetConstantValue(comp, &cns)) + { + printf(" (%lld)", (long long)cns); + } + break; + } + case ScevOper::ZeroExtend: + case ScevOper::SignExtend: + { + ScevUnop* unop = (ScevUnop*)this; + printf("%cext<%d>(", unop->Oper == ScevOper::ZeroExtend ? 'z' : 's', genTypeSize(unop->Type) * 8); + unop->Op1->Dump(comp); + printf(")"); + break; + } + case ScevOper::Add: + case ScevOper::Mul: + case ScevOper::Lsh: + { + ScevBinop* binop = (ScevBinop*)this; + printf("("); + binop->Op1->Dump(comp); + const char* op; + switch (binop->Oper) + { + case ScevOper::Add: + op = "+"; + break; + case ScevOper::Mul: + op = "*"; + break; + case ScevOper::Lsh: + op = "<<"; + break; + default: + unreached(); + } + printf(" %s ", op); + binop->Op2->Dump(comp); + printf(")"); + break; + } + case ScevOper::AddRec: + { + ScevAddRec* addRec = (ScevAddRec*)this; + printf("<" FMT_LP, addRec->Loop->GetIndex()); + printf(", "); + addRec->Start->Dump(comp); + printf(", "); + addRec->Step->Dump(comp); + printf(">"); + break; + } + default: + unreached(); + } +} +#endif + +//------------------------------------------------------------------------ +// ScalarEvolutionContext: Construct an instance of a context to do scalar evolution in. +// +// Parameters: +// comp - Compiler instance +// +// Remarks: +// After construction the context should be reset for a new loop by calling +// ResetForLoop. +// +ScalarEvolutionContext::ScalarEvolutionContext(Compiler* comp) + : m_comp(comp), m_cache(comp->getAllocator(CMK_LoopIVOpts)) +{ +} + +//------------------------------------------------------------------------ +// ResetForLoop: Reset the internal cache in preparation of scalar +// evolution analysis inside a new loop. +// +// Parameters: +// loop - The loop. +// +void ScalarEvolutionContext::ResetForLoop(FlowGraphNaturalLoop* loop) +{ + m_loop = loop; + m_cache.RemoveAll(); +} + +//------------------------------------------------------------------------ +// NewConstant: Create a SCEV node that represents a constant. +// +// Returns: +// The new node. +// +ScevConstant* ScalarEvolutionContext::NewConstant(var_types type, int64_t value) +{ + ScevConstant* constant = new (m_comp, CMK_LoopIVOpts) ScevConstant(type, value); + return constant; +} + +//------------------------------------------------------------------------ +// NewLocal: Create a SCEV node that represents an invariant local (i.e. a +// use of an SSA def from outside the loop). +// +// Parameters: +// lclNum - The local +// ssaNum - The SSA number of the def outside the loop that is being used. +// +// Returns: +// The new node. +// +ScevLocal* ScalarEvolutionContext::NewLocal(unsigned lclNum, unsigned ssaNum) +{ + var_types type = genActualType(m_comp->lvaGetDesc(lclNum)); + ScevLocal* invariantLocal = new (m_comp, CMK_LoopIVOpts) ScevLocal(type, lclNum, ssaNum); + return invariantLocal; +} + +//------------------------------------------------------------------------ +// NewExtension: Create a SCEV node that represents a zero or sign extension. +// +// Parameters: +// oper - The operation (ScevOper::ZeroExtend or ScevOper::SignExtend) +// targetType - The target type of the extension +// op - The operand being extended. +// +// Returns: +// The new node. +// +ScevUnop* ScalarEvolutionContext::NewExtension(ScevOper oper, var_types targetType, Scev* op) +{ + assert(op != nullptr); + ScevUnop* ext = new (m_comp, CMK_LoopIVOpts) ScevUnop(oper, targetType, op); + return ext; +} + +//------------------------------------------------------------------------ +// NewBinop: Create a SCEV node that represents a binary operation. +// +// Parameters: +// oper - The operation +// op1 - First operand +// op2 - Second operand +// +// Returns: +// The new node. +// +ScevBinop* ScalarEvolutionContext::NewBinop(ScevOper oper, Scev* op1, Scev* op2) +{ + assert((op1 != nullptr) && (op2 != nullptr)); + ScevBinop* binop = new (m_comp, CMK_LoopIVOpts) ScevBinop(oper, op1->Type, op1, op2); + return binop; +} + +//------------------------------------------------------------------------ +// NewAddRec: Create a SCEV node that represents a new add recurrence. +// +// Parameters: +// loop - The loop where this add recurrence is evolving +// start - Value of the recurrence at the first iteration +// step - Step value of the recurrence +// +// Returns: +// The new node. +// +ScevAddRec* ScalarEvolutionContext::NewAddRec(Scev* start, Scev* step) +{ + assert((start != nullptr) && (step != nullptr)); + ScevAddRec* addRec = new (m_comp, CMK_LoopIVOpts) ScevAddRec(start->Type, start, step DEBUGARG(m_loop)); + return addRec; +} + +//------------------------------------------------------------------------ +// CreateSimpleInvariantScev: Create a "simple invariant" SCEV node for a tree: +// either an invariant local use or a constant. +// +// Parameters: +// tree - The tree +// +// Returns: +// SCEV node or nullptr if the tree is not a simple invariant. +// +Scev* ScalarEvolutionContext::CreateSimpleInvariantScev(GenTree* tree) +{ + if (tree->OperIs(GT_CNS_INT, GT_CNS_LNG)) + { + return CreateScevForConstant(tree->AsIntConCommon()); + } + + if (tree->OperIs(GT_LCL_VAR) && tree->AsLclVarCommon()->HasSsaName()) + { + LclVarDsc* dsc = m_comp->lvaGetDesc(tree->AsLclVarCommon()); + LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(tree->AsLclVarCommon()->GetSsaNum()); + + if ((ssaDsc->GetBlock() == nullptr) || !m_loop->ContainsBlock(ssaDsc->GetBlock())) + { + return NewLocal(tree->AsLclVarCommon()->GetLclNum(), tree->AsLclVarCommon()->GetSsaNum()); + } + } + + return nullptr; +} + +//------------------------------------------------------------------------ +// CreateScevForConstant: Given an integer constant, create a SCEV node for it. +// +// Parameters: +// tree - The integer constant +// +// Returns: +// SCEV node or nullptr if the integer constant is not representable (e.g. a handle). +// +Scev* ScalarEvolutionContext::CreateScevForConstant(GenTreeIntConCommon* tree) +{ + if (tree->IsIconHandle() || !tree->TypeIs(TYP_INT, TYP_LONG)) + { + return nullptr; + } + + return NewConstant(tree->TypeGet(), tree->AsIntConCommon()->IntegralValue()); +} + +//------------------------------------------------------------------------ +// AnalyzeNew: Analyze the specified tree in the specified block, without going +// through the cache. +// +// Parameters: +// block - Block containing the tree +// tree - Tree node +// depth - Current analysis depth +// +// Returns: +// SCEV node if the tree was analyzable; otherwise nullptr if the value is +// cannot be described. +// +Scev* ScalarEvolutionContext::AnalyzeNew(BasicBlock* block, GenTree* tree, int depth) +{ + switch (tree->OperGet()) + { + case GT_CNS_INT: + case GT_CNS_LNG: + { + return CreateScevForConstant(tree->AsIntConCommon()); + } + case GT_LCL_VAR: + case GT_PHI_ARG: + { + if (!tree->AsLclVarCommon()->HasSsaName()) + { + return nullptr; + } + + assert(m_comp->lvaInSsa(tree->AsLclVarCommon()->GetLclNum())); + LclVarDsc* dsc = m_comp->lvaGetDesc(tree->AsLclVarCommon()); + LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(tree->AsLclVarCommon()->GetSsaNum()); + + if ((ssaDsc->GetBlock() == nullptr) || !m_loop->ContainsBlock(ssaDsc->GetBlock())) + { + return NewLocal(tree->AsLclVarCommon()->GetLclNum(), tree->AsLclVarCommon()->GetSsaNum()); + } + + if (ssaDsc->GetDefNode() == nullptr) + { + // GT_CALL retbuf def? + return nullptr; + } + + if (ssaDsc->GetDefNode()->GetLclNum() != tree->AsLclVarCommon()->GetLclNum()) + { + // Should be a def of the parent + assert(dsc->lvIsStructField && (ssaDsc->GetDefNode()->GetLclNum() == dsc->lvParentLcl)); + return nullptr; + } + + return Analyze(ssaDsc->GetBlock(), ssaDsc->GetDefNode(), depth + 1); + } + case GT_STORE_LCL_VAR: + { + GenTreeLclVarCommon* store = tree->AsLclVarCommon(); + GenTree* data = store->Data(); + if (!data->OperIs(GT_PHI)) + { + return Analyze(block, data, depth + 1); + } + + if (block != m_loop->GetHeader()) + { + return nullptr; + } + + // We have a phi def for the current loop. Look for a primary + // induction variable. + GenTreePhi* phi = data->AsPhi(); + GenTreePhiArg* enterSsa = nullptr; + GenTreePhiArg* backedgeSsa = nullptr; + + for (GenTreePhi::Use& use : phi->Uses()) + { + GenTreePhiArg* phiArg = use.GetNode()->AsPhiArg(); + GenTreePhiArg*& ssaArg = m_loop->ContainsBlock(phiArg->gtPredBB) ? backedgeSsa : enterSsa; + if ((ssaArg == nullptr) || (ssaArg->GetSsaNum() == phiArg->GetSsaNum())) + { + ssaArg = phiArg; + } + else + { + return nullptr; + } + } + + if ((enterSsa == nullptr) || (backedgeSsa == nullptr)) + { + return nullptr; + } + + ScevLocal* enterScev = NewLocal(enterSsa->GetLclNum(), enterSsa->GetSsaNum()); + + LclVarDsc* dsc = m_comp->lvaGetDesc(store); + LclSsaVarDsc* ssaDsc = dsc->GetPerSsaData(backedgeSsa->GetSsaNum()); + + if (ssaDsc->GetDefNode() == nullptr) + { + // GT_CALL retbuf def + return nullptr; + } + + if (ssaDsc->GetDefNode()->GetLclNum() != store->GetLclNum()) + { + assert(dsc->lvIsStructField && ssaDsc->GetDefNode()->GetLclNum() == dsc->lvParentLcl); + return nullptr; + } + + assert(ssaDsc->GetBlock() != nullptr); + + // We currently do not handle complicated addrecs. We can do this + // by inserting a symbolic node in the cache and analyzing while it + // is part of the cache. It would allow us to model + // + // int i = 0; + // while (i < n) + // { + // int j = i + 1; + // ... + // i = j; + // } + // => + // + // and chains of recurrences, such as + // + // int i = 0; + // int j = 0; + // while (i < n) + // { + // j++; + // i += j; + // } + // => > + // + // The main issue is that it requires cache invalidation afterwards + // and turning the recursive result into an addrec. + // + return CreateSimpleAddRec(store, enterScev, ssaDsc->GetBlock(), ssaDsc->GetDefNode()->Data()); + } + case GT_CAST: + { + GenTreeCast* cast = tree->AsCast(); + if (cast->gtCastType != TYP_LONG) + { + return nullptr; + } + + Scev* op = Analyze(block, cast->CastOp(), depth + 1); + if (op == nullptr) + { + return nullptr; + } + + return NewExtension(cast->IsUnsigned() ? ScevOper::ZeroExtend : ScevOper::SignExtend, TYP_LONG, op); + } + case GT_ADD: + case GT_MUL: + case GT_LSH: + { + Scev* op1 = Analyze(block, tree->gtGetOp1(), depth + 1); + if (op1 == nullptr) + return nullptr; + + Scev* op2 = Analyze(block, tree->gtGetOp2(), depth + 1); + if (op2 == nullptr) + return nullptr; + + ScevOper oper; + switch (tree->OperGet()) + { + case GT_ADD: + oper = ScevOper::Add; + break; + case GT_MUL: + oper = ScevOper::Mul; + break; + case GT_LSH: + oper = ScevOper::Lsh; + break; + default: + unreached(); + } + + return NewBinop(oper, op1, op2); + } + case GT_COMMA: + { + return Analyze(block, tree->gtGetOp2(), depth + 1); + } + case GT_ARR_ADDR: + { + return Analyze(block, tree->AsArrAddr()->Addr(), depth + 1); + } + default: + return nullptr; + } +} + +//------------------------------------------------------------------------ +// CreateSimpleAddRec: Create a "simple" add-recurrence. This handles the most +// common patterns for primary induction variables where we see a store like +// "i = i + 1". +// +// Parameters: +// headerStore - Phi definition of the candidate primary induction variable +// enterScev - SCEV describing start value of the primary induction variable +// stepDefBlock - Block containing the def of the step value +// stepDefData - Value of the def of the step value +// +// Returns: +// SCEV node if this is a simple addrec shape. Otherwise nullptr. +// +Scev* ScalarEvolutionContext::CreateSimpleAddRec(GenTreeLclVarCommon* headerStore, + ScevLocal* enterScev, + BasicBlock* stepDefBlock, + GenTree* stepDefData) +{ + if (!stepDefData->OperIs(GT_ADD)) + { + return nullptr; + } + + GenTree* stepTree; + GenTree* op1 = stepDefData->gtGetOp1(); + GenTree* op2 = stepDefData->gtGetOp2(); + if (op1->OperIs(GT_LCL_VAR) && (op1->AsLclVar()->GetLclNum() == headerStore->GetLclNum()) && + (op1->AsLclVar()->GetSsaNum() == headerStore->GetSsaNum())) + { + stepTree = op2; + } + else if (op2->OperIs(GT_LCL_VAR) && (op2->AsLclVar()->GetLclNum() == headerStore->GetLclNum()) && + (op2->AsLclVar()->GetSsaNum() == headerStore->GetSsaNum())) + { + stepTree = op1; + } + else + { + // Not a simple IV shape (i.e. more complex than "i = i + k") + return nullptr; + } + + Scev* stepScev = CreateSimpleInvariantScev(stepTree); + if (stepScev == nullptr) + { + return nullptr; + } + + return NewAddRec(enterScev, stepScev); +} + +//------------------------------------------------------------------------ +// Analyze: Analyze the specified tree in the specified block. +// +// Parameters: +// block - Block containing the tree +// tree - Tree node +// +// Returns: +// SCEV node if the tree was analyzable; otherwise nullptr if the value is +// cannot be described. +// +Scev* ScalarEvolutionContext::Analyze(BasicBlock* block, GenTree* tree) +{ + return Analyze(block, tree, 0); +} + +// Since the analysis follows SSA defs we have no upper bound on the potential +// depth of the analysis performed. We put an artificial limit on this for two +// reasons: +// 1. The analysis is recursive, and we should not stack overflow regardless of +// the input program. +// 2. If we produced arbitrarily deep SCEV trees then all algorithms over their +// structure would similarly be at risk of stack overflows if they were +// recursive. However, these algorithms are generally much more elegant when +// they make use of recursion. +const int SCALAR_EVOLUTION_ANALYSIS_MAX_DEPTH = 64; + +//------------------------------------------------------------------------ +// Analyze: Analyze the specified tree in the specified block. +// +// Parameters: +// block - Block containing the tree +// tree - Tree node +// depth - Current analysis depth +// +// Returns: +// SCEV node if the tree was analyzable; otherwise nullptr if the value is +// cannot be described. +// +Scev* ScalarEvolutionContext::Analyze(BasicBlock* block, GenTree* tree, int depth) +{ + Scev* result; + if (!m_cache.Lookup(tree, &result)) + { + if (depth >= SCALAR_EVOLUTION_ANALYSIS_MAX_DEPTH) + { + return nullptr; + } + + result = AnalyzeNew(block, tree, depth); + m_cache.Set(tree, result); + } + + return result; +} + +//------------------------------------------------------------------------ +// FoldBinop: Fold simple binops. +// +// Type parameters: +// T - Type that the binop is being evaluated in +// +// Parameters: +// oper - Binary operation +// op1 - First operand +// op2 - Second operand +// +// Returns: +// Folded value. +// +template +static T FoldBinop(ScevOper oper, T op1, T op2) +{ + switch (oper) + { + case ScevOper::Add: + return op1 + op2; + case ScevOper::Mul: + return op1 * op2; + case ScevOper::Lsh: + return op1 << op2; + default: + unreached(); + } +} + +//------------------------------------------------------------------------ +// Simplify: Try to simplify a SCEV node by folding and canonicalization. +// +// Parameters: +// scev - The node +// +// Returns: +// Simplified node. +// +// Remarks: +// Canonicalization is done for binops; constants are moved to the right and +// addrecs are moved to the left. +// +// Simple unops/binops on constants are folded. Operands are distributed into +// add recs whenever possible. +// +Scev* ScalarEvolutionContext::Simplify(Scev* scev) +{ + switch (scev->Oper) + { + case ScevOper::Constant: + case ScevOper::Local: + { + return scev; + } + case ScevOper::ZeroExtend: + case ScevOper::SignExtend: + { + ScevUnop* unop = (ScevUnop*)scev; + assert(genTypeSize(unop->Type) >= genTypeSize(unop->Op1->Type)); + + Scev* op1 = Simplify(unop->Op1); + + if (unop->Type == op1->Type) + { + return op1; + } + + assert((unop->Type == TYP_LONG) && (op1->Type == TYP_INT)); + + if (op1->OperIs(ScevOper::Constant)) + { + ScevConstant* cns = (ScevConstant*)op1; + return NewConstant(unop->Type, unop->OperIs(ScevOper::ZeroExtend) ? (uint64_t)(int32_t)cns->Value + : (int64_t)(int32_t)cns->Value); + } + + if (op1->OperIs(ScevOper::AddRec)) + { + // TODO-Cleanup: This requires some proof that it is ok, but + // currently we do not rely on this. + return op1; + } + + return (op1 == unop->Op1) ? unop : NewExtension(unop->Oper, unop->Type, op1); + } + case ScevOper::Add: + case ScevOper::Mul: + case ScevOper::Lsh: + { + ScevBinop* binop = (ScevBinop*)scev; + Scev* op1 = Simplify(binop->Op1); + Scev* op2 = Simplify(binop->Op2); + + if (binop->OperIs(ScevOper::Add, ScevOper::Mul)) + { + // Normalize addrecs to the left + if (op2->OperIs(ScevOper::AddRec) && !op1->OperIs(ScevOper::AddRec)) + { + std::swap(op1, op2); + } + // Normalize constants to the right + if (op1->OperIs(ScevOper::Constant) && !op2->OperIs(ScevOper::Constant)) + { + std::swap(op1, op2); + } + } + + if (op1->OperIs(ScevOper::AddRec)) + { + // + x => + // * x => + ScevAddRec* addRec = (ScevAddRec*)op1; + Scev* newStart = Simplify(NewBinop(binop->Oper, addRec->Start, op2)); + Scev* newStep = scev->OperIs(ScevOper::Mul, ScevOper::Lsh) + ? Simplify(NewBinop(binop->Oper, addRec->Step, op2)) + : addRec->Step; + return NewAddRec(newStart, newStep); + } + + if (op1->OperIs(ScevOper::Constant) && op2->OperIs(ScevOper::Constant)) + { + ScevConstant* cns1 = (ScevConstant*)op1; + ScevConstant* cns2 = (ScevConstant*)op2; + int64_t newValue; + if (binop->TypeIs(TYP_INT)) + { + newValue = FoldBinop(binop->Oper, static_cast(cns1->Value), + static_cast(cns2->Value)); + } + else + { + assert(binop->TypeIs(TYP_LONG)); + newValue = FoldBinop(binop->Oper, cns1->Value, cns2->Value); + } + + return NewConstant(binop->Type, newValue); + } + + return (op1 == binop->Op1) && (op2 == binop->Op2) ? binop : NewBinop(binop->Oper, op1, op2); + } + case ScevOper::AddRec: + { + ScevAddRec* addRec = (ScevAddRec*)scev; + Scev* start = Simplify(addRec->Start); + Scev* step = Simplify(addRec->Step); + return (start == addRec->Start) && (step == addRec->Step) ? addRec : NewAddRec(start, step); + } + default: + unreached(); + } +} diff --git a/src/coreclr/jit/scev.h b/src/coreclr/jit/scev.h new file mode 100644 index 0000000000000..603088d962366 --- /dev/null +++ b/src/coreclr/jit/scev.h @@ -0,0 +1,155 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma once + +// This file contains the definition of the scalar evolution IR. This IR allows +// representing the values of IR nodes inside loops in a closed form, taking +// into account that they are changing on each loop iteration. The IR is based +// around the following possible operations. At the core is ScevOper::AddRec, +// which represents a value that evolves by an add recurrence. In dumps it is +// described by where "loop" is the loop the value is +// evolving in, "start" is the initial value and "step" is the step by which +// the value evolves in every iteration. +// +// See scev.cpp for further documentation. +// +enum class ScevOper +{ + Constant, + Local, + ZeroExtend, + SignExtend, + Add, + Mul, + Lsh, + AddRec, +}; + +static bool ScevOperIs(ScevOper oper, ScevOper otherOper) +{ + return oper == otherOper; +} + +template +static bool ScevOperIs(ScevOper oper, ScevOper operFirst, Args... operTail) +{ + return oper == operFirst || ScevOperIs(oper, operTail...); +} + +struct Scev +{ + const ScevOper Oper; + const var_types Type; + + Scev(ScevOper oper, var_types type) : Oper(oper), Type(type) + { + } + + template + bool OperIs(Args... opers) + { + return ScevOperIs(Oper, opers...); + } + + bool TypeIs(var_types type) + { + return Type == type; + } + + bool GetConstantValue(Compiler* comp, int64_t* cns); + +#ifdef DEBUG + void Dump(Compiler* comp); +#endif +}; + +struct ScevConstant : Scev +{ + ScevConstant(var_types type, int64_t value) : Scev(ScevOper::Constant, type), Value(value) + { + } + + int64_t Value; +}; + +struct ScevLocal : Scev +{ + ScevLocal(var_types type, unsigned lclNum, unsigned ssaNum) + : Scev(ScevOper::Local, type), LclNum(lclNum), SsaNum(ssaNum) + { + } + + const unsigned LclNum; + const unsigned SsaNum; + + bool GetConstantValue(Compiler* comp, int64_t* cns); +}; + +struct ScevUnop : Scev +{ + ScevUnop(ScevOper oper, var_types type, Scev* op1) : Scev(oper, type), Op1(op1) + { + } + + Scev* const Op1; +}; + +struct ScevBinop : ScevUnop +{ + ScevBinop(ScevOper oper, var_types type, Scev* op1, Scev* op2) : ScevUnop(oper, type, op1), Op2(op2) + { + } + + Scev* const Op2; +}; + +// Represents a value that evolves by an add recurrence. +// The value at iteration N is Start + N * Step. +// "Start" and "Step" are guaranteed to be invariant in "Loop". +struct ScevAddRec : Scev +{ + ScevAddRec(var_types type, Scev* start, Scev* step DEBUGARG(FlowGraphNaturalLoop* loop)) + : Scev(ScevOper::AddRec, type), Start(start), Step(step) DEBUGARG(Loop(loop)) + { + } + + Scev* const Start; + Scev* const Step; + INDEBUG(FlowGraphNaturalLoop* const Loop); +}; + +typedef JitHashTable, Scev*> ScalarEvolutionMap; + +// Scalar evolution is analyzed in the context of a single loop, and are +// computed on-demand by the use of the "Analyze" method on this class, which +// also maintains a cache. +class ScalarEvolutionContext +{ + Compiler* m_comp; + FlowGraphNaturalLoop* m_loop = nullptr; + ScalarEvolutionMap m_cache; + + Scev* Analyze(BasicBlock* block, GenTree* tree, int depth); + Scev* AnalyzeNew(BasicBlock* block, GenTree* tree, int depth); + Scev* CreateSimpleAddRec(GenTreeLclVarCommon* headerStore, + ScevLocal* start, + BasicBlock* stepDefBlock, + GenTree* stepDefData); + Scev* CreateSimpleInvariantScev(GenTree* tree); + Scev* CreateScevForConstant(GenTreeIntConCommon* tree); + +public: + ScalarEvolutionContext(Compiler* comp); + + void ResetForLoop(FlowGraphNaturalLoop* loop); + + ScevConstant* NewConstant(var_types type, int64_t value); + ScevLocal* NewLocal(unsigned lclNum, unsigned ssaNum); + ScevUnop* NewExtension(ScevOper oper, var_types targetType, Scev* op); + ScevBinop* NewBinop(ScevOper oper, Scev* op1, Scev* op2); + ScevAddRec* NewAddRec(Scev* start, Scev* step); + + Scev* Analyze(BasicBlock* block, GenTree* tree); + Scev* Simplify(Scev* scev); +}; diff --git a/src/coreclr/jit/sideeffects.cpp b/src/coreclr/jit/sideeffects.cpp index d2c1de6c749a5..a2dd47c994ef4 100644 --- a/src/coreclr/jit/sideeffects.cpp +++ b/src/coreclr/jit/sideeffects.cpp @@ -174,7 +174,7 @@ AliasSet::NodeInfo::NodeInfo(Compiler* compiler, GenTree* node) // Is the operation a write? If so, set `node` to the location that is being written to. bool isWrite = false; - if (node->OperIsStore() || node->OperIs(GT_STORE_DYN_BLK, GT_MEMORYBARRIER)) + if (node->OperIsStore() || node->OperIs(GT_MEMORYBARRIER)) { isWrite = true; } diff --git a/src/coreclr/jit/simd.cpp b/src/coreclr/jit/simd.cpp index d69730ad520ed..48c23eb646411 100644 --- a/src/coreclr/jit/simd.cpp +++ b/src/coreclr/jit/simd.cpp @@ -549,8 +549,6 @@ bool areFieldAddressesTheSame(GenTreeFieldAddr* op1, GenTreeFieldAddr* op2) bool Compiler::areFieldsContiguous(GenTreeIndir* op1, GenTreeIndir* op2) { assert(op1->isIndir() && op2->isIndir()); - // TODO-1stClassStructs: delete once IND nodes are no more. - assert(!op1->TypeIs(TYP_STRUCT) && !op2->TypeIs(TYP_STRUCT)); var_types op1Type = op1->TypeGet(); var_types op2Type = op2->TypeGet(); diff --git a/src/coreclr/jit/switchrecognition.cpp b/src/coreclr/jit/switchrecognition.cpp index 1008b81194f8d..1a523bcfbeec7 100644 --- a/src/coreclr/jit/switchrecognition.cpp +++ b/src/coreclr/jit/switchrecognition.cpp @@ -363,16 +363,18 @@ bool Compiler::optSwitchConvert(BasicBlock* firstBlock, int testsCount, ssize_t* assert(lastBlock->FalseTargetIs(blockIfTrue)); fgRemoveRefPred(blockIfTrue, firstBlock); BasicBlock* targetBlock = blockIfTrue; - blockIfTrue = fgNewBBafter(BBJ_ALWAYS, firstBlock, true, targetBlock); + blockIfTrue = fgNewBBafter(BBJ_ALWAYS, firstBlock, true); FlowEdge* const newEdge = fgAddRefPred(targetBlock, blockIfTrue); skipPredRemoval = true; + blockIfTrue->SetTargetEdge(newEdge); } else { assert(lastBlock->FalseTargetIs(blockIfFalse)); BasicBlock* targetBlock = blockIfFalse; - blockIfFalse = fgNewBBafter(BBJ_ALWAYS, firstBlock, true, targetBlock); + blockIfFalse = fgNewBBafter(BBJ_ALWAYS, firstBlock, true); FlowEdge* const newEdge = fgAddRefPred(targetBlock, blockIfFalse); + blockIfFalse->SetTargetEdge(newEdge); } } diff --git a/src/coreclr/jit/targetamd64.h b/src/coreclr/jit/targetamd64.h index 4abe71984b57c..147355c2474a2 100644 --- a/src/coreclr/jit/targetamd64.h +++ b/src/coreclr/jit/targetamd64.h @@ -563,4 +563,9 @@ #define RBM_STACK_PROBE_HELPER_TRASH RBM_RAX #endif // !UNIX_AMD64_ABI + #define SWIFT_SUPPORT + #define REG_SWIFT_ERROR REG_R12 + #define RBM_SWIFT_ERROR RBM_R12 + #define SWIFT_SELF_REG REG_R13 + // clang-format on diff --git a/src/coreclr/jit/targetarm64.h b/src/coreclr/jit/targetarm64.h index 3646ecb4407bf..74a14535f1507 100644 --- a/src/coreclr/jit/targetarm64.h +++ b/src/coreclr/jit/targetarm64.h @@ -370,4 +370,9 @@ #define REG_ZERO_INIT_FRAME_REG2 REG_R10 #define REG_ZERO_INIT_FRAME_SIMD REG_V16 + #define SWIFT_SUPPORT + #define REG_SWIFT_ERROR REG_R21 + #define RBM_SWIFT_ERROR RBM_R21 + #define SWIFT_SELF_REG REG_R20 + // clang-format on diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 40ecc8fbf616f..4a8ca85aa3e58 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -9573,8 +9573,8 @@ const uint8_t ValueNumStore::s_vnfOpAttribs[VNF_COUNT] = { static genTreeOps genTreeOpsIllegalAsVNFunc[] = {GT_IND, // When we do heap memory. GT_NULLCHECK, GT_QMARK, GT_COLON, GT_LOCKADD, GT_XADD, GT_XCHG, - GT_CMPXCHG, GT_LCLHEAP, GT_BOX, GT_XORR, GT_XAND, GT_STORE_DYN_BLK, - GT_STORE_LCL_VAR, GT_STORE_LCL_FLD, GT_STOREIND, GT_STORE_BLK, + GT_CMPXCHG, GT_LCLHEAP, GT_BOX, GT_XORR, GT_XAND, GT_STORE_LCL_VAR, + GT_STORE_LCL_FLD, GT_STOREIND, GT_STORE_BLK, // These need special semantics: GT_COMMA, // == second argument (but with exception(s) from first). GT_ARR_ADDR, GT_BOUNDS_CHECK, @@ -9874,7 +9874,7 @@ class ValueNumberState return false; } - if (!predBlock->KindIs(BBJ_COND) || predBlock->TrueTargetIs(predBlock->GetFalseTarget())) + if (!predBlock->KindIs(BBJ_COND) || predBlock->TrueEdgeIs(predBlock->GetFalseEdge())) { return true; } @@ -11324,7 +11324,9 @@ void Compiler::fgValueNumberTree(GenTree* tree) break; case GT_CATCH_ARG: + case GT_SWIFT_ERROR: // We know nothing about the value of a caught expression. + // We also know nothing about the error register's value post-Swift call. tree->gtVNPair.SetBoth(vnStore->VNForExpr(compCurBB, tree->TypeGet())); break; @@ -11486,12 +11488,7 @@ void Compiler::fgValueNumberTree(GenTree* tree) unsigned loadSize = tree->AsIndir()->Size(); VNFuncApp funcApp{VNF_COUNT}; - // TODO-1stClassStructs: delete layout-less "IND(struct)" nodes and the "loadSize == 0" condition. - if (loadSize == 0) - { - tree->gtVNPair.SetBoth(vnStore->VNForExpr(compCurBB, loadType)); - } - else if (fgValueNumberConstLoad(tree->AsIndir())) + if (fgValueNumberConstLoad(tree->AsIndir())) { // VN is assigned inside fgValueNumberConstLoad } @@ -11758,30 +11755,6 @@ void Compiler::fgValueNumberTree(GenTree* tree) break; #endif // FEATURE_HW_INTRINSICS - case GT_STORE_DYN_BLK: - { - // Conservatively, mutate the heaps - we don't analyze these rare stores. - // Likewise, any locals possibly defined by them we mark as address-exposed. - fgMutateGcHeap(tree DEBUGARG("dynamic block store")); - - GenTreeStoreDynBlk* store = tree->AsStoreDynBlk(); - ValueNumPair vnpExcSet = ValueNumStore::VNPForEmptyExcSet(); - - // Propagate the exceptions... - vnpExcSet = vnStore->VNPUnionExcSet(store->Addr()->gtVNPair, vnpExcSet); - vnpExcSet = vnStore->VNPUnionExcSet(store->Data()->gtVNPair, vnpExcSet); - vnpExcSet = vnStore->VNPUnionExcSet(store->gtDynamicSize->gtVNPair, vnpExcSet); - - // This is a store, it produces no value. Thus we use VNPForVoid(). - store->gtVNPair = vnStore->VNPWithExc(vnStore->VNPForVoid(), vnpExcSet); - - // Note that we are only adding the exception for the destination address. - // Currently, "Data()" is an explicit indirection in case this is a "cpblk". - assert(store->Data()->gtEffectiveVal()->OperIsIndir() || store->OperIsInitBlkOp()); - fgValueNumberAddExceptionSetForIndirection(store, store->Addr()); - break; - } - case GT_CMPXCHG: // Specialop { // For CMPXCHG and other intrinsics add an arbitrary side effect on GcHeap/ByrefExposed. diff --git a/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs b/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs index 42e677737be96..d47cae0a18ed0 100644 --- a/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs +++ b/src/coreclr/nativeaot/Common/src/Internal/Runtime/CompilerHelpers/StartupCodeHelpers.cs @@ -290,12 +290,7 @@ private static unsafe void RehydrateData(IntPtr dehydratedData, int length) { // At the time of writing this, 90% of DehydratedDataCommand.Copy cases // would fall into the above specialized cases. 10% fall back to memmove. - memmove(pDest, pCurrent, (nuint)payload); - - // Not a DllImport - we don't need a GC transition since this is early startup - [MethodImplAttribute(MethodImplOptions.InternalCall)] - [RuntimeImport("*", "memmove")] - static extern unsafe void* memmove(byte* dmem, byte* smem, nuint size); + Unsafe.CopyBlock(pDest, pCurrent, (uint)payload); } pDest += payload; diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs index 136872edd4a47..7f3df7a7893a2 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/CompilerServices/Unsafe.cs @@ -119,5 +119,15 @@ public static T ReadUnaligned(void* source) { throw new PlatformNotSupportedException(); } + + /// + /// Copies bytes from the source address to the destination address. + /// + [Intrinsic] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void CopyBlock(void* destination, void* source, uint byteCount) + { + throw new PlatformNotSupportedException(); + } } } diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs index c8d9a0a74c264..89855a54113d2 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/ExceptionHandling.cs @@ -68,6 +68,14 @@ private struct EHEnum private IntPtr _dummy; // For alignment } + internal struct MethodRegionInfo + { + internal byte* _hotStartAddress; + internal nuint _hotSize; + internal byte* _coldStartAddress; + internal nuint _coldSize; + } + #pragma warning disable IDE0060 // This is a fail-fast function used by the runtime as a last resort that will terminate the process with // as little effort as possible. No guarantee is made about the semantics of this fail-fast. @@ -932,6 +940,20 @@ private static void DebugVerifyHandlingFrame(UIntPtr handlingFrameSP) "Handling frame must have a valid stack frame pointer"); } + // Caclulate the code offset from the start of the method as if the hot and cold regions were + // stored sequentially in memory. + private static uint CalculateCodeOffset(byte* pbControlPC, in MethodRegionInfo methodRegionInfo) + { + uint codeOffset = (uint)(pbControlPC - methodRegionInfo._hotStartAddress); + // If the PC is in the cold region, adjust the offset to be relative to the start of the method. + if ((methodRegionInfo._coldSize != 0) && (codeOffset >= methodRegionInfo._hotSize)) + { + codeOffset = (uint)(methodRegionInfo._hotSize + (nuint)(pbControlPC - methodRegionInfo._coldStartAddress)); + } + + return codeOffset; + } + private static void UpdateStackTrace(object exceptionObj, UIntPtr curFramePtr, IntPtr ip, UIntPtr sp, ref bool isFirstRethrowFrame, ref UIntPtr prevFramePtr, ref bool isFirstFrame, ref ExInfo exInfo) { @@ -958,13 +980,13 @@ private static bool FindFirstPassHandler(object exception, uint idxStart, tryRegionIdx = MaxTryRegionIdx; EHEnum ehEnum; - byte* pbMethodStartAddress; - if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref frameIter, &pbMethodStartAddress, &ehEnum)) + MethodRegionInfo methodRegionInfo; + if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref frameIter, out methodRegionInfo, &ehEnum)) return false; byte* pbControlPC = frameIter.ControlPC; - uint codeOffset = (uint)(pbControlPC - pbMethodStartAddress); + uint codeOffset = CalculateCodeOffset(pbControlPC, in methodRegionInfo); uint lastTryStart = 0, lastTryEnd = 0; @@ -1111,13 +1133,14 @@ private static void InvokeSecondPass(ref ExInfo exInfo, uint idxStart) private static void InvokeSecondPass(ref ExInfo exInfo, uint idxStart, uint idxLimit) { EHEnum ehEnum; - byte* pbMethodStartAddress; - if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref exInfo._frameIter, &pbMethodStartAddress, &ehEnum)) + MethodRegionInfo methodRegionInfo; + + if (!InternalCalls.RhpEHEnumInitFromStackFrameIterator(ref exInfo._frameIter, out methodRegionInfo, &ehEnum)) return; byte* pbControlPC = exInfo._frameIter.ControlPC; - uint codeOffset = (uint)(pbControlPC - pbMethodStartAddress); + uint codeOffset = CalculateCodeOffset(pbControlPC, in methodRegionInfo); uint lastTryStart = 0, lastTryEnd = 0; diff --git a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs index 806208187e92f..8cf281392abf0 100644 --- a/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs +++ b/src/coreclr/nativeaot/Runtime.Base/src/System/Runtime/InternalCalls.cs @@ -177,7 +177,7 @@ internal static int RhEndNoGCRegion() [RuntimeImport(Redhawk.BaseName, "RhpEHEnumInitFromStackFrameIterator")] [MethodImpl(MethodImplOptions.InternalCall)] - internal static extern unsafe bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, byte** pMethodStartAddress, void* pEHEnum); + internal static extern unsafe bool RhpEHEnumInitFromStackFrameIterator(ref StackFrameIterator pFrameIter, out EH.MethodRegionInfo pMethodRegionInfo, void* pEHEnum); [RuntimeImport(Redhawk.BaseName, "RhpEHEnumNext")] [MethodImpl(MethodImplOptions.InternalCall)] diff --git a/src/coreclr/nativeaot/Runtime/CMakeLists.txt b/src/coreclr/nativeaot/Runtime/CMakeLists.txt index 7ef6f223fdb58..a5e865ff749dc 100644 --- a/src/coreclr/nativeaot/Runtime/CMakeLists.txt +++ b/src/coreclr/nativeaot/Runtime/CMakeLists.txt @@ -263,11 +263,13 @@ add_definitions(-D_LIB) # there is a problem with undefined symbols when this is set # add_definitions(-DSTRESS_HEAP) -if(WIN32) +if(CLR_CMAKE_TARGET_WIN32) set(FEATURE_ETW 1) add_definitions(-DFEATURE_ETW) add_definitions(-DFEATURE_SUSPEND_REDIRECTION) - add_definitions(-DFEATURE_SPECIAL_USER_MODE_APC) + if (CLR_CMAKE_TARGET_ARCH_AMD64) + add_definitions(-DFEATURE_SPECIAL_USER_MODE_APC) + endif() else() if(NOT CLR_CMAKE_TARGET_MACCATALYST AND NOT CLR_CMAKE_TARGET_IOS AND NOT CLR_CMAKE_TARGET_TVOS) add_definitions(-DFEATURE_READONLY_GS_COOKIE) diff --git a/src/coreclr/nativeaot/Runtime/EHHelpers.cpp b/src/coreclr/nativeaot/Runtime/EHHelpers.cpp index 28cb9e617f09b..1a54b9bcc9b55 100644 --- a/src/coreclr/nativeaot/Runtime/EHHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/EHHelpers.cpp @@ -29,13 +29,25 @@ #include "MethodTable.inl" #include "CommonMacros.inl" +struct MethodRegionInfo +{ + void* hotStartAddress; + size_t hotSize; + void* coldStartAddress; + size_t coldSize; +}; + COOP_PINVOKE_HELPER(FC_BOOL_RET, RhpEHEnumInitFromStackFrameIterator, ( - StackFrameIterator* pFrameIter, void ** pMethodStartAddressOut, EHEnum* pEHEnum)) + StackFrameIterator* pFrameIter, MethodRegionInfo* pMethodRegionInfoOut, EHEnum* pEHEnum)) { ICodeManager * pCodeManager = pFrameIter->GetCodeManager(); pEHEnum->m_pCodeManager = pCodeManager; - FC_RETURN_BOOL(pCodeManager->EHEnumInit(pFrameIter->GetMethodInfo(), pMethodStartAddressOut, &pEHEnum->m_state)); + pMethodRegionInfoOut->hotSize = 0; // unknown + pMethodRegionInfoOut->coldStartAddress = nullptr; + pMethodRegionInfoOut->coldSize = 0; + + FC_RETURN_BOOL(pCodeManager->EHEnumInit(pFrameIter->GetMethodInfo(), &pMethodRegionInfoOut->hotStartAddress, &pEHEnum->m_state)); } COOP_PINVOKE_HELPER(FC_BOOL_RET, RhpEHEnumNext, (EHEnum* pEHEnum, EHClause* pEHClause)) diff --git a/src/coreclr/nativeaot/Runtime/MathHelpers.cpp b/src/coreclr/nativeaot/Runtime/MathHelpers.cpp index 39acda9d5f4f0..d1c867d183aa2 100644 --- a/src/coreclr/nativeaot/Runtime/MathHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/MathHelpers.cpp @@ -67,11 +67,6 @@ EXTERN_C NATIVEAOT_API int64_t REDHAWK_CALLCONV RhpLMul(int64_t i, int64_t j) return i * j; } -EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpULMul(uint64_t i, uint64_t j) -{ - return i * j; -} - EXTERN_C NATIVEAOT_API uint64_t REDHAWK_CALLCONV RhpLRsz(uint64_t i, int32_t j) { return i >> (j & 0x3f); diff --git a/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S b/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S index 79ffed2b05210..966b052a2b9f9 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S +++ b/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S @@ -90,9 +90,11 @@ LOCAL_LABEL(RhpNewFast_RarePath): // Set the new objects MethodTable pointer on success. cbz x0, LOCAL_LABEL(NewOutOfMemory) + .cfi_remember_state POP_COOP_PINVOKE_FRAME EPILOG_RETURN + .cfi_restore_state LOCAL_LABEL(NewOutOfMemory): // This is the OOM failure path. We are going to tail-call to a managed helper that will throw // an out of memory exception that the caller of this allocator understands. @@ -262,9 +264,11 @@ LOCAL_LABEL(RhpNewArray_Rare): // Set the new objects MethodTable pointer and length on success. cbz x0, LOCAL_LABEL(ArrayOutOfMemory) + .cfi_remember_state POP_COOP_PINVOKE_FRAME EPILOG_RETURN + .cfi_restore_state LOCAL_LABEL(ArrayOutOfMemory): // This is the OOM failure path. We are going to tail-call to a managed helper that will throw // an out of memory exception that the caller of this allocator understands. diff --git a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S index abe7555b76113..8075335ea0b2b 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S +++ b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S @@ -146,8 +146,11 @@ NESTED_ENTRY RhpWaitForGC, _TEXT, NoHandler ldr x2, [sp, #OFFSETOF__PInvokeTransitionFrame__m_Flags] tbnz x2, #PTFF_THREAD_ABORT_BIT, LOCAL_LABEL(ThrowThreadAbort) + .cfi_remember_state POP_PROBE_FRAME EPILOG_RETURN + + .cfi_restore_state LOCAL_LABEL(ThrowThreadAbort): POP_PROBE_FRAME mov w0, #STATUS_REDHAWK_THREAD_ABORT diff --git a/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp b/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp index 06706ed1e8a4b..2842caf78ef68 100644 --- a/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp +++ b/src/coreclr/nativeaot/Runtime/windows/PalRedhawkMinWin.cpp @@ -543,7 +543,7 @@ REDHAWK_PALEXPORT void REDHAWK_PALAPI PalHijack(HANDLE hThread, _In_opt_ void* p pThread->SetActivationPending(false); DWORD lastError = GetLastError(); - if (lastError != ERROR_INVALID_PARAMETER) + if (lastError != ERROR_INVALID_PARAMETER && lastError != ERROR_NOT_SUPPORTED) { // An unexpected failure has happened. It is a concern. ASSERT_UNCONDITIONALLY("Failed to queue an APC for unusual reason."); diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj index 4ca91458c70e7..86ce820851c1d 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj @@ -184,7 +184,6 @@ - diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs index fbf56e6e4603d..f2c15681c288f 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/Runtime/MethodInfos/CustomMethodInvoker.cs @@ -36,7 +36,7 @@ public CustomMethodInvoker(Type thisType, Type[] parameterTypes, InvokerOptions if (!(thisObject == null && 0 != (_options & InvokerOptions.AllowNullThis))) ValidateThis(thisObject, _thisType.TypeHandle); - int argCount = (arguments != null) ? arguments.Length : 0; + int argCount = arguments.Length; if (argCount != _parameterTypes.Length) throw new TargetParameterCountException(); diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs index 24a10f91faa7b..9587d6f2d67fe 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Reflection/RuntimeAssembly.cs @@ -1,10 +1,34 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Globalization; + +using Internal.Reflection.Augments; + namespace System.Reflection { // Base class for runtime implemented Assembly public abstract class RuntimeAssembly : Assembly { + internal static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, CultureInfo culture, Version? version, bool throwOnFileNotFound) + { + AssemblyName mainAssemblyAn = mainAssembly.GetName(); + AssemblyName an = new AssemblyName(); + + an.CultureInfo = culture; + an.Name = mainAssemblyAn.Name + ".resources"; + an.SetPublicKeyToken(mainAssemblyAn.GetPublicKeyToken()); + an.Flags = mainAssemblyAn.Flags; + an.Version = version ?? mainAssemblyAn.Version; + + Assembly? retAssembly = ReflectionAugments.ReflectionCoreCallbacks.Load(an, throwOnFileNotFound); + + if (retAssembly == mainAssembly) + { + retAssembly = null; + } + + return retAssembly; + } } } diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.NativeAot.cs deleted file mode 100644 index f16c1ba389238..0000000000000 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.NativeAot.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; -using System.Diagnostics; -using System.Globalization; -using System.IO; -using System.Reflection; -using System.Text; - -using Internal.Reflection.Augments; - -namespace System.Resources -{ - internal partial class ManifestBasedResourceGroveler - { - // Internal version of GetSatelliteAssembly that avoids throwing FileNotFoundException - private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, - CultureInfo culture, - Version? version) - { - AssemblyName mainAssemblyAn = mainAssembly.GetName(); - AssemblyName an = new AssemblyName(); - - an.CultureInfo = culture; - an.Name = mainAssemblyAn.Name + ".resources"; - an.SetPublicKeyToken(mainAssemblyAn.GetPublicKeyToken()); - an.Flags = mainAssemblyAn.Flags; - an.Version = version ?? mainAssemblyAn.Version; - - Assembly? retAssembly = ReflectionAugments.ReflectionCoreCallbacks.Load(an, false); - - if (retAssembly == mainAssembly) - { - retAssembly = null; - } - - return retAssembly; - } - } -} diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs index e8340e4119151..968e97c425cf8 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/ValueType.cs @@ -95,43 +95,28 @@ public override unsafe bool Equals([NotNullWhen(true)] object? obj) public override unsafe int GetHashCode() { - int hashCode = (int)this.GetMethodTable()->HashCode; + HashCode hashCode = default; + hashCode.Add((IntPtr)this.GetMethodTable()); - hashCode ^= GetHashCodeImpl(); - - return hashCode; - } - - private unsafe int GetHashCodeImpl() - { int numFields = __GetFieldHelper(GetNumFields, out _); if (numFields == UseFastHelper) - return FastGetValueTypeHashCodeHelper(this.GetMethodTable(), ref this.GetRawData()); + hashCode.AddBytes(GetSpanForField(this.GetMethodTable(), ref this.GetRawData())); + else + RegularGetValueTypeHashCode(ref hashCode, ref this.GetRawData(), numFields); - return RegularGetValueTypeHashCode(ref this.GetRawData(), numFields); + return hashCode.ToHashCode(); } - private static unsafe int FastGetValueTypeHashCodeHelper(MethodTable* type, ref byte data) + private static unsafe ReadOnlySpan GetSpanForField(MethodTable* type, ref byte data) { // Sanity check - if there are GC references, we should not be hashing bytes Debug.Assert(!type->ContainsGCPointers); - - int size = (int)type->ValueTypeSize; - int hashCode = 0; - - for (int i = 0; i < size / 4; i++) - { - hashCode ^= Unsafe.As(ref Unsafe.Add(ref data, i * 4)); - } - - return hashCode; + return new ReadOnlySpan(ref data, (int)type->ValueTypeSize); } - private unsafe int RegularGetValueTypeHashCode(ref byte data, int numFields) + private unsafe void RegularGetValueTypeHashCode(ref HashCode hashCode, ref byte data, int numFields) { - int hashCode = 0; - // We only take the hashcode for the first non-null field. That's what the CLR does. for (int i = 0; i < numFields; i++) { @@ -142,15 +127,15 @@ private unsafe int RegularGetValueTypeHashCode(ref byte data, int numFields) if (fieldType->ElementType == EETypeElementType.Single) { - hashCode = Unsafe.As(ref fieldData).GetHashCode(); + hashCode.Add(Unsafe.As(ref fieldData)); } else if (fieldType->ElementType == EETypeElementType.Double) { - hashCode = Unsafe.As(ref fieldData).GetHashCode(); + hashCode.Add(Unsafe.As(ref fieldData)); } else if (fieldType->IsPrimitive) { - hashCode = FastGetValueTypeHashCodeHelper(fieldType, ref fieldData); + hashCode.AddBytes(GetSpanForField(fieldType, ref fieldData)); } else if (fieldType->IsValueType) { @@ -164,7 +149,7 @@ private unsafe int RegularGetValueTypeHashCode(ref byte data, int numFields) var fieldValue = (ValueType)RuntimeImports.RhBox(fieldType, ref fieldData); if (fieldValue != null) { - hashCode = fieldValue.GetHashCodeImpl(); + hashCode.Add(fieldValue); } else { @@ -177,7 +162,7 @@ private unsafe int RegularGetValueTypeHashCode(ref byte data, int numFields) object fieldValue = Unsafe.As(ref fieldData); if (fieldValue != null) { - hashCode = fieldValue.GetHashCode(); + hashCode.Add(fieldValue); } else { @@ -187,8 +172,6 @@ private unsafe int RegularGetValueTypeHashCode(ref byte data, int numFields) } break; } - - return hashCode; } } } diff --git a/src/coreclr/runtime.proj b/src/coreclr/runtime.proj index a6442c16173ed..773b0290d523f 100644 --- a/src/coreclr/runtime.proj +++ b/src/coreclr/runtime.proj @@ -1,7 +1,6 @@ - ClrFullNativeBuild;ClrRuntimeSubset;ClrJitSubset;ClrPalTestsSubset;ClrAllJitsSubset;ClrILToolsSubset;ClrNativeAotSubset;ClrSpmiSubset;ClrCrossComponentsSubset;ClrDebugSubset;HostArchitecture;PgoInstrument;NativeOptimizationDataSupported;CMakeArgs <_IcuDir Condition="'$(PkgMicrosoft_NETCore_Runtime_ICU_Transport)' != ''">$(PkgMicrosoft_NETCore_Runtime_ICU_Transport)/runtimes/$(TargetOS)-$(TargetArchitecture)$(_RuntimeVariant)/native <_BuildNativeTargetOS>$(TargetOS) diff --git a/src/coreclr/scripts/emitUnitTests.sh b/src/coreclr/scripts/emitUnitTests.sh index 9738dc5c50e96..0d08f5bf8a490 100755 --- a/src/coreclr/scripts/emitUnitTests.sh +++ b/src/coreclr/scripts/emitUnitTests.sh @@ -109,6 +109,7 @@ cut -f 3- -d ' ' $output_dir/capstone_output.txt \ if [ -n "$verbose" ]; then egrep "$verbose" $output_dir/clr_instrs.txt + egrep "$verbose" $output_dir/capstone_output.txt else (head -n 5; tail -n 5) < $output_dir/clr_instrs.txt fi diff --git a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_X86/X86Emitter.cs b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_X86/X86Emitter.cs index e386e9a34da6d..8601f35180e4b 100644 --- a/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_X86/X86Emitter.cs +++ b/src/coreclr/tools/Common/Compiler/DependencyAnalysis/Target_X86/X86Emitter.cs @@ -17,6 +17,37 @@ public X86Emitter(NodeFactory factory, bool relocsOnly) public ObjectDataBuilder Builder; public TargetRegisterMap TargetRegister; + public void EmitMOV(ref AddrMode addrMode, Register reg) + { + Debug.Assert(addrMode.Size != AddrModeSize.Int8 && addrMode.Size != AddrModeSize.Int16); + EmitIndirInstruction(0x89, (byte)reg, ref addrMode); + } + + public void EmitMOV(Register reg, ref AddrMode addrMode) + { + Debug.Assert(addrMode.Size != AddrModeSize.Int8 && addrMode.Size != AddrModeSize.Int16); + EmitIndirInstruction(0x8B, (byte)reg, ref addrMode); + } + + public void EmitMOV(ref AddrMode addrMode, ISymbolNode symbol) + { + if (symbol.RepresentsIndirectionCell) + { + throw new NotImplementedException(); + } + else + { + EmitIndirInstruction(0xC7, (byte)0, ref addrMode); + Builder.EmitReloc(symbol, RelocType.IMAGE_REL_BASED_HIGHLOW); + } + } + + public void EmitLEA(Register reg, ref AddrMode addrMode) + { + Debug.Assert(addrMode.Size != AddrModeSize.Int8 && addrMode.Size != AddrModeSize.Int16); + EmitIndirInstruction(0x8D, (byte)reg, ref addrMode); + } + public void EmitCMP(ref AddrMode addrMode, sbyte immediate) { if (addrMode.Size == AddrModeSize.Int16) @@ -48,18 +79,55 @@ public void EmitJMP(ISymbolNode symbol) } } + public void EmitJE(ISymbolNode symbol) + { + if (symbol.RepresentsIndirectionCell) + { + throw new NotImplementedException(); + } + else + { + Builder.EmitByte(0x0f); + Builder.EmitByte(0x84); + Builder.EmitReloc(symbol, RelocType.IMAGE_REL_BASED_REL32); + } + } + public void EmitXOR(Register register1, Register register2) { Builder.EmitByte(0x33); Builder.EmitByte((byte)(0xC0 | ((byte)register1 << 3) | (byte)register2)); } + public void EmitZeroReg(Register reg) + { + EmitXOR(reg, reg); + } + + public void EmitPOP(Register reg) + { + Builder.EmitByte((byte)(0x58 + (byte)reg)); + } + + public void EmitStackDup() + { + // PUSH [ESP] + Builder.EmitByte(0xff); + Builder.EmitByte(0x34); + Builder.EmitByte(0x24); + } + public void EmitPUSH(sbyte imm8) { Builder.EmitByte(0x6A); Builder.EmitByte(unchecked((byte)imm8)); } + public void EmitPUSH(Register reg) + { + Builder.EmitByte((byte)(0x50 + (byte)reg)); + } + public void EmitPUSH(ISymbolNode node) { if (node.RepresentsIndirectionCell) @@ -103,6 +171,11 @@ public void EmitINT3() Builder.EmitByte(0xCC); } + public void EmitJmpToAddrMode(ref AddrMode addrMode) + { + EmitIndirInstruction(0xFF, 0x4, ref addrMode); + } + public void EmitRET() { Builder.EmitByte(0xC3); diff --git a/src/coreclr/tools/Common/JitInterface/JitConfigProvider.cs b/src/coreclr/tools/Common/JitInterface/JitConfigProvider.cs index eb7b71e870e26..1d9ef515d4e49 100644 --- a/src/coreclr/tools/Common/JitInterface/JitConfigProvider.cs +++ b/src/coreclr/tools/Common/JitInterface/JitConfigProvider.cs @@ -130,7 +130,6 @@ public string GetStringConfigValue(string name) private static string GetTargetSpec(TargetDetails target) { - string targetOSComponent = (target.OperatingSystem == TargetOS.Windows ? "win" : "unix"); string targetArchComponent = target.Architecture switch { TargetArchitecture.X86 => "x86", @@ -142,10 +141,21 @@ private static string GetTargetSpec(TargetDetails target) _ => throw new NotImplementedException(target.Architecture.ToString()) }; - if ((target.Architecture == TargetArchitecture.ARM64) || (target.Architecture == TargetArchitecture.ARM)) + string targetOSComponent; + if (target.Architecture is TargetArchitecture.ARM64 or TargetArchitecture.ARM) { targetOSComponent = "universal"; } +#if !READYTORUN + else if (target.OperatingSystem == TargetOS.Windows && target.Architecture == TargetArchitecture.X86) + { + targetOSComponent = "win_aot"; + } +#endif + else + { + targetOSComponent = target.OperatingSystem == TargetOS.Windows ? "win" : "unix"; + } return targetOSComponent + '_' + targetArchComponent + "_" + RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(); } diff --git a/src/coreclr/tools/Common/JitInterface/LoongArch64PassStructInRegister.cs b/src/coreclr/tools/Common/JitInterface/LoongArch64PassStructInRegister.cs index 8592386dbb7cb..b0c193d095007 100644 --- a/src/coreclr/tools/Common/JitInterface/LoongArch64PassStructInRegister.cs +++ b/src/coreclr/tools/Common/JitInterface/LoongArch64PassStructInRegister.cs @@ -55,11 +55,7 @@ public static uint GetLoongArch64PassStructInRegisterFlags(TypeDesc typeDesc) int fieldIndex = 0; foreach (FieldDesc field in typeDesc.GetFields()) { - if (fieldIndex > 1) - { - return (uint)StructFloatFieldInfoFlags.STRUCT_NO_FLOAT_FIELD; - } - else if (field.IsStatic) + if (field.IsStatic) { continue; } @@ -162,6 +158,11 @@ public static uint GetLoongArch64PassStructInRegisterFlags(TypeDesc typeDesc) default: { + if ((numIntroducedFields == 2) && (field.FieldType.Category == TypeFlags.Class)) + { + return (uint)StructFloatFieldInfoFlags.STRUCT_NO_FLOAT_FIELD; + } + if (field.FieldType.GetElementSize().AsInt == 8) { if (numIntroducedFields > 1) diff --git a/src/coreclr/tools/Common/TypeSystem/Common/ArrayType.cs b/src/coreclr/tools/Common/TypeSystem/Common/ArrayType.cs index 4bd5f2b49b321..768f5f7eaa4ff 100644 --- a/src/coreclr/tools/Common/TypeSystem/Common/ArrayType.cs +++ b/src/coreclr/tools/Common/TypeSystem/Common/ArrayType.cs @@ -260,9 +260,18 @@ public override MethodSignature Signature case ArrayMethodKind.AddressWithHiddenArg: { var parameters = new TypeDesc[_owningType.Rank + 1]; - parameters[0] = Context.GetPointerType(Context.GetWellKnownType(WellKnownType.Void)); - for (int i = 0; i < _owningType.Rank; i++) - parameters[i + 1] = _owningType.Context.GetWellKnownType(WellKnownType.Int32); + if (Context.Target.Architecture == TargetArchitecture.X86) + { + for (int i = 0; i < _owningType.Rank; i++) + parameters[i] = _owningType.Context.GetWellKnownType(WellKnownType.Int32); + parameters[_owningType.Rank] = Context.GetPointerType(Context.GetWellKnownType(WellKnownType.Void)); + } + else + { + parameters[0] = Context.GetPointerType(Context.GetWellKnownType(WellKnownType.Void)); + for (int i = 0; i < _owningType.Rank; i++) + parameters[i + 1] = _owningType.Context.GetWellKnownType(WellKnownType.Int32); + } _signature = new MethodSignature(0, 0, _owningType.ElementType.MakeByRefType(), parameters, MethodSignature.EmbeddedSignatureMismatchPermittedFlag); } break; diff --git a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/ArrayMethodILEmitter.cs b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/ArrayMethodILEmitter.cs index b90abeeb79ab9..69aec3abb713b 100644 --- a/src/coreclr/tools/Common/TypeSystem/IL/Stubs/ArrayMethodILEmitter.cs +++ b/src/coreclr/tools/Common/TypeSystem/IL/Stubs/ArrayMethodILEmitter.cs @@ -81,7 +81,9 @@ private void EmitILForAccessor() int pointerSize = context.Target.PointerSize; - int argStartOffset = _method.Kind == ArrayMethodKind.AddressWithHiddenArg ? 2 : 1; + bool isX86 = context.Target.Architecture == TargetArchitecture.X86; + int argStartOffset = !isX86 && _method.Kind == ArrayMethodKind.AddressWithHiddenArg ? 2 : 1; + int hiddenArg = !isX86 ? 1 : 1 + _rank; var rangeExceptionLabel = _emitter.NewCodeLabel(); ILCodeLabel typeMismatchExceptionLabel = null; @@ -112,7 +114,7 @@ private void EmitILForAccessor() // As per ECMA-335 III.2.3, the prefix suppresses the type check. // if (hiddenArg == IntPtr.Zero) // goto TypeCheckPassed; - codeStream.EmitLdArg(1); + codeStream.EmitLdArg(hiddenArg); codeStream.Emit(ILOpcode.brfalse, typeCheckPassedLabel); // MethodTable* actualElementType = this.m_pEEType->RelatedParameterType; // ArrayElementType @@ -122,7 +124,7 @@ private void EmitILForAccessor() _emitter.NewToken(eetypeType.GetKnownMethod("get_RelatedParameterType", null))); // MethodTable* expectedElementType = hiddenArg->RelatedParameterType; // ArrayElementType - codeStream.EmitLdArg(1); + codeStream.EmitLdArg(hiddenArg); codeStream.Emit(ILOpcode.call, _emitter.NewToken(eetypeType.GetKnownMethod("get_RelatedParameterType", null))); diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.BoxedTypes.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.BoxedTypes.cs index 9236e10d2c35a..85e9f484f44b8 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.BoxedTypes.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.BoxedTypes.cs @@ -442,21 +442,34 @@ public override MethodIL EmitIL() ILEmitter emit = new ILEmitter(); ILCodeStream codeStream = emit.NewCodeStream(); + bool isX86 = Context.Target.Architecture == TargetArchitecture.X86; + FieldDesc eeTypeField = Context.GetWellKnownType(WellKnownType.Object).GetKnownField("m_pEEType"); // Load ByRef to the field with the value of the boxed valuetype codeStream.EmitLdArg(0); codeStream.Emit(ILOpcode.ldflda, emit.NewToken(Context.SystemModule.GetKnownType("System.Runtime.CompilerServices", "RawData").GetField("Data"))); + if (isX86) + { + for (int i = 0; i < _targetMethod.Signature.Length; i++) + { + codeStream.EmitLdArg(i + 1); + } + } + // Load the MethodTable of the boxed valuetype (this is the hidden generic context parameter expected // by the (canonical) instance method, but normally not part of the signature in IL). codeStream.EmitLdArg(0); codeStream.Emit(ILOpcode.ldfld, emit.NewToken(eeTypeField)); // Load rest of the arguments - for (int i = 0; i < _targetMethod.Signature.Length; i++) + if (!isX86) { - codeStream.EmitLdArg(i + 1); + for (int i = 0; i < _targetMethod.Signature.Length; i++) + { + codeStream.EmitLdArg(i + 1); + } } // Call an instance method on the target valuetype that has a fake instantiation parameter @@ -608,9 +621,18 @@ public override MethodSignature Signature // Shared instance methods on generic valuetypes have a hidden parameter with the generic context. // We add it to the signature so that we can refer to it from IL. - parameters[0] = Context.GetWellKnownType(WellKnownType.Void).MakePointerType(); - for (int i = 0; i < _methodRepresented.Signature.Length; i++) - parameters[i + 1] = _methodRepresented.Signature[i]; + if (Context.Target.Architecture == TargetArchitecture.X86) + { + for (int i = 0; i < _methodRepresented.Signature.Length; i++) + parameters[i] = _methodRepresented.Signature[i]; + parameters[_methodRepresented.Signature.Length] = Context.GetWellKnownType(WellKnownType.Void).MakePointerType(); + } + else + { + parameters[0] = Context.GetWellKnownType(WellKnownType.Void).MakePointerType(); + for (int i = 0; i < _methodRepresented.Signature.Length; i++) + parameters[i + 1] = _methodRepresented.Signature[i]; + } _signature = new MethodSignature(_methodRepresented.Signature.Flags, _methodRepresented.Signature.GenericParameterCount, diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.InterfaceThunks.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.InterfaceThunks.cs index 53c34efc0ef70..92a5be3fed38c 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.InterfaceThunks.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/CompilerTypeSystemContext.InterfaceThunks.cs @@ -216,9 +216,19 @@ public override MethodIL EmitIL() MethodDesc getOrdinalInterfaceMethod = Context.GetHelperEntryPoint("SharedCodeHelpers", "GetOrdinalInterface"); MethodDesc getCurrentContext = Context.GetHelperEntryPoint("SharedCodeHelpers", "GetCurrentSharedThunkContext"); + bool isX86 = Context.Target.Architecture == TargetArchitecture.X86; + // Load "this" codeStream.EmitLdArg(0); + if (isX86) + { + for (int i = 0; i < _targetMethod.Signature.Length; i++) + { + codeStream.EmitLdArg(i + 1); + } + } + // Load the instantiating argument. if (_interfaceIndex == UseContextFromRuntime) { @@ -232,10 +242,13 @@ public override MethodIL EmitIL() codeStream.Emit(ILOpcode.call, emit.NewToken(getOrdinalInterfaceMethod)); } - // Load rest of the arguments - for (int i = 0; i < _targetMethod.Signature.Length; i++) + if (!isX86) { - codeStream.EmitLdArg(i + 1); + // Load rest of the arguments + for (int i = 0; i < _targetMethod.Signature.Length; i++) + { + codeStream.EmitLdArg(i + 1); + } } // Call an instance method on the target interface that has a fake instantiation parameter @@ -292,9 +305,18 @@ public override MethodSignature Signature // Shared instance methods on generic interfaces have a hidden parameter with the generic context. // We add it to the signature so that we can refer to it from IL. - parameters[0] = Context.GetWellKnownType(WellKnownType.IntPtr); - for (int i = 0; i < _methodRepresented.Signature.Length; i++) - parameters[i + 1] = _methodRepresented.Signature[i]; + if (Context.Target.Architecture == TargetArchitecture.X86) + { + for (int i = 0; i < _methodRepresented.Signature.Length; i++) + parameters[i] = _methodRepresented.Signature[i]; + parameters[_methodRepresented.Signature.Length] = Context.GetWellKnownType(WellKnownType.Void).MakePointerType(); + } + else + { + parameters[0] = Context.GetWellKnownType(WellKnownType.IntPtr); + for (int i = 0; i < _methodRepresented.Signature.Length; i++) + parameters[i + 1] = _methodRepresented.Signature[i]; + } _signature = new MethodSignature(_methodRepresented.Signature.Flags, _methodRepresented.Signature.GenericParameterCount, diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/GenericLookupResult.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/GenericLookupResult.cs index 0cf2a6bb44e4a..efbc64f88799d 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/GenericLookupResult.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/GenericLookupResult.cs @@ -11,42 +11,6 @@ namespace ILCompiler.DependencyAnalysis { - // Represents a generic lookup within a canonical method body. - // TODO: unify with NativeFormat.FixupSignatureKind - public enum LookupResultType - { - Invalid, - MethodTable, // a type - UnwrapNullable, // a type (The type T described by a type spec that is generic over Nullable) - NonGcStatic, // the non-gc statics of a type - GcStatic, // the gc statics of a type - Method, // a method - InterfaceDispatchCell, // the dispatch cell for calling an interface method - MethodDictionary, // a dictionary for calling a generic method - UnboxingStub, // the unboxing stub for a method - ArrayType, // an array of type - DefaultCtor, // default ctor of a type - AllocObject, // the allocator of a type - GvmVtableOffset, // vtable offset of a generic virtual method - ProfileCounter, // profiling counter cell - MethodLdToken, // a ldtoken result for a method - FieldLdToken, // a ldtoken result for a field - Field, // a field descriptor - IsInst, // isinst helper - CastClass, // castclass helper - AllocArray, // the array allocator of a type - TypeSize, // size of the type - FieldOffset, // field offset - CallingConvention_NoInstParam, // CallingConventionConverterThunk NO_INSTANTIATING_PARAM - CallingConvention_HasInstParam, // CallingConventionConverterThunk HAS_INSTANTIATING_PARAM - CallingConvention_MaybeInstParam, // CallingConventionConverterThunk MAYBE_INSTANTIATING_PARAM - VtableOffset, // Offset of a virtual method into the type's vtable - Constrained, // ConstrainedCallDesc - ConstrainedDirect, // Direct ConstrainedCallDesc - Integer, // Integer - UnboxingMethod, // UnboxingMethod - } - public struct GenericLookupResultContext { private readonly TypeSystemEntity _canonicalOwner; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunGenericHelperNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunGenericHelperNode.cs index 079a38cc7460a..e61f9c5b088cd 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunGenericHelperNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunGenericHelperNode.cs @@ -1,15 +1,254 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; + using ILCompiler.DependencyAnalysis.X86; +using Internal.TypeSystem; + +using Debug = System.Diagnostics.Debug; + namespace ILCompiler.DependencyAnalysis { public partial class ReadyToRunGenericHelperNode { + protected void EmitDictionaryLookup(NodeFactory factory, ref X86Emitter encoder, Register context, Register result, GenericLookupResult lookup, bool relocsOnly) + { + // INVARIANT: must not trash context register + + // Find the generic dictionary slot + int dictionarySlot = 0; + if (!relocsOnly) + { + // The concrete slot won't be known until we're emitting data - don't ask for it in relocsOnly. + if (!factory.GenericDictionaryLayout(_dictionaryOwner).TryGetSlotForEntry(lookup, out dictionarySlot)) + { + encoder.EmitZeroReg(result); + return; + } + } + + // Load the generic dictionary cell + AddrMode loadEntry = new AddrMode( + context, null, dictionarySlot * factory.Target.PointerSize, 0, AddrModeSize.Int32); + encoder.EmitMOV(result, ref loadEntry); + + // If there's any invalid entries, we need to test for them + // + // Skip this in relocsOnly to make it easier to weed out bugs - the _hasInvalidEntries + // flag can change over the course of compilation and the bad slot helper dependency + // should be reported by someone else - the system should not rely on it coming from here. + if (!relocsOnly && _hasInvalidEntries) + { + AddrMode resultAddr = new AddrMode(Register.RegDirect | result, null, 0, 0, AddrModeSize.Int32); + encoder.EmitCMP(ref resultAddr, 0); + encoder.EmitJE(GetBadSlotHelper(factory)); + } + } + protected sealed override void EmitCode(NodeFactory factory, ref X86Emitter encoder, bool relocsOnly) { - encoder.EmitINT3(); + // First load the generic context into the context register. + EmitLoadGenericContext(factory, ref encoder, relocsOnly); + + switch (_id) + { + case ReadyToRunHelperId.GetNonGCStaticBase: + { + if (!TriggersLazyStaticConstructor(factory)) + { + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Result, _lookupSignature, relocsOnly); + encoder.EmitRET(); + } + else + { + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg0, _lookupSignature, relocsOnly); + encoder.EmitMOV(encoder.TargetRegister.Result, encoder.TargetRegister.Arg0); + + // We need to trigger the cctor before returning the base. It is stored at the beginning of the non-GC statics region. + int cctorContextSize = NonGCStaticsNode.GetClassConstructorContextSize(factory.Target); + AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg0, null, -cctorContextSize, 0, AddrModeSize.Int32); + encoder.EmitCMP(ref initialized, 0); + encoder.EmitRETIfEqual(); + + AddrMode loadCctor = new AddrMode(encoder.TargetRegister.Arg0, null, -cctorContextSize, 0, AddrModeSize.Int32); + encoder.EmitLEA(encoder.TargetRegister.Arg0, ref loadCctor); + encoder.EmitMOV(encoder.TargetRegister.Arg1, encoder.TargetRegister.Result); + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnNonGCStaticBase)); + } + } + break; + + case ReadyToRunHelperId.GetGCStaticBase: + { + MetadataType target = (MetadataType)_target; + + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Result, _lookupSignature, relocsOnly); + + AddrMode loadFromResult = new AddrMode(encoder.TargetRegister.Result, null, 0, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadFromResult); + + if (!TriggersLazyStaticConstructor(factory)) + { + encoder.EmitRET(); + } + else + { + // We need to trigger the cctor before returning the base. It is stored at the beginning of the non-GC statics region. + GenericLookupResult nonGcRegionLookup = factory.GenericLookup.TypeNonGCStaticBase(target); + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg0, nonGcRegionLookup, relocsOnly); + + int cctorContextSize = NonGCStaticsNode.GetClassConstructorContextSize(factory.Target); + AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg0, null, -cctorContextSize, 0, AddrModeSize.Int32); + encoder.EmitCMP(ref initialized, 0); + encoder.EmitRETIfEqual(); + + encoder.EmitMOV(encoder.TargetRegister.Arg1, encoder.TargetRegister.Result); + AddrMode loadCctor = new AddrMode(encoder.TargetRegister.Arg0, null, -cctorContextSize, 0, AddrModeSize.Int32); + encoder.EmitLEA(encoder.TargetRegister.Arg0, ref loadCctor); + + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnGCStaticBase)); + } + } + break; + + case ReadyToRunHelperId.GetThreadStaticBase: + { + MetadataType target = (MetadataType)_target; + + // Look up the index cell + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Arg1, _lookupSignature, relocsOnly); + + ISymbolNode helperEntrypoint; + if (TriggersLazyStaticConstructor(factory)) + { + // There is a lazy class constructor. We need the non-GC static base because that's where the + // class constructor context lives. + GenericLookupResult nonGcRegionLookup = factory.GenericLookup.TypeNonGCStaticBase(target); + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Result, nonGcRegionLookup, relocsOnly); + int cctorContextSize = NonGCStaticsNode.GetClassConstructorContextSize(factory.Target); + AddrMode loadCctor = new AddrMode(encoder.TargetRegister.Result, null, -cctorContextSize, 0, AddrModeSize.Int32); + encoder.EmitLEA(encoder.TargetRegister.Result, ref loadCctor); + + AddrMode storeAtEspPlus4 = new AddrMode(Register.ESP, null, 4, 0, AddrModeSize.Int32); + encoder.EmitStackDup(); + encoder.EmitMOV(ref storeAtEspPlus4, encoder.TargetRegister.Result); + + helperEntrypoint = factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase); + } + else + { + helperEntrypoint = factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType); + } + + // First arg: address of the TypeManager slot that provides the helper with + // information about module index and the type manager instance (which is used + // for initialization on first access). + AddrMode loadFromArg1 = new AddrMode(encoder.TargetRegister.Arg1, null, 0, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Arg0, ref loadFromArg1); + + // Second arg: index of the type in the ThreadStatic section of the modules + AddrMode loadFromArg1AndDelta = new AddrMode(encoder.TargetRegister.Arg1, null, factory.Target.PointerSize, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Arg1, ref loadFromArg1AndDelta); + + encoder.EmitJMP(helperEntrypoint); + } + break; + + case ReadyToRunHelperId.DelegateCtor: + { + // This is a weird helper. Codegen populated Arg0 and Arg1 with the values that the constructor + // method expects. Codegen also passed us the generic context on stack. + // We now need to load the delegate target method on the stack (using a dictionary lookup) + // and the optional 4th parameter, and call the ctor. + + var target = (DelegateCreationInfo)_target; + + // EmitLoadGenericContext loaded the context from stack into Result + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Result, encoder.TargetRegister.Result, _lookupSignature, relocsOnly); + + AddrMode storeAtEspPlus4 = new AddrMode(Register.ESP, null, 4, 0, AddrModeSize.Int32); + if (target.Thunk != null) + { + Debug.Assert(target.Constructor.Method.Signature.Length == 3); + AddrMode storeAtEspPlus8 = new AddrMode(Register.ESP, null, 8, 0, AddrModeSize.Int32); + encoder.EmitStackDup(); + encoder.EmitMOV(ref storeAtEspPlus8, encoder.TargetRegister.Result); + encoder.EmitMOV(encoder.TargetRegister.Result, target.Thunk); + encoder.EmitMOV(ref storeAtEspPlus4, encoder.TargetRegister.Result); + } + else + { + Debug.Assert(target.Constructor.Method.Signature.Length == 2); + encoder.EmitMOV(ref storeAtEspPlus4, encoder.TargetRegister.Result); + } + + encoder.EmitJMP(target.Constructor); + } + break; + + // These are all simple: just get the thing from the dictionary and we're done + case ReadyToRunHelperId.TypeHandle: + case ReadyToRunHelperId.MethodHandle: + case ReadyToRunHelperId.FieldHandle: + case ReadyToRunHelperId.MethodDictionary: + case ReadyToRunHelperId.MethodEntry: + case ReadyToRunHelperId.VirtualDispatchCell: + case ReadyToRunHelperId.DefaultConstructor: + case ReadyToRunHelperId.ObjectAllocator: + case ReadyToRunHelperId.TypeHandleForCasting: + case ReadyToRunHelperId.ConstrainedDirectCall: + { + EmitDictionaryLookup(factory, ref encoder, encoder.TargetRegister.Arg0, encoder.TargetRegister.Result, _lookupSignature, relocsOnly); + encoder.EmitRET(); + } + break; + default: + throw new NotImplementedException(); + } + } + + protected virtual void EmitLoadGenericContext(NodeFactory factory, ref X86Emitter encoder, bool relocsOnly) + { + // Assume generic context is already loaded in the context register. + if (Id == ReadyToRunHelperId.DelegateCtor) + { + AddrMode loadAtEspPlus4 = new AddrMode(Register.ESP, null, 4, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadAtEspPlus4); + } + } + } + + public partial class ReadyToRunGenericLookupFromTypeNode + { + protected override void EmitLoadGenericContext(NodeFactory factory, ref X86Emitter encoder, bool relocsOnly) + { + // We start with context register pointing to the MethodTable + Register contextRegister = encoder.TargetRegister.Arg0; + + // Locate the VTable slot that points to the dictionary + int vtableSlot = 0; + if (!relocsOnly) + { + // The concrete slot won't be known until we're emitting data - don't ask for it in relocsOnly. + vtableSlot = VirtualMethodSlotHelper.GetGenericDictionarySlot(factory, (TypeDesc)_dictionaryOwner); + } + + int pointerSize = factory.Target.PointerSize; + int slotOffset = EETypeNode.GetVTableOffset(pointerSize) + (vtableSlot * pointerSize); + + // DelegateCtor is special, the context is on stack + if (Id == ReadyToRunHelperId.DelegateCtor) + { + AddrMode loadAtEspPlus4 = new AddrMode(Register.ESP, null, 4, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadAtEspPlus4); + contextRegister = encoder.TargetRegister.Result; + } + + // Load the dictionary pointer from the VTable + AddrMode loadDictionary = new AddrMode(contextRegister, null, slotOffset, 0, AddrModeSize.Int32); + encoder.EmitMOV(contextRegister, ref loadDictionary); } } } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunHelperNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunHelperNode.cs index 255ec2f0b0161..c6f6408ad6761 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunHelperNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/Target_X86/X86ReadyToRunHelperNode.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; using ILCompiler.DependencyAnalysis.X86; @@ -20,7 +21,25 @@ protected override void EmitCode(NodeFactory factory, ref X86Emitter encoder, bo { case ReadyToRunHelperId.VirtualCall: { - encoder.EmitINT3(); + MethodDesc targetMethod = (MethodDesc)Target; + + Debug.Assert(!targetMethod.OwningType.IsInterface); + Debug.Assert(!targetMethod.CanMethodBeInSealedVTable(factory)); + + AddrMode loadFromThisPtr = new AddrMode(encoder.TargetRegister.Arg0, null, 0, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadFromThisPtr); + + int pointerSize = factory.Target.PointerSize; + + int slot = 0; + if (!relocsOnly) + { + slot = VirtualMethodSlotHelper.GetVirtualMethodSlot(factory, targetMethod, targetMethod.OwningType); + Debug.Assert(slot != -1); + } + + AddrMode jmpAddrMode = new AddrMode(encoder.TargetRegister.Result, null, EETypeNode.GetVTableOffset(pointerSize) + (slot * pointerSize), 0, AddrModeSize.Int32); + encoder.EmitJmpToAddrMode(ref jmpAddrMode); } break; @@ -51,19 +70,119 @@ protected override void EmitCode(NodeFactory factory, ref X86Emitter encoder, bo case ReadyToRunHelperId.GetThreadStaticBase: { - encoder.EmitINT3(); + MetadataType target = (MetadataType)Target; + ISortableSymbolNode index = factory.TypeThreadStaticIndex(target); + if (index is TypeThreadStaticIndexNode ti && ti.IsInlined) + { + throw new NotImplementedException(); + } + else + { + encoder.EmitMOV(encoder.TargetRegister.Result, index); + + // First arg: address of the TypeManager slot that provides the helper with + // information about module index and the type manager instance (which is used + // for initialization on first access). + AddrMode loadFromEax = new AddrMode(encoder.TargetRegister.Result, null, 0, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Arg0, ref loadFromEax); + + // Second arg: index of the type in the ThreadStatic section of the modules + AddrMode loadFromEaxAndDelta = new AddrMode(encoder.TargetRegister.Result, null, factory.Target.PointerSize, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Arg1, ref loadFromEaxAndDelta); + + ISymbolNode helper = factory.HelperEntrypoint(HelperEntrypoint.GetThreadStaticBaseForType); + if (!factory.PreinitializationManager.HasLazyStaticConstructor(target)) + { + encoder.EmitJMP(helper); + } + else + { + encoder.EmitMOV(encoder.TargetRegister.Result, factory.TypeNonGCStaticsSymbol(target), -NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); + + AddrMode initialized = new AddrMode(encoder.TargetRegister.Result, null, 0, 0, AddrModeSize.Int32); + encoder.EmitCMP(ref initialized, 0); + encoder.EmitJE(helper); + + // Add extra parameter and tail call + encoder.EmitStackDup(); + AddrMode storeAtEspPlus4 = new AddrMode(Register.ESP, null, 4, 0, AddrModeSize.Int32); + encoder.EmitMOV(ref storeAtEspPlus4, encoder.TargetRegister.Result); + + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnThreadStaticBase)); + } + } } break; case ReadyToRunHelperId.GetGCStaticBase: { - encoder.EmitINT3(); + MetadataType target = (MetadataType)Target; + bool hasLazyStaticConstructor = factory.PreinitializationManager.HasLazyStaticConstructor(target); + encoder.EmitMOV(encoder.TargetRegister.Result, factory.TypeGCStaticsSymbol(target)); + AddrMode loadFromEax = new AddrMode(encoder.TargetRegister.Result, null, 0, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadFromEax); + + if (!hasLazyStaticConstructor) + { + encoder.EmitRET(); + } + else + { + // We need to trigger the cctor before returning the base. It is stored at the beginning of the non-GC statics region. + encoder.EmitMOV(encoder.TargetRegister.Arg0, factory.TypeNonGCStaticsSymbol(target), -NonGCStaticsNode.GetClassConstructorContextSize(factory.Target)); + + AddrMode initialized = new AddrMode(encoder.TargetRegister.Arg0, null, 0, 0, AddrModeSize.Int32); + encoder.EmitCMP(ref initialized, 0); + encoder.EmitRETIfEqual(); + + encoder.EmitMOV(encoder.TargetRegister.Arg1, encoder.TargetRegister.Result); + encoder.EmitJMP(factory.HelperEntrypoint(HelperEntrypoint.EnsureClassConstructorRunAndReturnGCStaticBase)); + } } break; case ReadyToRunHelperId.DelegateCtor: { - encoder.EmitINT3(); + DelegateCreationInfo target = (DelegateCreationInfo)Target; + + encoder.EmitStackDup(); + + if (target.TargetNeedsVTableLookup) + { + Debug.Assert(!target.TargetMethod.CanMethodBeInSealedVTable(factory)); + + AddrMode loadFromThisPtr = new AddrMode(encoder.TargetRegister.Arg1, null, 0, 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadFromThisPtr); + + int slot = 0; + if (!relocsOnly) + slot = VirtualMethodSlotHelper.GetVirtualMethodSlot(factory, target.TargetMethod, target.TargetMethod.OwningType); + + Debug.Assert(slot != -1); + AddrMode loadFromSlot = new AddrMode(encoder.TargetRegister.Result, null, EETypeNode.GetVTableOffset(factory.Target.PointerSize) + (slot * factory.Target.PointerSize), 0, AddrModeSize.Int32); + encoder.EmitMOV(encoder.TargetRegister.Result, ref loadFromSlot); + } + else + { + encoder.EmitMOV(encoder.TargetRegister.Result, target.GetTargetNode(factory)); + } + + AddrMode storeAtEspPlus4 = new AddrMode(Register.ESP, null, 4, 0, AddrModeSize.Int32); + if (target.Thunk != null) + { + Debug.Assert(target.Constructor.Method.Signature.Length == 3); + AddrMode storeAtEspPlus8 = new AddrMode(Register.ESP, null, 8, 0, AddrModeSize.Int32); + encoder.EmitStackDup(); + encoder.EmitMOV(ref storeAtEspPlus8, encoder.TargetRegister.Result); + encoder.EmitMOV(ref storeAtEspPlus4, target.Thunk); + } + else + { + Debug.Assert(target.Constructor.Method.Signature.Length == 2); + encoder.EmitMOV(ref storeAtEspPlus4, encoder.TargetRegister.Result); + } + + encoder.EmitJMP(target.Constructor); } break; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs index 5a5f208dfa4d6..8d5b1357d9e3e 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs @@ -359,13 +359,5 @@ public static string GetNewObjectHelperForType(TypeDesc type) return "RhpNewFast"; } - - public static string GetNewArrayHelperForType(TypeDesc type) - { - if (type.RequiresAlign8()) - return "RhpNewArrayAlign8"; - - return "RhpNewArray"; - } } } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectWriter/CoffObjectWriter.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectWriter/CoffObjectWriter.cs index 3198fafaf5d22..6211ca306c3e4 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectWriter/CoffObjectWriter.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectWriter/CoffObjectWriter.cs @@ -446,8 +446,8 @@ private protected override void EmitUnwindInfo( // Emit RUNTIME_FUNCTION pdataSectionWriter.EmitAlignment(4); pdataSectionWriter.EmitSymbolReference(IMAGE_REL_BASED_ADDR32NB, currentSymbolName, start); - // Only x64 has the End symbol - if (_machine == Machine.Amd64) + // Only x86/x64 has the End symbol + if (_machine is Machine.I386 or Machine.Amd64) { pdataSectionWriter.EmitSymbolReference(IMAGE_REL_BASED_ADDR32NB, currentSymbolName, end); } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/SubstitutedILProvider.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/SubstitutedILProvider.cs index 28e86abb80189..09c54b96c5e89 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/SubstitutedILProvider.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/SubstitutedILProvider.cs @@ -879,7 +879,7 @@ private static bool TryExpandTypeEquality(MethodIL methodIL, byte[] body, Opcode return false; TypeDesc type2 = ReadLdToken(ref reader, methodIL, flags); - if (type1 == null) + if (type2 == null) return false; if (!ReadGetTypeFromHandle(ref reader, methodIL, flags)) diff --git a/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj b/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj index 09e2a5bec4c8d..f7aa4cc8b93fd 100644 --- a/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj +++ b/src/coreclr/tools/aot/ILCompiler/repro/repro.csproj @@ -6,7 +6,9 @@ AnyCPU false false - linux-x64;win-x64;osx-x64;freebsd-x64;freebsd-arm64 + linux-x64;win-x64;osx-x64 + + $(RuntimeIdentifiers);freebsd-x64;freebsd-arm64 Debug;Release;Checked true false diff --git a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp index 7f9c64eedd185..74040dc5aa3be 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp @@ -878,7 +878,6 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b break; case IMAGE_REL_ARM64_PAGEOFFSET_12A: // ADD 12 bit page offset - case IMAGE_REL_AARCH64_TLSDESC_ADD_LO12: // TLSDESC ADD for corresponding ADRP { if ((section_begin <= address) && (address < section_end)) // A reloc for our section? { @@ -889,28 +888,16 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b } break; + case IMAGE_REL_AARCH64_TLSDESC_LD64_LO12: + case IMAGE_REL_AARCH64_TLSDESC_ADD_LO12: // TLSDESC ADD for corresponding ADRP case IMAGE_REL_AARCH64_TLSDESC_CALL: - case IMAGE_REL_TLSGD: { - DWORDLONG fixupLocation = tmp.location; - - size_t address = section_begin + (size_t)fixupLocation - (size_t)originalAddr; - if ((section_begin <= address) && (address < section_end)) // A reloc for our section? - { - LogDebug(" fixupLoc-%016" PRIX64 " (@%p) : %08X => %08X", fixupLocation, address, - *(DWORD*)address, (DWORD)tmp.target); - *(DWORD*)address = (DWORD)tmp.target; - } + // These are patched later by linker during actual execution + // and do not need relocation. wasRelocHandled = true; } break; - case IMAGE_REL_AARCH64_TLSDESC_LD64_LO12: - { - // not needed - } - break; - default: break; } @@ -938,6 +925,12 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b wasRelocHandled = true; } + else if (relocType == IMAGE_REL_TLSGD) + { + // These are patched later by linker during actual execution + // and do not need relocation. + wasRelocHandled = true; + } } if (wasRelocHandled) diff --git a/src/coreclr/vm/callcounting.cpp b/src/coreclr/vm/callcounting.cpp index e7c70dccf4096..c464949f7aeee 100644 --- a/src/coreclr/vm/callcounting.cpp +++ b/src/coreclr/vm/callcounting.cpp @@ -660,7 +660,7 @@ bool CallCountingManager::SetCodeEntryPoint( CallCount callCountThreshold = g_pConfig->TieredCompilation_CallCountThreshold(); _ASSERTE(callCountThreshold != 0); - // Let's tier up all cast and runtime helpers faster than other methods. This is because we want to import them as + // Let's tier up all cast helpers faster than other methods. This is because we want to import them as // direct calls in codegen and they need to be promoted earlier than their callers. if (methodDesc->GetMethodTable() == g_pCastHelpers) { diff --git a/src/coreclr/vm/comutilnative.cpp b/src/coreclr/vm/comutilnative.cpp index 612cb9d72dc0d..6c7e2468d2744 100644 --- a/src/coreclr/vm/comutilnative.cpp +++ b/src/coreclr/vm/comutilnative.cpp @@ -1703,9 +1703,10 @@ enum ValueTypeHashCodeStrategy DoubleField, SingleField, FastGetHashCode, + ValueTypeOverride, }; -static ValueTypeHashCodeStrategy GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize) +static ValueTypeHashCodeStrategy GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize, MethodTable** fieldMTOut) { CONTRACTL { @@ -1772,10 +1773,18 @@ static ValueTypeHashCodeStrategy GetHashCodeStrategy(MethodTable* mt, QCall::Obj *fieldSize = field->LoadSize(); ret = ValueTypeHashCodeStrategy::FastGetHashCode; } + else if (HasOverriddenMethod(fieldMT, + CoreLibBinder::GetClass(CLASS__VALUE_TYPE), + CoreLibBinder::GetMethod(METHOD__VALUE_TYPE__GET_HASH_CODE)->GetSlot())) + { + *fieldOffset += field->GetOffsetUnsafe(); + *fieldMTOut = fieldMT; + ret = ValueTypeHashCodeStrategy::ValueTypeOverride; + } else { *fieldOffset += field->GetOffsetUnsafe(); - ret = GetHashCodeStrategy(fieldMT, objHandle, fieldOffset, fieldSize); + ret = GetHashCodeStrategy(fieldMT, objHandle, fieldOffset, fieldSize, fieldMTOut); } } } @@ -1785,18 +1794,18 @@ static ValueTypeHashCodeStrategy GetHashCodeStrategy(MethodTable* mt, QCall::Obj return ret; } -extern "C" INT32 QCALLTYPE ValueType_GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize) +extern "C" INT32 QCALLTYPE ValueType_GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize, MethodTable** fieldMT) { QCALL_CONTRACT; ValueTypeHashCodeStrategy ret = ValueTypeHashCodeStrategy::None; *fieldOffset = 0; *fieldSize = 0; + *fieldMT = NULL; BEGIN_QCALL; - - ret = GetHashCodeStrategy(mt, objHandle, fieldOffset, fieldSize); + ret = GetHashCodeStrategy(mt, objHandle, fieldOffset, fieldSize, fieldMT); END_QCALL; diff --git a/src/coreclr/vm/comutilnative.h b/src/coreclr/vm/comutilnative.h index a3c5ea65c3ca7..0f305e0af9007 100644 --- a/src/coreclr/vm/comutilnative.h +++ b/src/coreclr/vm/comutilnative.h @@ -252,7 +252,7 @@ class MethodTableNative { extern "C" BOOL QCALLTYPE MethodTable_AreTypesEquivalent(MethodTable* mta, MethodTable* mtb); extern "C" BOOL QCALLTYPE MethodTable_CanCompareBitsOrUseFastGetHashCode(MethodTable* mt); -extern "C" INT32 QCALLTYPE ValueType_GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize); +extern "C" INT32 QCALLTYPE ValueType_GetHashCodeStrategy(MethodTable* mt, QCall::ObjectHandleOnStack objHandle, UINT32* fieldOffset, UINT32* fieldSize, MethodTable** fieldMT); class StreamNative { public: diff --git a/src/coreclr/vm/dllimport.cpp b/src/coreclr/vm/dllimport.cpp index bc106bf0f43d8..12ca187ecbeac 100644 --- a/src/coreclr/vm/dllimport.cpp +++ b/src/coreclr/vm/dllimport.cpp @@ -4264,8 +4264,7 @@ static void CreateNDirectStubAccessMetadata( { if (unmgdCallConv == CorInfoCallConvExtension::Managed || unmgdCallConv == CorInfoCallConvExtension::Fastcall || - unmgdCallConv == CorInfoCallConvExtension::FastcallMemberFunction || - unmgdCallConv == CorInfoCallConvExtension::Swift) + unmgdCallConv == CorInfoCallConvExtension::FastcallMemberFunction) { COMPlusThrow(kTypeLoadException, IDS_INVALID_PINVOKE_CALLCONV); } diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h index 55b93f1f1ab68..9da9ebdb102a6 100644 --- a/src/coreclr/vm/ecalllist.h +++ b/src/coreclr/vm/ecalllist.h @@ -471,8 +471,7 @@ FCFuncStart(gRuntimeHelpers) FCFuncElement("PrepareDelegate", ReflectionInvocation::PrepareDelegate) FCFuncElement("GetHashCode", ObjectNative::GetHashCode) FCFuncElement("TryGetHashCode", ObjectNative::TryGetHashCode) - FCFuncElement("Equals", ObjectNative::Equals) - FCFuncElement("AllocateUninitializedClone", ObjectNative::AllocateUninitializedClone) + FCFuncElement("ContentEquals", ObjectNative::ContentEquals) FCFuncElement("EnsureSufficientExecutionStack", ReflectionInvocation::EnsureSufficientExecutionStack) FCFuncElement("TryEnsureSufficientExecutionStack", ReflectionInvocation::TryEnsureSufficientExecutionStack) FCFuncElement("AllocTailCallArgBuffer", TailCallHelp::AllocTailCallArgBuffer) diff --git a/src/coreclr/vm/eetwain.cpp b/src/coreclr/vm/eetwain.cpp index b0886fcadebee..597fda3d45b5d 100644 --- a/src/coreclr/vm/eetwain.cpp +++ b/src/coreclr/vm/eetwain.cpp @@ -4072,6 +4072,7 @@ bool UnwindEbpDoubleAlignFrame( // Set baseSP as initial SP baseSP += GetPushedArgSize(info, table, curOffs); +#ifdef UNIX_X86_ABI // 16-byte stack alignment padding (allocated in genFuncletProlog) // Current funclet frame layout (see CodeGen::genFuncletProlog() and genFuncletEpilog()): // prolog: sub esp, 12 @@ -4082,6 +4083,7 @@ bool UnwindEbpDoubleAlignFrame( const TADDR funcletStart = pCodeInfo->GetJitManager()->GetFuncletStartAddress(pCodeInfo); if (funcletStart != pCodeInfo->GetCodeAddress() && methodStart[pCodeInfo->GetRelOffset()] != X86_INSTR_RETN) baseSP += 12; +#endif pContext->PCTAddr = baseSP; pContext->ControlPC = *PTR_PCODE(pContext->PCTAddr); @@ -4578,6 +4580,7 @@ bool EECodeManager::EnumGcRefs( PREGDISPLAY pContext, #endif // _DEBUG +#ifndef FEATURE_EH_FUNCLETS /* What kind of a frame is this ? */ FrameType frameType = FR_NORMAL; @@ -4622,6 +4625,7 @@ bool EECodeManager::EnumGcRefs( PREGDISPLAY pContext, &info); } } +#endif bool willContinueExecution = !(flags & ExecutionAborted); unsigned pushedSize = 0; @@ -4712,16 +4716,45 @@ bool EECodeManager::EnumGcRefs( PREGDISPLAY pContext, { _ASSERTE(willContinueExecution); +#ifdef FEATURE_EH_FUNCLETS + // Funclets' frame pointers(EBP) are always restored so they can access to main function's local variables. + // Therefore the value of EBP is invalid for unwinder so we should use ESP instead. + // See UnwindStackFrame for details. + if (pCodeInfo->IsFunclet()) + { + PTR_CBYTE methodStart = PTR_CBYTE(pCodeInfo->GetSavedMethodCode()); + TADDR baseSP = ESP; + // Set baseSP as initial SP + baseSP += GetPushedArgSize(&info, table, curOffs); + +#ifdef UNIX_X86_ABI + // 16-byte stack alignment padding (allocated in genFuncletProlog) + // Current funclet frame layout (see CodeGen::genFuncletProlog() and genFuncletEpilog()): + // prolog: sub esp, 12 + // epilog: add esp, 12 + // ret + // SP alignment padding should be added for all instructions except the first one and the last one. + // Epilog may not exist (unreachable), so we need to check the instruction code. + const PTR_CBYTE funcletStart = PTR_CBYTE(pCodeInfo->GetJitManager()->GetFuncletStartAddress(pCodeInfo)); + if (funcletStart != methodStart + curOffs && methodStart[curOffs] != X86_INSTR_RETN) + baseSP += 12; +#endif + + // -sizeof(void*) because we want to point *AT* first parameter + pPendingArgFirst = (DWORD *)(size_t)(baseSP - sizeof(void*)); + } +#else // FEATURE_EH_FUNCLETS if (info.handlers) { // -sizeof(void*) because we want to point *AT* first parameter pPendingArgFirst = (DWORD *)(size_t)(baseSP - sizeof(void*)); } +#endif else if (info.localloc) { - baseSP = *(DWORD *)(size_t)(EBP - GetLocallocSPOffset(&info)); + TADDR locallocBaseSP = *(DWORD *)(size_t)(EBP - GetLocallocSPOffset(&info)); // -sizeof(void*) because we want to point *AT* first parameter - pPendingArgFirst = (DWORD *)(size_t) (baseSP - sizeof(void*)); + pPendingArgFirst = (DWORD *)(size_t) (locallocBaseSP - sizeof(void*)); } else { diff --git a/src/coreclr/vm/exceptionhandling.cpp b/src/coreclr/vm/exceptionhandling.cpp index 38358918245ac..5399f2943e93e 100644 --- a/src/coreclr/vm/exceptionhandling.cpp +++ b/src/coreclr/vm/exceptionhandling.cpp @@ -6624,6 +6624,13 @@ bool ExceptionTracker::IsInStackRegionUnwoundBySpecifiedException(CrawlFrame * p // Remember that sfLowerBound and sfUpperBound are in the "OS format". // Refer to the comment for CallerStackFrame for more information. + + if (g_isNewExceptionHandlingEnabled) + { + // The new exception handling sets the ranges always to the SP of the unwound frame + return (sfLowerBound < csfToCheck) && (csfToCheck <= sfUpperBound); + } + #ifndef STACK_RANGE_BOUNDS_ARE_CALLER_SP if ((sfLowerBound < csfToCheck) && (csfToCheck <= sfUpperBound)) #else // !STACK_RANGE_BOUNDS_ARE_CALLER_SP @@ -7970,7 +7977,7 @@ struct ExtendedEHClauseEnumerator : EH_CLAUSE_ENUMERATOR unsigned EHCount; }; -extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, BYTE** pMethodStartAddress, EH_CLAUSE_ENUMERATOR * pEHEnum) +extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, IJitManager::MethodRegionInfo* pMethodRegionInfo, EH_CLAUSE_ENUMERATOR * pEHEnum) { QCALL_CONTRACT; @@ -7984,7 +7991,7 @@ extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *p IJitManager* pJitMan = pFrameIter->m_crawl.GetJitManager(); const METHODTOKEN& MethToken = pFrameIter->m_crawl.GetMethodToken(); - *pMethodStartAddress = (BYTE*)pJitMan->JitTokenToStartAddress(MethToken); + pJitMan->JitTokenToMethodRegionInfo(MethToken, pMethodRegionInfo); pExtendedEHEnum->EHCount = pJitMan->InitializeEHEnumeration(MethToken, pEHEnum); END_QCALL; diff --git a/src/coreclr/vm/exceptionhandlingqcalls.h b/src/coreclr/vm/exceptionhandlingqcalls.h index 7747c14f531d2..7054080cef3c0 100644 --- a/src/coreclr/vm/exceptionhandlingqcalls.h +++ b/src/coreclr/vm/exceptionhandlingqcalls.h @@ -17,7 +17,7 @@ extern "C" void QCALLTYPE CallFinallyFunclet(BYTE* pHandlerIP, REGDISPLAY* pvReg extern "C" BOOL QCALLTYPE CallFilterFunclet(QCall::ObjectHandleOnStack exceptionObj, BYTE* pFilterP, REGDISPLAY* pvRegDisplay); extern "C" void QCALLTYPE ResumeAtInterceptionLocation(REGDISPLAY* pvRegDisplay); extern "C" void QCALLTYPE AppendExceptionStackFrame(QCall::ObjectHandleOnStack exceptionObj, SIZE_T ip, SIZE_T sp, int flags, ExInfo *pExInfo); -extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, BYTE** pMethodStartAddress, EH_CLAUSE_ENUMERATOR * pEHEnum); +extern "C" BOOL QCALLTYPE EHEnumInitFromStackFrameIterator(StackFrameIterator *pFrameIter, IJitManager::MethodRegionInfo *pMethodRegionInfo, EH_CLAUSE_ENUMERATOR * pEHEnum); extern "C" BOOL QCALLTYPE EHEnumNext(EH_CLAUSE_ENUMERATOR* pEHEnum, RhEHClause* pEHClause); extern "C" bool QCALLTYPE SfiInit(StackFrameIterator* pThis, CONTEXT* pStackwalkCtx, bool instructionFault, bool* pIsExceptionIntercepted); extern "C" bool QCALLTYPE SfiNext(StackFrameIterator* pThis, unsigned int* uExCollideClauseIdx, bool* fUnwoundReversePInvoke, bool* pIsExceptionIntercepted); diff --git a/src/coreclr/vm/loongarch64/stubs.cpp b/src/coreclr/vm/loongarch64/stubs.cpp index da93bd587ed34..052d71ebc1e44 100644 --- a/src/coreclr/vm/loongarch64/stubs.cpp +++ b/src/coreclr/vm/loongarch64/stubs.cpp @@ -1928,7 +1928,7 @@ PCODE DynamicHelpers::CreateDictionaryLookupHelper(LoaderAllocator * pAllocator, } } - _ASSERTE(indirectionsDataSize == dataOffset); + _ASSERTE((indirectionsDataSize ? indirectionsDataSize : codeSize) == dataOffset); // No null test required if (!pLookup->testForNull) diff --git a/src/coreclr/vm/profdetach.cpp b/src/coreclr/vm/profdetach.cpp index bf138209ce6ad..09f11458c3887 100644 --- a/src/coreclr/vm/profdetach.cpp +++ b/src/coreclr/vm/profdetach.cpp @@ -326,7 +326,7 @@ void ProfilingAPIDetach::ExecuteEvacuationLoop() { CRITSEC_Holder csh(ProfilingAPIUtility::GetStatusCrst()); - for (SIZE_T pos = 0; pos < s_profilerDetachInfos.Size(); ++pos) + while (s_profilerDetachInfos.Size() > 0) { ProfilerDetachInfo current = s_profilerDetachInfos.Pop(); diff --git a/src/coreclr/vm/qcallentrypoints.cpp b/src/coreclr/vm/qcallentrypoints.cpp index 8e6a670297943..2d6466fc7d7c7 100644 --- a/src/coreclr/vm/qcallentrypoints.cpp +++ b/src/coreclr/vm/qcallentrypoints.cpp @@ -324,6 +324,7 @@ static const Entry s_QCall[] = DllImportEntry(GetFileLoadExceptionMessage) DllImportEntry(FileLoadException_GetMessageForHR) DllImportEntry(Interlocked_MemoryBarrierProcessWide) + DllImportEntry(ObjectNative_AllocateUninitializedClone) DllImportEntry(Monitor_Wait) DllImportEntry(Monitor_Pulse) DllImportEntry(Monitor_PulseAll) diff --git a/src/coreclr/vm/threads.cpp b/src/coreclr/vm/threads.cpp index 081f5c6cf7a73..15b3df1f6ae14 100644 --- a/src/coreclr/vm/threads.cpp +++ b/src/coreclr/vm/threads.cpp @@ -8276,12 +8276,7 @@ void Thread::InitializeSpecialUserModeApc() return; } - // In the future, once code paths using the special user-mode APC get some bake time, it should be used regardless of - // whether CET shadow stacks are enabled - if (AreCetShadowStacksEnabled()) - { - s_pfnQueueUserAPC2Proc = pfnQueueUserAPC2Proc; - } + s_pfnQueueUserAPC2Proc = pfnQueueUserAPC2Proc; } #endif // FEATURE_SPECIAL_USER_MODE_APC diff --git a/src/installer/Directory.Build.targets b/src/installer/Directory.Build.targets index dccb8277ba767..c4e8a8c8fb701 100644 --- a/src/installer/Directory.Build.targets +++ b/src/installer/Directory.Build.targets @@ -1,7 +1,7 @@ - $(InstallerName)-pgo + $(InstallerName) $(ArchiveName)-pgo diff --git a/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj b/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj index 4b2f49ec43568..fec324d46cccb 100644 --- a/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj +++ b/src/installer/managed/Microsoft.NET.HostModel/Microsoft.NET.HostModel.csproj @@ -4,7 +4,7 @@ netstandard2.0 Abstractions for modifying .NET host binaries false - true + true true true true diff --git a/src/installer/pkg/sfx/Directory.Build.props b/src/installer/pkg/sfx/Directory.Build.props index b0711d7f7ac91..dbf349249cefb 100644 --- a/src/installer/pkg/sfx/Directory.Build.props +++ b/src/installer/pkg/sfx/Directory.Build.props @@ -11,7 +11,7 @@ true true - + true true diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj index 337a0ce1ebee3..c3fd9eab49047 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Crossgen2.sfxproj @@ -7,12 +7,13 @@ true ToolPack $(SharedFrameworkName).Crossgen2 - .PGO $(SharedFrameworkName)$(PgoSuffix).$(RuntimeIdentifier) dotnet-crossgen2 crossgen2 - linux-x64;linux-musl-x64;linux-arm;linux-musl-arm;linux-arm64;linux-musl-arm64;freebsd-x64;freebsd-arm64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64 + linux-x64;linux-musl-x64;linux-arm;linux-musl-arm;linux-arm64;linux-musl-arm64;osx-x64;osx-arm64;win-x64;win-x86;win-arm64 + + $(RuntimeIdentifiers);freebsd-x64;freebsd-arm64 false tools/ true diff --git a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj index 8cd98f995ee0c..fc7b8b90fe907 100644 --- a/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj +++ b/src/installer/pkg/sfx/Microsoft.NETCore.App/Microsoft.NETCore.App.Host.sfxproj @@ -8,7 +8,7 @@ dotnet-apphost-pack dotnet-apphost-pack NetCore.AppHostPack - false + false - + + sos + https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/flat2/dotnet-sos/$(DotnetSosVersion)/dotnet-sos.$(DotnetSosVersion).nupkg + + + + + set _NT_SYMBOL_PATH=%25HELIX_CORRELATION_PAYLOAD%25%3B%25HELIX_CORRELATION_PAYLOAD%25\PDB%3B%25HELIX_CORRELATION_PAYLOAD%25\shared\$(MicrosoftNetCoreAppFrameworkName)\$(ProductVersion) + %25HELIX_CORRELATION_PAYLOAD%25\dotnet %25HELIX_CORRELATION_PAYLOAD%25\sos\tools\net$(DotnetSosTargetFrameworkVersion)\any\dotnet-sos.dll install --architecture $(TargetArchitecture) + $(HelixPreCommands);$(NtSymbolPathEnvVar);$(ExecuteDotNetSos) + + + + @@ -451,6 +454,10 @@ + + + @@ -611,7 +618,9 @@ - + + + @@ -762,7 +771,7 @@ BuildInParallel="$(Samples_BuildInParallel)" /> - + diff --git a/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj b/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj index 1c085ed36e446..c273e6ba4172f 100644 --- a/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj +++ b/src/mono/System.Private.CoreLib/System.Private.CoreLib.csproj @@ -246,7 +246,6 @@ - @@ -264,7 +263,6 @@ Condition="'$(FeatureObjCMarshal)' == 'true'"/> - diff --git a/src/mono/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.Mono.cs deleted file mode 100644 index 54871091da0f4..0000000000000 --- a/src/mono/System.Private.CoreLib/src/System/Resources/ManifestBasedResourceGroveler.Mono.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Globalization; -using System.Reflection; - -namespace System.Resources -{ - internal partial class ManifestBasedResourceGroveler - { - private static Assembly? InternalGetSatelliteAssembly(Assembly mainAssembly, CultureInfo culture, Version? version) - { - return (RuntimeAssembly.InternalGetSatelliteAssembly(mainAssembly, culture, version, throwOnFileNotFound: false)); - } - } -} diff --git a/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs b/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs index 391bfaec6a2a4..ca9ecdbe9f829 100644 --- a/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs +++ b/src/mono/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.Mono.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Diagnostics.Tracing; +using System.Runtime.InteropServices; using System.Runtime.Serialization; namespace System.Runtime.CompilerServices @@ -137,9 +138,15 @@ public static void RunModuleConstructor(ModuleHandle module) RunModuleConstructor(module.Value); } - public static IntPtr AllocateTypeAssociatedMemory(Type type, int size) + public static unsafe IntPtr AllocateTypeAssociatedMemory(Type type, int size) { - throw new PlatformNotSupportedException(); + if (type is not RuntimeType) + throw new ArgumentException(SR.Arg_MustBeType, nameof(type)); + + ArgumentOutOfRangeException.ThrowIfNegative(size); + + // We don't support unloading; the memory will never be freed. + return (IntPtr)NativeMemory.AllocZeroed((uint)size); } [Intrinsic] diff --git a/src/mono/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs b/src/mono/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs deleted file mode 100644 index e3dae854517e2..0000000000000 --- a/src/mono/System.Private.CoreLib/src/System/Security/DynamicSecurityMethodAttribute.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace System.Security -{ - // DynamicSecurityMethodAttribute: - // All methods that use StackCrawlMark should be marked with this attribute. This attribute - // disables inlining of the calling method to allow stackwalking to find the exact caller. - // - // This attribute used to indicate that the target method requires space for a security object - // to be allocated on the callers stack. It is not used for this purpose anymore because of security - // stackwalks are not ever done in CoreCLR. - [AttributeUsage(AttributeTargets.Method | AttributeTargets.Constructor, AllowMultiple = true, Inherited = false)] - internal sealed class DynamicSecurityMethodAttribute : Attribute - { - public DynamicSecurityMethodAttribute() { } - } -} diff --git a/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj b/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj index 87cfb17bbc047..7283603a955e1 100644 --- a/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj +++ b/src/mono/browser/debugger/DebuggerTestSuite/DebuggerTestSuite.csproj @@ -81,7 +81,7 @@ BeforeTargets="CopyTestZipForHelix" DependsOnTargets="_GenerateRunSettingsFile"> - + @@ -89,12 +89,15 @@ <_Regex>^ *(DebuggerTests[^\($]+) - <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', $(_Regex)))" /> - + <_TestLines0 Include="$([System.Text.RegularExpressions.Regex]::Match('%(_ListOfTestsLines.Identity)', '$(_Regex)'))" /> + + + + Lines="@(TestClassName->Distinct())" + Overwrite="true" /> diff --git a/src/mono/browser/runtime/debug.ts b/src/mono/browser/runtime/debug.ts index 74c0128f2e4e3..1cbe85aff718a 100644 --- a/src/mono/browser/runtime/debug.ts +++ b/src/mono/browser/runtime/debug.ts @@ -1,8 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -import BuildConfiguration from "consts:configuration"; - import { INTERNAL, Module, loaderHelpers, runtimeHelpers } from "./globals"; import { toBase64StringImpl } from "./base64"; import cwraps from "./cwraps"; @@ -364,11 +362,6 @@ export function mono_wasm_debugger_log(level: number, message_ptr: CharPtr): voi INTERNAL.logging.debugger(level, message); return; } - - if (BuildConfiguration === "Debug") { - // eslint-disable-next-line no-console - console.debug(`Debugger.Debug: ${message}`); - } } type CallDetails = { diff --git a/src/mono/browser/runtime/exports-binding.ts b/src/mono/browser/runtime/exports-binding.ts index 3e65378483df3..986e6f89a9c50 100644 --- a/src/mono/browser/runtime/exports-binding.ts +++ b/src/mono/browser/runtime/exports-binding.ts @@ -31,6 +31,7 @@ import { mono_wasm_pthread_on_pthread_attached, mono_wasm_pthread_on_pthread_unregistered, mono_wasm_pthread_on_pthread_registered, mono_wasm_pthread_set_name, mono_wasm_install_js_worker_interop, mono_wasm_uninstall_js_worker_interop } from "./pthreads"; +import { mono_wasm_dump_threads } from "./pthreads/ui-thread"; // the JS methods would be visible to EMCC linker and become imports of the WASM module @@ -45,6 +46,8 @@ export const mono_wasm_threads_imports = !WasmEnableThreads ? [] : [ // threads.c mono_wasm_eventloop_has_unsettled_interop_promises, + // mono-threads.c + mono_wasm_dump_threads, // diagnostics_server.c mono_wasm_diagnostic_server_on_server_thread_created, mono_wasm_diagnostic_server_on_runtime_server_init, diff --git a/src/mono/browser/runtime/exports-internal.ts b/src/mono/browser/runtime/exports-internal.ts index 80c52669fb3ad..61c6e1fe6e3f4 100644 --- a/src/mono/browser/runtime/exports-internal.ts +++ b/src/mono/browser/runtime/exports-internal.ts @@ -23,14 +23,14 @@ import { mono_wasm_get_func_id_to_name_mappings } from "./logging"; import { monoStringToStringUnsafe } from "./strings"; import { mono_wasm_bind_cs_function } from "./invoke-cs"; -import { dumpThreads, thread_available } from "./pthreads"; +import { mono_wasm_dump_threads, thread_available } from "./pthreads"; export function export_internal(): any { return { // tests mono_wasm_exit: (exit_code: number) => { Module.err("early exit " + exit_code); }, forceDisposeProxies, - dumpThreads: WasmEnableThreads ? dumpThreads : undefined, + mono_wasm_dump_threads: WasmEnableThreads ? mono_wasm_dump_threads : undefined, // with mono_wasm_debugger_log and mono_wasm_trace_logger logging: undefined, diff --git a/src/mono/browser/runtime/exports.ts b/src/mono/browser/runtime/exports.ts index 8b2984f393b1f..2a4dc08320a0b 100644 --- a/src/mono/browser/runtime/exports.ts +++ b/src/mono/browser/runtime/exports.ts @@ -10,7 +10,7 @@ import WasmEnableExceptionHandling from "consts:wasmEnableExceptionHandling"; import type { RuntimeAPI } from "./types"; import { Module, exportedRuntimeAPI, loaderHelpers, passEmscriptenInternals, runtimeHelpers, setRuntimeGlobals, } from "./globals"; -import { GlobalObjects } from "./types/internal"; +import { GlobalObjects, RuntimeHelpers } from "./types/internal"; import { configureEmscriptenStartup, configureRuntimeStartup, configureWorkerStartup } from "./startup"; import { create_weak_ref } from "./weak-ref"; @@ -22,7 +22,7 @@ import { mono_wasm_stringify_as_error_with_stack } from "./logging"; import { instantiate_asset, instantiate_symbols_asset, instantiate_segmentation_rules_asset } from "./assets"; import { jiterpreter_dump_stats } from "./jiterpreter"; import { forceDisposeProxies } from "./gc-handles"; -import { dumpThreads } from "./pthreads"; +import { mono_wasm_dump_threads } from "./pthreads"; export let runtimeList: RuntimeList; @@ -32,19 +32,19 @@ function initializeExports(globalObjects: GlobalObjects): RuntimeAPI { const globalThisAny = globalThis as any; Object.assign(globals.internal, export_internal()); - Object.assign(runtimeHelpers, { + const rh: Partial = { stringify_as_error_with_stack: mono_wasm_stringify_as_error_with_stack, instantiate_symbols_asset, instantiate_asset, jiterpreter_dump_stats, forceDisposeProxies, instantiate_segmentation_rules_asset, - }); + + }; if (WasmEnableThreads) { - Object.assign(runtimeHelpers, { - dumpThreads, - }); + rh.dumpThreads = mono_wasm_dump_threads; } + Object.assign(runtimeHelpers, rh); const API = export_api(); Object.assign(exportedRuntimeAPI, { diff --git a/src/mono/browser/runtime/jiterpreter-support.ts b/src/mono/browser/runtime/jiterpreter-support.ts index 998056d0aa145..6ca884459d950 100644 --- a/src/mono/browser/runtime/jiterpreter-support.ts +++ b/src/mono/browser/runtime/jiterpreter-support.ts @@ -1148,7 +1148,7 @@ class Cfg { blockStack: Array = []; backDispatchOffsets: Array = []; dispatchTable = new Map(); - observedBranchTargets = new Set(); + observedBackBranchTargets = new Set(); trace = 0; constructor(builder: WasmBuilder) { @@ -1165,7 +1165,7 @@ class Cfg { this.lastSegmentEnd = 0; this.overheadBytes = 10; // epilogue this.dispatchTable.clear(); - this.observedBranchTargets.clear(); + this.observedBackBranchTargets.clear(); this.trace = trace; this.backDispatchOffsets.length = 0; } @@ -1212,7 +1212,9 @@ class Cfg { } branch(target: MintOpcodePtr, isBackward: boolean, branchType: CfgBranchType) { - this.observedBranchTargets.add(target); + if (isBackward) + this.observedBackBranchTargets.add(target); + this.appendBlob(); this.segments.push({ type: "branch", @@ -1224,7 +1226,10 @@ class Cfg { // some branches will generate bailouts instead so we allocate 4 bytes per branch // to try and balance this out and avoid underestimating too much this.overheadBytes += 4; // forward branches are a constant br + depth (optimally 2 bytes) + if (isBackward) { + // TODO: Make this smaller by setting the flag inside the dispatcher when disp != 0, + // this will save space for any trace with more than one back-branch // get_local // i32_const 1 // i32_store 0 0 @@ -1298,7 +1303,7 @@ class Cfg { const breakDepth = this.blockStack.indexOf(offset); if (breakDepth < 0) continue; - if (!this.observedBranchTargets.has(offset)) + if (!this.observedBackBranchTargets.has(offset)) continue; this.dispatchTable.set(offset, this.backDispatchOffsets.length + 1); diff --git a/src/mono/browser/runtime/jiterpreter-trace-generator.ts b/src/mono/browser/runtime/jiterpreter-trace-generator.ts index a2edb88382028..f8ac833956ff6 100644 --- a/src/mono/browser/runtime/jiterpreter-trace-generator.ts +++ b/src/mono/browser/runtime/jiterpreter-trace-generator.ts @@ -142,6 +142,8 @@ export function generateBackwardBranchTable( // IP of the start of the trace in U16s, relative to startOfBody. const rbase16 = (ip - startOfBody) / 2; + // FIXME: This will potentially scan the entire method and record branches that won't + // ever run since the trace compilation will end before we reach them. while (ip < endOfBody) { // IP of the current opcode in U16s, relative to startOfBody. This is what the back branch table uses const rip16 = (ip - startOfBody) / 2; @@ -166,16 +168,23 @@ export function generateBackwardBranchTable( break; } - const rtarget16 = rip16 + (displacement); - if (rtarget16 < 0) { - mono_log_info(`opcode @${ip}'s displacement of ${displacement} goes before body: ${rtarget16}. aborting backbranch table generation`); - break; - } + // Only record *backward* branches + // We will filter this down further in the Cfg because it takes note of which branches it sees, + // but it is also beneficial to have a null table (further down) due to seeing no potential + // back branch targets at all, as it allows the Cfg to skip additional code generation entirely + // if it knows there will never be any backwards branches in a given trace + if (displacement < 0) { + const rtarget16 = rip16 + (displacement); + if (rtarget16 < 0) { + mono_log_info(`opcode @${ip}'s displacement of ${displacement} goes before body: ${rtarget16}. aborting backbranch table generation`); + break; + } - // If the relative target is before the start of the trace, don't record it. - // The trace will be unable to successfully branch to it so it would just make the table bigger. - if (rtarget16 >= rbase16) - table.push(rtarget16); + // If the relative target is before the start of the trace, don't record it. + // The trace will be unable to successfully branch to it so it would just make the table bigger. + if (rtarget16 >= rbase16) + table.push(rtarget16); + } switch (opcode) { case MintOpcode.MINT_CALL_HANDLER: diff --git a/src/mono/browser/runtime/jiterpreter.ts b/src/mono/browser/runtime/jiterpreter.ts index 9d47c2e39b1ea..5ef86e182905e 100644 --- a/src/mono/browser/runtime/jiterpreter.ts +++ b/src/mono/browser/runtime/jiterpreter.ts @@ -891,7 +891,7 @@ function generate_wasm( // suites or benchmarks if you've enabled stats const tracesCompiled = getCounter(JiterpCounter.TracesCompiled); if (builder.options.enableStats && tracesCompiled && (tracesCompiled % autoDumpInterval) === 0) - jiterpreter_dump_stats(false, true); + jiterpreter_dump_stats(true); return idx; } catch (exc: any) { @@ -1074,14 +1074,14 @@ export function mono_jiterp_free_method_data_js( mono_jiterp_free_method_data_jit_call(method); } -export function jiterpreter_dump_stats(b?: boolean, concise?: boolean) { +export function jiterpreter_dump_stats(concise?: boolean): void { if (!runtimeHelpers.runtimeReady) { return; } - if (!mostRecentOptions || (b !== undefined)) + if (!mostRecentOptions) mostRecentOptions = getOptions(); - if (!mostRecentOptions.enableStats && (b !== undefined)) + if (!mostRecentOptions.enableStats) return; const backBranchesEmitted = getCounter(JiterpCounter.BackBranchesEmitted), @@ -1243,10 +1243,4 @@ export function jiterpreter_dump_stats(b?: boolean, concise?: boolean) { for (const k in simdFallbackCounters) mono_log_info(`// simd ${k}: ${simdFallbackCounters[k]} fallback insn(s)`); - - if ((typeof (globalThis.setTimeout) === "function") && (b !== undefined)) - setTimeout( - () => jiterpreter_dump_stats(b), - 15000 - ); } diff --git a/src/mono/browser/runtime/pthreads/index.ts b/src/mono/browser/runtime/pthreads/index.ts index a97f0f0f06082..c3128d2be2842 100644 --- a/src/mono/browser/runtime/pthreads/index.ts +++ b/src/mono/browser/runtime/pthreads/index.ts @@ -6,7 +6,7 @@ export { mono_wasm_pthread_ptr, update_thread_info, isMonoThreadMessage, monoThreadInfo, } from "./shared"; export { - dumpThreads, thread_available, cancelThreads, is_thread_available, + mono_wasm_dump_threads, thread_available, cancelThreads, is_thread_available, populateEmscriptenPool, mono_wasm_init_threads, init_finalizer_thread, waitForThread, replaceEmscriptenPThreadUI } from "./ui-thread"; diff --git a/src/mono/browser/runtime/pthreads/ui-thread.ts b/src/mono/browser/runtime/pthreads/ui-thread.ts index c7fb54a66e48c..baa72e7a3b975 100644 --- a/src/mono/browser/runtime/pthreads/ui-thread.ts +++ b/src/mono/browser/runtime/pthreads/ui-thread.ts @@ -182,7 +182,7 @@ export function cancelThreads() { } } -export function dumpThreads(): void { +export function mono_wasm_dump_threads(): void { if (!WasmEnableThreads) return; mono_log_info("Dumping web worker info as seen by UI thread, it could be stale: "); const emptyInfo: PThreadInfo = { @@ -278,7 +278,7 @@ export function replaceEmscriptenPThreadUI(modulePThread: PThreadLibrary): void } }; if (BuildConfiguration === "Debug") { - (globalThis as any).dumpThreads = dumpThreads; + (globalThis as any).dumpThreads = mono_wasm_dump_threads; (globalThis as any).getModulePThread = getModulePThread; } } diff --git a/src/mono/browser/runtime/types/internal.ts b/src/mono/browser/runtime/types/internal.ts index 4c9fa7babc62f..7d4eb89d0f3c9 100644 --- a/src/mono/browser/runtime/types/internal.ts +++ b/src/mono/browser/runtime/types/internal.ts @@ -235,7 +235,7 @@ export type RuntimeHelpers = { instantiate_asset: (asset: AssetEntry, url: string, bytes: Uint8Array) => void, instantiate_symbols_asset: (pendingAsset: AssetEntryInternal) => Promise, instantiate_segmentation_rules_asset: (pendingAsset: AssetEntryInternal) => Promise, - jiterpreter_dump_stats?: (x: boolean) => string, + jiterpreter_dump_stats?: (concise?: boolean) => void, forceDisposeProxies: (disposeMethods: boolean, verbose: boolean) => void, dumpThreads: () => void, } diff --git a/src/mono/mono/mini/exceptions-ppc.c b/src/mono/mono/mini/exceptions-ppc.c index 146fece236928..c7537a258acfb 100644 --- a/src/mono/mono/mini/exceptions-ppc.c +++ b/src/mono/mono/mini/exceptions-ppc.c @@ -838,5 +838,8 @@ mono_arch_setup_resume_sighandler_ctx (MonoContext *ctx, gpointer func) ctx->regs[2] = (gulong)handler_ftnptr->toc; #else MONO_CONTEXT_SET_IP(ctx, (unsigned long) func); +#ifdef TARGET_POWERPC64 + ctx->regs[12] = (gulong)func; +#endif #endif } diff --git a/src/mono/mono/mini/interp/mintops.def b/src/mono/mono/mini/interp/mintops.def index d45579ce40fe3..afd4bf10dea60 100644 --- a/src/mono/mono/mini/interp/mintops.def +++ b/src/mono/mono/mini/interp/mintops.def @@ -655,9 +655,6 @@ OPDEF(MINT_RET_I8_IMM, "ret.i8.imm", 2, 0, 0, MintOpShortInt) OPDEF(MINT_ADD_I4_IMM, "add.i4.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_ADD_I8_IMM, "add.i8.imm", 4, 1, 1, MintOpShortInt) -OPDEF(MINT_ADD_MUL_I4_IMM, "add.mul.i4.imm", 5, 1, 1, MintOpTwoShorts) -OPDEF(MINT_ADD_MUL_I8_IMM, "add.mul.i8.imm", 5, 1, 1, MintOpTwoShorts) - OPDEF(MINT_MUL_I4_IMM, "mul.i4.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_MUL_I8_IMM, "mul.i8.imm", 4, 1, 1, MintOpShortInt) @@ -668,6 +665,9 @@ OPDEF(MINT_SHL_I8_IMM, "shl.i8.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_SHR_I4_IMM, "shr.i4.imm", 4, 1, 1, MintOpShortInt) OPDEF(MINT_SHR_I8_IMM, "shr.i8.imm", 4, 1, 1, MintOpShortInt) +OPDEF(MINT_ADD_MUL_I4_IMM, "add.mul.i4.imm", 5, 1, 1, MintOpTwoShorts) +OPDEF(MINT_ADD_MUL_I8_IMM, "add.mul.i8.imm", 5, 1, 1, MintOpTwoShorts) + OPDEF(MINT_SHL_AND_I4, "shl.i4.and", 4, 1, 2, MintOpNoArgs) OPDEF(MINT_SHL_AND_I8, "shl.i8.and", 4, 1, 2, MintOpNoArgs) diff --git a/src/mono/mono/mini/interp/mintops.h b/src/mono/mono/mini/interp/mintops.h index 27e3821dbccf8..a50a7d409215b 100644 --- a/src/mono/mono/mini/interp/mintops.h +++ b/src/mono/mono/mini/interp/mintops.h @@ -224,6 +224,7 @@ typedef enum { #define MINT_IS_LDC_I8(op) ((op) >= MINT_LDC_I8_0 && (op) <= MINT_LDC_I8) #define MINT_IS_UNOP(op) ((op) >= MINT_ADD1_I4 && (op) <= MINT_CEQ0_I4) #define MINT_IS_BINOP(op) ((op) >= MINT_ADD_I4 && (op) <= MINT_CLT_UN_R8) +#define MINT_IS_BINOP_IMM(op) ((op) >= MINT_ADD_I4_IMM && (op) <= MINT_SHR_I8_IMM) #define MINT_IS_BINOP_SHIFT(op) ((op) >= MINT_SHR_UN_I4 && (op) <= MINT_SHR_I8) #define MINT_IS_LDFLD(op) ((op) >= MINT_LDFLD_I1 && (op) <= MINT_LDFLD_O) #define MINT_IS_STFLD(op) ((op) >= MINT_STFLD_I1 && (op) <= MINT_STFLD_O) diff --git a/src/mono/mono/mini/interp/transform-opt.c b/src/mono/mono/mini/interp/transform-opt.c index 0e4e7be42065d..41473cabb0aab 100644 --- a/src/mono/mono/mini/interp/transform-opt.c +++ b/src/mono/mono/mini/interp/transform-opt.c @@ -1959,7 +1959,8 @@ interp_reorder_bblocks (TransformData *td) InterpInst *last_ins = interp_last_ins (in_bb); if (last_ins && (MINT_IS_CONDITIONAL_BRANCH (last_ins->opcode) || MINT_IS_UNCONDITIONAL_BRANCH (last_ins->opcode)) && - last_ins->info.target_bb == bb) { + last_ins->info.target_bb == bb && + in_bb != bb) { InterpBasicBlock *target_bb = first->info.target_bb; last_ins->info.target_bb = target_bb; interp_unlink_bblocks (in_bb, bb); @@ -2129,6 +2130,12 @@ interp_get_mt_for_ldind (int ldind_op) result.field = op val->field; \ break; +#define INTERP_FOLD_SHIFTOP_IMM(opcode,local_type,field,shift_op,cast_type) \ + case opcode: \ + result.type = local_type; \ + result.field = (cast_type)val->field shift_op ins->data [0]; \ + break; + #define INTERP_FOLD_CONV(opcode,val_type_dst,field_dst,val_type_src,field_src,cast_type) \ case opcode: \ result.type = val_type_dst; \ @@ -2168,6 +2175,19 @@ interp_fold_unop (TransformData *td, InterpInst *ins) INTERP_FOLD_UNOP (MINT_NOT_I8, VAR_VALUE_I8, l, ~); INTERP_FOLD_UNOP (MINT_CEQ0_I4, VAR_VALUE_I4, i, 0 ==); + INTERP_FOLD_UNOP (MINT_ADD_I4_IMM, VAR_VALUE_I4, i, ((gint32)(gint16)ins->data [0])+); + INTERP_FOLD_UNOP (MINT_ADD_I8_IMM, VAR_VALUE_I8, l, ((gint64)(gint16)ins->data [0])+); + + INTERP_FOLD_UNOP (MINT_MUL_I4_IMM, VAR_VALUE_I4, i, ((gint32)(gint16)ins->data [0])*); + INTERP_FOLD_UNOP (MINT_MUL_I8_IMM, VAR_VALUE_I8, l, ((gint64)(gint16)ins->data [0])*); + + INTERP_FOLD_SHIFTOP_IMM (MINT_SHR_UN_I4_IMM, VAR_VALUE_I4, i, >>, guint32); + INTERP_FOLD_SHIFTOP_IMM (MINT_SHR_UN_I8_IMM, VAR_VALUE_I8, l, >>, guint64); + INTERP_FOLD_SHIFTOP_IMM (MINT_SHL_I4_IMM, VAR_VALUE_I4, i, <<, gint32); + INTERP_FOLD_SHIFTOP_IMM (MINT_SHL_I8_IMM, VAR_VALUE_I8, l, <<, gint64); + INTERP_FOLD_SHIFTOP_IMM (MINT_SHR_I4_IMM, VAR_VALUE_I4, i, >>, gint32); + INTERP_FOLD_SHIFTOP_IMM (MINT_SHR_I8_IMM, VAR_VALUE_I8, l, >>, gint64); + INTERP_FOLD_CONV (MINT_CONV_I1_I4, VAR_VALUE_I4, i, VAR_VALUE_I4, i, gint8); INTERP_FOLD_CONV (MINT_CONV_I1_I8, VAR_VALUE_I4, i, VAR_VALUE_I8, l, gint8); INTERP_FOLD_CONV (MINT_CONV_U1_I4, VAR_VALUE_I4, i, VAR_VALUE_I4, i, guint8); @@ -2901,7 +2921,7 @@ interp_cprop (TransformData *td) td->var_values [dreg].type = VAR_VALUE_I4; td->var_values [dreg].i = (gint32)td->data_items [ins->data [0]]; #endif - } else if (MINT_IS_UNOP (opcode)) { + } else if (MINT_IS_UNOP (opcode) || MINT_IS_BINOP_IMM (opcode)) { ins = interp_fold_unop (td, ins); } else if (MINT_IS_UNOP_CONDITIONAL_BRANCH (opcode)) { ins = interp_fold_unop_cond_br (td, bb, ins); diff --git a/src/mono/mono/utils/mono-threads-wasm.c b/src/mono/mono/utils/mono-threads-wasm.c index 1746c7213e96d..ca11139ce43b3 100644 --- a/src/mono/mono/utils/mono-threads-wasm.c +++ b/src/mono/mono/utils/mono-threads-wasm.c @@ -541,9 +541,15 @@ mono_threads_wasm_on_thread_registered (void) } #ifndef DISABLE_THREADS +static pthread_t deputy_thread_tid; extern void mono_wasm_start_deputy_thread_async (void); extern void mono_wasm_trace_logger (const char *log_domain, const char *log_level, const char *message, mono_bool fatal, void *user_data); -static pthread_t deputy_thread_tid; +extern void mono_wasm_dump_threads (void); + +void mono_wasm_dump_threads_async (void) +{ + mono_threads_wasm_async_run_in_target_thread (mono_threads_wasm_ui_thread_tid (), mono_wasm_dump_threads); +} gboolean mono_threads_wasm_is_deputy_thread (void) diff --git a/src/mono/mono/utils/mono-threads-wasm.h b/src/mono/mono/utils/mono-threads-wasm.h index 13668709357db..1aa68ee7d329c 100644 --- a/src/mono/mono/utils/mono-threads-wasm.h +++ b/src/mono/mono/utils/mono-threads-wasm.h @@ -28,6 +28,9 @@ mono_threads_wasm_ui_thread_tid (void); #ifndef DISABLE_THREADS +void +mono_wasm_dump_threads_async (void); + gboolean mono_threads_wasm_is_deputy_thread (void); diff --git a/src/mono/mono/utils/mono-threads.c b/src/mono/mono/utils/mono-threads.c index 515cde6eebad5..14a00bc915429 100644 --- a/src/mono/mono/utils/mono-threads.c +++ b/src/mono/mono/utils/mono-threads.c @@ -281,6 +281,12 @@ mono_threads_end_global_suspend (void) static void dump_threads (void) { +#ifdef HOST_BROWSER +#ifndef DISABLE_THREADS + mono_wasm_dump_threads_async (); +#endif +#endif + MonoThreadInfo *cur = mono_thread_info_current (); g_async_safe_printf ("STATE CUE CARD: (? means a positive number, usually 1 or 2, * means any number)\n"); diff --git a/src/mono/sample/wasm/blazor-frame/blazor.csproj b/src/mono/sample/wasm/blazor-frame/blazor.csproj index 5592718e07144..3061f77e83ac3 100644 --- a/src/mono/sample/wasm/blazor-frame/blazor.csproj +++ b/src/mono/sample/wasm/blazor-frame/blazor.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/README.md b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/README.md new file mode 100644 index 0000000000000..996992663189f --- /dev/null +++ b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/README.md @@ -0,0 +1,15 @@ +## WasmBasicTestApp + +This is a test application used by various Wasm.Build.Tests. The idea is to share a common behavior (so that we don't have to maintain many test apps) and tweak it for the test case. +It typically suits scenario where you need more than a plain template app. If the test case is too different, feel free to create another app. + +### Usage + +The app reads `test` query parameter and uses it to switch between test cases. Entrypoint is `main.js`. +There is common unit, then switch based on test case for modifying app startup, then app starts and executes next switch based on test case for actually running code. + +Some test cases passes additional parameters to differentiate behavior, see `src/mono/wasm/Wasm.Build.Tests/TestAppScenarios`. + +### Running out side of WBT + +One of the benefits is that you can copy the app out of intree and run the app without running Wasm.Build.Tests with just `dotnet run`. \ No newline at end of file diff --git a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js index 2e1b7ff8c84fe..3a01053875c0a 100644 --- a/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js +++ b/src/mono/wasm/testassets/WasmBasicTestApp/App/wwwroot/main.js @@ -33,6 +33,7 @@ switch (testCase) { Math.floor(Math.random() * 5) + 5, Math.floor(Math.random() * 5) + 10 ]; + console.log(`Failing test at assembly indexes [${failAtAssemblyNumbers.join(", ")}]`); let alreadyFailed = []; dotnet.withDiagnosticTracing(true).withResourceLoader((type, name, defaultUri, integrity, behavior) => { if (type === "dotnetjs") { @@ -45,8 +46,8 @@ switch (testCase) { return defaultUri; } - assemblyCounter++; - if (!failAtAssemblyNumbers.includes(assemblyCounter) || alreadyFailed.includes(defaultUri)) + const currentCounter = assemblyCounter++; + if (!failAtAssemblyNumbers.includes(currentCounter) || alreadyFailed.includes(defaultUri)) return defaultUri; alreadyFailed.push(defaultUri); diff --git a/src/native/corehost/json_parser.h b/src/native/corehost/json_parser.h index 2c2845aac46bf..d7393b0ae678d 100644 --- a/src/native/corehost/json_parser.h +++ b/src/native/corehost/json_parser.h @@ -8,12 +8,22 @@ // https://github.com/Tencent/rapidjson/issues/1596#issuecomment-548774663 #define RAPIDJSON_48BITPOINTER_OPTIMIZATION 0 +// see https://github.com/Tencent/rapidjson/issues/1448 +// including windows.h on purpose to provoke a compile time problem as GetObject is a +// macro that gets defined when windows.h is included +#ifdef _WIN32 +#define NOMINMAX +#include +#endif + #include "pal.h" #include #include #include #include "bundle/info.h" +#undef GetObject + class json_parser_t { public: #ifdef _WIN32 diff --git a/src/native/external/rapidjson-version.txt b/src/native/external/rapidjson-version.txt index 0ccc08a3c2238..b6f5f9532a7db 100644 --- a/src/native/external/rapidjson-version.txt +++ b/src/native/external/rapidjson-version.txt @@ -1,6 +1,6 @@ -d87b698d0fcc10a5f632ecbc80a9cb2a8fa094a5 +3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d -https://github.com/Tencent/rapidjson/commit/d87b698d0fcc10a5f632ecbc80a9cb2a8fa094a5 +https://github.com/Tencent/rapidjson/commit/3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d Note: This library is not using a proper release lifecycle. v1.1.0 was the last version released in 2016. - Therefore, we are pointing to a random commit from 2019 rather than a version tag. + Therefore, we are pointing to a random commit from 2024 rather than a version tag. diff --git a/src/native/external/rapidjson/README.TXT b/src/native/external/rapidjson/README.TXT index bc0a70382f4a4..9eff509a934d3 100644 --- a/src/native/external/rapidjson/README.TXT +++ b/src/native/external/rapidjson/README.TXT @@ -1,2 +1,2 @@ -This directory contains the contents of `include/rapidjson` from -, commit hash d87b698d0fcc10. +This directory contains selective files from +https://github.com/Tencent/rapidjson/tree/3f73edae00aba5b0112a80b4d41e6f1ff7d92a3d/include/rapidjson diff --git a/src/native/external/rapidjson/allocators.h b/src/native/external/rapidjson/allocators.h index cc67c8971323c..275417bd8b377 100644 --- a/src/native/external/rapidjson/allocators.h +++ b/src/native/external/rapidjson/allocators.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -16,6 +16,14 @@ #define RAPIDJSON_ALLOCATORS_H_ #include "rapidjson.h" +#include "internal/meta.h" + +#include +#include + +#if RAPIDJSON_HAS_CXX11 +#include +#endif RAPIDJSON_NAMESPACE_BEGIN @@ -77,19 +85,26 @@ class CrtAllocator { static const bool kNeedFree = true; void* Malloc(size_t size) { if (size) // behavior of malloc(0) is implementation defined. - return std::malloc(size); + return RAPIDJSON_MALLOC(size); else return NULL; // standardize to returning NULL. } void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; if (newSize == 0) { - std::free(originalPtr); + RAPIDJSON_FREE(originalPtr); return NULL; } - return std::realloc(originalPtr, newSize); + return RAPIDJSON_REALLOC(originalPtr, newSize); + } + static void Free(void *ptr) RAPIDJSON_NOEXCEPT { RAPIDJSON_FREE(ptr); } + + bool operator==(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { + return true; + } + bool operator!=(const CrtAllocator&) const RAPIDJSON_NOEXCEPT { + return false; } - static void Free(void *ptr) { std::free(ptr); } }; /////////////////////////////////////////////////////////////////////////////// @@ -113,16 +128,64 @@ class CrtAllocator { */ template class MemoryPoolAllocator { + //! Chunk header for perpending to each chunk. + /*! Chunks are stored as a singly linked list. + */ + struct ChunkHeader { + size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). + size_t size; //!< Current size of allocated memory in bytes. + ChunkHeader *next; //!< Next chunk in the linked list. + }; + + struct SharedData { + ChunkHeader *chunkHead; //!< Head of the chunk linked-list. Only the head chunk serves allocation. + BaseAllocator* ownBaseAllocator; //!< base allocator created by this object. + size_t refcount; + bool ownBuffer; + }; + + static const size_t SIZEOF_SHARED_DATA = RAPIDJSON_ALIGN(sizeof(SharedData)); + static const size_t SIZEOF_CHUNK_HEADER = RAPIDJSON_ALIGN(sizeof(ChunkHeader)); + + static inline ChunkHeader *GetChunkHead(SharedData *shared) + { + return reinterpret_cast(reinterpret_cast(shared) + SIZEOF_SHARED_DATA); + } + static inline uint8_t *GetChunkBuffer(SharedData *shared) + { + return reinterpret_cast(shared->chunkHead) + SIZEOF_CHUNK_HEADER; + } + + static const size_t kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. + public: static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) + static const bool kRefCounted = true; //!< Tell users that this allocator is reference counted on copy //! Constructor with chunkSize. /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. \param baseAllocator The allocator for allocating memory chunks. */ + explicit MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + chunk_capacity_(chunkSize), + baseAllocator_(baseAllocator ? baseAllocator : RAPIDJSON_NEW(BaseAllocator)()), + shared_(static_cast(baseAllocator_ ? baseAllocator_->Malloc(SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER) : 0)) { + RAPIDJSON_ASSERT(baseAllocator_ != 0); + RAPIDJSON_ASSERT(shared_ != 0); + if (baseAllocator) { + shared_->ownBaseAllocator = 0; + } + else { + shared_->ownBaseAllocator = baseAllocator_; + } + shared_->chunkHead = GetChunkHead(shared_); + shared_->chunkHead->capacity = 0; + shared_->chunkHead->size = 0; + shared_->chunkHead->next = 0; + shared_->ownBuffer = true; + shared_->refcount = 1; } //! Constructor with user-supplied buffer. @@ -136,41 +199,101 @@ class MemoryPoolAllocator { \param baseAllocator The allocator for allocating memory chunks. */ MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : - chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) + chunk_capacity_(chunkSize), + baseAllocator_(baseAllocator), + shared_(static_cast(AlignBuffer(buffer, size))) + { + RAPIDJSON_ASSERT(size >= SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER); + shared_->chunkHead = GetChunkHead(shared_); + shared_->chunkHead->capacity = size - SIZEOF_SHARED_DATA - SIZEOF_CHUNK_HEADER; + shared_->chunkHead->size = 0; + shared_->chunkHead->next = 0; + shared_->ownBaseAllocator = 0; + shared_->ownBuffer = false; + shared_->refcount = 1; + } + + MemoryPoolAllocator(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT : + chunk_capacity_(rhs.chunk_capacity_), + baseAllocator_(rhs.baseAllocator_), + shared_(rhs.shared_) + { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + ++shared_->refcount; + } + MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT { - RAPIDJSON_ASSERT(buffer != 0); - RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); - chunkHead_ = reinterpret_cast(buffer); - chunkHead_->capacity = size - sizeof(ChunkHeader); - chunkHead_->size = 0; - chunkHead_->next = 0; + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + ++rhs.shared_->refcount; + this->~MemoryPoolAllocator(); + baseAllocator_ = rhs.baseAllocator_; + chunk_capacity_ = rhs.chunk_capacity_; + shared_ = rhs.shared_; + return *this; } +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + MemoryPoolAllocator(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT : + chunk_capacity_(rhs.chunk_capacity_), + baseAllocator_(rhs.baseAllocator_), + shared_(rhs.shared_) + { + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + rhs.shared_ = 0; + } + MemoryPoolAllocator& operator=(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT + { + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + this->~MemoryPoolAllocator(); + baseAllocator_ = rhs.baseAllocator_; + chunk_capacity_ = rhs.chunk_capacity_; + shared_ = rhs.shared_; + rhs.shared_ = 0; + return *this; + } +#endif + //! Destructor. /*! This deallocates all memory chunks, excluding the user-supplied buffer. */ - ~MemoryPoolAllocator() { + ~MemoryPoolAllocator() RAPIDJSON_NOEXCEPT { + if (!shared_) { + // do nothing if moved + return; + } + if (shared_->refcount > 1) { + --shared_->refcount; + return; + } Clear(); - RAPIDJSON_DELETE(ownBaseAllocator_); + BaseAllocator *a = shared_->ownBaseAllocator; + if (shared_->ownBuffer) { + baseAllocator_->Free(shared_); + } + RAPIDJSON_DELETE(a); } - //! Deallocates all memory chunks, excluding the user-supplied buffer. - void Clear() { - while (chunkHead_ && chunkHead_ != userBuffer_) { - ChunkHeader* next = chunkHead_->next; - baseAllocator_->Free(chunkHead_); - chunkHead_ = next; + //! Deallocates all memory chunks, excluding the first/user one. + void Clear() RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + for (;;) { + ChunkHeader* c = shared_->chunkHead; + if (!c->next) { + break; + } + shared_->chunkHead = c->next; + baseAllocator_->Free(c); } - if (chunkHead_ && chunkHead_ == userBuffer_) - chunkHead_->size = 0; // Clear user buffer + shared_->chunkHead->size = 0; } //! Computes the total capacity of allocated memory chunks. /*! \return total capacity in bytes. */ - size_t Capacity() const { + size_t Capacity() const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); size_t capacity = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) capacity += c->capacity; return capacity; } @@ -178,25 +301,35 @@ class MemoryPoolAllocator { //! Computes the memory blocks allocated. /*! \return total used bytes. */ - size_t Size() const { + size_t Size() const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); size_t size = 0; - for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) + for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next) size += c->size; return size; } + //! Whether the allocator is shared. + /*! \return true or false. + */ + bool Shared() const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + return shared_->refcount > 1; + } + //! Allocates a memory block. (concept Allocator) void* Malloc(size_t size) { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); if (!size) return NULL; size = RAPIDJSON_ALIGN(size); - if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity) + if (RAPIDJSON_UNLIKELY(shared_->chunkHead->size + size > shared_->chunkHead->capacity)) if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size)) return NULL; - void *buffer = reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size; - chunkHead_->size += size; + void *buffer = GetChunkBuffer(shared_) + shared_->chunkHead->size; + shared_->chunkHead->size += size; return buffer; } @@ -205,6 +338,7 @@ class MemoryPoolAllocator { if (originalPtr == 0) return Malloc(newSize); + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); if (newSize == 0) return NULL; @@ -216,10 +350,10 @@ class MemoryPoolAllocator { return originalPtr; // Simply expand it if it is the last allocation and there is sufficient space - if (originalPtr == reinterpret_cast(chunkHead_) + RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) { + if (originalPtr == GetChunkBuffer(shared_) + shared_->chunkHead->size - originalSize) { size_t increment = static_cast(newSize - originalSize); - if (chunkHead_->size + increment <= chunkHead_->capacity) { - chunkHead_->size += increment; + if (shared_->chunkHead->size + increment <= shared_->chunkHead->capacity) { + shared_->chunkHead->size += increment; return originalPtr; } } @@ -235,50 +369,325 @@ class MemoryPoolAllocator { } //! Frees a memory block (concept Allocator) - static void Free(void *ptr) { (void)ptr; } // Do nothing + static void Free(void *ptr) RAPIDJSON_NOEXCEPT { (void)ptr; } // Do nothing -private: - //! Copy constructor is not permitted. - MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */; - //! Copy assignment operator is not permitted. - MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */; + //! Compare (equality) with another MemoryPoolAllocator + bool operator==(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT { + RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0); + RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0); + return shared_ == rhs.shared_; + } + //! Compare (inequality) with another MemoryPoolAllocator + bool operator!=(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT { + return !operator==(rhs); + } +private: //! Creates a new chunk. /*! \param capacity Capacity of the chunk in bytes. \return true if success. */ bool AddChunk(size_t capacity) { if (!baseAllocator_) - ownBaseAllocator_ = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); - if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) { + shared_->ownBaseAllocator = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)(); + if (ChunkHeader* chunk = static_cast(baseAllocator_->Malloc(SIZEOF_CHUNK_HEADER + capacity))) { chunk->capacity = capacity; chunk->size = 0; - chunk->next = chunkHead_; - chunkHead_ = chunk; + chunk->next = shared_->chunkHead; + shared_->chunkHead = chunk; return true; } else return false; } - static const int kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity. - - //! Chunk header for perpending to each chunk. - /*! Chunks are stored as a singly linked list. - */ - struct ChunkHeader { - size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). - size_t size; //!< Current size of allocated memory in bytes. - ChunkHeader *next; //!< Next chunk in the linked list. - }; + static inline void* AlignBuffer(void* buf, size_t &size) + { + RAPIDJSON_NOEXCEPT_ASSERT(buf != 0); + const uintptr_t mask = sizeof(void*) - 1; + const uintptr_t ubuf = reinterpret_cast(buf); + if (RAPIDJSON_UNLIKELY(ubuf & mask)) { + const uintptr_t abuf = (ubuf + mask) & ~mask; + RAPIDJSON_ASSERT(size >= abuf - ubuf); + buf = reinterpret_cast(abuf); + size -= abuf - ubuf; + } + return buf; + } - ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. - void *userBuffer_; //!< User supplied buffer. BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. - BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. + SharedData *shared_; //!< The shared data of the allocator }; +namespace internal { + template + struct IsRefCounted : + public FalseType + { }; + template + struct IsRefCounted::Type> : + public TrueType + { }; +} + +template +inline T* Realloc(A& a, T* old_p, size_t old_n, size_t new_n) +{ + RAPIDJSON_NOEXCEPT_ASSERT(old_n <= (std::numeric_limits::max)() / sizeof(T) && new_n <= (std::numeric_limits::max)() / sizeof(T)); + return static_cast(a.Realloc(old_p, old_n * sizeof(T), new_n * sizeof(T))); +} + +template +inline T *Malloc(A& a, size_t n = 1) +{ + return Realloc(a, NULL, 0, n); +} + +template +inline void Free(A& a, T *p, size_t n = 1) +{ + static_cast(Realloc(a, p, n, 0)); +} + +#ifdef __GNUC__ +RAPIDJSON_DIAG_PUSH +RAPIDJSON_DIAG_OFF(effc++) // std::allocator can safely be inherited +#endif + +template +class StdAllocator : + public std::allocator +{ + typedef std::allocator allocator_type; +#if RAPIDJSON_HAS_CXX11 + typedef std::allocator_traits traits_type; +#else + typedef allocator_type traits_type; +#endif + +public: + typedef BaseAllocator BaseAllocatorType; + + StdAllocator() RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_() + { } + + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + template + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + StdAllocator(StdAllocator&& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(std::move(rhs)), + baseAllocator_(std::move(rhs.baseAllocator_)) + { } +#endif +#if RAPIDJSON_HAS_CXX11 + using propagate_on_container_move_assignment = std::true_type; + using propagate_on_container_swap = std::true_type; +#endif + + /* implicit */ + StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_(baseAllocator) + { } + + ~StdAllocator() RAPIDJSON_NOEXCEPT + { } + + template + struct rebind { + typedef StdAllocator other; + }; + + typedef typename traits_type::size_type size_type; + typedef typename traits_type::difference_type difference_type; + + typedef typename traits_type::value_type value_type; + typedef typename traits_type::pointer pointer; + typedef typename traits_type::const_pointer const_pointer; + +#if RAPIDJSON_HAS_CXX11 + + typedef typename std::add_lvalue_reference::type &reference; + typedef typename std::add_lvalue_reference::type>::type &const_reference; + + pointer address(reference r) const RAPIDJSON_NOEXCEPT + { + return std::addressof(r); + } + const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT + { + return std::addressof(r); + } + + size_type max_size() const RAPIDJSON_NOEXCEPT + { + return traits_type::max_size(*this); + } + + template + void construct(pointer p, Args&&... args) + { + traits_type::construct(*this, p, std::forward(args)...); + } + void destroy(pointer p) + { + traits_type::destroy(*this, p); + } + +#else // !RAPIDJSON_HAS_CXX11 + + typedef typename allocator_type::reference reference; + typedef typename allocator_type::const_reference const_reference; + + pointer address(reference r) const RAPIDJSON_NOEXCEPT + { + return allocator_type::address(r); + } + const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT + { + return allocator_type::address(r); + } + + size_type max_size() const RAPIDJSON_NOEXCEPT + { + return allocator_type::max_size(); + } + + void construct(pointer p, const_reference r) + { + allocator_type::construct(p, r); + } + void destroy(pointer p) + { + allocator_type::destroy(p); + } + +#endif // !RAPIDJSON_HAS_CXX11 + + template + U* allocate(size_type n = 1, const void* = 0) + { + return RAPIDJSON_NAMESPACE::Malloc(baseAllocator_, n); + } + template + void deallocate(U* p, size_type n = 1) + { + RAPIDJSON_NAMESPACE::Free(baseAllocator_, p, n); + } + + pointer allocate(size_type n = 1, const void* = 0) + { + return allocate(n); + } + void deallocate(pointer p, size_type n = 1) + { + deallocate(p, n); + } + +#if RAPIDJSON_HAS_CXX11 + using is_always_equal = std::is_empty; +#endif + + template + bool operator==(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT + { + return baseAllocator_ == rhs.baseAllocator_; + } + template + bool operator!=(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT + { + return !operator==(rhs); + } + + //! rapidjson Allocator concept + static const bool kNeedFree = BaseAllocator::kNeedFree; + static const bool kRefCounted = internal::IsRefCounted::Value; + void* Malloc(size_t size) + { + return baseAllocator_.Malloc(size); + } + void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) + { + return baseAllocator_.Realloc(originalPtr, originalSize, newSize); + } + static void Free(void *ptr) RAPIDJSON_NOEXCEPT + { + BaseAllocator::Free(ptr); + } + +private: + template + friend class StdAllocator; // access to StdAllocator.* + + BaseAllocator baseAllocator_; +}; + +#if !RAPIDJSON_HAS_CXX17 // std::allocator deprecated in C++17 +template +class StdAllocator : + public std::allocator +{ + typedef std::allocator allocator_type; + +public: + typedef BaseAllocator BaseAllocatorType; + + StdAllocator() RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_() + { } + + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + template + StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT : + allocator_type(rhs), + baseAllocator_(rhs.baseAllocator_) + { } + + /* implicit */ + StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT : + allocator_type(), + baseAllocator_(baseAllocator) + { } + + ~StdAllocator() RAPIDJSON_NOEXCEPT + { } + + template + struct rebind { + typedef StdAllocator other; + }; + + typedef typename allocator_type::value_type value_type; + +private: + template + friend class StdAllocator; // access to StdAllocator.* + + BaseAllocator baseAllocator_; +}; +#endif + +#ifdef __GNUC__ +RAPIDJSON_DIAG_POP +#endif + RAPIDJSON_NAMESPACE_END #endif // RAPIDJSON_ENCODINGS_H_ diff --git a/src/native/external/rapidjson/cursorstreamwrapper.h b/src/native/external/rapidjson/cursorstreamwrapper.h deleted file mode 100644 index 52c11a7c01d7d..0000000000000 --- a/src/native/external/rapidjson/cursorstreamwrapper.h +++ /dev/null @@ -1,78 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_CURSORSTREAMWRAPPER_H_ -#define RAPIDJSON_CURSORSTREAMWRAPPER_H_ - -#include "stream.h" - -#if defined(__GNUC__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#if defined(_MSC_VER) && _MSC_VER <= 1800 -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4702) // unreachable code -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -RAPIDJSON_NAMESPACE_BEGIN - - -//! Cursor stream wrapper for counting line and column number if error exists. -/*! - \tparam InputStream Any stream that implements Stream Concept -*/ -template > -class CursorStreamWrapper : public GenericStreamWrapper { -public: - typedef typename Encoding::Ch Ch; - - CursorStreamWrapper(InputStream& is): - GenericStreamWrapper(is), line_(1), col_(0) {} - - // counting line and column number - Ch Take() { - Ch ch = this->is_.Take(); - if(ch == '\n') { - line_ ++; - col_ = 0; - } else { - col_ ++; - } - return ch; - } - - //! Get the error line number, if error exists. - size_t GetLine() const { return line_; } - //! Get the error column number, if error exists. - size_t GetColumn() const { return col_; } - -private: - size_t line_; //!< Current Line - size_t col_; //!< Current Column -}; - -#if defined(_MSC_VER) && _MSC_VER <= 1800 -RAPIDJSON_DIAG_POP -#endif - -#if defined(__GNUC__) -RAPIDJSON_DIAG_POP -#endif - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_CURSORSTREAMWRAPPER_H_ diff --git a/src/native/external/rapidjson/document.h b/src/native/external/rapidjson/document.h index 74666e3423ee7..2cd9a70a60037 100644 --- a/src/native/external/rapidjson/document.h +++ b/src/native/external/rapidjson/document.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -24,6 +24,9 @@ #include "encodedstream.h" #include // placement new #include +#ifdef __cpp_lib_three_way_comparison +#include +#endif RAPIDJSON_DIAG_PUSH #ifdef __clang__ @@ -39,12 +42,21 @@ RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible lo RAPIDJSON_DIAG_OFF(effc++) #endif // __GNUC__ +#ifdef GetObject +// see https://github.com/Tencent/rapidjson/issues/1448 +// a former included windows.h might have defined a macro called GetObject, which affects +// GetObject defined here. This ensures the macro does not get applied +#pragma push_macro("GetObject") +#define RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED +#undef GetObject +#endif + #ifndef RAPIDJSON_NOMEMBERITERATORCLASS #include // std::random_access_iterator_tag #endif -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS -#include // std::move +#if RAPIDJSON_USE_MEMBERSMAP +#include // std::multimap #endif RAPIDJSON_NAMESPACE_BEGIN @@ -56,6 +68,48 @@ class GenericValue; template class GenericDocument; +/*! \def RAPIDJSON_DEFAULT_ALLOCATOR + \ingroup RAPIDJSON_CONFIG + \brief Allows to choose default allocator. + + User can define this to use CrtAllocator or MemoryPoolAllocator. +*/ +#ifndef RAPIDJSON_DEFAULT_ALLOCATOR +#define RAPIDJSON_DEFAULT_ALLOCATOR ::RAPIDJSON_NAMESPACE::MemoryPoolAllocator<::RAPIDJSON_NAMESPACE::CrtAllocator> +#endif + +/*! \def RAPIDJSON_DEFAULT_STACK_ALLOCATOR + \ingroup RAPIDJSON_CONFIG + \brief Allows to choose default stack allocator for Document. + + User can define this to use CrtAllocator or MemoryPoolAllocator. +*/ +#ifndef RAPIDJSON_DEFAULT_STACK_ALLOCATOR +#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR ::RAPIDJSON_NAMESPACE::CrtAllocator +#endif + +/*! \def RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY + \ingroup RAPIDJSON_CONFIG + \brief User defined kDefaultObjectCapacity value. + + User can define this as any natural number. +*/ +#ifndef RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY +// number of objects that rapidjson::Value allocates memory for by default +#define RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY 16 +#endif + +/*! \def RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY + \ingroup RAPIDJSON_CONFIG + \brief User defined kDefaultArrayCapacity value. + + User can define this as any natural number. +*/ +#ifndef RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY +// number of array elements that rapidjson::Value allocates memory for by default +#define RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY 16 +#endif + //! Name-value pair in a JSON object value. /*! This class was internal to GenericValue. It used to be a inner struct. @@ -63,15 +117,45 @@ class GenericDocument; https://code.google.com/p/rapidjson/issues/detail?id=64 */ template -struct GenericMember { +class GenericMember { +public: GenericValue name; //!< name of member (must be a string) GenericValue value; //!< value of member. +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS + //! Move constructor in C++11 + GenericMember(GenericMember&& rhs) RAPIDJSON_NOEXCEPT + : name(std::move(rhs.name)), + value(std::move(rhs.value)) + { + } + + //! Move assignment in C++11 + GenericMember& operator=(GenericMember&& rhs) RAPIDJSON_NOEXCEPT { + return *this = static_cast(rhs); + } +#endif + + //! Assignment with move semantics. + /*! \param rhs Source of the assignment. Its name and value will become a null value after assignment. + */ + GenericMember& operator=(GenericMember& rhs) RAPIDJSON_NOEXCEPT { + if (RAPIDJSON_LIKELY(this != &rhs)) { + name = rhs.name; + value = rhs.value; + } + return *this; + } + // swap() for std::sort() and other potential use in STL. friend inline void swap(GenericMember& a, GenericMember& b) RAPIDJSON_NOEXCEPT { a.name.Swap(b.name); a.value.Swap(b.value); } + +private: + //! Copy constructor is not permitted. + GenericMember(const GenericMember& rhs); }; /////////////////////////////////////////////////////////////////////////////// @@ -175,12 +259,16 @@ class GenericMemberIterator { //! @name relations //@{ - bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; } - bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; } - bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; } - bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; } - bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; } - bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; } + template bool operator==(const GenericMemberIterator& that) const { return ptr_ == that.ptr_; } + template bool operator!=(const GenericMemberIterator& that) const { return ptr_ != that.ptr_; } + template bool operator<=(const GenericMemberIterator& that) const { return ptr_ <= that.ptr_; } + template bool operator>=(const GenericMemberIterator& that) const { return ptr_ >= that.ptr_; } + template bool operator< (const GenericMemberIterator& that) const { return ptr_ < that.ptr_; } + template bool operator> (const GenericMemberIterator& that) const { return ptr_ > that.ptr_; } + +#ifdef __cpp_lib_three_way_comparison + template std::strong_ordering operator<=>(const GenericMemberIterator& that) const { return ptr_ <=> that.ptr_; } +#endif //@} //! @name dereference @@ -210,12 +298,14 @@ class GenericMemberIterator; //! non-const GenericMemberIterator template class GenericMemberIterator { +public: //! use plain pointer as iterator type typedef GenericMember* Iterator; }; //! const GenericMemberIterator template class GenericMemberIterator { +public: //! use plain const pointer as iterator type typedef const GenericMember* Iterator; }; @@ -574,7 +664,7 @@ template class GenericObject; \tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document) \tparam Allocator Allocator type for allocating memory of object, array and string. */ -template > +template class GenericValue { public: //! Name-value pair in an object. @@ -651,18 +741,8 @@ class GenericValue { template GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) { switch (rhs.GetType()) { - case kObjectType: { - SizeType count = rhs.data_.o.size; - Member* lm = reinterpret_cast(allocator.Malloc(count * sizeof(Member))); - const typename GenericValue::Member* rm = rhs.GetMembersPointer(); - for (SizeType i = 0; i < count; i++) { - new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); - new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); - } - data_.f.flags = kObjectFlag; - data_.o.size = data_.o.capacity = count; - SetMembersPointer(lm); - } + case kObjectType: + DoCopyMembers(rhs, allocator, copyConstStrings); break; case kArrayType: { SizeType count = rhs.data_.a.size; @@ -798,25 +878,30 @@ class GenericValue { /*! Need to destruct elements of array, members of object, or copy-string. */ ~GenericValue() { - if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + // With RAPIDJSON_USE_MEMBERSMAP, the maps need to be destroyed to release + // their Allocator if it's refcounted (e.g. MemoryPoolAllocator). + if (Allocator::kNeedFree || (RAPIDJSON_USE_MEMBERSMAP+0 && + internal::IsRefCounted::Value)) { switch(data_.f.flags) { case kArrayFlag: { GenericValue* e = GetElementsPointer(); for (GenericValue* v = e; v != e + data_.a.size; ++v) v->~GenericValue(); - Allocator::Free(e); + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + Allocator::Free(e); + } } break; case kObjectFlag: - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - Allocator::Free(GetMembersPointer()); + DoFreeMembers(); break; case kCopyStringFlag: - Allocator::Free(const_cast(GetStringPointer())); + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + Allocator::Free(const_cast(GetStringPointer())); + } break; default: @@ -835,8 +920,13 @@ class GenericValue { */ GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT { if (RAPIDJSON_LIKELY(this != &rhs)) { + // Can't destroy "this" before assigning "rhs", otherwise "rhs" + // could be used after free if it's an sub-Value of "this", + // hence the temporary danse. + GenericValue temp; + temp.RawAssign(rhs); this->~GenericValue(); - RawAssign(rhs); + RawAssign(temp); } return *this; } @@ -988,6 +1078,7 @@ class GenericValue { */ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); } +#ifndef __cpp_impl_three_way_comparison //! Not-equal-to operator /*! \return !(*this == rhs) */ @@ -1012,6 +1103,7 @@ class GenericValue { */ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); } //@} +#endif //!@name Type //@{ @@ -1138,13 +1230,28 @@ class GenericValue { else { RAPIDJSON_ASSERT(false); // see above note - // This will generate -Wexit-time-destructors in clang - // static GenericValue NullValue; - // return NullValue; - - // Use static buffer and placement-new to prevent destruction - static char buffer[sizeof(GenericValue)]; +#if RAPIDJSON_HAS_CXX11 + // Use thread-local storage to prevent races between threads. + // Use static buffer and placement-new to prevent destruction, with + // alignas() to ensure proper alignment. + alignas(GenericValue) thread_local static char buffer[sizeof(GenericValue)]; + return *new (buffer) GenericValue(); +#elif defined(_MSC_VER) && _MSC_VER < 1900 + // There's no way to solve both thread locality and proper alignment + // simultaneously. + __declspec(thread) static char buffer[sizeof(GenericValue)]; return *new (buffer) GenericValue(); +#elif defined(__GNUC__) || defined(__clang__) + // This will generate -Wexit-time-destructors in clang, but that's + // better than having under-alignment. + __thread static GenericValue buffer; + return buffer; +#else + // Don't know what compiler this is, so don't know how to ensure + // thread-locality. + static GenericValue buffer; + return buffer; +#endif } } template @@ -1177,10 +1284,7 @@ class GenericValue { */ GenericValue& MemberReserve(SizeType newCapacity, Allocator &allocator) { RAPIDJSON_ASSERT(IsObject()); - if (newCapacity > data_.o.capacity) { - SetMembersPointer(reinterpret_cast(allocator.Realloc(GetMembersPointer(), data_.o.capacity * sizeof(Member), newCapacity * sizeof(Member)))); - data_.o.capacity = newCapacity; - } + DoReserveMembers(newCapacity, allocator); return *this; } @@ -1254,11 +1358,7 @@ class GenericValue { MemberIterator FindMember(const GenericValue& name) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - MemberIterator member = MemberBegin(); - for ( ; member != MemberEnd(); ++member) - if (name.StringEqual(member->name)) - break; - return member; + return DoFindMember(name); } template ConstMemberIterator FindMember(const GenericValue& name) const { return const_cast(*this).FindMember(name); } @@ -1287,14 +1387,7 @@ class GenericValue { GenericValue& AddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { RAPIDJSON_ASSERT(IsObject()); RAPIDJSON_ASSERT(name.IsString()); - - ObjectData& o = data_.o; - if (o.size >= o.capacity) - MemberReserve(o.capacity == 0 ? kDefaultObjectCapacity : (o.capacity + (o.capacity + 1) / 2), allocator); - Member* members = GetMembersPointer(); - members[o.size].name.RawAssign(name); - members[o.size].value.RawAssign(value); - o.size++; + DoAddMember(name, value, allocator); return *this; } @@ -1428,9 +1521,7 @@ class GenericValue { */ void RemoveAllMembers() { RAPIDJSON_ASSERT(IsObject()); - for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) - m->~Member(); - data_.o.size = 0; + DoClearMembers(); } //! Remove a member in object by its name. @@ -1474,14 +1565,7 @@ class GenericValue { RAPIDJSON_ASSERT(data_.o.size > 0); RAPIDJSON_ASSERT(GetMembersPointer() != 0); RAPIDJSON_ASSERT(m >= MemberBegin() && m < MemberEnd()); - - MemberIterator last(GetMembersPointer() + (data_.o.size - 1)); - if (data_.o.size > 1 && m != last) - *m = *last; // Move the last one to this place - else - m->~Member(); // Only one left, just destroy - --data_.o.size; - return m; + return DoRemoveMember(m); } //! Remove a member from an object by iterator. @@ -1513,13 +1597,7 @@ class GenericValue { RAPIDJSON_ASSERT(first >= MemberBegin()); RAPIDJSON_ASSERT(first <= last); RAPIDJSON_ASSERT(last <= MemberEnd()); - - MemberIterator pos = MemberBegin() + (first - MemberBegin()); - for (MemberIterator itr = pos; itr != last; ++itr) - itr->~Member(); - std::memmove(static_cast(&*pos), &*last, static_cast(MemberEnd() - last) * sizeof(Member)); - data_.o.size -= static_cast(last - first); - return pos; + return DoEraseMembers(first, last); } //! Erase a member in object by its name. @@ -1548,7 +1626,9 @@ class GenericValue { } Object GetObject() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } + Object GetObj() { RAPIDJSON_ASSERT(IsObject()); return Object(*this); } ConstObject GetObject() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } + ConstObject GetObj() const { RAPIDJSON_ASSERT(IsObject()); return ConstObject(*this); } //@} @@ -1770,12 +1850,12 @@ class GenericValue { //!@name String //@{ - const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return (data_.f.flags & kInlineStrFlag) ? data_.ss.str : GetStringPointer(); } + const Ch* GetString() const { RAPIDJSON_ASSERT(IsString()); return DataString(data_); } //! Get the length of string. /*! Since rapidjson permits "\\u0000" in the json string, strlen(v.GetString()) may not equal to v.GetStringLength(). */ - SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return ((data_.f.flags & kInlineStrFlag) ? (data_.ss.GetLength()) : data_.s.length); } + SizeType GetStringLength() const { RAPIDJSON_ASSERT(IsString()); return DataStringLength(data_); } //! Set this value as a string without copying source string. /*! This version has better performance with supplied length, and also support string containing null character. @@ -1886,7 +1966,7 @@ class GenericValue { case kArrayType: if (RAPIDJSON_UNLIKELY(!handler.StartArray())) return false; - for (const GenericValue* v = Begin(); v != End(); ++v) + for (ConstValueIterator v = Begin(); v != End(); ++v) if (RAPIDJSON_UNLIKELY(!v->Accept(handler))) return false; return handler.EndArray(data_.a.size); @@ -1922,25 +2002,26 @@ class GenericValue { // Initial flags of different types. kNullFlag = kNullType, - kTrueFlag = kTrueType | kBoolFlag, - kFalseFlag = kFalseType | kBoolFlag, - kNumberIntFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag, - kNumberUintFlag = kNumberType | kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag, - kNumberInt64Flag = kNumberType | kNumberFlag | kInt64Flag, - kNumberUint64Flag = kNumberType | kNumberFlag | kUint64Flag, - kNumberDoubleFlag = kNumberType | kNumberFlag | kDoubleFlag, - kNumberAnyFlag = kNumberType | kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag, - kConstStringFlag = kStringType | kStringFlag, - kCopyStringFlag = kStringType | kStringFlag | kCopyFlag, - kShortStringFlag = kStringType | kStringFlag | kCopyFlag | kInlineStrFlag, + // These casts are added to suppress the warning on MSVC about bitwise operations between enums of different types. + kTrueFlag = static_cast(kTrueType) | static_cast(kBoolFlag), + kFalseFlag = static_cast(kFalseType) | static_cast(kBoolFlag), + kNumberIntFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kIntFlag | kInt64Flag), + kNumberUintFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kUintFlag | kUint64Flag | kInt64Flag), + kNumberInt64Flag = static_cast(kNumberType) | static_cast(kNumberFlag | kInt64Flag), + kNumberUint64Flag = static_cast(kNumberType) | static_cast(kNumberFlag | kUint64Flag), + kNumberDoubleFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kDoubleFlag), + kNumberAnyFlag = static_cast(kNumberType) | static_cast(kNumberFlag | kIntFlag | kInt64Flag | kUintFlag | kUint64Flag | kDoubleFlag), + kConstStringFlag = static_cast(kStringType) | static_cast(kStringFlag), + kCopyStringFlag = static_cast(kStringType) | static_cast(kStringFlag | kCopyFlag), + kShortStringFlag = static_cast(kStringType) | static_cast(kStringFlag | kCopyFlag | kInlineStrFlag), kObjectFlag = kObjectType, kArrayFlag = kArrayType, kTypeMask = 0x07 }; - static const SizeType kDefaultArrayCapacity = 16; - static const SizeType kDefaultObjectCapacity = 16; + static const SizeType kDefaultArrayCapacity = RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY; + static const SizeType kDefaultObjectCapacity = RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY; struct Flag { #if RAPIDJSON_48BITPOINTER_OPTIMIZATION @@ -2023,6 +2104,13 @@ class GenericValue { Flag f; }; // 16 bytes in 32-bit mode, 24 bytes in 64-bit mode, 16 bytes in 64-bit with RAPIDJSON_48BITPOINTER_OPTIMIZATION + static RAPIDJSON_FORCEINLINE const Ch* DataString(const Data& data) { + return (data.f.flags & kInlineStrFlag) ? data.ss.str : RAPIDJSON_GETPOINTER(Ch, data.s.str); + } + static RAPIDJSON_FORCEINLINE SizeType DataStringLength(const Data& data) { + return (data.f.flags & kInlineStrFlag) ? data.ss.GetLength() : data.s.length; + } + RAPIDJSON_FORCEINLINE const Ch* GetStringPointer() const { return RAPIDJSON_GETPOINTER(Ch, data_.s.str); } RAPIDJSON_FORCEINLINE const Ch* SetStringPointer(const Ch* str) { return RAPIDJSON_SETPOINTER(Ch, data_.s.str, str); } RAPIDJSON_FORCEINLINE GenericValue* GetElementsPointer() const { return RAPIDJSON_GETPOINTER(GenericValue, data_.a.elements); } @@ -2030,6 +2118,286 @@ class GenericValue { RAPIDJSON_FORCEINLINE Member* GetMembersPointer() const { return RAPIDJSON_GETPOINTER(Member, data_.o.members); } RAPIDJSON_FORCEINLINE Member* SetMembersPointer(Member* members) { return RAPIDJSON_SETPOINTER(Member, data_.o.members, members); } +#if RAPIDJSON_USE_MEMBERSMAP + + struct MapTraits { + struct Less { + bool operator()(const Data& s1, const Data& s2) const { + SizeType n1 = DataStringLength(s1), n2 = DataStringLength(s2); + int cmp = std::memcmp(DataString(s1), DataString(s2), sizeof(Ch) * (n1 < n2 ? n1 : n2)); + return cmp < 0 || (cmp == 0 && n1 < n2); + } + }; + typedef std::pair Pair; + typedef std::multimap > Map; + typedef typename Map::iterator Iterator; + }; + typedef typename MapTraits::Map Map; + typedef typename MapTraits::Less MapLess; + typedef typename MapTraits::Pair MapPair; + typedef typename MapTraits::Iterator MapIterator; + + // + // Layout of the members' map/array, re(al)located according to the needed capacity: + // + // {Map*}<>{capacity}<>{Member[capacity]}<>{MapIterator[capacity]} + // + // (where <> stands for the RAPIDJSON_ALIGN-ment, if needed) + // + + static RAPIDJSON_FORCEINLINE size_t GetMapLayoutSize(SizeType capacity) { + return RAPIDJSON_ALIGN(sizeof(Map*)) + + RAPIDJSON_ALIGN(sizeof(SizeType)) + + RAPIDJSON_ALIGN(capacity * sizeof(Member)) + + capacity * sizeof(MapIterator); + } + + static RAPIDJSON_FORCEINLINE SizeType &GetMapCapacity(Map* &map) { + return *reinterpret_cast(reinterpret_cast(&map) + + RAPIDJSON_ALIGN(sizeof(Map*))); + } + + static RAPIDJSON_FORCEINLINE Member* GetMapMembers(Map* &map) { + return reinterpret_cast(reinterpret_cast(&map) + + RAPIDJSON_ALIGN(sizeof(Map*)) + + RAPIDJSON_ALIGN(sizeof(SizeType))); + } + + static RAPIDJSON_FORCEINLINE MapIterator* GetMapIterators(Map* &map) { + return reinterpret_cast(reinterpret_cast(&map) + + RAPIDJSON_ALIGN(sizeof(Map*)) + + RAPIDJSON_ALIGN(sizeof(SizeType)) + + RAPIDJSON_ALIGN(GetMapCapacity(map) * sizeof(Member))); + } + + static RAPIDJSON_FORCEINLINE Map* &GetMap(Member* members) { + RAPIDJSON_ASSERT(members != 0); + return *reinterpret_cast(reinterpret_cast(members) - + RAPIDJSON_ALIGN(sizeof(SizeType)) - + RAPIDJSON_ALIGN(sizeof(Map*))); + } + + // Some compilers' debug mechanisms want all iterators to be destroyed, for their accounting.. + RAPIDJSON_FORCEINLINE MapIterator DropMapIterator(MapIterator& rhs) { +#if RAPIDJSON_HAS_CXX11 + MapIterator ret = std::move(rhs); +#else + MapIterator ret = rhs; +#endif + rhs.~MapIterator(); + return ret; + } + + Map* &DoReallocMap(Map** oldMap, SizeType newCapacity, Allocator& allocator) { + Map **newMap = static_cast(allocator.Malloc(GetMapLayoutSize(newCapacity))); + GetMapCapacity(*newMap) = newCapacity; + if (!oldMap) { + *newMap = new (allocator.Malloc(sizeof(Map))) Map(MapLess(), allocator); + } + else { + *newMap = *oldMap; + size_t count = (*oldMap)->size(); + std::memcpy(static_cast(GetMapMembers(*newMap)), + static_cast(GetMapMembers(*oldMap)), + count * sizeof(Member)); + MapIterator *oldIt = GetMapIterators(*oldMap), + *newIt = GetMapIterators(*newMap); + while (count--) { + new (&newIt[count]) MapIterator(DropMapIterator(oldIt[count])); + } + Allocator::Free(oldMap); + } + return *newMap; + } + + RAPIDJSON_FORCEINLINE Member* DoAllocMembers(SizeType capacity, Allocator& allocator) { + return GetMapMembers(DoReallocMap(0, capacity, allocator)); + } + + void DoReserveMembers(SizeType newCapacity, Allocator& allocator) { + ObjectData& o = data_.o; + if (newCapacity > o.capacity) { + Member* oldMembers = GetMembersPointer(); + Map **oldMap = oldMembers ? &GetMap(oldMembers) : 0, + *&newMap = DoReallocMap(oldMap, newCapacity, allocator); + RAPIDJSON_SETPOINTER(Member, o.members, GetMapMembers(newMap)); + o.capacity = newCapacity; + } + } + + template + MemberIterator DoFindMember(const GenericValue& name) { + if (Member* members = GetMembersPointer()) { + Map* &map = GetMap(members); + MapIterator mit = map->find(reinterpret_cast(name.data_)); + if (mit != map->end()) { + return MemberIterator(&members[mit->second]); + } + } + return MemberEnd(); + } + + void DoClearMembers() { + if (Member* members = GetMembersPointer()) { + Map* &map = GetMap(members); + MapIterator* mit = GetMapIterators(map); + for (SizeType i = 0; i < data_.o.size; i++) { + map->erase(DropMapIterator(mit[i])); + members[i].~Member(); + } + data_.o.size = 0; + } + } + + void DoFreeMembers() { + if (Member* members = GetMembersPointer()) { + GetMap(members)->~Map(); + for (SizeType i = 0; i < data_.o.size; i++) { + members[i].~Member(); + } + if (Allocator::kNeedFree) { // Shortcut by Allocator's trait + Map** map = &GetMap(members); + Allocator::Free(*map); + Allocator::Free(map); + } + } + } + +#else // !RAPIDJSON_USE_MEMBERSMAP + + RAPIDJSON_FORCEINLINE Member* DoAllocMembers(SizeType capacity, Allocator& allocator) { + return Malloc(allocator, capacity); + } + + void DoReserveMembers(SizeType newCapacity, Allocator& allocator) { + ObjectData& o = data_.o; + if (newCapacity > o.capacity) { + Member* newMembers = Realloc(allocator, GetMembersPointer(), o.capacity, newCapacity); + RAPIDJSON_SETPOINTER(Member, o.members, newMembers); + o.capacity = newCapacity; + } + } + + template + MemberIterator DoFindMember(const GenericValue& name) { + MemberIterator member = MemberBegin(); + for ( ; member != MemberEnd(); ++member) + if (name.StringEqual(member->name)) + break; + return member; + } + + void DoClearMembers() { + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + data_.o.size = 0; + } + + void DoFreeMembers() { + for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m) + m->~Member(); + Allocator::Free(GetMembersPointer()); + } + +#endif // !RAPIDJSON_USE_MEMBERSMAP + + void DoAddMember(GenericValue& name, GenericValue& value, Allocator& allocator) { + ObjectData& o = data_.o; + if (o.size >= o.capacity) + DoReserveMembers(o.capacity ? (o.capacity + (o.capacity + 1) / 2) : kDefaultObjectCapacity, allocator); + Member* members = GetMembersPointer(); + Member* m = members + o.size; + m->name.RawAssign(name); + m->value.RawAssign(value); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(members); + MapIterator* mit = GetMapIterators(map); + new (&mit[o.size]) MapIterator(map->insert(MapPair(m->name.data_, o.size))); +#endif + ++o.size; + } + + MemberIterator DoRemoveMember(MemberIterator m) { + ObjectData& o = data_.o; + Member* members = GetMembersPointer(); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(members); + MapIterator* mit = GetMapIterators(map); + SizeType mpos = static_cast(&*m - members); + map->erase(DropMapIterator(mit[mpos])); +#endif + MemberIterator last(members + (o.size - 1)); + if (o.size > 1 && m != last) { +#if RAPIDJSON_USE_MEMBERSMAP + new (&mit[mpos]) MapIterator(DropMapIterator(mit[&*last - members])); + mit[mpos]->second = mpos; +#endif + *m = *last; // Move the last one to this place + } + else { + m->~Member(); // Only one left, just destroy + } + --o.size; + return m; + } + + MemberIterator DoEraseMembers(ConstMemberIterator first, ConstMemberIterator last) { + ObjectData& o = data_.o; + MemberIterator beg = MemberBegin(), + pos = beg + (first - beg), + end = MemberEnd(); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(GetMembersPointer()); + MapIterator* mit = GetMapIterators(map); +#endif + for (MemberIterator itr = pos; itr != last; ++itr) { +#if RAPIDJSON_USE_MEMBERSMAP + map->erase(DropMapIterator(mit[itr - beg])); +#endif + itr->~Member(); + } +#if RAPIDJSON_USE_MEMBERSMAP + if (first != last) { + // Move remaining members/iterators + MemberIterator next = pos + (last - first); + for (MemberIterator itr = pos; next != end; ++itr, ++next) { + std::memcpy(static_cast(&*itr), &*next, sizeof(Member)); + SizeType mpos = static_cast(itr - beg); + new (&mit[mpos]) MapIterator(DropMapIterator(mit[next - beg])); + mit[mpos]->second = mpos; + } + } +#else + std::memmove(static_cast(&*pos), &*last, + static_cast(end - last) * sizeof(Member)); +#endif + o.size -= static_cast(last - first); + return pos; + } + + template + void DoCopyMembers(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings) { + RAPIDJSON_ASSERT(rhs.GetType() == kObjectType); + + data_.f.flags = kObjectFlag; + SizeType count = rhs.data_.o.size; + Member* lm = DoAllocMembers(count, allocator); + const typename GenericValue::Member* rm = rhs.GetMembersPointer(); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(lm); + MapIterator* mit = GetMapIterators(map); +#endif + for (SizeType i = 0; i < count; i++) { + new (&lm[i].name) GenericValue(rm[i].name, allocator, copyConstStrings); + new (&lm[i].value) GenericValue(rm[i].value, allocator, copyConstStrings); +#if RAPIDJSON_USE_MEMBERSMAP + new (&mit[i]) MapIterator(map->insert(MapPair(lm[i].name.data_, i))); +#endif + } + data_.o.size = data_.o.capacity = count; + SetMembersPointer(lm); + } + // Initialize this value as array with initial data, without calling destructor. void SetArrayRaw(GenericValue* values, SizeType count, Allocator& allocator) { data_.f.flags = kArrayFlag; @@ -2047,9 +2415,16 @@ class GenericValue { void SetObjectRaw(Member* members, SizeType count, Allocator& allocator) { data_.f.flags = kObjectFlag; if (count) { - Member* m = static_cast(allocator.Malloc(count * sizeof(Member))); + Member* m = DoAllocMembers(count, allocator); SetMembersPointer(m); std::memcpy(static_cast(m), members, count * sizeof(Member)); +#if RAPIDJSON_USE_MEMBERSMAP + Map* &map = GetMap(m); + MapIterator* mit = GetMapIterators(map); + for (SizeType i = 0; i < count; i++) { + new (&mit[i]) MapIterator(map->insert(MapPair(m[i].name.data_, i))); + } +#endif } else SetMembersPointer(0); @@ -2094,11 +2469,11 @@ class GenericValue { const SizeType len1 = GetStringLength(); const SizeType len2 = rhs.GetStringLength(); - if (len1 != len2) { return false; } + if(len1 != len2) { return false; } const Ch* const str1 = GetString(); const Ch* const str2 = rhs.GetString(); - if (str1 == str2) { return true; } // fast path for constant string + if(str1 == str2) { return true; } // fast path for constant string return (std::memcmp(str1, str2, sizeof(Ch) * len1) == 0); } @@ -2120,12 +2495,13 @@ typedef GenericValue > Value; \tparam StackAllocator Allocator for allocating memory for stack during parsing. \warning Although GenericDocument inherits from GenericValue, the API does \b not provide any virtual functions, especially no virtual destructor. To avoid memory leaks, do not \c delete a GenericDocument object via a pointer to a GenericValue. */ -template , typename StackAllocator = CrtAllocator> +template class GenericDocument : public GenericValue { public: typedef typename Encoding::Ch Ch; //!< Character type derived from Encoding. typedef GenericValue ValueType; //!< Value type of the document. typedef Allocator AllocatorType; //!< Allocator type from template parameter. + typedef StackAllocator StackAllocatorType; //!< StackAllocator type from template parameter. //! Constructor /*! Creates an empty document of specified type. @@ -2170,6 +2546,13 @@ class GenericDocument : public GenericValue { #endif ~GenericDocument() { + // Clear the ::ValueType before ownAllocator is destroyed, ~ValueType() + // runs last and may access its elements or members which would be freed + // with an allocator like MemoryPoolAllocator (CrtAllocator does not + // free its data when destroyed, but MemoryPoolAllocator does). + if (ownAllocator_) { + ValueType::SetNull(); + } Destroy(); } @@ -2505,6 +2888,7 @@ class GenericDocument : public GenericValue { //! GenericDocument with UTF8 encoding typedef GenericDocument > Document; + //! Helper class for accessing Value of array type. /*! Instance of this helper class is obtained by \c GenericValue::GetArray(). @@ -2529,6 +2913,7 @@ class GenericArray { GenericArray& operator=(const GenericArray& rhs) { value_ = rhs.value_; return *this; } ~GenericArray() {} + operator ValueType&() const { return value_; } SizeType Size() const { return value_.Size(); } SizeType Capacity() const { return value_.Capacity(); } bool Empty() const { return value_.Empty(); } @@ -2584,6 +2969,7 @@ class GenericObject { GenericObject& operator=(const GenericObject& rhs) { value_ = rhs.value_; return *this; } ~GenericObject() {} + operator ValueType&() const { return value_; } SizeType MemberCount() const { return value_.MemberCount(); } SizeType MemberCapacity() const { return value_.MemberCapacity(); } bool ObjectEmpty() const { return value_.ObjectEmpty(); } @@ -2649,4 +3035,9 @@ class GenericObject { RAPIDJSON_NAMESPACE_END RAPIDJSON_DIAG_POP +#ifdef RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED +#pragma pop_macro("GetObject") +#undef RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED +#endif + #endif // RAPIDJSON_DOCUMENT_H_ diff --git a/src/native/external/rapidjson/encodedstream.h b/src/native/external/rapidjson/encodedstream.h index 223601c0599b4..cf046b89235f5 100644 --- a/src/native/external/rapidjson/encodedstream.h +++ b/src/native/external/rapidjson/encodedstream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/encodings.h b/src/native/external/rapidjson/encodings.h index 0b24467950158..50ad18bdc08cd 100644 --- a/src/native/external/rapidjson/encodings.h +++ b/src/native/external/rapidjson/encodings.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/error/en.h b/src/native/external/rapidjson/error/en.h index 2db838bff2399..c87b04eb133ed 100644 --- a/src/native/external/rapidjson/error/en.h +++ b/src/native/external/rapidjson/error/en.h @@ -1,15 +1,15 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_EN_H_ @@ -39,13 +39,13 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not be followed by other values."); - + case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); - + case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); - + case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); @@ -65,6 +65,108 @@ inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErro } } +//! Maps error code of validation into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param validateErrorCode Error code obtained from validator. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const RAPIDJSON_ERROR_CHARTYPE* GetValidateError_En(ValidateErrorCode validateErrorCode) { + switch (validateErrorCode) { + case kValidateErrors: return RAPIDJSON_ERROR_STRING("One or more validation errors have occurred"); + case kValidateErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kValidateErrorMultipleOf: return RAPIDJSON_ERROR_STRING("Number '%actual' is not a multiple of the 'multipleOf' value '%expected'."); + case kValidateErrorMaximum: return RAPIDJSON_ERROR_STRING("Number '%actual' is greater than the 'maximum' value '%expected'."); + case kValidateErrorExclusiveMaximum: return RAPIDJSON_ERROR_STRING("Number '%actual' is greater than or equal to the 'exclusiveMaximum' value '%expected'."); + case kValidateErrorMinimum: return RAPIDJSON_ERROR_STRING("Number '%actual' is less than the 'minimum' value '%expected'."); + case kValidateErrorExclusiveMinimum: return RAPIDJSON_ERROR_STRING("Number '%actual' is less than or equal to the 'exclusiveMinimum' value '%expected'."); + + case kValidateErrorMaxLength: return RAPIDJSON_ERROR_STRING("String '%actual' is longer than the 'maxLength' value '%expected'."); + case kValidateErrorMinLength: return RAPIDJSON_ERROR_STRING("String '%actual' is shorter than the 'minLength' value '%expected'."); + case kValidateErrorPattern: return RAPIDJSON_ERROR_STRING("String '%actual' does not match the 'pattern' regular expression."); + + case kValidateErrorMaxItems: return RAPIDJSON_ERROR_STRING("Array of length '%actual' is longer than the 'maxItems' value '%expected'."); + case kValidateErrorMinItems: return RAPIDJSON_ERROR_STRING("Array of length '%actual' is shorter than the 'minItems' value '%expected'."); + case kValidateErrorUniqueItems: return RAPIDJSON_ERROR_STRING("Array has duplicate items at indices '%duplicates' but 'uniqueItems' is true."); + case kValidateErrorAdditionalItems: return RAPIDJSON_ERROR_STRING("Array has an additional item at index '%disallowed' that is not allowed by the schema."); + + case kValidateErrorMaxProperties: return RAPIDJSON_ERROR_STRING("Object has '%actual' members which is more than 'maxProperties' value '%expected'."); + case kValidateErrorMinProperties: return RAPIDJSON_ERROR_STRING("Object has '%actual' members which is less than 'minProperties' value '%expected'."); + case kValidateErrorRequired: return RAPIDJSON_ERROR_STRING("Object is missing the following members required by the schema: '%missing'."); + case kValidateErrorAdditionalProperties: return RAPIDJSON_ERROR_STRING("Object has an additional member '%disallowed' that is not allowed by the schema."); + case kValidateErrorPatternProperties: return RAPIDJSON_ERROR_STRING("Object has 'patternProperties' that are not allowed by the schema."); + case kValidateErrorDependencies: return RAPIDJSON_ERROR_STRING("Object has missing property or schema dependencies, refer to following errors."); + + case kValidateErrorEnum: return RAPIDJSON_ERROR_STRING("Property has a value that is not one of its allowed enumerated values."); + case kValidateErrorType: return RAPIDJSON_ERROR_STRING("Property has a type '%actual' that is not in the following list: '%expected'."); + + case kValidateErrorOneOf: return RAPIDJSON_ERROR_STRING("Property did not match any of the sub-schemas specified by 'oneOf', refer to following errors."); + case kValidateErrorOneOfMatch: return RAPIDJSON_ERROR_STRING("Property matched more than one of the sub-schemas specified by 'oneOf', indices '%matches'."); + case kValidateErrorAllOf: return RAPIDJSON_ERROR_STRING("Property did not match all of the sub-schemas specified by 'allOf', refer to following errors."); + case kValidateErrorAnyOf: return RAPIDJSON_ERROR_STRING("Property did not match any of the sub-schemas specified by 'anyOf', refer to following errors."); + case kValidateErrorNot: return RAPIDJSON_ERROR_STRING("Property matched the sub-schema specified by 'not'."); + + case kValidateErrorReadOnly: return RAPIDJSON_ERROR_STRING("Property is read-only but has been provided when validation is for writing."); + case kValidateErrorWriteOnly: return RAPIDJSON_ERROR_STRING("Property is write-only but has been provided when validation is for reading."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + +//! Maps error code of schema document compilation into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param schemaErrorCode Error code obtained from compiling the schema document. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ + inline const RAPIDJSON_ERROR_CHARTYPE* GetSchemaError_En(SchemaErrorCode schemaErrorCode) { + switch (schemaErrorCode) { + case kSchemaErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kSchemaErrorStartUnknown: return RAPIDJSON_ERROR_STRING("Pointer '%value' to start of schema does not resolve to a location in the document."); + case kSchemaErrorRefPlainName: return RAPIDJSON_ERROR_STRING("$ref fragment '%value' must be a JSON pointer."); + case kSchemaErrorRefInvalid: return RAPIDJSON_ERROR_STRING("$ref must not be an empty string."); + case kSchemaErrorRefPointerInvalid: return RAPIDJSON_ERROR_STRING("$ref fragment '%value' is not a valid JSON pointer at offset '%offset'."); + case kSchemaErrorRefUnknown: return RAPIDJSON_ERROR_STRING("$ref '%value' does not resolve to a location in the target document."); + case kSchemaErrorRefCyclical: return RAPIDJSON_ERROR_STRING("$ref '%value' is cyclical."); + case kSchemaErrorRefNoRemoteProvider: return RAPIDJSON_ERROR_STRING("$ref is remote but there is no remote provider."); + case kSchemaErrorRefNoRemoteSchema: return RAPIDJSON_ERROR_STRING("$ref '%value' is remote but the remote provider did not return a schema."); + case kSchemaErrorRegexInvalid: return RAPIDJSON_ERROR_STRING("Invalid regular expression '%value' in 'pattern' or 'patternProperties'."); + case kSchemaErrorSpecUnknown: return RAPIDJSON_ERROR_STRING("JSON schema draft or OpenAPI version is not recognized."); + case kSchemaErrorSpecUnsupported: return RAPIDJSON_ERROR_STRING("JSON schema draft or OpenAPI version is not supported."); + case kSchemaErrorSpecIllegal: return RAPIDJSON_ERROR_STRING("Both JSON schema draft and OpenAPI version found in document."); + case kSchemaErrorReadOnlyAndWriteOnly: return RAPIDJSON_ERROR_STRING("Property must not be both 'readOnly' and 'writeOnly'."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } + } + +//! Maps error code of pointer parse into error message. +/*! + \ingroup RAPIDJSON_ERRORS + \param pointerParseErrorCode Error code obtained from pointer parse. + \return the error message. + \note User can make a copy of this function for localization. + Using switch-case is safer for future modification of error codes. +*/ +inline const RAPIDJSON_ERROR_CHARTYPE* GetPointerParseError_En(PointerParseErrorCode pointerParseErrorCode) { + switch (pointerParseErrorCode) { + case kPointerParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); + + case kPointerParseErrorTokenMustBeginWithSolidus: return RAPIDJSON_ERROR_STRING("A token must begin with a '/'."); + case kPointerParseErrorInvalidEscape: return RAPIDJSON_ERROR_STRING("Invalid escape."); + case kPointerParseErrorInvalidPercentEncoding: return RAPIDJSON_ERROR_STRING("Invalid percent encoding in URI fragment."); + case kPointerParseErrorCharacterMustPercentEncode: return RAPIDJSON_ERROR_STRING("A character must be percent encoded in a URI fragment."); + + default: return RAPIDJSON_ERROR_STRING("Unknown error."); + } +} + RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/src/native/external/rapidjson/error/error.h b/src/native/external/rapidjson/error/error.h index 9311d2f03bffe..cae345db36d2c 100644 --- a/src/native/external/rapidjson/error/error.h +++ b/src/native/external/rapidjson/error/error.h @@ -1,15 +1,15 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_ERROR_ERROR_H_ @@ -42,7 +42,7 @@ RAPIDJSON_DIAG_OFF(padded) /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_ERROR_STRING -//! Macro for converting string literial to \ref RAPIDJSON_ERROR_CHARTYPE[]. +//! Macro for converting string literal to \ref RAPIDJSON_ERROR_CHARTYPE[]. /*! \ingroup RAPIDJSON_ERRORS By default this conversion macro does nothing. On Windows, user can define this macro as \c _T(x) for supporting both @@ -152,6 +152,130 @@ struct ParseResult { */ typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetParseErrorFunc)(ParseErrorCode); +/////////////////////////////////////////////////////////////////////////////// +// ValidateErrorCode + +//! Error codes when validating. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericSchemaValidator +*/ +enum ValidateErrorCode { + kValidateErrors = -1, //!< Top level error code when kValidateContinueOnErrorsFlag set. + kValidateErrorNone = 0, //!< No error. + + kValidateErrorMultipleOf, //!< Number is not a multiple of the 'multipleOf' value. + kValidateErrorMaximum, //!< Number is greater than the 'maximum' value. + kValidateErrorExclusiveMaximum, //!< Number is greater than or equal to the 'maximum' value. + kValidateErrorMinimum, //!< Number is less than the 'minimum' value. + kValidateErrorExclusiveMinimum, //!< Number is less than or equal to the 'minimum' value. + + kValidateErrorMaxLength, //!< String is longer than the 'maxLength' value. + kValidateErrorMinLength, //!< String is longer than the 'maxLength' value. + kValidateErrorPattern, //!< String does not match the 'pattern' regular expression. + + kValidateErrorMaxItems, //!< Array is longer than the 'maxItems' value. + kValidateErrorMinItems, //!< Array is shorter than the 'minItems' value. + kValidateErrorUniqueItems, //!< Array has duplicate items but 'uniqueItems' is true. + kValidateErrorAdditionalItems, //!< Array has additional items that are not allowed by the schema. + + kValidateErrorMaxProperties, //!< Object has more members than 'maxProperties' value. + kValidateErrorMinProperties, //!< Object has less members than 'minProperties' value. + kValidateErrorRequired, //!< Object is missing one or more members required by the schema. + kValidateErrorAdditionalProperties, //!< Object has additional members that are not allowed by the schema. + kValidateErrorPatternProperties, //!< See other errors. + kValidateErrorDependencies, //!< Object has missing property or schema dependencies. + + kValidateErrorEnum, //!< Property has a value that is not one of its allowed enumerated values. + kValidateErrorType, //!< Property has a type that is not allowed by the schema. + + kValidateErrorOneOf, //!< Property did not match any of the sub-schemas specified by 'oneOf'. + kValidateErrorOneOfMatch, //!< Property matched more than one of the sub-schemas specified by 'oneOf'. + kValidateErrorAllOf, //!< Property did not match all of the sub-schemas specified by 'allOf'. + kValidateErrorAnyOf, //!< Property did not match any of the sub-schemas specified by 'anyOf'. + kValidateErrorNot, //!< Property matched the sub-schema specified by 'not'. + + kValidateErrorReadOnly, //!< Property is read-only but has been provided when validation is for writing + kValidateErrorWriteOnly //!< Property is write-only but has been provided when validation is for reading +}; + +//! Function pointer type of GetValidateError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetValidateError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetValidateErrorFunc GetValidateError = GetValidateError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetValidateError(validator.GetInvalidSchemaCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetValidateErrorFunc)(ValidateErrorCode); + +/////////////////////////////////////////////////////////////////////////////// +// SchemaErrorCode + +//! Error codes when validating. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericSchemaValidator +*/ +enum SchemaErrorCode { + kSchemaErrorNone = 0, //!< No error. + + kSchemaErrorStartUnknown, //!< Pointer to start of schema does not resolve to a location in the document + kSchemaErrorRefPlainName, //!< $ref fragment must be a JSON pointer + kSchemaErrorRefInvalid, //!< $ref must not be an empty string + kSchemaErrorRefPointerInvalid, //!< $ref fragment is not a valid JSON pointer at offset + kSchemaErrorRefUnknown, //!< $ref does not resolve to a location in the target document + kSchemaErrorRefCyclical, //!< $ref is cyclical + kSchemaErrorRefNoRemoteProvider, //!< $ref is remote but there is no remote provider + kSchemaErrorRefNoRemoteSchema, //!< $ref is remote but the remote provider did not return a schema + kSchemaErrorRegexInvalid, //!< Invalid regular expression in 'pattern' or 'patternProperties' + kSchemaErrorSpecUnknown, //!< JSON schema draft or OpenAPI version is not recognized + kSchemaErrorSpecUnsupported, //!< JSON schema draft or OpenAPI version is not supported + kSchemaErrorSpecIllegal, //!< Both JSON schema draft and OpenAPI version found in document + kSchemaErrorReadOnlyAndWriteOnly //!< Property must not be both 'readOnly' and 'writeOnly' +}; + +//! Function pointer type of GetSchemaError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetSchemaError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetSchemaErrorFunc GetSchemaError = GetSchemaError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetSchemaError(validator.GetInvalidSchemaCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetSchemaErrorFunc)(SchemaErrorCode); + +/////////////////////////////////////////////////////////////////////////////// +// PointerParseErrorCode + +//! Error code of JSON pointer parsing. +/*! \ingroup RAPIDJSON_ERRORS + \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode +*/ +enum PointerParseErrorCode { + kPointerParseErrorNone = 0, //!< The parse is successful + + kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' + kPointerParseErrorInvalidEscape, //!< Invalid escape + kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment + kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment +}; + +//! Function pointer type of GetPointerParseError(). +/*! \ingroup RAPIDJSON_ERRORS + + This is the prototype for \c GetPointerParseError_X(), where \c X is a locale. + User can dynamically change locale in runtime, e.g.: +\code + GetPointerParseErrorFunc GetPointerParseError = GetPointerParseError_En; // or whatever + const RAPIDJSON_ERROR_CHARTYPE* s = GetPointerParseError(pointer.GetParseErrorCode()); +\endcode +*/ +typedef const RAPIDJSON_ERROR_CHARTYPE* (*GetPointerParseErrorFunc)(PointerParseErrorCode); + + RAPIDJSON_NAMESPACE_END #ifdef __clang__ diff --git a/src/native/external/rapidjson/filereadstream.h b/src/native/external/rapidjson/filereadstream.h deleted file mode 100644 index 6b343707ade08..0000000000000 --- a/src/native/external/rapidjson/filereadstream.h +++ /dev/null @@ -1,99 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_FILEREADSTREAM_H_ -#define RAPIDJSON_FILEREADSTREAM_H_ - -#include "stream.h" -#include - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(unreachable-code) -RAPIDJSON_DIAG_OFF(missing-noreturn) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! File byte stream for input using fread(). -/*! - \note implements Stream concept -*/ -class FileReadStream { -public: - typedef char Ch; //!< Character type (byte). - - //! Constructor. - /*! - \param fp File pointer opened for read. - \param buffer user-supplied buffer. - \param bufferSize size of buffer in bytes. Must >=4 bytes. - */ - FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { - RAPIDJSON_ASSERT(fp_ != 0); - RAPIDJSON_ASSERT(bufferSize >= 4); - Read(); - } - - Ch Peek() const { return *current_; } - Ch Take() { Ch c = *current_; Read(); return c; } - size_t Tell() const { return count_ + static_cast(current_ - buffer_); } - - // Not implemented - void Put(Ch) { RAPIDJSON_ASSERT(false); } - void Flush() { RAPIDJSON_ASSERT(false); } - Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } - - // For encoding detection only. - const Ch* Peek4() const { - return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0; - } - -private: - void Read() { - if (current_ < bufferLast_) - ++current_; - else if (!eof_) { - count_ += readCount_; - readCount_ = std::fread(buffer_, 1, bufferSize_, fp_); - bufferLast_ = buffer_ + readCount_ - 1; - current_ = buffer_; - - if (readCount_ < bufferSize_) { - buffer_[readCount_] = '\0'; - ++bufferLast_; - eof_ = true; - } - } - } - - std::FILE* fp_; - Ch *buffer_; - size_t bufferSize_; - Ch *bufferLast_; - Ch *current_; - size_t readCount_; - size_t count_; //!< Number of characters read - bool eof_; -}; - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/src/native/external/rapidjson/filewritestream.h b/src/native/external/rapidjson/filewritestream.h deleted file mode 100644 index 8b48fee197c43..0000000000000 --- a/src/native/external/rapidjson/filewritestream.h +++ /dev/null @@ -1,104 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_FILEWRITESTREAM_H_ -#define RAPIDJSON_FILEWRITESTREAM_H_ - -#include "stream.h" -#include - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(unreachable-code) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of C file stream for output using fwrite(). -/*! - \note implements Stream concept -*/ -class FileWriteStream { -public: - typedef char Ch; //!< Character type. Only support char. - - FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { - RAPIDJSON_ASSERT(fp_ != 0); - } - - void Put(char c) { - if (current_ >= bufferEnd_) - Flush(); - - *current_++ = c; - } - - void PutN(char c, size_t n) { - size_t avail = static_cast(bufferEnd_ - current_); - while (n > avail) { - std::memset(current_, c, avail); - current_ += avail; - Flush(); - n -= avail; - avail = static_cast(bufferEnd_ - current_); - } - - if (n > 0) { - std::memset(current_, c, n); - current_ += n; - } - } - - void Flush() { - if (current_ != buffer_) { - size_t result = std::fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); - if (result < static_cast(current_ - buffer_)) { - // failure deliberately ignored at this time - // added to avoid warn_unused_result build errors - } - current_ = buffer_; - } - } - - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - // Prohibit copy constructor & assignment operator. - FileWriteStream(const FileWriteStream&); - FileWriteStream& operator=(const FileWriteStream&); - - std::FILE* fp_; - char *buffer_; - char *bufferEnd_; - char *current_; -}; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(FileWriteStream& stream, char c, size_t n) { - stream.PutN(c, n); -} - -RAPIDJSON_NAMESPACE_END - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_FILESTREAM_H_ diff --git a/src/native/external/rapidjson/fwd.h b/src/native/external/rapidjson/fwd.h index e8104e841bcdc..d62f77f0ecfaf 100644 --- a/src/native/external/rapidjson/fwd.h +++ b/src/native/external/rapidjson/fwd.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -102,7 +102,7 @@ class PrettyWriter; // document.h template -struct GenericMember; +class GenericMember; template class GenericMemberIterator; diff --git a/src/native/external/rapidjson/internal/biginteger.h b/src/native/external/rapidjson/internal/biginteger.h index a31c8a88d6eb4..4930043dc7c5f 100644 --- a/src/native/external/rapidjson/internal/biginteger.h +++ b/src/native/external/rapidjson/internal/biginteger.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -17,9 +17,13 @@ #include "../rapidjson.h" -#if defined(_MSC_VER) && !__INTEL_COMPILER && defined(_M_AMD64) +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && defined(_M_AMD64) #include // for _umul128 +#if !defined(_ARM64EC_) #pragma intrinsic(_umul128) +#else +#pragma comment(lib,"softintrin") +#endif #endif RAPIDJSON_NAMESPACE_BEGIN @@ -37,7 +41,8 @@ class BigInteger { digits_[0] = u; } - BigInteger(const char* decimals, size_t length) : count_(1) { + template + BigInteger(const Ch* decimals, size_t length) : count_(1) { RAPIDJSON_ASSERT(length > 0); digits_[0] = 0; size_t i = 0; @@ -221,7 +226,8 @@ class BigInteger { bool IsZero() const { return count_ == 1 && digits_[0] == 0; } private: - void AppendDecimal64(const char* begin, const char* end) { + template + void AppendDecimal64(const Ch* begin, const Ch* end) { uint64_t u = ParseUint64(begin, end); if (IsZero()) *this = u; @@ -236,11 +242,12 @@ class BigInteger { digits_[count_++] = digit; } - static uint64_t ParseUint64(const char* begin, const char* end) { + template + static uint64_t ParseUint64(const Ch* begin, const Ch* end) { uint64_t r = 0; - for (const char* p = begin; p != end; ++p) { - RAPIDJSON_ASSERT(*p >= '0' && *p <= '9'); - r = r * 10u + static_cast(*p - '0'); + for (const Ch* p = begin; p != end; ++p) { + RAPIDJSON_ASSERT(*p >= Ch('0') && *p <= Ch('9')); + r = r * 10u + static_cast(*p - Ch('0')); } return r; } @@ -252,7 +259,7 @@ class BigInteger { if (low < k) (*outHigh)++; return low; -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(a) * static_cast(b); p += k; diff --git a/src/native/external/rapidjson/internal/clzll.h b/src/native/external/rapidjson/internal/clzll.h new file mode 100644 index 0000000000000..8fc5118aa47b8 --- /dev/null +++ b/src/native/external/rapidjson/internal/clzll.h @@ -0,0 +1,71 @@ +// Tencent is pleased to support the open source community by making RapidJSON available. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. +// +// Licensed under the MIT License (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at +// +// http://opensource.org/licenses/MIT +// +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +#ifndef RAPIDJSON_CLZLL_H_ +#define RAPIDJSON_CLZLL_H_ + +#include "../rapidjson.h" + +#if defined(_MSC_VER) && !defined(UNDER_CE) +#include +#if defined(_WIN64) +#pragma intrinsic(_BitScanReverse64) +#else +#pragma intrinsic(_BitScanReverse) +#endif +#endif + +RAPIDJSON_NAMESPACE_BEGIN +namespace internal { + +inline uint32_t clzll(uint64_t x) { + // Passing 0 to __builtin_clzll is UB in GCC and results in an + // infinite loop in the software implementation. + RAPIDJSON_ASSERT(x != 0); + +#if defined(_MSC_VER) && !defined(UNDER_CE) + unsigned long r = 0; +#if defined(_WIN64) + _BitScanReverse64(&r, x); +#else + // Scan the high 32 bits. + if (_BitScanReverse(&r, static_cast(x >> 32))) + return 63 - (r + 32); + + // Scan the low 32 bits. + _BitScanReverse(&r, static_cast(x & 0xFFFFFFFF)); +#endif // _WIN64 + + return 63 - r; +#elif (defined(__GNUC__) && __GNUC__ >= 4) || RAPIDJSON_HAS_BUILTIN(__builtin_clzll) + // __builtin_clzll wrapper + return static_cast(__builtin_clzll(x)); +#else + // naive version + uint32_t r = 0; + while (!(x & (static_cast(1) << 63))) { + x <<= 1; + ++r; + } + + return r; +#endif // _MSC_VER +} + +#define RAPIDJSON_CLZLL RAPIDJSON_NAMESPACE::internal::clzll + +} // namespace internal +RAPIDJSON_NAMESPACE_END + +#endif // RAPIDJSON_CLZLL_H_ diff --git a/src/native/external/rapidjson/internal/diyfp.h b/src/native/external/rapidjson/internal/diyfp.h index b6c2cf5618d45..1f60fb60ca043 100644 --- a/src/native/external/rapidjson/internal/diyfp.h +++ b/src/native/external/rapidjson/internal/diyfp.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -20,12 +20,16 @@ #define RAPIDJSON_DIYFP_H_ #include "../rapidjson.h" +#include "clzll.h" #include #if defined(_MSC_VER) && defined(_M_AMD64) && !defined(__INTEL_COMPILER) #include -#pragma intrinsic(_BitScanReverse64) +#if !defined(_ARM64EC_) #pragma intrinsic(_umul128) +#else +#pragma comment(lib,"softintrin") +#endif #endif RAPIDJSON_NAMESPACE_BEGIN @@ -75,7 +79,7 @@ struct DiyFp { if (l & (uint64_t(1) << 63)) // rounding h++; return DiyFp(h, e + rhs.e + 64); -#elif (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6)) && defined(__x86_64__) __extension__ typedef unsigned __int128 uint128; uint128 p = static_cast(f) * static_cast(rhs.f); uint64_t h = static_cast(p >> 64); @@ -100,22 +104,8 @@ struct DiyFp { } DiyFp Normalize() const { - RAPIDJSON_ASSERT(f != 0); // https://stackoverflow.com/a/26809183/291737 -#if defined(_MSC_VER) && defined(_M_AMD64) - unsigned long index; - _BitScanReverse64(&index, f); - return DiyFp(f << (63 - index), e - (63 - index)); -#elif defined(__GNUC__) && __GNUC__ >= 4 - int s = __builtin_clzll(f); + int s = static_cast(clzll(f)); return DiyFp(f << s, e - s); -#else - DiyFp res = *this; - while (!(res.f & (static_cast(1) << 63))) { - res.f <<= 1; - res.e--; - } - return res; -#endif } DiyFp NormalizeBoundary() const { diff --git a/src/native/external/rapidjson/internal/dtoa.h b/src/native/external/rapidjson/internal/dtoa.h index bf2e9b2e59a49..cd456721a71c0 100644 --- a/src/native/external/rapidjson/internal/dtoa.h +++ b/src/native/external/rapidjson/internal/dtoa.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -58,7 +58,11 @@ inline int CountDecimalDigit32(uint32_t n) { } inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buffer, int* len, int* K) { - static const uint32_t kPow10[] = { 1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 }; + static const uint64_t kPow10[] = { 1ULL, 10ULL, 100ULL, 1000ULL, 10000ULL, 100000ULL, 1000000ULL, 10000000ULL, 100000000ULL, + 1000000000ULL, 10000000000ULL, 100000000000ULL, 1000000000000ULL, + 10000000000000ULL, 100000000000000ULL, 1000000000000000ULL, + 10000000000000000ULL, 100000000000000000ULL, 1000000000000000000ULL, + 10000000000000000000ULL }; const DiyFp one(uint64_t(1) << -Mp.e, Mp.e); const DiyFp wp_w = Mp - W; uint32_t p1 = static_cast(Mp.f >> -one.e); @@ -86,7 +90,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff uint64_t tmp = (static_cast(p1) << -one.e) + p2; if (tmp <= delta) { *K += kappa; - GrisuRound(buffer, *len, delta, tmp, static_cast(kPow10[kappa]) << -one.e, wp_w.f); + GrisuRound(buffer, *len, delta, tmp, kPow10[kappa] << -one.e, wp_w.f); return; } } @@ -103,7 +107,7 @@ inline void DigitGen(const DiyFp& W, const DiyFp& Mp, uint64_t delta, char* buff if (p2 < delta) { *K += kappa; int index = -kappa; - GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 9 ? kPow10[index] : 0)); + GrisuRound(buffer, *len, delta, p2, one.f, wp_w.f * (index < 20 ? kPow10[index] : 0)); return; } } diff --git a/src/native/external/rapidjson/internal/ieee754.h b/src/native/external/rapidjson/internal/ieee754.h index c2684ba2a35ff..68c9e96649b8a 100644 --- a/src/native/external/rapidjson/internal/ieee754.h +++ b/src/native/external/rapidjson/internal/ieee754.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/itoa.h b/src/native/external/rapidjson/internal/itoa.h index 9b1c45cc1b4a8..9fe8c932ffa6d 100644 --- a/src/native/external/rapidjson/internal/itoa.h +++ b/src/native/external/rapidjson/internal/itoa.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/meta.h b/src/native/external/rapidjson/internal/meta.h index d401edf85150d..27092dc0d69c4 100644 --- a/src/native/external/rapidjson/internal/meta.h +++ b/src/native/external/rapidjson/internal/meta.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/pow10.h b/src/native/external/rapidjson/internal/pow10.h index 02f475d705fcb..eae1a43ed1a06 100644 --- a/src/native/external/rapidjson/internal/pow10.h +++ b/src/native/external/rapidjson/internal/pow10.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/regex.h b/src/native/external/rapidjson/internal/regex.h deleted file mode 100644 index 16e355921f884..0000000000000 --- a/src/native/external/rapidjson/internal/regex.h +++ /dev/null @@ -1,740 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_INTERNAL_REGEX_H_ -#define RAPIDJSON_INTERNAL_REGEX_H_ - -#include "../allocators.h" -#include "../stream.h" -#include "stack.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -RAPIDJSON_DIAG_OFF(switch-enum) -RAPIDJSON_DIAG_OFF(implicit-fallthrough) -#elif defined(_MSC_VER) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#if __GNUC__ >= 7 -RAPIDJSON_DIAG_OFF(implicit-fallthrough) -#endif -#endif - -#ifndef RAPIDJSON_REGEX_VERBOSE -#define RAPIDJSON_REGEX_VERBOSE 0 -#endif - -RAPIDJSON_NAMESPACE_BEGIN -namespace internal { - -/////////////////////////////////////////////////////////////////////////////// -// DecodedStream - -template -class DecodedStream { -public: - DecodedStream(SourceStream& ss) : ss_(ss), codepoint_() { Decode(); } - unsigned Peek() { return codepoint_; } - unsigned Take() { - unsigned c = codepoint_; - if (c) // No further decoding when '\0' - Decode(); - return c; - } - -private: - void Decode() { - if (!Encoding::Decode(ss_, &codepoint_)) - codepoint_ = 0; - } - - SourceStream& ss_; - unsigned codepoint_; -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericRegex - -static const SizeType kRegexInvalidState = ~SizeType(0); //!< Represents an invalid index in GenericRegex::State::out, out1 -static const SizeType kRegexInvalidRange = ~SizeType(0); - -template -class GenericRegexSearch; - -//! Regular expression engine with subset of ECMAscript grammar. -/*! - Supported regular expression syntax: - - \c ab Concatenation - - \c a|b Alternation - - \c a? Zero or one - - \c a* Zero or more - - \c a+ One or more - - \c a{3} Exactly 3 times - - \c a{3,} At least 3 times - - \c a{3,5} 3 to 5 times - - \c (ab) Grouping - - \c ^a At the beginning - - \c a$ At the end - - \c . Any character - - \c [abc] Character classes - - \c [a-c] Character class range - - \c [a-z0-9_] Character class combination - - \c [^abc] Negated character classes - - \c [^a-c] Negated character class range - - \c [\b] Backspace (U+0008) - - \c \\| \\\\ ... Escape characters - - \c \\f Form feed (U+000C) - - \c \\n Line feed (U+000A) - - \c \\r Carriage return (U+000D) - - \c \\t Tab (U+0009) - - \c \\v Vertical tab (U+000B) - - \note This is a Thompson NFA engine, implemented with reference to - Cox, Russ. "Regular Expression Matching Can Be Simple And Fast (but is slow in Java, Perl, PHP, Python, Ruby,...).", - https://swtch.com/~rsc/regexp/regexp1.html -*/ -template -class GenericRegex { -public: - typedef Encoding EncodingType; - typedef typename Encoding::Ch Ch; - template friend class GenericRegexSearch; - - GenericRegex(const Ch* source, Allocator* allocator = 0) : - ownAllocator_(allocator ? 0 : RAPIDJSON_NEW(Allocator)()), allocator_(allocator ? allocator : ownAllocator_), - states_(allocator_, 256), ranges_(allocator_, 256), root_(kRegexInvalidState), stateCount_(), rangeCount_(), - anchorBegin_(), anchorEnd_() - { - GenericStringStream ss(source); - DecodedStream, Encoding> ds(ss); - Parse(ds); - } - - ~GenericRegex() - { - RAPIDJSON_DELETE(ownAllocator_); - } - - bool IsValid() const { - return root_ != kRegexInvalidState; - } - -private: - enum Operator { - kZeroOrOne, - kZeroOrMore, - kOneOrMore, - kConcatenation, - kAlternation, - kLeftParenthesis - }; - - static const unsigned kAnyCharacterClass = 0xFFFFFFFF; //!< For '.' - static const unsigned kRangeCharacterClass = 0xFFFFFFFE; - static const unsigned kRangeNegationFlag = 0x80000000; - - struct Range { - unsigned start; // - unsigned end; - SizeType next; - }; - - struct State { - SizeType out; //!< Equals to kInvalid for matching state - SizeType out1; //!< Equals to non-kInvalid for split - SizeType rangeStart; - unsigned codepoint; - }; - - struct Frag { - Frag(SizeType s, SizeType o, SizeType m) : start(s), out(o), minIndex(m) {} - SizeType start; - SizeType out; //!< link-list of all output states - SizeType minIndex; - }; - - State& GetState(SizeType index) { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom()[index]; - } - - const State& GetState(SizeType index) const { - RAPIDJSON_ASSERT(index < stateCount_); - return states_.template Bottom()[index]; - } - - Range& GetRange(SizeType index) { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom()[index]; - } - - const Range& GetRange(SizeType index) const { - RAPIDJSON_ASSERT(index < rangeCount_); - return ranges_.template Bottom()[index]; - } - - template - void Parse(DecodedStream& ds) { - Stack operandStack(allocator_, 256); // Frag - Stack operatorStack(allocator_, 256); // Operator - Stack atomCountStack(allocator_, 256); // unsigned (Atom per parenthesis) - - *atomCountStack.template Push() = 0; - - unsigned codepoint; - while (ds.Peek() != 0) { - switch (codepoint = ds.Take()) { - case '^': - anchorBegin_ = true; - break; - - case '$': - anchorEnd_ = true; - break; - - case '|': - while (!operatorStack.Empty() && *operatorStack.template Top() < kAlternation) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - *operatorStack.template Push() = kAlternation; - *atomCountStack.template Top() = 0; - break; - - case '(': - *operatorStack.template Push() = kLeftParenthesis; - *atomCountStack.template Push() = 0; - break; - - case ')': - while (!operatorStack.Empty() && *operatorStack.template Top() != kLeftParenthesis) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - if (operatorStack.Empty()) - return; - operatorStack.template Pop(1); - atomCountStack.template Pop(1); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '?': - if (!Eval(operandStack, kZeroOrOne)) - return; - break; - - case '*': - if (!Eval(operandStack, kZeroOrMore)) - return; - break; - - case '+': - if (!Eval(operandStack, kOneOrMore)) - return; - break; - - case '{': - { - unsigned n, m; - if (!ParseUnsigned(ds, &n)) - return; - - if (ds.Peek() == ',') { - ds.Take(); - if (ds.Peek() == '}') - m = kInfinityQuantifier; - else if (!ParseUnsigned(ds, &m) || m < n) - return; - } - else - m = n; - - if (!EvalQuantifier(operandStack, n, m) || ds.Peek() != '}') - return; - ds.Take(); - } - break; - - case '.': - PushOperand(operandStack, kAnyCharacterClass); - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '[': - { - SizeType range; - if (!ParseRange(ds, &range)) - return; - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, kRangeCharacterClass); - GetState(s).rangeStart = range; - *operandStack.template Push() = Frag(s, s, s); - } - ImplicitConcatenation(atomCountStack, operatorStack); - break; - - case '\\': // Escape character - if (!CharacterEscape(ds, &codepoint)) - return; // Unsupported escape character - // fall through to default - - default: // Pattern character - PushOperand(operandStack, codepoint); - ImplicitConcatenation(atomCountStack, operatorStack); - } - } - - while (!operatorStack.Empty()) - if (!Eval(operandStack, *operatorStack.template Pop(1))) - return; - - // Link the operand to matching state. - if (operandStack.GetSize() == sizeof(Frag)) { - Frag* e = operandStack.template Pop(1); - Patch(e->out, NewState(kRegexInvalidState, kRegexInvalidState, 0)); - root_ = e->start; - -#if RAPIDJSON_REGEX_VERBOSE - printf("root: %d\n", root_); - for (SizeType i = 0; i < stateCount_ ; i++) { - State& s = GetState(i); - printf("[%2d] out: %2d out1: %2d c: '%c'\n", i, s.out, s.out1, (char)s.codepoint); - } - printf("\n"); -#endif - } - } - - SizeType NewState(SizeType out, SizeType out1, unsigned codepoint) { - State* s = states_.template Push(); - s->out = out; - s->out1 = out1; - s->codepoint = codepoint; - s->rangeStart = kRegexInvalidRange; - return stateCount_++; - } - - void PushOperand(Stack& operandStack, unsigned codepoint) { - SizeType s = NewState(kRegexInvalidState, kRegexInvalidState, codepoint); - *operandStack.template Push() = Frag(s, s, s); - } - - void ImplicitConcatenation(Stack& atomCountStack, Stack& operatorStack) { - if (*atomCountStack.template Top()) - *operatorStack.template Push() = kConcatenation; - (*atomCountStack.template Top())++; - } - - SizeType Append(SizeType l1, SizeType l2) { - SizeType old = l1; - while (GetState(l1).out != kRegexInvalidState) - l1 = GetState(l1).out; - GetState(l1).out = l2; - return old; - } - - void Patch(SizeType l, SizeType s) { - for (SizeType next; l != kRegexInvalidState; l = next) { - next = GetState(l).out; - GetState(l).out = s; - } - } - - bool Eval(Stack& operandStack, Operator op) { - switch (op) { - case kConcatenation: - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag) * 2); - { - Frag e2 = *operandStack.template Pop(1); - Frag e1 = *operandStack.template Pop(1); - Patch(e1.out, e2.start); - *operandStack.template Push() = Frag(e1.start, e2.out, Min(e1.minIndex, e2.minIndex)); - } - return true; - - case kAlternation: - if (operandStack.GetSize() >= sizeof(Frag) * 2) { - Frag e2 = *operandStack.template Pop(1); - Frag e1 = *operandStack.template Pop(1); - SizeType s = NewState(e1.start, e2.start, 0); - *operandStack.template Push() = Frag(s, Append(e1.out, e2.out), Min(e1.minIndex, e2.minIndex)); - return true; - } - return false; - - case kZeroOrOne: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - *operandStack.template Push() = Frag(s, Append(e.out, s), e.minIndex); - return true; - } - return false; - - case kZeroOrMore: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push() = Frag(s, s, e.minIndex); - return true; - } - return false; - - case kOneOrMore: - if (operandStack.GetSize() >= sizeof(Frag)) { - Frag e = *operandStack.template Pop(1); - SizeType s = NewState(kRegexInvalidState, e.start, 0); - Patch(e.out, s); - *operandStack.template Push() = Frag(e.start, s, e.minIndex); - return true; - } - return false; - - default: - // syntax error (e.g. unclosed kLeftParenthesis) - return false; - } - } - - bool EvalQuantifier(Stack& operandStack, unsigned n, unsigned m) { - RAPIDJSON_ASSERT(n <= m); - RAPIDJSON_ASSERT(operandStack.GetSize() >= sizeof(Frag)); - - if (n == 0) { - if (m == 0) // a{0} not support - return false; - else if (m == kInfinityQuantifier) - Eval(operandStack, kZeroOrMore); // a{0,} -> a* - else { - Eval(operandStack, kZeroOrOne); // a{0,5} -> a? - for (unsigned i = 0; i < m - 1; i++) - CloneTopOperand(operandStack); // a{0,5} -> a? a? a? a? a? - for (unsigned i = 0; i < m - 1; i++) - Eval(operandStack, kConcatenation); // a{0,5} -> a?a?a?a?a? - } - return true; - } - - for (unsigned i = 0; i < n - 1; i++) // a{3} -> a a a - CloneTopOperand(operandStack); - - if (m == kInfinityQuantifier) - Eval(operandStack, kOneOrMore); // a{3,} -> a a a+ - else if (m > n) { - CloneTopOperand(operandStack); // a{3,5} -> a a a a - Eval(operandStack, kZeroOrOne); // a{3,5} -> a a a a? - for (unsigned i = n; i < m - 1; i++) - CloneTopOperand(operandStack); // a{3,5} -> a a a a? a? - for (unsigned i = n; i < m; i++) - Eval(operandStack, kConcatenation); // a{3,5} -> a a aa?a? - } - - for (unsigned i = 0; i < n - 1; i++) - Eval(operandStack, kConcatenation); // a{3} -> aaa, a{3,} -> aaa+, a{3.5} -> aaaa?a? - - return true; - } - - static SizeType Min(SizeType a, SizeType b) { return a < b ? a : b; } - - void CloneTopOperand(Stack& operandStack) { - const Frag src = *operandStack.template Top(); // Copy constructor to prevent invalidation - SizeType count = stateCount_ - src.minIndex; // Assumes top operand contains states in [src->minIndex, stateCount_) - State* s = states_.template Push(count); - memcpy(s, &GetState(src.minIndex), count * sizeof(State)); - for (SizeType j = 0; j < count; j++) { - if (s[j].out != kRegexInvalidState) - s[j].out += count; - if (s[j].out1 != kRegexInvalidState) - s[j].out1 += count; - } - *operandStack.template Push() = Frag(src.start + count, src.out + count, src.minIndex + count); - stateCount_ += count; - } - - template - bool ParseUnsigned(DecodedStream& ds, unsigned* u) { - unsigned r = 0; - if (ds.Peek() < '0' || ds.Peek() > '9') - return false; - while (ds.Peek() >= '0' && ds.Peek() <= '9') { - if (r >= 429496729 && ds.Peek() > '5') // 2^32 - 1 = 4294967295 - return false; // overflow - r = r * 10 + (ds.Take() - '0'); - } - *u = r; - return true; - } - - template - bool ParseRange(DecodedStream& ds, SizeType* range) { - bool isBegin = true; - bool negate = false; - int step = 0; - SizeType start = kRegexInvalidRange; - SizeType current = kRegexInvalidRange; - unsigned codepoint; - while ((codepoint = ds.Take()) != 0) { - if (isBegin) { - isBegin = false; - if (codepoint == '^') { - negate = true; - continue; - } - } - - switch (codepoint) { - case ']': - if (start == kRegexInvalidRange) - return false; // Error: nothing inside [] - if (step == 2) { // Add trailing '-' - SizeType r = NewRange('-'); - RAPIDJSON_ASSERT(current != kRegexInvalidRange); - GetRange(current).next = r; - } - if (negate) - GetRange(start).start |= kRangeNegationFlag; - *range = start; - return true; - - case '\\': - if (ds.Peek() == 'b') { - ds.Take(); - codepoint = 0x0008; // Escape backspace character - } - else if (!CharacterEscape(ds, &codepoint)) - return false; - // fall through to default - - default: - switch (step) { - case 1: - if (codepoint == '-') { - step++; - break; - } - // fall through to step 0 for other characters - - case 0: - { - SizeType r = NewRange(codepoint); - if (current != kRegexInvalidRange) - GetRange(current).next = r; - if (start == kRegexInvalidRange) - start = r; - current = r; - } - step = 1; - break; - - default: - RAPIDJSON_ASSERT(step == 2); - GetRange(current).end = codepoint; - step = 0; - } - } - } - return false; - } - - SizeType NewRange(unsigned codepoint) { - Range* r = ranges_.template Push(); - r->start = r->end = codepoint; - r->next = kRegexInvalidRange; - return rangeCount_++; - } - - template - bool CharacterEscape(DecodedStream& ds, unsigned* escapedCodepoint) { - unsigned codepoint; - switch (codepoint = ds.Take()) { - case '^': - case '$': - case '|': - case '(': - case ')': - case '?': - case '*': - case '+': - case '.': - case '[': - case ']': - case '{': - case '}': - case '\\': - *escapedCodepoint = codepoint; return true; - case 'f': *escapedCodepoint = 0x000C; return true; - case 'n': *escapedCodepoint = 0x000A; return true; - case 'r': *escapedCodepoint = 0x000D; return true; - case 't': *escapedCodepoint = 0x0009; return true; - case 'v': *escapedCodepoint = 0x000B; return true; - default: - return false; // Unsupported escape character - } - } - - Allocator* ownAllocator_; - Allocator* allocator_; - Stack states_; - Stack ranges_; - SizeType root_; - SizeType stateCount_; - SizeType rangeCount_; - - static const unsigned kInfinityQuantifier = ~0u; - - // For SearchWithAnchoring() - bool anchorBegin_; - bool anchorEnd_; -}; - -template -class GenericRegexSearch { -public: - typedef typename RegexType::EncodingType Encoding; - typedef typename Encoding::Ch Ch; - - GenericRegexSearch(const RegexType& regex, Allocator* allocator = 0) : - regex_(regex), allocator_(allocator), ownAllocator_(0), - state0_(allocator, 0), state1_(allocator, 0), stateSet_() - { - RAPIDJSON_ASSERT(regex_.IsValid()); - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - stateSet_ = static_cast(allocator_->Malloc(GetStateSetSize())); - state0_.template Reserve(regex_.stateCount_); - state1_.template Reserve(regex_.stateCount_); - } - - ~GenericRegexSearch() { - Allocator::Free(stateSet_); - RAPIDJSON_DELETE(ownAllocator_); - } - - template - bool Match(InputStream& is) { - return SearchWithAnchoring(is, true, true); - } - - bool Match(const Ch* s) { - GenericStringStream is(s); - return Match(is); - } - - template - bool Search(InputStream& is) { - return SearchWithAnchoring(is, regex_.anchorBegin_, regex_.anchorEnd_); - } - - bool Search(const Ch* s) { - GenericStringStream is(s); - return Search(is); - } - -private: - typedef typename RegexType::State State; - typedef typename RegexType::Range Range; - - template - bool SearchWithAnchoring(InputStream& is, bool anchorBegin, bool anchorEnd) { - DecodedStream ds(is); - - state0_.Clear(); - Stack *current = &state0_, *next = &state1_; - const size_t stateSetSize = GetStateSetSize(); - std::memset(stateSet_, 0, stateSetSize); - - bool matched = AddState(*current, regex_.root_); - unsigned codepoint; - while (!current->Empty() && (codepoint = ds.Take()) != 0) { - std::memset(stateSet_, 0, stateSetSize); - next->Clear(); - matched = false; - for (const SizeType* s = current->template Bottom(); s != current->template End(); ++s) { - const State& sr = regex_.GetState(*s); - if (sr.codepoint == codepoint || - sr.codepoint == RegexType::kAnyCharacterClass || - (sr.codepoint == RegexType::kRangeCharacterClass && MatchRange(sr.rangeStart, codepoint))) - { - matched = AddState(*next, sr.out) || matched; - if (!anchorEnd && matched) - return true; - } - if (!anchorBegin) - AddState(*next, regex_.root_); - } - internal::Swap(current, next); - } - - return matched; - } - - size_t GetStateSetSize() const { - return (regex_.stateCount_ + 31) / 32 * 4; - } - - // Return whether the added states is a match state - bool AddState(Stack& l, SizeType index) { - RAPIDJSON_ASSERT(index != kRegexInvalidState); - - const State& s = regex_.GetState(index); - if (s.out1 != kRegexInvalidState) { // Split - bool matched = AddState(l, s.out); - return AddState(l, s.out1) || matched; - } - else if (!(stateSet_[index >> 5] & (1u << (index & 31)))) { - stateSet_[index >> 5] |= (1u << (index & 31)); - *l.template PushUnsafe() = index; - } - return s.out == kRegexInvalidState; // by using PushUnsafe() above, we can ensure s is not validated due to reallocation. - } - - bool MatchRange(SizeType rangeIndex, unsigned codepoint) const { - bool yes = (regex_.GetRange(rangeIndex).start & RegexType::kRangeNegationFlag) == 0; - while (rangeIndex != kRegexInvalidRange) { - const Range& r = regex_.GetRange(rangeIndex); - if (codepoint >= (r.start & ~RegexType::kRangeNegationFlag) && codepoint <= r.end) - return yes; - rangeIndex = r.next; - } - return !yes; - } - - const RegexType& regex_; - Allocator* allocator_; - Allocator* ownAllocator_; - Stack state0_; - Stack state1_; - uint32_t* stateSet_; -}; - -typedef GenericRegex > Regex; -typedef GenericRegexSearch RegexSearch; - -} // namespace internal -RAPIDJSON_NAMESPACE_END - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#if defined(__clang__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_INTERNAL_REGEX_H_ diff --git a/src/native/external/rapidjson/internal/stack.h b/src/native/external/rapidjson/internal/stack.h index 45dca6a8b09eb..73abd706e9769 100644 --- a/src/native/external/rapidjson/internal/stack.h +++ b/src/native/external/rapidjson/internal/stack.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/internal/strfunc.h b/src/native/external/rapidjson/internal/strfunc.h index 226439a767364..b698a8f43fa63 100644 --- a/src/native/external/rapidjson/internal/strfunc.h +++ b/src/native/external/rapidjson/internal/strfunc.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -45,6 +45,20 @@ inline SizeType StrLen(const wchar_t* s) { return SizeType(std::wcslen(s)); } +//! Custom strcmpn() which works on different character types. +/*! \tparam Ch Character type (e.g. char, wchar_t, short) + \param s1 Null-terminated input string. + \param s2 Null-terminated input string. + \return 0 if equal +*/ +template +inline int StrCmp(const Ch* s1, const Ch* s2) { + RAPIDJSON_ASSERT(s1 != 0); + RAPIDJSON_ASSERT(s2 != 0); + while(*s1 && (*s1 == *s2)) { s1++; s2++; } + return static_cast(*s1) < static_cast(*s2) ? -1 : static_cast(*s1) > static_cast(*s2); +} + //! Returns number of code points in a encoded string. template bool CountStringCodePoint(const typename Encoding::Ch* s, SizeType length, SizeType* outCount) { diff --git a/src/native/external/rapidjson/internal/strtod.h b/src/native/external/rapidjson/internal/strtod.h index dfca22b65ac03..55f0e380bfaa3 100644 --- a/src/native/external/rapidjson/internal/strtod.h +++ b/src/native/external/rapidjson/internal/strtod.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -128,17 +128,18 @@ inline bool StrtodFast(double d, int p, double* result) { } // Compute an approximation and see if it is within 1/2 ULP -inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result) { +template +inline bool StrtodDiyFp(const Ch* decimals, int dLen, int dExp, double* result) { uint64_t significand = 0; int i = 0; // 2^64 - 1 = 18446744073709551615, 1844674407370955161 = 0x1999999999999999 for (; i < dLen; i++) { if (significand > RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) || - (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > '5')) + (significand == RAPIDJSON_UINT64_C2(0x19999999, 0x99999999) && decimals[i] > Ch('5'))) break; - significand = significand * 10u + static_cast(decimals[i] - '0'); + significand = significand * 10u + static_cast(decimals[i] - Ch('0')); } - if (i < dLen && decimals[i] >= '5') // Rounding + if (i < dLen && decimals[i] >= Ch('5')) // Rounding significand++; int remaining = dLen - i; @@ -205,7 +206,8 @@ inline bool StrtodDiyFp(const char* decimals, int dLen, int dExp, double* result return halfWay - static_cast(error) >= precisionBits || precisionBits >= halfWay + static_cast(error); } -inline double StrtodBigInteger(double approx, const char* decimals, int dLen, int dExp) { +template +inline double StrtodBigInteger(double approx, const Ch* decimals, int dLen, int dExp) { RAPIDJSON_ASSERT(dLen >= 0); const BigInteger dInt(decimals, static_cast(dLen)); Double a(approx); @@ -223,7 +225,8 @@ inline double StrtodBigInteger(double approx, const char* decimals, int dLen, in return a.NextPositiveDouble(); } -inline double StrtodFullPrecision(double d, int p, const char* decimals, size_t length, size_t decimalPosition, int exp) { +template +inline double StrtodFullPrecision(double d, int p, const Ch* decimals, size_t length, size_t decimalPosition, int exp) { RAPIDJSON_ASSERT(d >= 0.0); RAPIDJSON_ASSERT(length >= 1); diff --git a/src/native/external/rapidjson/internal/swap.h b/src/native/external/rapidjson/internal/swap.h index 666e49f97b68d..2cf92f93a1d37 100644 --- a/src/native/external/rapidjson/internal/swap.h +++ b/src/native/external/rapidjson/internal/swap.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/istreamwrapper.h b/src/native/external/rapidjson/istreamwrapper.h index c4950b9dcf828..01437ec0127a0 100644 --- a/src/native/external/rapidjson/istreamwrapper.h +++ b/src/native/external/rapidjson/istreamwrapper.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/memorybuffer.h b/src/native/external/rapidjson/memorybuffer.h deleted file mode 100644 index 39bee1dec1c03..0000000000000 --- a/src/native/external/rapidjson/memorybuffer.h +++ /dev/null @@ -1,70 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_MEMORYBUFFER_H_ -#define RAPIDJSON_MEMORYBUFFER_H_ - -#include "stream.h" -#include "internal/stack.h" - -RAPIDJSON_NAMESPACE_BEGIN - -//! Represents an in-memory output byte stream. -/*! - This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. - - It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. - - Differences between MemoryBuffer and StringBuffer: - 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. - 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. - - \tparam Allocator type for allocating memory buffer. - \note implements Stream concept -*/ -template -struct GenericMemoryBuffer { - typedef char Ch; // byte - - GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} - - void Put(Ch c) { *stack_.template Push() = c; } - void Flush() {} - - void Clear() { stack_.Clear(); } - void ShrinkToFit() { stack_.ShrinkToFit(); } - Ch* Push(size_t count) { return stack_.template Push(count); } - void Pop(size_t count) { stack_.template Pop(count); } - - const Ch* GetBuffer() const { - return stack_.template Bottom(); - } - - size_t GetSize() const { return stack_.GetSize(); } - - static const size_t kDefaultCapacity = 256; - mutable internal::Stack stack_; -}; - -typedef GenericMemoryBuffer<> MemoryBuffer; - -//! Implement specialized version of PutN() with memset() for better performance. -template<> -inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { - std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); -} - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_MEMORYBUFFER_H_ diff --git a/src/native/external/rapidjson/memorystream.h b/src/native/external/rapidjson/memorystream.h index 1d71d8a4f0e0a..77af6c999e977 100644 --- a/src/native/external/rapidjson/memorystream.h +++ b/src/native/external/rapidjson/memorystream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/msinttypes/inttypes.h b/src/native/external/rapidjson/msinttypes/inttypes.h deleted file mode 100644 index 18111286bf55b..0000000000000 --- a/src/native/external/rapidjson/msinttypes/inttypes.h +++ /dev/null @@ -1,316 +0,0 @@ -// ISO C9x compliant inttypes.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -// The above software in this distribution may have been modified by -// THL A29 Limited ("Tencent Modifications"). -// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_INTTYPES_H_ // [ -#define _MSC_INTTYPES_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#include "stdint.h" - -// miloyip: VC supports inttypes.h since VC2013 -#if _MSC_VER >= 1800 -#include -#else - -// 7.8 Format conversion of integer types - -typedef struct { - intmax_t quot; - intmax_t rem; -} imaxdiv_t; - -// 7.8.1 Macros for format specifiers - -#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 - -// The fprintf macros for signed integers are: -#define PRId8 "d" -#define PRIi8 "i" -#define PRIdLEAST8 "d" -#define PRIiLEAST8 "i" -#define PRIdFAST8 "d" -#define PRIiFAST8 "i" - -#define PRId16 "hd" -#define PRIi16 "hi" -#define PRIdLEAST16 "hd" -#define PRIiLEAST16 "hi" -#define PRIdFAST16 "hd" -#define PRIiFAST16 "hi" - -#define PRId32 "I32d" -#define PRIi32 "I32i" -#define PRIdLEAST32 "I32d" -#define PRIiLEAST32 "I32i" -#define PRIdFAST32 "I32d" -#define PRIiFAST32 "I32i" - -#define PRId64 "I64d" -#define PRIi64 "I64i" -#define PRIdLEAST64 "I64d" -#define PRIiLEAST64 "I64i" -#define PRIdFAST64 "I64d" -#define PRIiFAST64 "I64i" - -#define PRIdMAX "I64d" -#define PRIiMAX "I64i" - -#define PRIdPTR "Id" -#define PRIiPTR "Ii" - -// The fprintf macros for unsigned integers are: -#define PRIo8 "o" -#define PRIu8 "u" -#define PRIx8 "x" -#define PRIX8 "X" -#define PRIoLEAST8 "o" -#define PRIuLEAST8 "u" -#define PRIxLEAST8 "x" -#define PRIXLEAST8 "X" -#define PRIoFAST8 "o" -#define PRIuFAST8 "u" -#define PRIxFAST8 "x" -#define PRIXFAST8 "X" - -#define PRIo16 "ho" -#define PRIu16 "hu" -#define PRIx16 "hx" -#define PRIX16 "hX" -#define PRIoLEAST16 "ho" -#define PRIuLEAST16 "hu" -#define PRIxLEAST16 "hx" -#define PRIXLEAST16 "hX" -#define PRIoFAST16 "ho" -#define PRIuFAST16 "hu" -#define PRIxFAST16 "hx" -#define PRIXFAST16 "hX" - -#define PRIo32 "I32o" -#define PRIu32 "I32u" -#define PRIx32 "I32x" -#define PRIX32 "I32X" -#define PRIoLEAST32 "I32o" -#define PRIuLEAST32 "I32u" -#define PRIxLEAST32 "I32x" -#define PRIXLEAST32 "I32X" -#define PRIoFAST32 "I32o" -#define PRIuFAST32 "I32u" -#define PRIxFAST32 "I32x" -#define PRIXFAST32 "I32X" - -#define PRIo64 "I64o" -#define PRIu64 "I64u" -#define PRIx64 "I64x" -#define PRIX64 "I64X" -#define PRIoLEAST64 "I64o" -#define PRIuLEAST64 "I64u" -#define PRIxLEAST64 "I64x" -#define PRIXLEAST64 "I64X" -#define PRIoFAST64 "I64o" -#define PRIuFAST64 "I64u" -#define PRIxFAST64 "I64x" -#define PRIXFAST64 "I64X" - -#define PRIoMAX "I64o" -#define PRIuMAX "I64u" -#define PRIxMAX "I64x" -#define PRIXMAX "I64X" - -#define PRIoPTR "Io" -#define PRIuPTR "Iu" -#define PRIxPTR "Ix" -#define PRIXPTR "IX" - -// The fscanf macros for signed integers are: -#define SCNd8 "d" -#define SCNi8 "i" -#define SCNdLEAST8 "d" -#define SCNiLEAST8 "i" -#define SCNdFAST8 "d" -#define SCNiFAST8 "i" - -#define SCNd16 "hd" -#define SCNi16 "hi" -#define SCNdLEAST16 "hd" -#define SCNiLEAST16 "hi" -#define SCNdFAST16 "hd" -#define SCNiFAST16 "hi" - -#define SCNd32 "ld" -#define SCNi32 "li" -#define SCNdLEAST32 "ld" -#define SCNiLEAST32 "li" -#define SCNdFAST32 "ld" -#define SCNiFAST32 "li" - -#define SCNd64 "I64d" -#define SCNi64 "I64i" -#define SCNdLEAST64 "I64d" -#define SCNiLEAST64 "I64i" -#define SCNdFAST64 "I64d" -#define SCNiFAST64 "I64i" - -#define SCNdMAX "I64d" -#define SCNiMAX "I64i" - -#ifdef _WIN64 // [ -# define SCNdPTR "I64d" -# define SCNiPTR "I64i" -#else // _WIN64 ][ -# define SCNdPTR "ld" -# define SCNiPTR "li" -#endif // _WIN64 ] - -// The fscanf macros for unsigned integers are: -#define SCNo8 "o" -#define SCNu8 "u" -#define SCNx8 "x" -#define SCNX8 "X" -#define SCNoLEAST8 "o" -#define SCNuLEAST8 "u" -#define SCNxLEAST8 "x" -#define SCNXLEAST8 "X" -#define SCNoFAST8 "o" -#define SCNuFAST8 "u" -#define SCNxFAST8 "x" -#define SCNXFAST8 "X" - -#define SCNo16 "ho" -#define SCNu16 "hu" -#define SCNx16 "hx" -#define SCNX16 "hX" -#define SCNoLEAST16 "ho" -#define SCNuLEAST16 "hu" -#define SCNxLEAST16 "hx" -#define SCNXLEAST16 "hX" -#define SCNoFAST16 "ho" -#define SCNuFAST16 "hu" -#define SCNxFAST16 "hx" -#define SCNXFAST16 "hX" - -#define SCNo32 "lo" -#define SCNu32 "lu" -#define SCNx32 "lx" -#define SCNX32 "lX" -#define SCNoLEAST32 "lo" -#define SCNuLEAST32 "lu" -#define SCNxLEAST32 "lx" -#define SCNXLEAST32 "lX" -#define SCNoFAST32 "lo" -#define SCNuFAST32 "lu" -#define SCNxFAST32 "lx" -#define SCNXFAST32 "lX" - -#define SCNo64 "I64o" -#define SCNu64 "I64u" -#define SCNx64 "I64x" -#define SCNX64 "I64X" -#define SCNoLEAST64 "I64o" -#define SCNuLEAST64 "I64u" -#define SCNxLEAST64 "I64x" -#define SCNXLEAST64 "I64X" -#define SCNoFAST64 "I64o" -#define SCNuFAST64 "I64u" -#define SCNxFAST64 "I64x" -#define SCNXFAST64 "I64X" - -#define SCNoMAX "I64o" -#define SCNuMAX "I64u" -#define SCNxMAX "I64x" -#define SCNXMAX "I64X" - -#ifdef _WIN64 // [ -# define SCNoPTR "I64o" -# define SCNuPTR "I64u" -# define SCNxPTR "I64x" -# define SCNXPTR "I64X" -#else // _WIN64 ][ -# define SCNoPTR "lo" -# define SCNuPTR "lu" -# define SCNxPTR "lx" -# define SCNXPTR "lX" -#endif // _WIN64 ] - -#endif // __STDC_FORMAT_MACROS ] - -// 7.8.2 Functions for greatest-width integer types - -// 7.8.2.1 The imaxabs function -#define imaxabs _abs64 - -// 7.8.2.2 The imaxdiv function - -// This is modified version of div() function from Microsoft's div.c found -// in %MSVC.NET%\crt\src\div.c -#ifdef STATIC_IMAXDIV // [ -static -#else // STATIC_IMAXDIV ][ -_inline -#endif // STATIC_IMAXDIV ] -imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) -{ - imaxdiv_t result; - - result.quot = numer / denom; - result.rem = numer % denom; - - if (numer < 0 && result.rem > 0) { - // did division wrong; must fix up - ++result.quot; - result.rem -= denom; - } - - return result; -} - -// 7.8.2.3 The strtoimax and strtoumax functions -#define strtoimax _strtoi64 -#define strtoumax _strtoui64 - -// 7.8.2.4 The wcstoimax and wcstoumax functions -#define wcstoimax _wcstoi64 -#define wcstoumax _wcstoui64 - -#endif // _MSC_VER >= 1800 - -#endif // _MSC_INTTYPES_H_ ] diff --git a/src/native/external/rapidjson/msinttypes/stdint.h b/src/native/external/rapidjson/msinttypes/stdint.h deleted file mode 100644 index 3d4477b9a024a..0000000000000 --- a/src/native/external/rapidjson/msinttypes/stdint.h +++ /dev/null @@ -1,300 +0,0 @@ -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -// The above software in this distribution may have been modified by -// THL A29 Limited ("Tencent Modifications"). -// All Tencent Modifications are Copyright (C) 2015 THL A29 Limited. - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_STDINT_H_ // [ -#define _MSC_STDINT_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -// miloyip: Originally Visual Studio 2010 uses its own stdint.h. However it generates warning with INT64_C(), so change to use this file for vs2010. -#if _MSC_VER >= 1600 // [ -#include - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -#undef INT8_C -#undef INT16_C -#undef INT32_C -#undef INT64_C -#undef UINT8_C -#undef UINT16_C -#undef UINT32_C -#undef UINT64_C - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with . -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#else // ] _MSC_VER >= 1700 [ - -#include - -// For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we have to wrap include with 'extern "C++" {}' -// or compiler would give many errors like this: -// error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#if defined(__cplusplus) && !defined(_M_ARM) -extern "C" { -#endif -# include -#if defined(__cplusplus) && !defined(_M_ARM) -} -#endif - -// Define _W64 macros to mark types changing their size, like intptr_t. -#ifndef _W64 -# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 -# define _W64 __w64 -# else -# define _W64 -# endif -#endif - - -// 7.18.1 Integer types - -// 7.18.1.1 Exact-width integer types - -// Visual Studio 6 and Embedded Visual C++ 4 doesn't -// realize that, e.g. char has the same size as __int8 -// so we give up on __intX for them. -#if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; -#else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; -#endif -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; - - -// 7.18.1.2 Minimum-width integer types -typedef int8_t int_least8_t; -typedef int16_t int_least16_t; -typedef int32_t int_least32_t; -typedef int64_t int_least64_t; -typedef uint8_t uint_least8_t; -typedef uint16_t uint_least16_t; -typedef uint32_t uint_least32_t; -typedef uint64_t uint_least64_t; - -// 7.18.1.3 Fastest minimum-width integer types -typedef int8_t int_fast8_t; -typedef int16_t int_fast16_t; -typedef int32_t int_fast32_t; -typedef int64_t int_fast64_t; -typedef uint8_t uint_fast8_t; -typedef uint16_t uint_fast16_t; -typedef uint32_t uint_fast32_t; -typedef uint64_t uint_fast64_t; - -// 7.18.1.4 Integer types capable of holding object pointers -#ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; -#else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; -#endif // _WIN64 ] - -// 7.18.1.5 Greatest-width integer types -typedef int64_t intmax_t; -typedef uint64_t uintmax_t; - - -// 7.18.2 Limits of specified-width integer types - -#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 - -// 7.18.2.1 Limits of exact-width integer types -#define INT8_MIN ((int8_t)_I8_MIN) -#define INT8_MAX _I8_MAX -#define INT16_MIN ((int16_t)_I16_MIN) -#define INT16_MAX _I16_MAX -#define INT32_MIN ((int32_t)_I32_MIN) -#define INT32_MAX _I32_MAX -#define INT64_MIN ((int64_t)_I64_MIN) -#define INT64_MAX _I64_MAX -#define UINT8_MAX _UI8_MAX -#define UINT16_MAX _UI16_MAX -#define UINT32_MAX _UI32_MAX -#define UINT64_MAX _UI64_MAX - -// 7.18.2.2 Limits of minimum-width integer types -#define INT_LEAST8_MIN INT8_MIN -#define INT_LEAST8_MAX INT8_MAX -#define INT_LEAST16_MIN INT16_MIN -#define INT_LEAST16_MAX INT16_MAX -#define INT_LEAST32_MIN INT32_MIN -#define INT_LEAST32_MAX INT32_MAX -#define INT_LEAST64_MIN INT64_MIN -#define INT_LEAST64_MAX INT64_MAX -#define UINT_LEAST8_MAX UINT8_MAX -#define UINT_LEAST16_MAX UINT16_MAX -#define UINT_LEAST32_MAX UINT32_MAX -#define UINT_LEAST64_MAX UINT64_MAX - -// 7.18.2.3 Limits of fastest minimum-width integer types -#define INT_FAST8_MIN INT8_MIN -#define INT_FAST8_MAX INT8_MAX -#define INT_FAST16_MIN INT16_MIN -#define INT_FAST16_MAX INT16_MAX -#define INT_FAST32_MIN INT32_MIN -#define INT_FAST32_MAX INT32_MAX -#define INT_FAST64_MIN INT64_MIN -#define INT_FAST64_MAX INT64_MAX -#define UINT_FAST8_MAX UINT8_MAX -#define UINT_FAST16_MAX UINT16_MAX -#define UINT_FAST32_MAX UINT32_MAX -#define UINT_FAST64_MAX UINT64_MAX - -// 7.18.2.4 Limits of integer types capable of holding object pointers -#ifdef _WIN64 // [ -# define INTPTR_MIN INT64_MIN -# define INTPTR_MAX INT64_MAX -# define UINTPTR_MAX UINT64_MAX -#else // _WIN64 ][ -# define INTPTR_MIN INT32_MIN -# define INTPTR_MAX INT32_MAX -# define UINTPTR_MAX UINT32_MAX -#endif // _WIN64 ] - -// 7.18.2.5 Limits of greatest-width integer types -#define INTMAX_MIN INT64_MIN -#define INTMAX_MAX INT64_MAX -#define UINTMAX_MAX UINT64_MAX - -// 7.18.3 Limits of other integer types - -#ifdef _WIN64 // [ -# define PTRDIFF_MIN _I64_MIN -# define PTRDIFF_MAX _I64_MAX -#else // _WIN64 ][ -# define PTRDIFF_MIN _I32_MIN -# define PTRDIFF_MAX _I32_MAX -#endif // _WIN64 ] - -#define SIG_ATOMIC_MIN INT_MIN -#define SIG_ATOMIC_MAX INT_MAX - -#ifndef SIZE_MAX // [ -# ifdef _WIN64 // [ -# define SIZE_MAX _UI64_MAX -# else // _WIN64 ][ -# define SIZE_MAX _UI32_MAX -# endif // _WIN64 ] -#endif // SIZE_MAX ] - -// WCHAR_MIN and WCHAR_MAX are also defined in -#ifndef WCHAR_MIN // [ -# define WCHAR_MIN 0 -#endif // WCHAR_MIN ] -#ifndef WCHAR_MAX // [ -# define WCHAR_MAX _UI16_MAX -#endif // WCHAR_MAX ] - -#define WINT_MIN 0 -#define WINT_MAX _UI16_MAX - -#endif // __STDC_LIMIT_MACROS ] - - -// 7.18.4 Limits of other integer types - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with . -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#endif // _MSC_VER >= 1600 ] - -#endif // _MSC_STDINT_H_ ] diff --git a/src/native/external/rapidjson/ostreamwrapper.h b/src/native/external/rapidjson/ostreamwrapper.h deleted file mode 100644 index 6f4667c08ad7b..0000000000000 --- a/src/native/external/rapidjson/ostreamwrapper.h +++ /dev/null @@ -1,81 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_OSTREAMWRAPPER_H_ -#define RAPIDJSON_OSTREAMWRAPPER_H_ - -#include "stream.h" -#include - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(padded) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Wrapper of \c std::basic_ostream into RapidJSON's Stream concept. -/*! - The classes can be wrapped including but not limited to: - - - \c std::ostringstream - - \c std::stringstream - - \c std::wpstringstream - - \c std::wstringstream - - \c std::ifstream - - \c std::fstream - - \c std::wofstream - - \c std::wfstream - - \tparam StreamType Class derived from \c std::basic_ostream. -*/ - -template -class BasicOStreamWrapper { -public: - typedef typename StreamType::char_type Ch; - BasicOStreamWrapper(StreamType& stream) : stream_(stream) {} - - void Put(Ch c) { - stream_.put(c); - } - - void Flush() { - stream_.flush(); - } - - // Not implemented - char Peek() const { RAPIDJSON_ASSERT(false); return 0; } - char Take() { RAPIDJSON_ASSERT(false); return 0; } - size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } - char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } - size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } - -private: - BasicOStreamWrapper(const BasicOStreamWrapper&); - BasicOStreamWrapper& operator=(const BasicOStreamWrapper&); - - StreamType& stream_; -}; - -typedef BasicOStreamWrapper OStreamWrapper; -typedef BasicOStreamWrapper WOStreamWrapper; - -#ifdef __clang__ -RAPIDJSON_DIAG_POP -#endif - -RAPIDJSON_NAMESPACE_END - -#endif // RAPIDJSON_OSTREAMWRAPPER_H_ diff --git a/src/native/external/rapidjson/pointer.h b/src/native/external/rapidjson/pointer.h deleted file mode 100644 index 063abab9a1703..0000000000000 --- a/src/native/external/rapidjson/pointer.h +++ /dev/null @@ -1,1414 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_POINTER_H_ -#define RAPIDJSON_POINTER_H_ - -#include "document.h" -#include "internal/itoa.h" - -#ifdef __clang__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(switch-enum) -#elif defined(_MSC_VER) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -static const SizeType kPointerInvalidIndex = ~SizeType(0); //!< Represents an invalid index in GenericPointer::Token - -//! Error code of parsing. -/*! \ingroup RAPIDJSON_ERRORS - \see GenericPointer::GenericPointer, GenericPointer::GetParseErrorCode -*/ -enum PointerParseErrorCode { - kPointerParseErrorNone = 0, //!< The parse is successful - - kPointerParseErrorTokenMustBeginWithSolidus, //!< A token must begin with a '/' - kPointerParseErrorInvalidEscape, //!< Invalid escape - kPointerParseErrorInvalidPercentEncoding, //!< Invalid percent encoding in URI fragment - kPointerParseErrorCharacterMustPercentEncode //!< A character must percent encoded in URI fragment -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericPointer - -//! Represents a JSON Pointer. Use Pointer for UTF8 encoding and default allocator. -/*! - This class implements RFC 6901 "JavaScript Object Notation (JSON) Pointer" - (https://tools.ietf.org/html/rfc6901). - - A JSON pointer is for identifying a specific value in a JSON document - (GenericDocument). It can simplify coding of DOM tree manipulation, because it - can access multiple-level depth of DOM tree with single API call. - - After it parses a string representation (e.g. "/foo/0" or URI fragment - representation (e.g. "#/foo/0") into its internal representation (tokens), - it can be used to resolve a specific value in multiple documents, or sub-tree - of documents. - - Contrary to GenericValue, Pointer can be copy constructed and copy assigned. - Apart from assignment, a Pointer cannot be modified after construction. - - Although Pointer is very convenient, please aware that constructing Pointer - involves parsing and dynamic memory allocation. A special constructor with user- - supplied tokens eliminates these. - - GenericPointer depends on GenericDocument and GenericValue. - - \tparam ValueType The value type of the DOM tree. E.g. GenericValue > - \tparam Allocator The allocator type for allocating memory for internal representation. - - \note GenericPointer uses same encoding of ValueType. - However, Allocator of GenericPointer is independent of Allocator of Value. -*/ -template -class GenericPointer { -public: - typedef typename ValueType::EncodingType EncodingType; //!< Encoding type from Value - typedef typename ValueType::Ch Ch; //!< Character type from Value - - //! A token is the basic units of internal representation. - /*! - A JSON pointer string representation "/foo/123" is parsed to two tokens: - "foo" and 123. 123 will be represented in both numeric form and string form. - They are resolved according to the actual value type (object or array). - - For token that are not numbers, or the numeric value is out of bound - (greater than limits of SizeType), they are only treated as string form - (i.e. the token's index will be equal to kPointerInvalidIndex). - - This struct is public so that user can create a Pointer without parsing and - allocation, using a special constructor. - */ - struct Token { - const Ch* name; //!< Name of the token. It has null character at the end but it can contain null character. - SizeType length; //!< Length of the name. - SizeType index; //!< A valid array index, if it is not equal to kPointerInvalidIndex. - }; - - //!@name Constructors and destructor. - //@{ - - //! Default constructor. - GenericPointer(Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} - - //! Constructor that parses a string or URI fragment representation. - /*! - \param source A null-terminated, string or URI fragment representation of JSON pointer. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - */ - explicit GenericPointer(const Ch* source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source, internal::StrLen(source)); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Constructor that parses a string or URI fragment representation. - /*! - \param source A string or URI fragment representation of JSON pointer. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING. - */ - explicit GenericPointer(const std::basic_string& source, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source.c_str(), source.size()); - } -#endif - - //! Constructor that parses a string or URI fragment representation, with length of the source string. - /*! - \param source A string or URI fragment representation of JSON pointer. - \param length Length of source. - \param allocator User supplied allocator for this pointer. If no allocator is provided, it creates a self-owned one. - \note Slightly faster than the overload without length. - */ - GenericPointer(const Ch* source, size_t length, Allocator* allocator = 0) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - Parse(source, length); - } - - //! Constructor with user-supplied tokens. - /*! - This constructor let user supplies const array of tokens. - This prevents the parsing process and eliminates allocation. - This is preferred for memory constrained environments. - - \param tokens An constant array of tokens representing the JSON pointer. - \param tokenCount Number of tokens. - - \b Example - \code - #define NAME(s) { s, sizeof(s) / sizeof(s[0]) - 1, kPointerInvalidIndex } - #define INDEX(i) { #i, sizeof(#i) - 1, i } - - static const Pointer::Token kTokens[] = { NAME("foo"), INDEX(123) }; - static const Pointer p(kTokens, sizeof(kTokens) / sizeof(kTokens[0])); - // Equivalent to static const Pointer p("/foo/123"); - - #undef NAME - #undef INDEX - \endcode - */ - GenericPointer(const Token* tokens, size_t tokenCount) : allocator_(), ownAllocator_(), nameBuffer_(), tokens_(const_cast(tokens)), tokenCount_(tokenCount), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) {} - - //! Copy constructor. - GenericPointer(const GenericPointer& rhs) : allocator_(rhs.allocator_), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - *this = rhs; - } - - //! Copy constructor. - GenericPointer(const GenericPointer& rhs, Allocator* allocator) : allocator_(allocator), ownAllocator_(), nameBuffer_(), tokens_(), tokenCount_(), parseErrorOffset_(), parseErrorCode_(kPointerParseErrorNone) { - *this = rhs; - } - - //! Destructor. - ~GenericPointer() { - if (nameBuffer_) // If user-supplied tokens constructor is used, nameBuffer_ is nullptr and tokens_ are not deallocated. - Allocator::Free(tokens_); - RAPIDJSON_DELETE(ownAllocator_); - } - - //! Assignment operator. - GenericPointer& operator=(const GenericPointer& rhs) { - if (this != &rhs) { - // Do not delete ownAllcator - if (nameBuffer_) - Allocator::Free(tokens_); - - tokenCount_ = rhs.tokenCount_; - parseErrorOffset_ = rhs.parseErrorOffset_; - parseErrorCode_ = rhs.parseErrorCode_; - - if (rhs.nameBuffer_) - CopyFromRaw(rhs); // Normally parsed tokens. - else { - tokens_ = rhs.tokens_; // User supplied const tokens. - nameBuffer_ = 0; - } - } - return *this; - } - - //! Swap the content of this pointer with an other. - /*! - \param other The pointer to swap with. - \note Constant complexity. - */ - GenericPointer& Swap(GenericPointer& other) RAPIDJSON_NOEXCEPT { - internal::Swap(allocator_, other.allocator_); - internal::Swap(ownAllocator_, other.ownAllocator_); - internal::Swap(nameBuffer_, other.nameBuffer_); - internal::Swap(tokens_, other.tokens_); - internal::Swap(tokenCount_, other.tokenCount_); - internal::Swap(parseErrorOffset_, other.parseErrorOffset_); - internal::Swap(parseErrorCode_, other.parseErrorCode_); - return *this; - } - - //! free-standing swap function helper - /*! - Helper function to enable support for common swap implementation pattern based on \c std::swap: - \code - void swap(MyClass& a, MyClass& b) { - using std::swap; - swap(a.pointer, b.pointer); - // ... - } - \endcode - \see Swap() - */ - friend inline void swap(GenericPointer& a, GenericPointer& b) RAPIDJSON_NOEXCEPT { a.Swap(b); } - - //@} - - //!@name Append token - //@{ - - //! Append a token and return a new Pointer - /*! - \param token Token to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const Token& token, Allocator* allocator = 0) const { - GenericPointer r; - r.allocator_ = allocator; - Ch *p = r.CopyFromRaw(*this, 1, token.length + 1); - std::memcpy(p, token.name, (token.length + 1) * sizeof(Ch)); - r.tokens_[tokenCount_].name = p; - r.tokens_[tokenCount_].length = token.length; - r.tokens_[tokenCount_].index = token.index; - return r; - } - - //! Append a name token with length, and return a new Pointer - /*! - \param name Name to be appended. - \param length Length of name. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const Ch* name, SizeType length, Allocator* allocator = 0) const { - Token token = { name, length, kPointerInvalidIndex }; - return Append(token, allocator); - } - - //! Append a name token without length, and return a new Pointer - /*! - \param name Name (const Ch*) to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >), (GenericPointer)) - Append(T* name, Allocator* allocator = 0) const { - return Append(name, internal::StrLen(name), allocator); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Append a name token, and return a new Pointer - /*! - \param name Name to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const std::basic_string& name, Allocator* allocator = 0) const { - return Append(name.c_str(), static_cast(name.size()), allocator); - } -#endif - - //! Append a index token, and return a new Pointer - /*! - \param index Index to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(SizeType index, Allocator* allocator = 0) const { - char buffer[21]; - char* end = sizeof(SizeType) == 4 ? internal::u32toa(index, buffer) : internal::u64toa(index, buffer); - SizeType length = static_cast(end - buffer); - buffer[length] = '\0'; - - if (sizeof(Ch) == 1) { - Token token = { reinterpret_cast(buffer), length, index }; - return Append(token, allocator); - } - else { - Ch name[21]; - for (size_t i = 0; i <= length; i++) - name[i] = static_cast(buffer[i]); - Token token = { name, length, index }; - return Append(token, allocator); - } - } - - //! Append a token by value, and return a new Pointer - /*! - \param token token to be appended. - \param allocator Allocator for the newly return Pointer. - \return A new Pointer with appended token. - */ - GenericPointer Append(const ValueType& token, Allocator* allocator = 0) const { - if (token.IsString()) - return Append(token.GetString(), token.GetStringLength(), allocator); - else { - RAPIDJSON_ASSERT(token.IsUint64()); - RAPIDJSON_ASSERT(token.GetUint64() <= SizeType(~0)); - return Append(static_cast(token.GetUint64()), allocator); - } - } - - //!@name Handling Parse Error - //@{ - - //! Check whether this is a valid pointer. - bool IsValid() const { return parseErrorCode_ == kPointerParseErrorNone; } - - //! Get the parsing error offset in code unit. - size_t GetParseErrorOffset() const { return parseErrorOffset_; } - - //! Get the parsing error code. - PointerParseErrorCode GetParseErrorCode() const { return parseErrorCode_; } - - //@} - - //! Get the allocator of this pointer. - Allocator& GetAllocator() { return *allocator_; } - - //!@name Tokens - //@{ - - //! Get the token array (const version only). - const Token* GetTokens() const { return tokens_; } - - //! Get the number of tokens. - size_t GetTokenCount() const { return tokenCount_; } - - //@} - - //!@name Equality/inequality operators - //@{ - - //! Equality operator. - /*! - \note When any pointers are invalid, always returns false. - */ - bool operator==(const GenericPointer& rhs) const { - if (!IsValid() || !rhs.IsValid() || tokenCount_ != rhs.tokenCount_) - return false; - - for (size_t i = 0; i < tokenCount_; i++) { - if (tokens_[i].index != rhs.tokens_[i].index || - tokens_[i].length != rhs.tokens_[i].length || - (tokens_[i].length != 0 && std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch)* tokens_[i].length) != 0)) - { - return false; - } - } - - return true; - } - - //! Inequality operator. - /*! - \note When any pointers are invalid, always returns true. - */ - bool operator!=(const GenericPointer& rhs) const { return !(*this == rhs); } - - //! Less than operator. - /*! - \note Invalid pointers are always greater than valid ones. - */ - bool operator<(const GenericPointer& rhs) const { - if (!IsValid()) - return false; - if (!rhs.IsValid()) - return true; - - if (tokenCount_ != rhs.tokenCount_) - return tokenCount_ < rhs.tokenCount_; - - for (size_t i = 0; i < tokenCount_; i++) { - if (tokens_[i].index != rhs.tokens_[i].index) - return tokens_[i].index < rhs.tokens_[i].index; - - if (tokens_[i].length != rhs.tokens_[i].length) - return tokens_[i].length < rhs.tokens_[i].length; - - if (int cmp = std::memcmp(tokens_[i].name, rhs.tokens_[i].name, sizeof(Ch) * tokens_[i].length)) - return cmp < 0; - } - - return false; - } - - //@} - - //!@name Stringify - //@{ - - //! Stringify the pointer into string representation. - /*! - \tparam OutputStream Type of output stream. - \param os The output stream. - */ - template - bool Stringify(OutputStream& os) const { - return Stringify(os); - } - - //! Stringify the pointer into URI fragment representation. - /*! - \tparam OutputStream Type of output stream. - \param os The output stream. - */ - template - bool StringifyUriFragment(OutputStream& os) const { - return Stringify(os); - } - - //@} - - //!@name Create value - //@{ - - //! Create a value in a subtree. - /*! - If the value is not exist, it creates all parent values and a JSON Null value. - So it always succeed and return the newly created or existing value. - - Remind that it may change types of parents according to tokens, so it - potentially removes previously stored values. For example, if a document - was an array, and "/foo" is used to create a value, then the document - will be changed to an object, and all existing array elements are lost. - - \param root Root value of a DOM subtree to be resolved. It can be any value other than document root. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \param alreadyExist If non-null, it stores whether the resolved value is already exist. - \return The resolved newly created (a JSON Null value), or already exists value. - */ - ValueType& Create(ValueType& root, typename ValueType::AllocatorType& allocator, bool* alreadyExist = 0) const { - RAPIDJSON_ASSERT(IsValid()); - ValueType* v = &root; - bool exist = true; - for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - if (v->IsArray() && t->name[0] == '-' && t->length == 1) { - v->PushBack(ValueType().Move(), allocator); - v = &((*v)[v->Size() - 1]); - exist = false; - } - else { - if (t->index == kPointerInvalidIndex) { // must be object name - if (!v->IsObject()) - v->SetObject(); // Change to Object - } - else { // object name or array index - if (!v->IsArray() && !v->IsObject()) - v->SetArray(); // Change to Array - } - - if (v->IsArray()) { - if (t->index >= v->Size()) { - v->Reserve(t->index + 1, allocator); - while (t->index >= v->Size()) - v->PushBack(ValueType().Move(), allocator); - exist = false; - } - v = &((*v)[t->index]); - } - else { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); - if (m == v->MemberEnd()) { - v->AddMember(ValueType(t->name, t->length, allocator).Move(), ValueType().Move(), allocator); - v = &(--v->MemberEnd())->value; // Assumes AddMember() appends at the end - exist = false; - } - else - v = &m->value; - } - } - } - - if (alreadyExist) - *alreadyExist = exist; - - return *v; - } - - //! Creates a value in a document. - /*! - \param document A document to be resolved. - \param alreadyExist If non-null, it stores whether the resolved value is already exist. - \return The resolved newly created, or already exists value. - */ - template - ValueType& Create(GenericDocument& document, bool* alreadyExist = 0) const { - return Create(document, document.GetAllocator(), alreadyExist); - } - - //@} - - //!@name Query value - //@{ - - //! Query a value in a subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param unresolvedTokenIndex If the pointer cannot resolve a token in the pointer, this parameter can obtain the index of unresolved token. - \return Pointer to the value if it can be resolved. Otherwise null. - - \note - There are only 3 situations when a value cannot be resolved: - 1. A value in the path is not an array nor object. - 2. An object value does not contain the token. - 3. A token is out of range of an array value. - - Use unresolvedTokenIndex to retrieve the token index. - */ - ValueType* Get(ValueType& root, size_t* unresolvedTokenIndex = 0) const { - RAPIDJSON_ASSERT(IsValid()); - ValueType* v = &root; - for (const Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - switch (v->GetType()) { - case kObjectType: - { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); - if (m == v->MemberEnd()) - break; - v = &m->value; - } - continue; - case kArrayType: - if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - break; - v = &((*v)[t->index]); - continue; - default: - break; - } - - // Error: unresolved token - if (unresolvedTokenIndex) - *unresolvedTokenIndex = static_cast(t - tokens_); - return 0; - } - return v; - } - - //! Query a const value in a const subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \return Pointer to the value if it can be resolved. Otherwise null. - */ - const ValueType* Get(const ValueType& root, size_t* unresolvedTokenIndex = 0) const { - return Get(const_cast(root), unresolvedTokenIndex); - } - - //@} - - //!@name Query a value with default - //@{ - - //! Query a value in a subtree with default value. - /*! - Similar to Get(), but if the specified value do not exists, it creates all parents and clone the default value. - So that this function always succeed. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param defaultValue Default value to be cloned if the value was not exists. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& GetWithDefault(ValueType& root, const ValueType& defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - ValueType& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.CopyFrom(defaultValue, allocator); - } - - //! Query a value in a subtree with default null-terminated string. - ValueType& GetWithDefault(ValueType& root, const Ch* defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - ValueType& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.SetString(defaultValue, allocator); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Query a value in a subtree with default std::basic_string. - ValueType& GetWithDefault(ValueType& root, const std::basic_string& defaultValue, typename ValueType::AllocatorType& allocator) const { - bool alreadyExist; - ValueType& v = Create(root, allocator, &alreadyExist); - return alreadyExist ? v : v.SetString(defaultValue, allocator); - } -#endif - - //! Query a value in a subtree with default primitive value. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - GetWithDefault(ValueType& root, T defaultValue, typename ValueType::AllocatorType& allocator) const { - return GetWithDefault(root, ValueType(defaultValue).Move(), allocator); - } - - //! Query a value in a document with default value. - template - ValueType& GetWithDefault(GenericDocument& document, const ValueType& defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - - //! Query a value in a document with default null-terminated string. - template - ValueType& GetWithDefault(GenericDocument& document, const Ch* defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Query a value in a document with default std::basic_string. - template - ValueType& GetWithDefault(GenericDocument& document, const std::basic_string& defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } -#endif - - //! Query a value in a document with default primitive value. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - GetWithDefault(GenericDocument& document, T defaultValue) const { - return GetWithDefault(document, defaultValue, document.GetAllocator()); - } - - //@} - - //!@name Set a value - //@{ - - //! Set a value in a subtree, with move semantics. - /*! - It creates all parents if they are not exist or types are different to the tokens. - So this function always succeeds but potentially remove existing values. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param value Value to be set. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& Set(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = value; - } - - //! Set a value in a subtree, with copy semantics. - ValueType& Set(ValueType& root, const ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator).CopyFrom(value, allocator); - } - - //! Set a null-terminated string in a subtree. - ValueType& Set(ValueType& root, const Ch* value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value, allocator).Move(); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Set a std::basic_string in a subtree. - ValueType& Set(ValueType& root, const std::basic_string& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value, allocator).Move(); - } -#endif - - //! Set a primitive value in a subtree. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - Set(ValueType& root, T value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator) = ValueType(value).Move(); - } - - //! Set a value in a document, with move semantics. - template - ValueType& Set(GenericDocument& document, ValueType& value) const { - return Create(document) = value; - } - - //! Set a value in a document, with copy semantics. - template - ValueType& Set(GenericDocument& document, const ValueType& value) const { - return Create(document).CopyFrom(value, document.GetAllocator()); - } - - //! Set a null-terminated string in a document. - template - ValueType& Set(GenericDocument& document, const Ch* value) const { - return Create(document) = ValueType(value, document.GetAllocator()).Move(); - } - -#if RAPIDJSON_HAS_STDSTRING - //! Sets a std::basic_string in a document. - template - ValueType& Set(GenericDocument& document, const std::basic_string& value) const { - return Create(document) = ValueType(value, document.GetAllocator()).Move(); - } -#endif - - //! Set a primitive value in a document. - /*! - \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c bool - */ - template - RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (ValueType&)) - Set(GenericDocument& document, T value) const { - return Create(document) = value; - } - - //@} - - //!@name Swap a value - //@{ - - //! Swap a value with a value in a subtree. - /*! - It creates all parents if they are not exist or types are different to the tokens. - So this function always succeeds but potentially remove existing values. - - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \param value Value to be swapped. - \param allocator Allocator for creating the values if the specified value or its parents are not exist. - \see Create() - */ - ValueType& Swap(ValueType& root, ValueType& value, typename ValueType::AllocatorType& allocator) const { - return Create(root, allocator).Swap(value); - } - - //! Swap a value with a value in a document. - template - ValueType& Swap(GenericDocument& document, ValueType& value) const { - return Create(document).Swap(value); - } - - //@} - - //! Erase a value in a subtree. - /*! - \param root Root value of a DOM sub-tree to be resolved. It can be any value other than document root. - \return Whether the resolved value is found and erased. - - \note Erasing with an empty pointer \c Pointer(""), i.e. the root, always fail and return false. - */ - bool Erase(ValueType& root) const { - RAPIDJSON_ASSERT(IsValid()); - if (tokenCount_ == 0) // Cannot erase the root - return false; - - ValueType* v = &root; - const Token* last = tokens_ + (tokenCount_ - 1); - for (const Token *t = tokens_; t != last; ++t) { - switch (v->GetType()) { - case kObjectType: - { - typename ValueType::MemberIterator m = v->FindMember(GenericStringRef(t->name, t->length)); - if (m == v->MemberEnd()) - return false; - v = &m->value; - } - break; - case kArrayType: - if (t->index == kPointerInvalidIndex || t->index >= v->Size()) - return false; - v = &((*v)[t->index]); - break; - default: - return false; - } - } - - switch (v->GetType()) { - case kObjectType: - return v->EraseMember(GenericStringRef(last->name, last->length)); - case kArrayType: - if (last->index == kPointerInvalidIndex || last->index >= v->Size()) - return false; - v->Erase(v->Begin() + last->index); - return true; - default: - return false; - } - } - -private: - //! Clone the content from rhs to this. - /*! - \param rhs Source pointer. - \param extraToken Extra tokens to be allocated. - \param extraNameBufferSize Extra name buffer size (in number of Ch) to be allocated. - \return Start of non-occupied name buffer, for storing extra names. - */ - Ch* CopyFromRaw(const GenericPointer& rhs, size_t extraToken = 0, size_t extraNameBufferSize = 0) { - if (!allocator_) // allocator is independently owned. - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - - size_t nameBufferSize = rhs.tokenCount_; // null terminators for tokens - for (Token *t = rhs.tokens_; t != rhs.tokens_ + rhs.tokenCount_; ++t) - nameBufferSize += t->length; - - tokenCount_ = rhs.tokenCount_ + extraToken; - tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + (nameBufferSize + extraNameBufferSize) * sizeof(Ch))); - nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - if (rhs.tokenCount_ > 0) { - std::memcpy(tokens_, rhs.tokens_, rhs.tokenCount_ * sizeof(Token)); - } - if (nameBufferSize > 0) { - std::memcpy(nameBuffer_, rhs.nameBuffer_, nameBufferSize * sizeof(Ch)); - } - - // Adjust pointers to name buffer - std::ptrdiff_t diff = nameBuffer_ - rhs.nameBuffer_; - for (Token *t = tokens_; t != tokens_ + rhs.tokenCount_; ++t) - t->name += diff; - - return nameBuffer_ + nameBufferSize; - } - - //! Check whether a character should be percent-encoded. - /*! - According to RFC 3986 2.3 Unreserved Characters. - \param c The character (code unit) to be tested. - */ - bool NeedPercentEncode(Ch c) const { - return !((c >= '0' && c <= '9') || (c >= 'A' && c <='Z') || (c >= 'a' && c <= 'z') || c == '-' || c == '.' || c == '_' || c =='~'); - } - - //! Parse a JSON String or its URI fragment representation into tokens. -#ifndef __clang__ // -Wdocumentation - /*! - \param source Either a JSON Pointer string, or its URI fragment representation. Not need to be null terminated. - \param length Length of the source string. - \note Source cannot be JSON String Representation of JSON Pointer, e.g. In "/\u0000", \u0000 will not be unescaped. - */ -#endif - void Parse(const Ch* source, size_t length) { - RAPIDJSON_ASSERT(source != NULL); - RAPIDJSON_ASSERT(nameBuffer_ == 0); - RAPIDJSON_ASSERT(tokens_ == 0); - - // Create own allocator if user did not supply. - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - - // Count number of '/' as tokenCount - tokenCount_ = 0; - for (const Ch* s = source; s != source + length; s++) - if (*s == '/') - tokenCount_++; - - Token* token = tokens_ = static_cast(allocator_->Malloc(tokenCount_ * sizeof(Token) + length * sizeof(Ch))); - Ch* name = nameBuffer_ = reinterpret_cast(tokens_ + tokenCount_); - size_t i = 0; - - // Detect if it is a URI fragment - bool uriFragment = false; - if (source[i] == '#') { - uriFragment = true; - i++; - } - - if (i != length && source[i] != '/') { - parseErrorCode_ = kPointerParseErrorTokenMustBeginWithSolidus; - goto error; - } - - while (i < length) { - RAPIDJSON_ASSERT(source[i] == '/'); - i++; // consumes '/' - - token->name = name; - bool isNumber = true; - - while (i < length && source[i] != '/') { - Ch c = source[i]; - if (uriFragment) { - // Decoding percent-encoding for URI fragment - if (c == '%') { - PercentDecodeStream is(&source[i], source + length); - GenericInsituStringStream os(name); - Ch* begin = os.PutBegin(); - if (!Transcoder, EncodingType>().Validate(is, os) || !is.IsValid()) { - parseErrorCode_ = kPointerParseErrorInvalidPercentEncoding; - goto error; - } - size_t len = os.PutEnd(begin); - i += is.Tell() - 1; - if (len == 1) - c = *name; - else { - name += len; - isNumber = false; - i++; - continue; - } - } - else if (NeedPercentEncode(c)) { - parseErrorCode_ = kPointerParseErrorCharacterMustPercentEncode; - goto error; - } - } - - i++; - - // Escaping "~0" -> '~', "~1" -> '/' - if (c == '~') { - if (i < length) { - c = source[i]; - if (c == '0') c = '~'; - else if (c == '1') c = '/'; - else { - parseErrorCode_ = kPointerParseErrorInvalidEscape; - goto error; - } - i++; - } - else { - parseErrorCode_ = kPointerParseErrorInvalidEscape; - goto error; - } - } - - // First check for index: all of characters are digit - if (c < '0' || c > '9') - isNumber = false; - - *name++ = c; - } - token->length = static_cast(name - token->name); - if (token->length == 0) - isNumber = false; - *name++ = '\0'; // Null terminator - - // Second check for index: more than one digit cannot have leading zero - if (isNumber && token->length > 1 && token->name[0] == '0') - isNumber = false; - - // String to SizeType conversion - SizeType n = 0; - if (isNumber) { - for (size_t j = 0; j < token->length; j++) { - SizeType m = n * 10 + static_cast(token->name[j] - '0'); - if (m < n) { // overflow detection - isNumber = false; - break; - } - n = m; - } - } - - token->index = isNumber ? n : kPointerInvalidIndex; - token++; - } - - RAPIDJSON_ASSERT(name <= nameBuffer_ + length); // Should not overflow buffer - parseErrorCode_ = kPointerParseErrorNone; - return; - - error: - Allocator::Free(tokens_); - nameBuffer_ = 0; - tokens_ = 0; - tokenCount_ = 0; - parseErrorOffset_ = i; - return; - } - - //! Stringify to string or URI fragment representation. - /*! - \tparam uriFragment True for stringifying to URI fragment representation. False for string representation. - \tparam OutputStream type of output stream. - \param os The output stream. - */ - template - bool Stringify(OutputStream& os) const { - RAPIDJSON_ASSERT(IsValid()); - - if (uriFragment) - os.Put('#'); - - for (Token *t = tokens_; t != tokens_ + tokenCount_; ++t) { - os.Put('/'); - for (size_t j = 0; j < t->length; j++) { - Ch c = t->name[j]; - if (c == '~') { - os.Put('~'); - os.Put('0'); - } - else if (c == '/') { - os.Put('~'); - os.Put('1'); - } - else if (uriFragment && NeedPercentEncode(c)) { - // Transcode to UTF8 sequence - GenericStringStream source(&t->name[j]); - PercentEncodeStream target(os); - if (!Transcoder >().Validate(source, target)) - return false; - j += source.Tell() - 1; - } - else - os.Put(c); - } - } - return true; - } - - //! A helper stream for decoding a percent-encoded sequence into code unit. - /*! - This stream decodes %XY triplet into code unit (0-255). - If it encounters invalid characters, it sets output code unit as 0 and - mark invalid, and to be checked by IsValid(). - */ - class PercentDecodeStream { - public: - typedef typename ValueType::Ch Ch; - - //! Constructor - /*! - \param source Start of the stream - \param end Past-the-end of the stream. - */ - PercentDecodeStream(const Ch* source, const Ch* end) : src_(source), head_(source), end_(end), valid_(true) {} - - Ch Take() { - if (*src_ != '%' || src_ + 3 > end_) { // %XY triplet - valid_ = false; - return 0; - } - src_++; - Ch c = 0; - for (int j = 0; j < 2; j++) { - c = static_cast(c << 4); - Ch h = *src_; - if (h >= '0' && h <= '9') c = static_cast(c + h - '0'); - else if (h >= 'A' && h <= 'F') c = static_cast(c + h - 'A' + 10); - else if (h >= 'a' && h <= 'f') c = static_cast(c + h - 'a' + 10); - else { - valid_ = false; - return 0; - } - src_++; - } - return c; - } - - size_t Tell() const { return static_cast(src_ - head_); } - bool IsValid() const { return valid_; } - - private: - const Ch* src_; //!< Current read position. - const Ch* head_; //!< Original head of the string. - const Ch* end_; //!< Past-the-end position. - bool valid_; //!< Whether the parsing is valid. - }; - - //! A helper stream to encode character (UTF-8 code unit) into percent-encoded sequence. - template - class PercentEncodeStream { - public: - PercentEncodeStream(OutputStream& os) : os_(os) {} - void Put(char c) { // UTF-8 must be byte - unsigned char u = static_cast(c); - static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; - os_.Put('%'); - os_.Put(static_cast(hexDigits[u >> 4])); - os_.Put(static_cast(hexDigits[u & 15])); - } - private: - OutputStream& os_; - }; - - Allocator* allocator_; //!< The current allocator. It is either user-supplied or equal to ownAllocator_. - Allocator* ownAllocator_; //!< Allocator owned by this Pointer. - Ch* nameBuffer_; //!< A buffer containing all names in tokens. - Token* tokens_; //!< A list of tokens. - size_t tokenCount_; //!< Number of tokens in tokens_. - size_t parseErrorOffset_; //!< Offset in code unit when parsing fail. - PointerParseErrorCode parseErrorCode_; //!< Parsing error code. -}; - -//! GenericPointer for Value (UTF-8, default allocator). -typedef GenericPointer Pointer; - -//!@name Helper functions for GenericPointer -//@{ - -////////////////////////////////////////////////////////////////////////////// - -template -typename T::ValueType& CreateValueByPointer(T& root, const GenericPointer& pointer, typename T::AllocatorType& a) { - return pointer.Create(root, a); -} - -template -typename T::ValueType& CreateValueByPointer(T& root, const CharType(&source)[N], typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Create(root, a); -} - -// No allocator parameter - -template -typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const GenericPointer& pointer) { - return pointer.Create(document); -} - -template -typename DocumentType::ValueType& CreateValueByPointer(DocumentType& document, const CharType(&source)[N]) { - return GenericPointer(source, N - 1).Create(document); -} - -////////////////////////////////////////////////////////////////////////////// - -template -typename T::ValueType* GetValueByPointer(T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); -} - -template -const typename T::ValueType* GetValueByPointer(const T& root, const GenericPointer& pointer, size_t* unresolvedTokenIndex = 0) { - return pointer.Get(root, unresolvedTokenIndex); -} - -template -typename T::ValueType* GetValueByPointer(T& root, const CharType (&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); -} - -template -const typename T::ValueType* GetValueByPointer(const T& root, const CharType(&source)[N], size_t* unresolvedTokenIndex = 0) { - return GenericPointer(source, N - 1).Get(root, unresolvedTokenIndex); -} - -////////////////////////////////////////////////////////////////////////////// - -template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} - -template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const typename T::Ch* defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, const std::basic_string& defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -GetValueByPointerWithDefault(T& root, const GenericPointer& pointer, T2 defaultValue, typename T::AllocatorType& a) { - return pointer.GetWithDefault(root, defaultValue, a); -} - -template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::ValueType& defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); -} - -template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const typename T::Ch* defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename T::ValueType& GetValueByPointerWithDefault(T& root, const CharType(&source)[N], const std::basic_string& defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -GetValueByPointerWithDefault(T& root, const CharType(&source)[N], T2 defaultValue, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).GetWithDefault(root, defaultValue, a); -} - -// No allocator parameter - -template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} - -template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, const std::basic_string& defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -GetValueByPointerWithDefault(DocumentType& document, const GenericPointer& pointer, T2 defaultValue) { - return pointer.GetWithDefault(document, defaultValue); -} - -template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); -} - -template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename DocumentType::ValueType& GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], const std::basic_string& defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -GetValueByPointerWithDefault(DocumentType& document, const CharType(&source)[N], T2 defaultValue) { - return GenericPointer(source, N - 1).GetWithDefault(document, defaultValue); -} - -////////////////////////////////////////////////////////////////////////////// - -template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const typename T::Ch* value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename T::ValueType& SetValueByPointer(T& root, const GenericPointer& pointer, const std::basic_string& value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -SetValueByPointer(T& root, const GenericPointer& pointer, T2 value, typename T::AllocatorType& a) { - return pointer.Set(root, value, a); -} - -template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); -} - -template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); -} - -template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const typename T::Ch* value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename T::ValueType& SetValueByPointer(T& root, const CharType(&source)[N], const std::basic_string& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename T::ValueType&)) -SetValueByPointer(T& root, const CharType(&source)[N], T2 value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Set(root, value, a); -} - -// No allocator parameter - -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { - return pointer.Set(document, value); -} - -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::ValueType& value) { - return pointer.Set(document, value); -} - -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const typename DocumentType::Ch* value) { - return pointer.Set(document, value); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const GenericPointer& pointer, const std::basic_string& value) { - return pointer.Set(document, value); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -SetValueByPointer(DocumentType& document, const GenericPointer& pointer, T2 value) { - return pointer.Set(document, value); -} - -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { - return GenericPointer(source, N - 1).Set(document, value); -} - -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::ValueType& value) { - return GenericPointer(source, N - 1).Set(document, value); -} - -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const typename DocumentType::Ch* value) { - return GenericPointer(source, N - 1).Set(document, value); -} - -#if RAPIDJSON_HAS_STDSTRING -template -typename DocumentType::ValueType& SetValueByPointer(DocumentType& document, const CharType(&source)[N], const std::basic_string& value) { - return GenericPointer(source, N - 1).Set(document, value); -} -#endif - -template -RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr, internal::IsGenericValue >), (typename DocumentType::ValueType&)) -SetValueByPointer(DocumentType& document, const CharType(&source)[N], T2 value) { - return GenericPointer(source, N - 1).Set(document, value); -} - -////////////////////////////////////////////////////////////////////////////// - -template -typename T::ValueType& SwapValueByPointer(T& root, const GenericPointer& pointer, typename T::ValueType& value, typename T::AllocatorType& a) { - return pointer.Swap(root, value, a); -} - -template -typename T::ValueType& SwapValueByPointer(T& root, const CharType(&source)[N], typename T::ValueType& value, typename T::AllocatorType& a) { - return GenericPointer(source, N - 1).Swap(root, value, a); -} - -template -typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const GenericPointer& pointer, typename DocumentType::ValueType& value) { - return pointer.Swap(document, value); -} - -template -typename DocumentType::ValueType& SwapValueByPointer(DocumentType& document, const CharType(&source)[N], typename DocumentType::ValueType& value) { - return GenericPointer(source, N - 1).Swap(document, value); -} - -////////////////////////////////////////////////////////////////////////////// - -template -bool EraseValueByPointer(T& root, const GenericPointer& pointer) { - return pointer.Erase(root); -} - -template -bool EraseValueByPointer(T& root, const CharType(&source)[N]) { - return GenericPointer(source, N - 1).Erase(root); -} - -//@} - -RAPIDJSON_NAMESPACE_END - -#if defined(__clang__) || defined(_MSC_VER) -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_POINTER_H_ diff --git a/src/native/external/rapidjson/prettywriter.h b/src/native/external/rapidjson/prettywriter.h deleted file mode 100644 index 45afb6949deb4..0000000000000 --- a/src/native/external/rapidjson/prettywriter.h +++ /dev/null @@ -1,277 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License. You may obtain a copy of the License at -// -// http://opensource.org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. - -#ifndef RAPIDJSON_PRETTYWRITER_H_ -#define RAPIDJSON_PRETTYWRITER_H_ - -#include "writer.h" - -#ifdef __GNUC__ -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#if defined(__clang__) -RAPIDJSON_DIAG_PUSH -RAPIDJSON_DIAG_OFF(c++98-compat) -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -//! Combination of PrettyWriter format flags. -/*! \see PrettyWriter::SetFormatOptions - */ -enum PrettyFormatOptions { - kFormatDefault = 0, //!< Default pretty formatting. - kFormatSingleLineArray = 1 //!< Format arrays on a single line. -}; - -//! Writer with indentation and spacing. -/*! - \tparam OutputStream Type of output os. - \tparam SourceEncoding Encoding of source string. - \tparam TargetEncoding Encoding of output stream. - \tparam StackAllocator Type of allocator for allocating memory of stack. -*/ -template, typename TargetEncoding = UTF8<>, typename StackAllocator = CrtAllocator, unsigned writeFlags = kWriteDefaultFlags> -class PrettyWriter : public Writer { -public: - typedef Writer Base; - typedef typename Base::Ch Ch; - - //! Constructor - /*! \param os Output stream. - \param allocator User supplied allocator. If it is null, it will create a private one. - \param levelDepth Initial capacity of stack. - */ - explicit PrettyWriter(OutputStream& os, StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4), formatOptions_(kFormatDefault) {} - - - explicit PrettyWriter(StackAllocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : - Base(allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - PrettyWriter(PrettyWriter&& rhs) : - Base(std::forward(rhs)), indentChar_(rhs.indentChar_), indentCharCount_(rhs.indentCharCount_), formatOptions_(rhs.formatOptions_) {} -#endif - - //! Set custom indentation. - /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\\t', '\\n', '\\r'). - \param indentCharCount Number of indent characters for each indentation level. - \note The default indentation is 4 spaces. - */ - PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { - RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); - indentChar_ = indentChar; - indentCharCount_ = indentCharCount; - return *this; - } - - //! Set pretty writer formatting options. - /*! \param options Formatting options. - */ - PrettyWriter& SetFormatOptions(PrettyFormatOptions options) { - formatOptions_ = options; - return *this; - } - - /*! @name Implementation of Handler - \see Handler - */ - //@{ - - bool Null() { PrettyPrefix(kNullType); return Base::EndValue(Base::WriteNull()); } - bool Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); return Base::EndValue(Base::WriteBool(b)); } - bool Int(int i) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt(i)); } - bool Uint(unsigned u) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint(u)); } - bool Int64(int64_t i64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteInt64(i64)); } - bool Uint64(uint64_t u64) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteUint64(u64)); } - bool Double(double d) { PrettyPrefix(kNumberType); return Base::EndValue(Base::WriteDouble(d)); } - - bool RawNumber(const Ch* str, SizeType length, bool copy = false) { - RAPIDJSON_ASSERT(str != 0); - (void)copy; - PrettyPrefix(kNumberType); - return Base::EndValue(Base::WriteString(str, length)); - } - - bool String(const Ch* str, SizeType length, bool copy = false) { - RAPIDJSON_ASSERT(str != 0); - (void)copy; - PrettyPrefix(kStringType); - return Base::EndValue(Base::WriteString(str, length)); - } - -#if RAPIDJSON_HAS_STDSTRING - bool String(const std::basic_string& str) { - return String(str.data(), SizeType(str.size())); - } -#endif - - bool StartObject() { - PrettyPrefix(kObjectType); - new (Base::level_stack_.template Push()) typename Base::Level(false); - return Base::WriteStartObject(); - } - - bool Key(const Ch* str, SizeType length, bool copy = false) { return String(str, length, copy); } - -#if RAPIDJSON_HAS_STDSTRING - bool Key(const std::basic_string& str) { - return Key(str.data(), SizeType(str.size())); - } -#endif - - bool EndObject(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); // not inside an Object - RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); // currently inside an Array, not Object - RAPIDJSON_ASSERT(0 == Base::level_stack_.template Top()->valueCount % 2); // Object has a Key without a Value - - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty) { - Base::os_->Put('\n'); - WriteIndent(); - } - bool ret = Base::EndValue(Base::WriteEndObject()); - (void)ret; - RAPIDJSON_ASSERT(ret == true); - if (Base::level_stack_.Empty()) // end of json text - Base::Flush(); - return true; - } - - bool StartArray() { - PrettyPrefix(kArrayType); - new (Base::level_stack_.template Push()) typename Base::Level(true); - return Base::WriteStartArray(); - } - - bool EndArray(SizeType memberCount = 0) { - (void)memberCount; - RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); - RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); - bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; - - if (!empty && !(formatOptions_ & kFormatSingleLineArray)) { - Base::os_->Put('\n'); - WriteIndent(); - } - bool ret = Base::EndValue(Base::WriteEndArray()); - (void)ret; - RAPIDJSON_ASSERT(ret == true); - if (Base::level_stack_.Empty()) // end of json text - Base::Flush(); - return true; - } - - //@} - - /*! @name Convenience extensions */ - //@{ - - //! Simpler but slower overload. - bool String(const Ch* str) { return String(str, internal::StrLen(str)); } - bool Key(const Ch* str) { return Key(str, internal::StrLen(str)); } - - //@} - - //! Write a raw JSON value. - /*! - For user to write a stringified JSON as a value. - - \param json A well-formed JSON value. It should not contain null character within [0, length - 1] range. - \param length Length of the json. - \param type Type of the root of json. - \note When using PrettyWriter::RawValue(), the result json may not be indented correctly. - */ - bool RawValue(const Ch* json, size_t length, Type type) { - RAPIDJSON_ASSERT(json != 0); - PrettyPrefix(type); - return Base::EndValue(Base::WriteRawValue(json, length)); - } - -protected: - void PrettyPrefix(Type type) { - (void)type; - if (Base::level_stack_.GetSize() != 0) { // this value is not at root - typename Base::Level* level = Base::level_stack_.template Top(); - - if (level->inArray) { - if (level->valueCount > 0) { - Base::os_->Put(','); // add comma if it is not the first element in array - if (formatOptions_ & kFormatSingleLineArray) - Base::os_->Put(' '); - } - - if (!(formatOptions_ & kFormatSingleLineArray)) { - Base::os_->Put('\n'); - WriteIndent(); - } - } - else { // in object - if (level->valueCount > 0) { - if (level->valueCount % 2 == 0) { - Base::os_->Put(','); - Base::os_->Put('\n'); - } - else { - Base::os_->Put(':'); - Base::os_->Put(' '); - } - } - else - Base::os_->Put('\n'); - - if (level->valueCount % 2 == 0) - WriteIndent(); - } - if (!level->inArray && level->valueCount % 2 == 0) - RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name - level->valueCount++; - } - else { - RAPIDJSON_ASSERT(!Base::hasRoot_); // Should only has one and only one root. - Base::hasRoot_ = true; - } - } - - void WriteIndent() { - size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; - PutN(*Base::os_, static_cast(indentChar_), count); - } - - Ch indentChar_; - unsigned indentCharCount_; - PrettyFormatOptions formatOptions_; - -private: - // Prohibit copy constructor & assignment operator. - PrettyWriter(const PrettyWriter&); - PrettyWriter& operator=(const PrettyWriter&); -}; - -RAPIDJSON_NAMESPACE_END - -#if defined(__clang__) -RAPIDJSON_DIAG_POP -#endif - -#ifdef __GNUC__ -RAPIDJSON_DIAG_POP -#endif - -#endif // RAPIDJSON_RAPIDJSON_H_ diff --git a/src/native/external/rapidjson/rapidjson.h b/src/native/external/rapidjson/rapidjson.h index 549936ffe06c4..5ea69479501a3 100644 --- a/src/native/external/rapidjson/rapidjson.h +++ b/src/native/external/rapidjson/rapidjson.h @@ -1,15 +1,15 @@ // Tencent is pleased to support the open source community by making RapidJSON available. -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at // // http://opensource.org/licenses/MIT // -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the // specific language governing permissions and limitations under the License. #ifndef RAPIDJSON_RAPIDJSON_H_ @@ -17,7 +17,7 @@ /*!\file rapidjson.h \brief common definitions and configuration - + \see RAPIDJSON_CONFIG */ @@ -124,6 +124,19 @@ #define RAPIDJSON_NAMESPACE_END } #endif +/////////////////////////////////////////////////////////////////////////////// +// __cplusplus macro + +//!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN + +#if defined(_MSC_VER) +#define RAPIDJSON_CPLUSPLUS _MSVC_LANG +#else +#define RAPIDJSON_CPLUSPLUS __cplusplus +#endif + +//!@endcond + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_HAS_STDSTRING @@ -149,6 +162,24 @@ #include #endif // RAPIDJSON_HAS_STDSTRING +/////////////////////////////////////////////////////////////////////////////// +// RAPIDJSON_USE_MEMBERSMAP + +/*! \def RAPIDJSON_USE_MEMBERSMAP + \ingroup RAPIDJSON_CONFIG + \brief Enable RapidJSON support for object members handling in a \c std::multimap + + By defining this preprocessor symbol to \c 1, \ref rapidjson::GenericValue object + members are stored in a \c std::multimap for faster lookup and deletion times, a + trade off with a slightly slower insertion time and a small object allocat(or)ed + memory overhead. + + \hideinitializer +*/ +#ifndef RAPIDJSON_USE_MEMBERSMAP +#define RAPIDJSON_USE_MEMBERSMAP 0 // not by default +#endif + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_NO_INT64DEFINE @@ -164,7 +195,7 @@ */ #ifndef RAPIDJSON_NO_INT64DEFINE //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 +#if defined(_MSC_VER) && (_MSC_VER < 1800) // Visual Studio 2013 #include "msinttypes/stdint.h" #include "msinttypes/inttypes.h" #else @@ -246,7 +277,7 @@ # elif defined(RAPIDJSON_DOXYGEN_RUNNING) # define RAPIDJSON_ENDIAN # else -# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. +# error Unknown machine endianness detected. User needs to define RAPIDJSON_ENDIAN. # endif #endif // RAPIDJSON_ENDIAN @@ -411,7 +442,7 @@ RAPIDJSON_NAMESPACE_END // Prefer C++11 static_assert, if available #ifndef RAPIDJSON_STATIC_ASSERT -#if __cplusplus >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) +#if RAPIDJSON_CPLUSPLUS >= 201103L || ( defined(_MSC_VER) && _MSC_VER >= 1800 ) #define RAPIDJSON_STATIC_ASSERT(x) \ static_assert(x, RAPIDJSON_STRINGIFY(x)) #endif // C++11 @@ -482,7 +513,7 @@ RAPIDJSON_NAMESPACE_END //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN -#define RAPIDJSON_MULTILINEMACRO_BEGIN do { +#define RAPIDJSON_MULTILINEMACRO_BEGIN do { #define RAPIDJSON_MULTILINEMACRO_END \ } while((void)0, 0) @@ -490,6 +521,12 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_VERSION_CODE(x,y,z) \ (((x)*100000) + ((y)*100) + (z)) +#if defined(__has_builtin) +#define RAPIDJSON_HAS_BUILTIN(x) __has_builtin(x) +#else +#define RAPIDJSON_HAS_BUILTIN(x) 0 +#endif + /////////////////////////////////////////////////////////////////////////////// // RAPIDJSON_DIAG_PUSH/POP, RAPIDJSON_DIAG_OFF @@ -535,8 +572,14 @@ RAPIDJSON_NAMESPACE_END /////////////////////////////////////////////////////////////////////////////// // C++11 features +#ifndef RAPIDJSON_HAS_CXX11 +#define RAPIDJSON_HAS_CXX11 (RAPIDJSON_CPLUSPLUS >= 201103L) +#endif + #ifndef RAPIDJSON_HAS_CXX11_RVALUE_REFS -#if defined(__clang__) +#if RAPIDJSON_HAS_CXX11 +#define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 +#elif defined(__clang__) #if __has_feature(cxx_rvalue_references) && \ (defined(_MSC_VER) || defined(_LIBCPP_VERSION) || defined(__GLIBCXX__) && __GLIBCXX__ >= 20080306) #define RAPIDJSON_HAS_CXX11_RVALUE_REFS 1 @@ -553,8 +596,14 @@ RAPIDJSON_NAMESPACE_END #endif #endif // RAPIDJSON_HAS_CXX11_RVALUE_REFS +#if RAPIDJSON_HAS_CXX11_RVALUE_REFS +#include // std::move +#endif + #ifndef RAPIDJSON_HAS_CXX11_NOEXCEPT -#if defined(__clang__) +#if RAPIDJSON_HAS_CXX11 +#define RAPIDJSON_HAS_CXX11_NOEXCEPT 1 +#elif defined(__clang__) #define RAPIDJSON_HAS_CXX11_NOEXCEPT __has_feature(cxx_noexcept) #elif (defined(RAPIDJSON_GNUC) && (RAPIDJSON_GNUC >= RAPIDJSON_VERSION_CODE(4,6,0)) && defined(__GXX_EXPERIMENTAL_CXX0X__)) || \ (defined(_MSC_VER) && _MSC_VER >= 1900) || \ @@ -564,11 +613,13 @@ RAPIDJSON_NAMESPACE_END #define RAPIDJSON_HAS_CXX11_NOEXCEPT 0 #endif #endif +#ifndef RAPIDJSON_NOEXCEPT #if RAPIDJSON_HAS_CXX11_NOEXCEPT #define RAPIDJSON_NOEXCEPT noexcept #else -#define RAPIDJSON_NOEXCEPT /* noexcept */ +#define RAPIDJSON_NOEXCEPT throw() #endif // RAPIDJSON_HAS_CXX11_NOEXCEPT +#endif // no automatic detection, yet #ifndef RAPIDJSON_HAS_CXX11_TYPETRAITS @@ -591,6 +642,27 @@ RAPIDJSON_NAMESPACE_END #endif #endif // RAPIDJSON_HAS_CXX11_RANGE_FOR +/////////////////////////////////////////////////////////////////////////////// +// C++17 features + +#ifndef RAPIDJSON_HAS_CXX17 +#define RAPIDJSON_HAS_CXX17 (RAPIDJSON_CPLUSPLUS >= 201703L) +#endif + +#if RAPIDJSON_HAS_CXX17 +# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[fallthrough]] +#elif defined(__has_cpp_attribute) +# if __has_cpp_attribute(clang::fallthrough) +# define RAPIDJSON_DELIBERATE_FALLTHROUGH [[clang::fallthrough]] +# elif __has_cpp_attribute(fallthrough) +# define RAPIDJSON_DELIBERATE_FALLTHROUGH __attribute__((fallthrough)) +# else +# define RAPIDJSON_DELIBERATE_FALLTHROUGH +# endif +#else +# define RAPIDJSON_DELIBERATE_FALLTHROUGH +#endif + //!@endcond //! Assertion (in non-throwing contexts). @@ -609,16 +681,29 @@ RAPIDJSON_NAMESPACE_END #ifndef RAPIDJSON_NOEXCEPT_ASSERT #ifdef RAPIDJSON_ASSERT_THROWS -#if RAPIDJSON_HAS_CXX11_NOEXCEPT -#define RAPIDJSON_NOEXCEPT_ASSERT(x) -#else -#define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) -#endif // RAPIDJSON_HAS_CXX11_NOEXCEPT +#include +#define RAPIDJSON_NOEXCEPT_ASSERT(x) assert(x) #else #define RAPIDJSON_NOEXCEPT_ASSERT(x) RAPIDJSON_ASSERT(x) #endif // RAPIDJSON_ASSERT_THROWS #endif // RAPIDJSON_NOEXCEPT_ASSERT +/////////////////////////////////////////////////////////////////////////////// +// malloc/realloc/free + +#ifndef RAPIDJSON_MALLOC +///! customization point for global \c malloc +#define RAPIDJSON_MALLOC(size) std::malloc(size) +#endif +#ifndef RAPIDJSON_REALLOC +///! customization point for global \c realloc +#define RAPIDJSON_REALLOC(ptr, new_size) std::realloc(ptr, new_size) +#endif +#ifndef RAPIDJSON_FREE +///! customization point for global \c free +#define RAPIDJSON_FREE(ptr) std::free(ptr) +#endif + /////////////////////////////////////////////////////////////////////////////// // new/delete @@ -646,7 +731,7 @@ enum Type { kFalseType = 1, //!< false kTrueType = 2, //!< true kObjectType = 3, //!< object - kArrayType = 4, //!< array + kArrayType = 4, //!< array kStringType = 5, //!< string kNumberType = 6 //!< number }; diff --git a/src/native/external/rapidjson/reader.h b/src/native/external/rapidjson/reader.h index 44a6bcd30cf24..55546601e29bd 100644 --- a/src/native/external/rapidjson/reader.h +++ b/src/native/external/rapidjson/reader.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -20,6 +20,7 @@ #include "allocators.h" #include "stream.h" #include "encodedstream.h" +#include "internal/clzll.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strtod.h" @@ -153,6 +154,7 @@ enum ParseFlag { kParseNumbersAsStringsFlag = 64, //!< Parse all numbers (ints/doubles) as strings. kParseTrailingCommasFlag = 128, //!< Allow trailing commas at the end of objects and arrays. kParseNanAndInfFlag = 256, //!< Allow parsing NaN, Inf, Infinity, -Inf and -Infinity as doubles. + kParseEscapedApostropheFlag = 512, //!< Allow escaped apostrophe in strings. kParseDefaultFlags = RAPIDJSON_PARSE_DEFAULT_FLAGS //!< Default parse flags. Can be customized by defining RAPIDJSON_PARSE_DEFAULT_FLAGS }; @@ -443,16 +445,16 @@ inline const char *SkipWhitespace_SIMD(const char* p) { x = vmvnq_u8(x); // Negate x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract + uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract if (low == 0) { if (high != 0) { - int lz =__builtin_clzll(high);; + uint32_t lz = internal::clzll(high); return p + 8 + (lz >> 3); } } else { - int lz = __builtin_clzll(low);; + uint32_t lz = internal::clzll(low); return p + (lz >> 3); } } @@ -479,16 +481,16 @@ inline const char *SkipWhitespace_SIMD(const char* p, const char* end) { x = vmvnq_u8(x); // Negate x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract + uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract if (low == 0) { if (high != 0) { - int lz = __builtin_clzll(high); + uint32_t lz = internal::clzll(high); return p + 8 + (lz >> 3); } } else { - int lz = __builtin_clzll(low); + uint32_t lz = internal::clzll(low); return p + (lz >> 3); } } @@ -990,7 +992,7 @@ class GenericReader { //!@cond RAPIDJSON_HIDDEN_FROM_DOXYGEN #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 static const char escape[256] = { - Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'/', + Z16, Z16, 0, 0,'\"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, '/', Z16, Z16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, 0, 0,'\b', 0, 0, 0,'\f', 0, 0, 0, 0, 0, 0, 0,'\n', 0, 0, 0,'\r', 0,'\t', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1013,19 +1015,31 @@ class GenericReader { is.Take(); os.Put(static_cast(escape[static_cast(e)])); } + else if ((parseFlags & kParseEscapedApostropheFlag) && RAPIDJSON_LIKELY(e == '\'')) { // Allow escaped apostrophe + is.Take(); + os.Put('\''); + } else if (RAPIDJSON_LIKELY(e == 'u')) { // Unicode is.Take(); unsigned codepoint = ParseHex4(is, escapeOffset); RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDBFF)) { - // Handle UTF-16 surrogate pair - if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) - RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - unsigned codepoint2 = ParseHex4(is, escapeOffset); - RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; - if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + if (RAPIDJSON_UNLIKELY(codepoint >= 0xD800 && codepoint <= 0xDFFF)) { + // high surrogate, check if followed by valid low surrogate + if (RAPIDJSON_LIKELY(codepoint <= 0xDBFF)) { + // Handle UTF-16 surrogate pair + if (RAPIDJSON_UNLIKELY(!Consume(is, '\\') || !Consume(is, 'u'))) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + unsigned codepoint2 = ParseHex4(is, escapeOffset); + RAPIDJSON_PARSE_ERROR_EARLY_RETURN_VOID; + if (RAPIDJSON_UNLIKELY(codepoint2 < 0xDC00 || codepoint2 > 0xDFFF)) + RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); + codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } + // single low surrogate + else + { RAPIDJSON_PARSE_ERROR(kParseErrorStringUnicodeSurrogateInvalid, escapeOffset); - codepoint = (((codepoint - 0xD800) << 10) | (codepoint2 - 0xDC00)) + 0x10000; + } } TEncoding::Encode(os, codepoint); } @@ -1244,19 +1258,19 @@ class GenericReader { x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract + uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract SizeType length = 0; bool escaped = false; if (low == 0) { if (high != 0) { - unsigned lz = (unsigned)__builtin_clzll(high);; + uint32_t lz = internal::clzll(high); length = 8 + (lz >> 3); escaped = true; } } else { - unsigned lz = (unsigned)__builtin_clzll(low);; + uint32_t lz = internal::clzll(low); length = lz >> 3; escaped = true; } @@ -1314,19 +1328,19 @@ class GenericReader { x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract + uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract SizeType length = 0; bool escaped = false; if (low == 0) { if (high != 0) { - unsigned lz = (unsigned)__builtin_clzll(high); + uint32_t lz = internal::clzll(high); length = 8 + (lz >> 3); escaped = true; } } else { - unsigned lz = (unsigned)__builtin_clzll(low); + uint32_t lz = internal::clzll(low); length = lz >> 3; escaped = true; } @@ -1370,17 +1384,17 @@ class GenericReader { x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract + uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract if (low == 0) { if (high != 0) { - int lz = __builtin_clzll(high); + uint32_t lz = internal::clzll(high); p += 8 + (lz >> 3); break; } } else { - int lz = __builtin_clzll(low); + uint32_t lz = internal::clzll(low); p += lz >> 3; break; } @@ -1390,11 +1404,11 @@ class GenericReader { } #endif // RAPIDJSON_NEON - template + template class NumberStream; - template - class NumberStream { + template + class NumberStream { public: typedef typename InputStream::Ch Ch; @@ -1403,11 +1417,11 @@ class GenericReader { RAPIDJSON_FORCEINLINE Ch Peek() const { return is.Peek(); } RAPIDJSON_FORCEINLINE Ch TakePush() { return is.Take(); } RAPIDJSON_FORCEINLINE Ch Take() { return is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char) {} + RAPIDJSON_FORCEINLINE void Push(char) {} size_t Tell() { return is.Tell(); } size_t Length() { return 0; } - const char* Pop() { return 0; } + const StackCharacter* Pop() { return 0; } protected: NumberStream& operator=(const NumberStream&); @@ -1415,45 +1429,47 @@ class GenericReader { InputStream& is; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is), stackStream(reader.stack_) {} + NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s), stackStream(reader.stack_) {} RAPIDJSON_FORCEINLINE Ch TakePush() { - stackStream.Put(static_cast(Base::is.Peek())); + stackStream.Put(static_cast(Base::is.Peek())); return Base::is.Take(); } - RAPIDJSON_FORCEINLINE void Push(char c) { + RAPIDJSON_FORCEINLINE void Push(StackCharacter c) { stackStream.Put(c); } size_t Length() { return stackStream.Length(); } - const char* Pop() { + const StackCharacter* Pop() { stackStream.Put('\0'); return stackStream.Pop(); } private: - StackStream stackStream; + StackStream stackStream; }; - template - class NumberStream : public NumberStream { - typedef NumberStream Base; + template + class NumberStream : public NumberStream { + typedef NumberStream Base; public: - NumberStream(GenericReader& reader, InputStream& is) : Base(reader, is) {} + NumberStream(GenericReader& reader, InputStream& s) : Base(reader, s) {} RAPIDJSON_FORCEINLINE Ch Take() { return Base::TakePush(); } }; template void ParseNumber(InputStream& is, Handler& handler) { + typedef typename internal::SelectIf, typename TargetEncoding::Ch, char>::Type NumberCharacter; + internal::StreamLocalCopy copy(is); - NumberStream(s.Length()); - StringStream srcStream(s.Pop()); + GenericStringStream > srcStream(s.Pop()); StackStream dstStream(stack_); while (numCharsToCopy--) { - Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); + Transcoder, TargetEncoding>::Transcode(srcStream, dstStream); } dstStream.Put('\0'); const typename TargetEncoding::Ch* str = dstStream.Pop(); @@ -1691,7 +1707,7 @@ class GenericReader { } else { size_t length = s.Length(); - const char* decimal = s.Pop(); // Pop stack no matter if it will be used or not. + const NumberCharacter* decimal = s.Pop(); // Pop stack no matter if it will be used or not. if (useDouble) { int p = exp + expFrac; diff --git a/src/native/external/rapidjson/schema.h b/src/native/external/rapidjson/schema.h deleted file mode 100644 index 26ae947480634..0000000000000 --- a/src/native/external/rapidjson/schema.h +++ /dev/null @@ -1,2497 +0,0 @@ -// Tencent is pleased to support the open source community by making RapidJSON available-> -// -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip-> All rights reserved-> -// -// Licensed under the MIT License (the "License"); you may not use this file except -// in compliance with the License-> You may obtain a copy of the License at -// -// http://opensource->org/licenses/MIT -// -// Unless required by applicable law or agreed to in writing, software distributed -// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR -// CONDITIONS OF ANY KIND, either express or implied-> See the License for the -// specific language governing permissions and limitations under the License-> - -#ifndef RAPIDJSON_SCHEMA_H_ -#define RAPIDJSON_SCHEMA_H_ - -#include "document.h" -#include "pointer.h" -#include "stringbuffer.h" -#include // abs, floor - -#if !defined(RAPIDJSON_SCHEMA_USE_INTERNALREGEX) -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_INTERNALREGEX 0 -#endif - -#if !RAPIDJSON_SCHEMA_USE_INTERNALREGEX && defined(RAPIDJSON_SCHEMA_USE_STDREGEX) && (__cplusplus >=201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)) -#define RAPIDJSON_SCHEMA_USE_STDREGEX 1 -#else -#define RAPIDJSON_SCHEMA_USE_STDREGEX 0 -#endif - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX -#include "internal/regex.h" -#elif RAPIDJSON_SCHEMA_USE_STDREGEX -#include -#endif - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX || RAPIDJSON_SCHEMA_USE_STDREGEX -#define RAPIDJSON_SCHEMA_HAS_REGEX 1 -#else -#define RAPIDJSON_SCHEMA_HAS_REGEX 0 -#endif - -#ifndef RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_VERBOSE 0 -#endif - -#if RAPIDJSON_SCHEMA_VERBOSE -#include "stringbuffer.h" -#endif - -RAPIDJSON_DIAG_PUSH - -#if defined(__GNUC__) -RAPIDJSON_DIAG_OFF(effc++) -#endif - -#ifdef __clang__ -RAPIDJSON_DIAG_OFF(weak-vtables) -RAPIDJSON_DIAG_OFF(exit-time-destructors) -RAPIDJSON_DIAG_OFF(c++98-compat-pedantic) -RAPIDJSON_DIAG_OFF(variadic-macros) -#elif defined(_MSC_VER) -RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated -#endif - -RAPIDJSON_NAMESPACE_BEGIN - -/////////////////////////////////////////////////////////////////////////////// -// Verbose Utilities - -#if RAPIDJSON_SCHEMA_VERBOSE - -namespace internal { - -inline void PrintInvalidKeyword(const char* keyword) { - printf("Fail keyword: %s\n", keyword); -} - -inline void PrintInvalidKeyword(const wchar_t* keyword) { - wprintf(L"Fail keyword: %ls\n", keyword); -} - -inline void PrintInvalidDocument(const char* document) { - printf("Fail document: %s\n\n", document); -} - -inline void PrintInvalidDocument(const wchar_t* document) { - wprintf(L"Fail document: %ls\n\n", document); -} - -inline void PrintValidatorPointers(unsigned depth, const char* s, const char* d) { - printf("S: %*s%s\nD: %*s%s\n\n", depth * 4, " ", s, depth * 4, " ", d); -} - -inline void PrintValidatorPointers(unsigned depth, const wchar_t* s, const wchar_t* d) { - wprintf(L"S: %*ls%ls\nD: %*ls%ls\n\n", depth * 4, L" ", s, depth * 4, L" ", d); -} - -} // namespace internal - -#endif // RAPIDJSON_SCHEMA_VERBOSE - -/////////////////////////////////////////////////////////////////////////////// -// RAPIDJSON_INVALID_KEYWORD_RETURN - -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) internal::PrintInvalidKeyword(keyword) -#else -#define RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword) -#endif - -#define RAPIDJSON_INVALID_KEYWORD_RETURN(keyword)\ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - context.invalidKeyword = keyword.GetString();\ - RAPIDJSON_INVALID_KEYWORD_VERBOSE(keyword.GetString());\ - return false;\ -RAPIDJSON_MULTILINEMACRO_END - -/////////////////////////////////////////////////////////////////////////////// -// Forward declarations - -template -class GenericSchemaDocument; - -namespace internal { - -template -class Schema; - -/////////////////////////////////////////////////////////////////////////////// -// ISchemaValidator - -class ISchemaValidator { -public: - virtual ~ISchemaValidator() {} - virtual bool IsValid() const = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// ISchemaStateFactory - -template -class ISchemaStateFactory { -public: - virtual ~ISchemaStateFactory() {} - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType&) = 0; - virtual void DestroySchemaValidator(ISchemaValidator* validator) = 0; - virtual void* CreateHasher() = 0; - virtual uint64_t GetHashCode(void* hasher) = 0; - virtual void DestroryHasher(void* hasher) = 0; - virtual void* MallocState(size_t size) = 0; - virtual void FreeState(void* p) = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// IValidationErrorHandler - -template -class IValidationErrorHandler { -public: - typedef typename SchemaType::Ch Ch; - typedef typename SchemaType::SValue SValue; - - virtual ~IValidationErrorHandler() {} - - virtual void NotMultipleOf(int64_t actual, const SValue& expected) = 0; - virtual void NotMultipleOf(uint64_t actual, const SValue& expected) = 0; - virtual void NotMultipleOf(double actual, const SValue& expected) = 0; - virtual void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) = 0; - virtual void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) = 0; - virtual void AboveMaximum(double actual, const SValue& expected, bool exclusive) = 0; - virtual void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) = 0; - virtual void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) = 0; - virtual void BelowMinimum(double actual, const SValue& expected, bool exclusive) = 0; - - virtual void TooLong(const Ch* str, SizeType length, SizeType expected) = 0; - virtual void TooShort(const Ch* str, SizeType length, SizeType expected) = 0; - virtual void DoesNotMatch(const Ch* str, SizeType length) = 0; - - virtual void DisallowedItem(SizeType index) = 0; - virtual void TooFewItems(SizeType actualCount, SizeType expectedCount) = 0; - virtual void TooManyItems(SizeType actualCount, SizeType expectedCount) = 0; - virtual void DuplicateItems(SizeType index1, SizeType index2) = 0; - - virtual void TooManyProperties(SizeType actualCount, SizeType expectedCount) = 0; - virtual void TooFewProperties(SizeType actualCount, SizeType expectedCount) = 0; - virtual void StartMissingProperties() = 0; - virtual void AddMissingProperty(const SValue& name) = 0; - virtual bool EndMissingProperties() = 0; - virtual void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) = 0; - virtual void DisallowedProperty(const Ch* name, SizeType length) = 0; - - virtual void StartDependencyErrors() = 0; - virtual void StartMissingDependentProperties() = 0; - virtual void AddMissingDependentProperty(const SValue& targetName) = 0; - virtual void EndMissingDependentProperties(const SValue& sourceName) = 0; - virtual void AddDependencySchemaError(const SValue& souceName, ISchemaValidator* subvalidator) = 0; - virtual bool EndDependencyErrors() = 0; - - virtual void DisallowedValue() = 0; - virtual void StartDisallowedType() = 0; - virtual void AddExpectedType(const typename SchemaType::ValueType& expectedType) = 0; - virtual void EndDisallowedType(const typename SchemaType::ValueType& actualType) = 0; - virtual void NotAllOf(ISchemaValidator** subvalidators, SizeType count) = 0; - virtual void NoneOf(ISchemaValidator** subvalidators, SizeType count) = 0; - virtual void NotOneOf(ISchemaValidator** subvalidators, SizeType count) = 0; - virtual void Disallowed() = 0; -}; - - -/////////////////////////////////////////////////////////////////////////////// -// Hasher - -// For comparison of compound value -template -class Hasher { -public: - typedef typename Encoding::Ch Ch; - - Hasher(Allocator* allocator = 0, size_t stackCapacity = kDefaultSize) : stack_(allocator, stackCapacity) {} - - bool Null() { return WriteType(kNullType); } - bool Bool(bool b) { return WriteType(b ? kTrueType : kFalseType); } - bool Int(int i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } - bool Uint(unsigned u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } - bool Int64(int64_t i) { Number n; n.u.i = i; n.d = static_cast(i); return WriteNumber(n); } - bool Uint64(uint64_t u) { Number n; n.u.u = u; n.d = static_cast(u); return WriteNumber(n); } - bool Double(double d) { - Number n; - if (d < 0) n.u.i = static_cast(d); - else n.u.u = static_cast(d); - n.d = d; - return WriteNumber(n); - } - - bool RawNumber(const Ch* str, SizeType len, bool) { - WriteBuffer(kNumberType, str, len * sizeof(Ch)); - return true; - } - - bool String(const Ch* str, SizeType len, bool) { - WriteBuffer(kStringType, str, len * sizeof(Ch)); - return true; - } - - bool StartObject() { return true; } - bool Key(const Ch* str, SizeType len, bool copy) { return String(str, len, copy); } - bool EndObject(SizeType memberCount) { - uint64_t h = Hash(0, kObjectType); - uint64_t* kv = stack_.template Pop(memberCount * 2); - for (SizeType i = 0; i < memberCount; i++) - h ^= Hash(kv[i * 2], kv[i * 2 + 1]); // Use xor to achieve member order insensitive - *stack_.template Push() = h; - return true; - } - - bool StartArray() { return true; } - bool EndArray(SizeType elementCount) { - uint64_t h = Hash(0, kArrayType); - uint64_t* e = stack_.template Pop(elementCount); - for (SizeType i = 0; i < elementCount; i++) - h = Hash(h, e[i]); // Use hash to achieve element order sensitive - *stack_.template Push() = h; - return true; - } - - bool IsValid() const { return stack_.GetSize() == sizeof(uint64_t); } - - uint64_t GetHashCode() const { - RAPIDJSON_ASSERT(IsValid()); - return *stack_.template Top(); - } - -private: - static const size_t kDefaultSize = 256; - struct Number { - union U { - uint64_t u; - int64_t i; - }u; - double d; - }; - - bool WriteType(Type type) { return WriteBuffer(type, 0, 0); } - - bool WriteNumber(const Number& n) { return WriteBuffer(kNumberType, &n, sizeof(n)); } - - bool WriteBuffer(Type type, const void* data, size_t len) { - // FNV-1a from http://isthe.com/chongo/tech/comp/fnv/ - uint64_t h = Hash(RAPIDJSON_UINT64_C2(0x84222325, 0xcbf29ce4), type); - const unsigned char* d = static_cast(data); - for (size_t i = 0; i < len; i++) - h = Hash(h, d[i]); - *stack_.template Push() = h; - return true; - } - - static uint64_t Hash(uint64_t h, uint64_t d) { - static const uint64_t kPrime = RAPIDJSON_UINT64_C2(0x00000100, 0x000001b3); - h ^= d; - h *= kPrime; - return h; - } - - Stack stack_; -}; - -/////////////////////////////////////////////////////////////////////////////// -// SchemaValidationContext - -template -struct SchemaValidationContext { - typedef Schema SchemaType; - typedef ISchemaStateFactory SchemaValidatorFactoryType; - typedef IValidationErrorHandler ErrorHandlerType; - typedef typename SchemaType::ValueType ValueType; - typedef typename ValueType::Ch Ch; - - enum PatternValidatorType { - kPatternValidatorOnly, - kPatternValidatorWithProperty, - kPatternValidatorWithAdditionalProperty - }; - - SchemaValidationContext(SchemaValidatorFactoryType& f, ErrorHandlerType& eh, const SchemaType* s) : - factory(f), - error_handler(eh), - schema(s), - valueSchema(), - invalidKeyword(), - hasher(), - arrayElementHashCodes(), - validators(), - validatorCount(), - patternPropertiesValidators(), - patternPropertiesValidatorCount(), - patternPropertiesSchemas(), - patternPropertiesSchemaCount(), - valuePatternValidatorType(kPatternValidatorOnly), - propertyExist(), - inArray(false), - valueUniqueness(false), - arrayUniqueness(false) - { - } - - ~SchemaValidationContext() { - if (hasher) - factory.DestroryHasher(hasher); - if (validators) { - for (SizeType i = 0; i < validatorCount; i++) - factory.DestroySchemaValidator(validators[i]); - factory.FreeState(validators); - } - if (patternPropertiesValidators) { - for (SizeType i = 0; i < patternPropertiesValidatorCount; i++) - factory.DestroySchemaValidator(patternPropertiesValidators[i]); - factory.FreeState(patternPropertiesValidators); - } - if (patternPropertiesSchemas) - factory.FreeState(patternPropertiesSchemas); - if (propertyExist) - factory.FreeState(propertyExist); - } - - SchemaValidatorFactoryType& factory; - ErrorHandlerType& error_handler; - const SchemaType* schema; - const SchemaType* valueSchema; - const Ch* invalidKeyword; - void* hasher; // Only validator access - void* arrayElementHashCodes; // Only validator access this - ISchemaValidator** validators; - SizeType validatorCount; - ISchemaValidator** patternPropertiesValidators; - SizeType patternPropertiesValidatorCount; - const SchemaType** patternPropertiesSchemas; - SizeType patternPropertiesSchemaCount; - PatternValidatorType valuePatternValidatorType; - PatternValidatorType objectPatternValidatorType; - SizeType arrayElementIndex; - bool* propertyExist; - bool inArray; - bool valueUniqueness; - bool arrayUniqueness; -}; - -/////////////////////////////////////////////////////////////////////////////// -// Schema - -template -class Schema { -public: - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename SchemaDocumentType::AllocatorType AllocatorType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef SchemaValidationContext Context; - typedef Schema SchemaType; - typedef GenericValue SValue; - typedef IValidationErrorHandler ErrorHandler; - friend class GenericSchemaDocument; - - Schema(SchemaDocumentType* schemaDocument, const PointerType& p, const ValueType& value, const ValueType& document, AllocatorType* allocator) : - allocator_(allocator), - uri_(schemaDocument->GetURI(), *allocator), - pointer_(p, allocator), - typeless_(schemaDocument->GetTypeless()), - enum_(), - enumCount_(), - not_(), - type_((1 << kTotalSchemaType) - 1), // typeless - validatorCount_(), - notValidatorIndex_(), - properties_(), - additionalPropertiesSchema_(), - patternProperties_(), - patternPropertyCount_(), - propertyCount_(), - minProperties_(), - maxProperties_(SizeType(~0)), - additionalProperties_(true), - hasDependencies_(), - hasRequired_(), - hasSchemaDependencies_(), - additionalItemsSchema_(), - itemsList_(), - itemsTuple_(), - itemsTupleCount_(), - minItems_(), - maxItems_(SizeType(~0)), - additionalItems_(true), - uniqueItems_(false), - pattern_(), - minLength_(0), - maxLength_(~SizeType(0)), - exclusiveMinimum_(false), - exclusiveMaximum_(false), - defaultValueLength_(0) - { - typedef typename SchemaDocumentType::ValueType ValueType; - typedef typename ValueType::ConstValueIterator ConstValueIterator; - typedef typename ValueType::ConstMemberIterator ConstMemberIterator; - - if (!value.IsObject()) - return; - - if (const ValueType* v = GetMember(value, GetTypeString())) { - type_ = 0; - if (v->IsString()) - AddType(*v); - else if (v->IsArray()) - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) - AddType(*itr); - } - - if (const ValueType* v = GetMember(value, GetEnumString())) - if (v->IsArray() && v->Size() > 0) { - enum_ = static_cast(allocator_->Malloc(sizeof(uint64_t) * v->Size())); - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr) { - typedef Hasher > EnumHasherType; - char buffer[256u + 24]; - MemoryPoolAllocator<> hasherAllocator(buffer, sizeof(buffer)); - EnumHasherType h(&hasherAllocator, 256); - itr->Accept(h); - enum_[enumCount_++] = h.GetHashCode(); - } - } - - if (schemaDocument) { - AssignIfExist(allOf_, *schemaDocument, p, value, GetAllOfString(), document); - AssignIfExist(anyOf_, *schemaDocument, p, value, GetAnyOfString(), document); - AssignIfExist(oneOf_, *schemaDocument, p, value, GetOneOfString(), document); - } - - if (const ValueType* v = GetMember(value, GetNotString())) { - schemaDocument->CreateSchema(¬_, p.Append(GetNotString(), allocator_), *v, document); - notValidatorIndex_ = validatorCount_; - validatorCount_++; - } - - // Object - - const ValueType* properties = GetMember(value, GetPropertiesString()); - const ValueType* required = GetMember(value, GetRequiredString()); - const ValueType* dependencies = GetMember(value, GetDependenciesString()); - { - // Gather properties from properties/required/dependencies - SValue allProperties(kArrayType); - - if (properties && properties->IsObject()) - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) - AddUniqueElement(allProperties, itr->name); - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) - AddUniqueElement(allProperties, *itr); - - if (dependencies && dependencies->IsObject()) - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - AddUniqueElement(allProperties, itr->name); - if (itr->value.IsArray()) - for (ConstValueIterator i = itr->value.Begin(); i != itr->value.End(); ++i) - if (i->IsString()) - AddUniqueElement(allProperties, *i); - } - - if (allProperties.Size() > 0) { - propertyCount_ = allProperties.Size(); - properties_ = static_cast(allocator_->Malloc(sizeof(Property) * propertyCount_)); - for (SizeType i = 0; i < propertyCount_; i++) { - new (&properties_[i]) Property(); - properties_[i].name = allProperties[i]; - properties_[i].schema = typeless_; - } - } - } - - if (properties && properties->IsObject()) { - PointerType q = p.Append(GetPropertiesString(), allocator_); - for (ConstMemberIterator itr = properties->MemberBegin(); itr != properties->MemberEnd(); ++itr) { - SizeType index; - if (FindPropertyIndex(itr->name, &index)) - schemaDocument->CreateSchema(&properties_[index].schema, q.Append(itr->name, allocator_), itr->value, document); - } - } - - if (const ValueType* v = GetMember(value, GetPatternPropertiesString())) { - PointerType q = p.Append(GetPatternPropertiesString(), allocator_); - patternProperties_ = static_cast(allocator_->Malloc(sizeof(PatternProperty) * v->MemberCount())); - patternPropertyCount_ = 0; - - for (ConstMemberIterator itr = v->MemberBegin(); itr != v->MemberEnd(); ++itr) { - new (&patternProperties_[patternPropertyCount_]) PatternProperty(); - patternProperties_[patternPropertyCount_].pattern = CreatePattern(itr->name); - schemaDocument->CreateSchema(&patternProperties_[patternPropertyCount_].schema, q.Append(itr->name, allocator_), itr->value, document); - patternPropertyCount_++; - } - } - - if (required && required->IsArray()) - for (ConstValueIterator itr = required->Begin(); itr != required->End(); ++itr) - if (itr->IsString()) { - SizeType index; - if (FindPropertyIndex(*itr, &index)) { - properties_[index].required = true; - hasRequired_ = true; - } - } - - if (dependencies && dependencies->IsObject()) { - PointerType q = p.Append(GetDependenciesString(), allocator_); - hasDependencies_ = true; - for (ConstMemberIterator itr = dependencies->MemberBegin(); itr != dependencies->MemberEnd(); ++itr) { - SizeType sourceIndex; - if (FindPropertyIndex(itr->name, &sourceIndex)) { - if (itr->value.IsArray()) { - properties_[sourceIndex].dependencies = static_cast(allocator_->Malloc(sizeof(bool) * propertyCount_)); - std::memset(properties_[sourceIndex].dependencies, 0, sizeof(bool)* propertyCount_); - for (ConstValueIterator targetItr = itr->value.Begin(); targetItr != itr->value.End(); ++targetItr) { - SizeType targetIndex; - if (FindPropertyIndex(*targetItr, &targetIndex)) - properties_[sourceIndex].dependencies[targetIndex] = true; - } - } - else if (itr->value.IsObject()) { - hasSchemaDependencies_ = true; - schemaDocument->CreateSchema(&properties_[sourceIndex].dependenciesSchema, q.Append(itr->name, allocator_), itr->value, document); - properties_[sourceIndex].dependenciesValidatorIndex = validatorCount_; - validatorCount_++; - } - } - } - } - - if (const ValueType* v = GetMember(value, GetAdditionalPropertiesString())) { - if (v->IsBool()) - additionalProperties_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalPropertiesSchema_, p.Append(GetAdditionalPropertiesString(), allocator_), *v, document); - } - - AssignIfExist(minProperties_, value, GetMinPropertiesString()); - AssignIfExist(maxProperties_, value, GetMaxPropertiesString()); - - // Array - if (const ValueType* v = GetMember(value, GetItemsString())) { - PointerType q = p.Append(GetItemsString(), allocator_); - if (v->IsObject()) // List validation - schemaDocument->CreateSchema(&itemsList_, q, *v, document); - else if (v->IsArray()) { // Tuple validation - itemsTuple_ = static_cast(allocator_->Malloc(sizeof(const Schema*) * v->Size())); - SizeType index = 0; - for (ConstValueIterator itr = v->Begin(); itr != v->End(); ++itr, index++) - schemaDocument->CreateSchema(&itemsTuple_[itemsTupleCount_++], q.Append(index, allocator_), *itr, document); - } - } - - AssignIfExist(minItems_, value, GetMinItemsString()); - AssignIfExist(maxItems_, value, GetMaxItemsString()); - - if (const ValueType* v = GetMember(value, GetAdditionalItemsString())) { - if (v->IsBool()) - additionalItems_ = v->GetBool(); - else if (v->IsObject()) - schemaDocument->CreateSchema(&additionalItemsSchema_, p.Append(GetAdditionalItemsString(), allocator_), *v, document); - } - - AssignIfExist(uniqueItems_, value, GetUniqueItemsString()); - - // String - AssignIfExist(minLength_, value, GetMinLengthString()); - AssignIfExist(maxLength_, value, GetMaxLengthString()); - - if (const ValueType* v = GetMember(value, GetPatternString())) - pattern_ = CreatePattern(*v); - - // Number - if (const ValueType* v = GetMember(value, GetMinimumString())) - if (v->IsNumber()) - minimum_.CopyFrom(*v, *allocator_); - - if (const ValueType* v = GetMember(value, GetMaximumString())) - if (v->IsNumber()) - maximum_.CopyFrom(*v, *allocator_); - - AssignIfExist(exclusiveMinimum_, value, GetExclusiveMinimumString()); - AssignIfExist(exclusiveMaximum_, value, GetExclusiveMaximumString()); - - if (const ValueType* v = GetMember(value, GetMultipleOfString())) - if (v->IsNumber() && v->GetDouble() > 0.0) - multipleOf_.CopyFrom(*v, *allocator_); - - // Default - if (const ValueType* v = GetMember(value, GetDefaultValueString())) - if (v->IsString()) - defaultValueLength_ = v->GetStringLength(); - - } - - ~Schema() { - AllocatorType::Free(enum_); - if (properties_) { - for (SizeType i = 0; i < propertyCount_; i++) - properties_[i].~Property(); - AllocatorType::Free(properties_); - } - if (patternProperties_) { - for (SizeType i = 0; i < patternPropertyCount_; i++) - patternProperties_[i].~PatternProperty(); - AllocatorType::Free(patternProperties_); - } - AllocatorType::Free(itemsTuple_); -#if RAPIDJSON_SCHEMA_HAS_REGEX - if (pattern_) { - pattern_->~RegexType(); - AllocatorType::Free(pattern_); - } -#endif - } - - const SValue& GetURI() const { - return uri_; - } - - const PointerType& GetPointer() const { - return pointer_; - } - - bool BeginValue(Context& context) const { - if (context.inArray) { - if (uniqueItems_) - context.valueUniqueness = true; - - if (itemsList_) - context.valueSchema = itemsList_; - else if (itemsTuple_) { - if (context.arrayElementIndex < itemsTupleCount_) - context.valueSchema = itemsTuple_[context.arrayElementIndex]; - else if (additionalItemsSchema_) - context.valueSchema = additionalItemsSchema_; - else if (additionalItems_) - context.valueSchema = typeless_; - else { - context.error_handler.DisallowedItem(context.arrayElementIndex); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetItemsString()); - } - } - else - context.valueSchema = typeless_; - - context.arrayElementIndex++; - } - return true; - } - - RAPIDJSON_FORCEINLINE bool EndValue(Context& context) const { - if (context.patternPropertiesValidatorCount > 0) { - bool otherValid = false; - SizeType count = context.patternPropertiesValidatorCount; - if (context.objectPatternValidatorType != Context::kPatternValidatorOnly) - otherValid = context.patternPropertiesValidators[--count]->IsValid(); - - bool patternValid = true; - for (SizeType i = 0; i < count; i++) - if (!context.patternPropertiesValidators[i]->IsValid()) { - patternValid = false; - break; - } - - if (context.objectPatternValidatorType == Context::kPatternValidatorOnly) { - if (!patternValid) { - context.error_handler.PropertyViolations(context.patternPropertiesValidators, count); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - } - else if (context.objectPatternValidatorType == Context::kPatternValidatorWithProperty) { - if (!patternValid || !otherValid) { - context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - } - else if (!patternValid && !otherValid) { // kPatternValidatorWithAdditionalProperty) - context.error_handler.PropertyViolations(context.patternPropertiesValidators, count + 1); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternPropertiesString()); - } - } - - if (enum_) { - const uint64_t h = context.factory.GetHashCode(context.hasher); - for (SizeType i = 0; i < enumCount_; i++) - if (enum_[i] == h) - goto foundEnum; - context.error_handler.DisallowedValue(); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetEnumString()); - foundEnum:; - } - - if (allOf_.schemas) - for (SizeType i = allOf_.begin; i < allOf_.begin + allOf_.count; i++) - if (!context.validators[i]->IsValid()) { - context.error_handler.NotAllOf(&context.validators[allOf_.begin], allOf_.count); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAllOfString()); - } - - if (anyOf_.schemas) { - for (SizeType i = anyOf_.begin; i < anyOf_.begin + anyOf_.count; i++) - if (context.validators[i]->IsValid()) - goto foundAny; - context.error_handler.NoneOf(&context.validators[anyOf_.begin], anyOf_.count); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAnyOfString()); - foundAny:; - } - - if (oneOf_.schemas) { - bool oneValid = false; - for (SizeType i = oneOf_.begin; i < oneOf_.begin + oneOf_.count; i++) - if (context.validators[i]->IsValid()) { - if (oneValid) { - context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - } else - oneValid = true; - } - if (!oneValid) { - context.error_handler.NotOneOf(&context.validators[oneOf_.begin], oneOf_.count); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetOneOfString()); - } - } - - if (not_ && context.validators[notValidatorIndex_]->IsValid()) { - context.error_handler.Disallowed(); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetNotString()); - } - - return true; - } - - bool Null(Context& context) const { - if (!(type_ & (1 << kNullSchemaType))) { - DisallowedType(context, GetNullString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - return CreateParallelValidator(context); - } - - bool Bool(Context& context, bool) const { - if (!(type_ & (1 << kBooleanSchemaType))) { - DisallowedType(context, GetBooleanString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - return CreateParallelValidator(context); - } - - bool Int(Context& context, int i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); - } - - bool Uint(Context& context, unsigned u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } - - bool Int64(Context& context, int64_t i) const { - if (!CheckInt(context, i)) - return false; - return CreateParallelValidator(context); - } - - bool Uint64(Context& context, uint64_t u) const { - if (!CheckUint(context, u)) - return false; - return CreateParallelValidator(context); - } - - bool Double(Context& context, double d) const { - if (!(type_ & (1 << kNumberSchemaType))) { - DisallowedType(context, GetNumberString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - - if (!minimum_.IsNull() && !CheckDoubleMinimum(context, d)) - return false; - - if (!maximum_.IsNull() && !CheckDoubleMaximum(context, d)) - return false; - - if (!multipleOf_.IsNull() && !CheckDoubleMultipleOf(context, d)) - return false; - - return CreateParallelValidator(context); - } - - bool String(Context& context, const Ch* str, SizeType length, bool) const { - if (!(type_ & (1 << kStringSchemaType))) { - DisallowedType(context, GetStringString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - - if (minLength_ != 0 || maxLength_ != SizeType(~0)) { - SizeType count; - if (internal::CountStringCodePoint(str, length, &count)) { - if (count < minLength_) { - context.error_handler.TooShort(str, length, minLength_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinLengthString()); - } - if (count > maxLength_) { - context.error_handler.TooLong(str, length, maxLength_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxLengthString()); - } - } - } - - if (pattern_ && !IsPatternMatch(pattern_, str, length)) { - context.error_handler.DoesNotMatch(str, length); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetPatternString()); - } - - return CreateParallelValidator(context); - } - - bool StartObject(Context& context) const { - if (!(type_ & (1 << kObjectSchemaType))) { - DisallowedType(context, GetObjectString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - - if (hasDependencies_ || hasRequired_) { - context.propertyExist = static_cast(context.factory.MallocState(sizeof(bool) * propertyCount_)); - std::memset(context.propertyExist, 0, sizeof(bool) * propertyCount_); - } - - if (patternProperties_) { // pre-allocate schema array - SizeType count = patternPropertyCount_ + 1; // extra for valuePatternValidatorType - context.patternPropertiesSchemas = static_cast(context.factory.MallocState(sizeof(const SchemaType*) * count)); - context.patternPropertiesSchemaCount = 0; - std::memset(context.patternPropertiesSchemas, 0, sizeof(SchemaType*) * count); - } - - return CreateParallelValidator(context); - } - - bool Key(Context& context, const Ch* str, SizeType len, bool) const { - if (patternProperties_) { - context.patternPropertiesSchemaCount = 0; - for (SizeType i = 0; i < patternPropertyCount_; i++) - if (patternProperties_[i].pattern && IsPatternMatch(patternProperties_[i].pattern, str, len)) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = patternProperties_[i].schema; - context.valueSchema = typeless_; - } - } - - SizeType index; - if (FindPropertyIndex(ValueType(str, len).Move(), &index)) { - if (context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = properties_[index].schema; - context.valueSchema = typeless_; - context.valuePatternValidatorType = Context::kPatternValidatorWithProperty; - } - else - context.valueSchema = properties_[index].schema; - - if (context.propertyExist) - context.propertyExist[index] = true; - - return true; - } - - if (additionalPropertiesSchema_) { - if (additionalPropertiesSchema_ && context.patternPropertiesSchemaCount > 0) { - context.patternPropertiesSchemas[context.patternPropertiesSchemaCount++] = additionalPropertiesSchema_; - context.valueSchema = typeless_; - context.valuePatternValidatorType = Context::kPatternValidatorWithAdditionalProperty; - } - else - context.valueSchema = additionalPropertiesSchema_; - return true; - } - else if (additionalProperties_) { - context.valueSchema = typeless_; - return true; - } - - if (context.patternPropertiesSchemaCount == 0) { // patternProperties are not additional properties - context.error_handler.DisallowedProperty(str, len); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetAdditionalPropertiesString()); - } - - return true; - } - - bool EndObject(Context& context, SizeType memberCount) const { - if (hasRequired_) { - context.error_handler.StartMissingProperties(); - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].required && !context.propertyExist[index]) - if (properties_[index].schema->defaultValueLength_ == 0 ) - context.error_handler.AddMissingProperty(properties_[index].name); - if (context.error_handler.EndMissingProperties()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetRequiredString()); - } - - if (memberCount < minProperties_) { - context.error_handler.TooFewProperties(memberCount, minProperties_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinPropertiesString()); - } - - if (memberCount > maxProperties_) { - context.error_handler.TooManyProperties(memberCount, maxProperties_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxPropertiesString()); - } - - if (hasDependencies_) { - context.error_handler.StartDependencyErrors(); - for (SizeType sourceIndex = 0; sourceIndex < propertyCount_; sourceIndex++) { - const Property& source = properties_[sourceIndex]; - if (context.propertyExist[sourceIndex]) { - if (source.dependencies) { - context.error_handler.StartMissingDependentProperties(); - for (SizeType targetIndex = 0; targetIndex < propertyCount_; targetIndex++) - if (source.dependencies[targetIndex] && !context.propertyExist[targetIndex]) - context.error_handler.AddMissingDependentProperty(properties_[targetIndex].name); - context.error_handler.EndMissingDependentProperties(source.name); - } - else if (source.dependenciesSchema) { - ISchemaValidator* dependenciesValidator = context.validators[source.dependenciesValidatorIndex]; - if (!dependenciesValidator->IsValid()) - context.error_handler.AddDependencySchemaError(source.name, dependenciesValidator); - } - } - } - if (context.error_handler.EndDependencyErrors()) - RAPIDJSON_INVALID_KEYWORD_RETURN(GetDependenciesString()); - } - - return true; - } - - bool StartArray(Context& context) const { - if (!(type_ & (1 << kArraySchemaType))) { - DisallowedType(context, GetArrayString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - - context.arrayElementIndex = 0; - context.inArray = true; - - return CreateParallelValidator(context); - } - - bool EndArray(Context& context, SizeType elementCount) const { - context.inArray = false; - - if (elementCount < minItems_) { - context.error_handler.TooFewItems(elementCount, minItems_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinItemsString()); - } - - if (elementCount > maxItems_) { - context.error_handler.TooManyItems(elementCount, maxItems_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaxItemsString()); - } - - return true; - } - - // Generate functions for string literal according to Ch -#define RAPIDJSON_STRING_(name, ...) \ - static const ValueType& Get##name##String() {\ - static const Ch s[] = { __VA_ARGS__, '\0' };\ - static const ValueType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1));\ - return v;\ - } - - RAPIDJSON_STRING_(Null, 'n', 'u', 'l', 'l') - RAPIDJSON_STRING_(Boolean, 'b', 'o', 'o', 'l', 'e', 'a', 'n') - RAPIDJSON_STRING_(Object, 'o', 'b', 'j', 'e', 'c', 't') - RAPIDJSON_STRING_(Array, 'a', 'r', 'r', 'a', 'y') - RAPIDJSON_STRING_(String, 's', 't', 'r', 'i', 'n', 'g') - RAPIDJSON_STRING_(Number, 'n', 'u', 'm', 'b', 'e', 'r') - RAPIDJSON_STRING_(Integer, 'i', 'n', 't', 'e', 'g', 'e', 'r') - RAPIDJSON_STRING_(Type, 't', 'y', 'p', 'e') - RAPIDJSON_STRING_(Enum, 'e', 'n', 'u', 'm') - RAPIDJSON_STRING_(AllOf, 'a', 'l', 'l', 'O', 'f') - RAPIDJSON_STRING_(AnyOf, 'a', 'n', 'y', 'O', 'f') - RAPIDJSON_STRING_(OneOf, 'o', 'n', 'e', 'O', 'f') - RAPIDJSON_STRING_(Not, 'n', 'o', 't') - RAPIDJSON_STRING_(Properties, 'p', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Required, 'r', 'e', 'q', 'u', 'i', 'r', 'e', 'd') - RAPIDJSON_STRING_(Dependencies, 'd', 'e', 'p', 'e', 'n', 'd', 'e', 'n', 'c', 'i', 'e', 's') - RAPIDJSON_STRING_(PatternProperties, 'p', 'a', 't', 't', 'e', 'r', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(AdditionalProperties, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MinProperties, 'm', 'i', 'n', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(MaxProperties, 'm', 'a', 'x', 'P', 'r', 'o', 'p', 'e', 'r', 't', 'i', 'e', 's') - RAPIDJSON_STRING_(Items, 'i', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinItems, 'm', 'i', 'n', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MaxItems, 'm', 'a', 'x', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(AdditionalItems, 'a', 'd', 'd', 'i', 't', 'i', 'o', 'n', 'a', 'l', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(UniqueItems, 'u', 'n', 'i', 'q', 'u', 'e', 'I', 't', 'e', 'm', 's') - RAPIDJSON_STRING_(MinLength, 'm', 'i', 'n', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(MaxLength, 'm', 'a', 'x', 'L', 'e', 'n', 'g', 't', 'h') - RAPIDJSON_STRING_(Pattern, 'p', 'a', 't', 't', 'e', 'r', 'n') - RAPIDJSON_STRING_(Minimum, 'm', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(Maximum, 'm', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMinimum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'i', 'n', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(ExclusiveMaximum, 'e', 'x', 'c', 'l', 'u', 's', 'i', 'v', 'e', 'M', 'a', 'x', 'i', 'm', 'u', 'm') - RAPIDJSON_STRING_(MultipleOf, 'm', 'u', 'l', 't', 'i', 'p', 'l', 'e', 'O', 'f') - RAPIDJSON_STRING_(DefaultValue, 'd', 'e', 'f', 'a', 'u', 'l', 't') - -#undef RAPIDJSON_STRING_ - -private: - enum SchemaValueType { - kNullSchemaType, - kBooleanSchemaType, - kObjectSchemaType, - kArraySchemaType, - kStringSchemaType, - kNumberSchemaType, - kIntegerSchemaType, - kTotalSchemaType - }; - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - typedef internal::GenericRegex RegexType; -#elif RAPIDJSON_SCHEMA_USE_STDREGEX - typedef std::basic_regex RegexType; -#else - typedef char RegexType; -#endif - - struct SchemaArray { - SchemaArray() : schemas(), count() {} - ~SchemaArray() { AllocatorType::Free(schemas); } - const SchemaType** schemas; - SizeType begin; // begin index of context.validators - SizeType count; - }; - - template - void AddUniqueElement(V1& a, const V2& v) { - for (typename V1::ConstValueIterator itr = a.Begin(); itr != a.End(); ++itr) - if (*itr == v) - return; - V1 c(v, *allocator_); - a.PushBack(c, *allocator_); - } - - static const ValueType* GetMember(const ValueType& value, const ValueType& name) { - typename ValueType::ConstMemberIterator itr = value.FindMember(name); - return itr != value.MemberEnd() ? &(itr->value) : 0; - } - - static void AssignIfExist(bool& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsBool()) - out = v->GetBool(); - } - - static void AssignIfExist(SizeType& out, const ValueType& value, const ValueType& name) { - if (const ValueType* v = GetMember(value, name)) - if (v->IsUint64() && v->GetUint64() <= SizeType(~0)) - out = static_cast(v->GetUint64()); - } - - void AssignIfExist(SchemaArray& out, SchemaDocumentType& schemaDocument, const PointerType& p, const ValueType& value, const ValueType& name, const ValueType& document) { - if (const ValueType* v = GetMember(value, name)) { - if (v->IsArray() && v->Size() > 0) { - PointerType q = p.Append(name, allocator_); - out.count = v->Size(); - out.schemas = static_cast(allocator_->Malloc(out.count * sizeof(const Schema*))); - memset(out.schemas, 0, sizeof(Schema*)* out.count); - for (SizeType i = 0; i < out.count; i++) - schemaDocument.CreateSchema(&out.schemas[i], q.Append(i, allocator_), (*v)[i], document); - out.begin = validatorCount_; - validatorCount_ += out.count; - } - } - } - -#if RAPIDJSON_SCHEMA_USE_INTERNALREGEX - template - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) { - RegexType* r = new (allocator_->Malloc(sizeof(RegexType))) RegexType(value.GetString(), allocator_); - if (!r->IsValid()) { - r->~RegexType(); - AllocatorType::Free(r); - r = 0; - } - return r; - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType) { - GenericRegexSearch rs(*pattern); - return rs.Search(str); - } -#elif RAPIDJSON_SCHEMA_USE_STDREGEX - template - RegexType* CreatePattern(const ValueType& value) { - if (value.IsString()) { - RegexType *r = static_cast(allocator_->Malloc(sizeof(RegexType))); - try { - return new (r) RegexType(value.GetString(), std::size_t(value.GetStringLength()), std::regex_constants::ECMAScript); - } - catch (const std::regex_error&) { - AllocatorType::Free(r); - } - } - return 0; - } - - static bool IsPatternMatch(const RegexType* pattern, const Ch *str, SizeType length) { - std::match_results r; - return std::regex_search(str, str + length, r, *pattern); - } -#else - template - RegexType* CreatePattern(const ValueType&) { return 0; } - - static bool IsPatternMatch(const RegexType*, const Ch *, SizeType) { return true; } -#endif // RAPIDJSON_SCHEMA_USE_STDREGEX - - void AddType(const ValueType& type) { - if (type == GetNullString() ) type_ |= 1 << kNullSchemaType; - else if (type == GetBooleanString()) type_ |= 1 << kBooleanSchemaType; - else if (type == GetObjectString() ) type_ |= 1 << kObjectSchemaType; - else if (type == GetArrayString() ) type_ |= 1 << kArraySchemaType; - else if (type == GetStringString() ) type_ |= 1 << kStringSchemaType; - else if (type == GetIntegerString()) type_ |= 1 << kIntegerSchemaType; - else if (type == GetNumberString() ) type_ |= (1 << kNumberSchemaType) | (1 << kIntegerSchemaType); - } - - bool CreateParallelValidator(Context& context) const { - if (enum_ || context.arrayUniqueness) - context.hasher = context.factory.CreateHasher(); - - if (validatorCount_) { - RAPIDJSON_ASSERT(context.validators == 0); - context.validators = static_cast(context.factory.MallocState(sizeof(ISchemaValidator*) * validatorCount_)); - context.validatorCount = validatorCount_; - - if (allOf_.schemas) - CreateSchemaValidators(context, allOf_); - - if (anyOf_.schemas) - CreateSchemaValidators(context, anyOf_); - - if (oneOf_.schemas) - CreateSchemaValidators(context, oneOf_); - - if (not_) - context.validators[notValidatorIndex_] = context.factory.CreateSchemaValidator(*not_); - - if (hasSchemaDependencies_) { - for (SizeType i = 0; i < propertyCount_; i++) - if (properties_[i].dependenciesSchema) - context.validators[properties_[i].dependenciesValidatorIndex] = context.factory.CreateSchemaValidator(*properties_[i].dependenciesSchema); - } - } - - return true; - } - - void CreateSchemaValidators(Context& context, const SchemaArray& schemas) const { - for (SizeType i = 0; i < schemas.count; i++) - context.validators[schemas.begin + i] = context.factory.CreateSchemaValidator(*schemas.schemas[i]); - } - - // O(n) - bool FindPropertyIndex(const ValueType& name, SizeType* outIndex) const { - SizeType len = name.GetStringLength(); - const Ch* str = name.GetString(); - for (SizeType index = 0; index < propertyCount_; index++) - if (properties_[index].name.GetStringLength() == len && - (std::memcmp(properties_[index].name.GetString(), str, sizeof(Ch) * len) == 0)) - { - *outIndex = index; - return true; - } - return false; - } - - bool CheckInt(Context& context, int64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { - DisallowedType(context, GetIntegerString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - - if (!minimum_.IsNull()) { - if (minimum_.IsInt64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetInt64() : i < minimum_.GetInt64()) { - context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - } - else if (minimum_.IsUint64()) { - context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); // i <= max(int64_t) < minimum.GetUint64() - } - else if (!CheckDoubleMinimum(context, static_cast(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsInt64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetInt64() : i > maximum_.GetInt64()) { - context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - } - else if (maximum_.IsUint64()) { } - /* do nothing */ // i <= max(int64_t) < maximum_.GetUint64() - else if (!CheckDoubleMaximum(context, static_cast(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (static_cast(i >= 0 ? i : -i) % multipleOf_.GetUint64() != 0) { - context.error_handler.NotMultipleOf(i, multipleOf_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - } - else if (!CheckDoubleMultipleOf(context, static_cast(i))) - return false; - } - - return true; - } - - bool CheckUint(Context& context, uint64_t i) const { - if (!(type_ & ((1 << kIntegerSchemaType) | (1 << kNumberSchemaType)))) { - DisallowedType(context, GetIntegerString()); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetTypeString()); - } - - if (!minimum_.IsNull()) { - if (minimum_.IsUint64()) { - if (exclusiveMinimum_ ? i <= minimum_.GetUint64() : i < minimum_.GetUint64()) { - context.error_handler.BelowMinimum(i, minimum_, exclusiveMinimum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - } - else if (minimum_.IsInt64()) - /* do nothing */; // i >= 0 > minimum.Getint64() - else if (!CheckDoubleMinimum(context, static_cast(i))) - return false; - } - - if (!maximum_.IsNull()) { - if (maximum_.IsUint64()) { - if (exclusiveMaximum_ ? i >= maximum_.GetUint64() : i > maximum_.GetUint64()) { - context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - } - else if (maximum_.IsInt64()) { - context.error_handler.AboveMaximum(i, maximum_, exclusiveMaximum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); // i >= 0 > maximum_ - } - else if (!CheckDoubleMaximum(context, static_cast(i))) - return false; - } - - if (!multipleOf_.IsNull()) { - if (multipleOf_.IsUint64()) { - if (i % multipleOf_.GetUint64() != 0) { - context.error_handler.NotMultipleOf(i, multipleOf_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - } - else if (!CheckDoubleMultipleOf(context, static_cast(i))) - return false; - } - - return true; - } - - bool CheckDoubleMinimum(Context& context, double d) const { - if (exclusiveMinimum_ ? d <= minimum_.GetDouble() : d < minimum_.GetDouble()) { - context.error_handler.BelowMinimum(d, minimum_, exclusiveMinimum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMinimumString()); - } - return true; - } - - bool CheckDoubleMaximum(Context& context, double d) const { - if (exclusiveMaximum_ ? d >= maximum_.GetDouble() : d > maximum_.GetDouble()) { - context.error_handler.AboveMaximum(d, maximum_, exclusiveMaximum_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMaximumString()); - } - return true; - } - - bool CheckDoubleMultipleOf(Context& context, double d) const { - double a = std::abs(d), b = std::abs(multipleOf_.GetDouble()); - double q = std::floor(a / b); - double r = a - q * b; - if (r > 0.0) { - context.error_handler.NotMultipleOf(d, multipleOf_); - RAPIDJSON_INVALID_KEYWORD_RETURN(GetMultipleOfString()); - } - return true; - } - - void DisallowedType(Context& context, const ValueType& actualType) const { - ErrorHandler& eh = context.error_handler; - eh.StartDisallowedType(); - - if (type_ & (1 << kNullSchemaType)) eh.AddExpectedType(GetNullString()); - if (type_ & (1 << kBooleanSchemaType)) eh.AddExpectedType(GetBooleanString()); - if (type_ & (1 << kObjectSchemaType)) eh.AddExpectedType(GetObjectString()); - if (type_ & (1 << kArraySchemaType)) eh.AddExpectedType(GetArrayString()); - if (type_ & (1 << kStringSchemaType)) eh.AddExpectedType(GetStringString()); - - if (type_ & (1 << kNumberSchemaType)) eh.AddExpectedType(GetNumberString()); - else if (type_ & (1 << kIntegerSchemaType)) eh.AddExpectedType(GetIntegerString()); - - eh.EndDisallowedType(actualType); - } - - struct Property { - Property() : schema(), dependenciesSchema(), dependenciesValidatorIndex(), dependencies(), required(false) {} - ~Property() { AllocatorType::Free(dependencies); } - SValue name; - const SchemaType* schema; - const SchemaType* dependenciesSchema; - SizeType dependenciesValidatorIndex; - bool* dependencies; - bool required; - }; - - struct PatternProperty { - PatternProperty() : schema(), pattern() {} - ~PatternProperty() { - if (pattern) { - pattern->~RegexType(); - AllocatorType::Free(pattern); - } - } - const SchemaType* schema; - RegexType* pattern; - }; - - AllocatorType* allocator_; - SValue uri_; - PointerType pointer_; - const SchemaType* typeless_; - uint64_t* enum_; - SizeType enumCount_; - SchemaArray allOf_; - SchemaArray anyOf_; - SchemaArray oneOf_; - const SchemaType* not_; - unsigned type_; // bitmask of kSchemaType - SizeType validatorCount_; - SizeType notValidatorIndex_; - - Property* properties_; - const SchemaType* additionalPropertiesSchema_; - PatternProperty* patternProperties_; - SizeType patternPropertyCount_; - SizeType propertyCount_; - SizeType minProperties_; - SizeType maxProperties_; - bool additionalProperties_; - bool hasDependencies_; - bool hasRequired_; - bool hasSchemaDependencies_; - - const SchemaType* additionalItemsSchema_; - const SchemaType* itemsList_; - const SchemaType** itemsTuple_; - SizeType itemsTupleCount_; - SizeType minItems_; - SizeType maxItems_; - bool additionalItems_; - bool uniqueItems_; - - RegexType* pattern_; - SizeType minLength_; - SizeType maxLength_; - - SValue minimum_; - SValue maximum_; - SValue multipleOf_; - bool exclusiveMinimum_; - bool exclusiveMaximum_; - - SizeType defaultValueLength_; -}; - -template -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - *documentStack.template Push() = '/'; - char buffer[21]; - size_t length = static_cast((sizeof(SizeType) == 4 ? u32toa(index, buffer) : u64toa(index, buffer)) - buffer); - for (size_t i = 0; i < length; i++) - *documentStack.template Push() = static_cast(buffer[i]); - } -}; - -// Partial specialized version for char to prevent buffer copying. -template -struct TokenHelper { - RAPIDJSON_FORCEINLINE static void AppendIndexToken(Stack& documentStack, SizeType index) { - if (sizeof(SizeType) == 4) { - char *buffer = documentStack.template Push(1 + 10); // '/' + uint - *buffer++ = '/'; - const char* end = internal::u32toa(index, buffer); - documentStack.template Pop(static_cast(10 - (end - buffer))); - } - else { - char *buffer = documentStack.template Push(1 + 20); // '/' + uint64 - *buffer++ = '/'; - const char* end = internal::u64toa(index, buffer); - documentStack.template Pop(static_cast(20 - (end - buffer))); - } - } -}; - -} // namespace internal - -/////////////////////////////////////////////////////////////////////////////// -// IGenericRemoteSchemaDocumentProvider - -template -class IGenericRemoteSchemaDocumentProvider { -public: - typedef typename SchemaDocumentType::Ch Ch; - - virtual ~IGenericRemoteSchemaDocumentProvider() {} - virtual const SchemaDocumentType* GetRemoteDocument(const Ch* uri, SizeType length) = 0; -}; - -/////////////////////////////////////////////////////////////////////////////// -// GenericSchemaDocument - -//! JSON schema document. -/*! - A JSON schema document is a compiled version of a JSON schema. - It is basically a tree of internal::Schema. - - \note This is an immutable class (i.e. its instance cannot be modified after construction). - \tparam ValueT Type of JSON value (e.g. \c Value ), which also determine the encoding. - \tparam Allocator Allocator type for allocating memory of this document. -*/ -template -class GenericSchemaDocument { -public: - typedef ValueT ValueType; - typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProviderType; - typedef Allocator AllocatorType; - typedef typename ValueType::EncodingType EncodingType; - typedef typename EncodingType::Ch Ch; - typedef internal::Schema SchemaType; - typedef GenericPointer PointerType; - typedef GenericValue URIType; - friend class internal::Schema; - template - friend class GenericSchemaValidator; - - //! Constructor. - /*! - Compile a JSON document into schema document. - - \param document A JSON document as source. - \param uri The base URI of this schema document for purposes of violation reporting. - \param uriLength Length of \c name, in code points. - \param remoteProvider An optional remote schema document provider for resolving remote reference. Can be null. - \param allocator An optional allocator instance for allocating memory. Can be null. - */ - explicit GenericSchemaDocument(const ValueType& document, const Ch* uri = 0, SizeType uriLength = 0, - IRemoteSchemaDocumentProviderType* remoteProvider = 0, Allocator* allocator = 0) : - remoteProvider_(remoteProvider), - allocator_(allocator), - ownAllocator_(), - root_(), - typeless_(), - schemaMap_(allocator, kInitialSchemaMapSize), - schemaRef_(allocator, kInitialSchemaRefSize) - { - if (!allocator_) - ownAllocator_ = allocator_ = RAPIDJSON_NEW(Allocator)(); - - Ch noUri[1] = {0}; - uri_.SetString(uri ? uri : noUri, uriLength, *allocator_); - - typeless_ = static_cast(allocator_->Malloc(sizeof(SchemaType))); - new (typeless_) SchemaType(this, PointerType(), ValueType(kObjectType).Move(), ValueType(kObjectType).Move(), allocator_); - - // Generate root schema, it will call CreateSchema() to create sub-schemas, - // And call AddRefSchema() if there are $ref. - CreateSchemaRecursive(&root_, PointerType(), document, document); - - // Resolve $ref - while (!schemaRef_.Empty()) { - SchemaRefEntry* refEntry = schemaRef_.template Pop(1); - if (const SchemaType* s = GetSchema(refEntry->target)) { - if (refEntry->schema) - *refEntry->schema = s; - - // Create entry in map if not exist - if (!GetSchema(refEntry->source)) { - new (schemaMap_.template Push()) SchemaEntry(refEntry->source, const_cast(s), false, allocator_); - } - } - else if (refEntry->schema) - *refEntry->schema = typeless_; - - refEntry->~SchemaRefEntry(); - } - - RAPIDJSON_ASSERT(root_ != 0); - - schemaRef_.ShrinkToFit(); // Deallocate all memory for ref - } - -#if RAPIDJSON_HAS_CXX11_RVALUE_REFS - //! Move constructor in C++11 - GenericSchemaDocument(GenericSchemaDocument&& rhs) RAPIDJSON_NOEXCEPT : - remoteProvider_(rhs.remoteProvider_), - allocator_(rhs.allocator_), - ownAllocator_(rhs.ownAllocator_), - root_(rhs.root_), - typeless_(rhs.typeless_), - schemaMap_(std::move(rhs.schemaMap_)), - schemaRef_(std::move(rhs.schemaRef_)), - uri_(std::move(rhs.uri_)) - { - rhs.remoteProvider_ = 0; - rhs.allocator_ = 0; - rhs.ownAllocator_ = 0; - rhs.typeless_ = 0; - } -#endif - - //! Destructor - ~GenericSchemaDocument() { - while (!schemaMap_.Empty()) - schemaMap_.template Pop(1)->~SchemaEntry(); - - if (typeless_) { - typeless_->~SchemaType(); - Allocator::Free(typeless_); - } - - RAPIDJSON_DELETE(ownAllocator_); - } - - const URIType& GetURI() const { return uri_; } - - //! Get the root schema. - const SchemaType& GetRoot() const { return *root_; } - -private: - //! Prohibit copying - GenericSchemaDocument(const GenericSchemaDocument&); - //! Prohibit assignment - GenericSchemaDocument& operator=(const GenericSchemaDocument&); - - struct SchemaRefEntry { - SchemaRefEntry(const PointerType& s, const PointerType& t, const SchemaType** outSchema, Allocator *allocator) : source(s, allocator), target(t, allocator), schema(outSchema) {} - PointerType source; - PointerType target; - const SchemaType** schema; - }; - - struct SchemaEntry { - SchemaEntry(const PointerType& p, SchemaType* s, bool o, Allocator* allocator) : pointer(p, allocator), schema(s), owned(o) {} - ~SchemaEntry() { - if (owned) { - schema->~SchemaType(); - Allocator::Free(schema); - } - } - PointerType pointer; - SchemaType* schema; - bool owned; - }; - - void CreateSchemaRecursive(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - if (schema) - *schema = typeless_; - - if (v.GetType() == kObjectType) { - const SchemaType* s = GetSchema(pointer); - if (!s) - CreateSchema(schema, pointer, v, document); - - for (typename ValueType::ConstMemberIterator itr = v.MemberBegin(); itr != v.MemberEnd(); ++itr) - CreateSchemaRecursive(0, pointer.Append(itr->name, allocator_), itr->value, document); - } - else if (v.GetType() == kArrayType) - for (SizeType i = 0; i < v.Size(); i++) - CreateSchemaRecursive(0, pointer.Append(i, allocator_), v[i], document); - } - - void CreateSchema(const SchemaType** schema, const PointerType& pointer, const ValueType& v, const ValueType& document) { - RAPIDJSON_ASSERT(pointer.IsValid()); - if (v.IsObject()) { - if (!HandleRefSchema(pointer, schema, v, document)) { - SchemaType* s = new (allocator_->Malloc(sizeof(SchemaType))) SchemaType(this, pointer, v, document, allocator_); - new (schemaMap_.template Push()) SchemaEntry(pointer, s, true, allocator_); - if (schema) - *schema = s; - } - } - } - - bool HandleRefSchema(const PointerType& source, const SchemaType** schema, const ValueType& v, const ValueType& document) { - static const Ch kRefString[] = { '$', 'r', 'e', 'f', '\0' }; - static const ValueType kRefValue(kRefString, 4); - - typename ValueType::ConstMemberIterator itr = v.FindMember(kRefValue); - if (itr == v.MemberEnd()) - return false; - - if (itr->value.IsString()) { - SizeType len = itr->value.GetStringLength(); - if (len > 0) { - const Ch* s = itr->value.GetString(); - SizeType i = 0; - while (i < len && s[i] != '#') // Find the first # - i++; - - if (i > 0) { // Remote reference, resolve immediately - if (remoteProvider_) { - if (const GenericSchemaDocument* remoteDocument = remoteProvider_->GetRemoteDocument(s, i)) { - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const SchemaType* sc = remoteDocument->GetSchema(pointer)) { - if (schema) - *schema = sc; - new (schemaMap_.template Push()) SchemaEntry(source, const_cast(sc), false, allocator_); - return true; - } - } - } - } - } - else if (s[i] == '#') { // Local reference, defer resolution - PointerType pointer(&s[i], len - i, allocator_); - if (pointer.IsValid()) { - if (const ValueType* nv = pointer.Get(document)) - if (HandleRefSchema(source, schema, *nv, document)) - return true; - - new (schemaRef_.template Push()) SchemaRefEntry(source, pointer, schema, allocator_); - return true; - } - } - } - } - return false; - } - - const SchemaType* GetSchema(const PointerType& pointer) const { - for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) - if (pointer == target->pointer) - return target->schema; - return 0; - } - - PointerType GetPointer(const SchemaType* schema) const { - for (const SchemaEntry* target = schemaMap_.template Bottom(); target != schemaMap_.template End(); ++target) - if (schema == target->schema) - return target->pointer; - return PointerType(); - } - - const SchemaType* GetTypeless() const { return typeless_; } - - static const size_t kInitialSchemaMapSize = 64; - static const size_t kInitialSchemaRefSize = 64; - - IRemoteSchemaDocumentProviderType* remoteProvider_; - Allocator *allocator_; - Allocator *ownAllocator_; - const SchemaType* root_; //!< Root schema. - SchemaType* typeless_; - internal::Stack schemaMap_; // Stores created Pointer -> Schemas - internal::Stack schemaRef_; // Stores Pointer from $ref and schema which holds the $ref - URIType uri_; -}; - -//! GenericSchemaDocument using Value type. -typedef GenericSchemaDocument SchemaDocument; -//! IGenericRemoteSchemaDocumentProvider using SchemaDocument. -typedef IGenericRemoteSchemaDocumentProvider IRemoteSchemaDocumentProvider; - -/////////////////////////////////////////////////////////////////////////////// -// GenericSchemaValidator - -//! JSON Schema Validator. -/*! - A SAX style JSON schema validator. - It uses a \c GenericSchemaDocument to validate SAX events. - It delegates the incoming SAX events to an output handler. - The default output handler does nothing. - It can be reused multiple times by calling \c Reset(). - - \tparam SchemaDocumentType Type of schema document. - \tparam OutputHandler Type of output handler. Default handler does nothing. - \tparam StateAllocator Allocator for storing the internal validation states. -*/ -template < - typename SchemaDocumentType, - typename OutputHandler = BaseReaderHandler, - typename StateAllocator = CrtAllocator> -class GenericSchemaValidator : - public internal::ISchemaStateFactory, - public internal::ISchemaValidator, - public internal::IValidationErrorHandler -{ -public: - typedef typename SchemaDocumentType::SchemaType SchemaType; - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename SchemaType::EncodingType EncodingType; - typedef typename SchemaType::SValue SValue; - typedef typename EncodingType::Ch Ch; - typedef GenericStringRef StringRefType; - typedef GenericValue ValueType; - - //! Constructor without output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - outputHandler_(0), - error_(kObjectType), - currentError_(), - missingDependents_(), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif - { - } - - //! Constructor with output handler. - /*! - \param schemaDocument The schema document to conform to. - \param allocator Optional allocator for storing internal validation states. - \param schemaStackCapacity Optional initial capacity of schema path stack. - \param documentStackCapacity Optional initial capacity of document path stack. - */ - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - OutputHandler& outputHandler, - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(schemaDocument.GetRoot()), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - outputHandler_(&outputHandler), - error_(kObjectType), - currentError_(), - missingDependents_(), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(0) -#endif - { - } - - //! Destructor. - ~GenericSchemaValidator() { - Reset(); - RAPIDJSON_DELETE(ownStateAllocator_); - } - - //! Reset the internal states. - void Reset() { - while (!schemaStack_.Empty()) - PopSchema(); - documentStack_.Clear(); - error_.SetObject(); - currentError_.SetNull(); - missingDependents_.SetNull(); - valid_ = true; - } - - //! Checks whether the current state is valid. - // Implementation of ISchemaValidator - virtual bool IsValid() const { return valid_; } - - //! Gets the error object. - ValueType& GetError() { return error_; } - const ValueType& GetError() const { return error_; } - - //! Gets the JSON pointer pointed to the invalid schema. - PointerType GetInvalidSchemaPointer() const { - return schemaStack_.Empty() ? PointerType() : CurrentSchema().GetPointer(); - } - - //! Gets the keyword of invalid schema. - const Ch* GetInvalidSchemaKeyword() const { - return schemaStack_.Empty() ? 0 : CurrentContext().invalidKeyword; - } - - //! Gets the JSON pointer pointed to the invalid value. - PointerType GetInvalidDocumentPointer() const { - if (documentStack_.Empty()) { - return PointerType(); - } - else { - return PointerType(documentStack_.template Bottom(), documentStack_.GetSize() / sizeof(Ch)); - } - } - - void NotMultipleOf(int64_t actual, const SValue& expected) { - AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); - } - void NotMultipleOf(uint64_t actual, const SValue& expected) { - AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); - } - void NotMultipleOf(double actual, const SValue& expected) { - AddNumberError(SchemaType::GetMultipleOfString(), ValueType(actual).Move(), expected); - } - void AboveMaximum(int64_t actual, const SValue& expected, bool exclusive) { - AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, - exclusive ? &SchemaType::GetExclusiveMaximumString : 0); - } - void AboveMaximum(uint64_t actual, const SValue& expected, bool exclusive) { - AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, - exclusive ? &SchemaType::GetExclusiveMaximumString : 0); - } - void AboveMaximum(double actual, const SValue& expected, bool exclusive) { - AddNumberError(SchemaType::GetMaximumString(), ValueType(actual).Move(), expected, - exclusive ? &SchemaType::GetExclusiveMaximumString : 0); - } - void BelowMinimum(int64_t actual, const SValue& expected, bool exclusive) { - AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, - exclusive ? &SchemaType::GetExclusiveMinimumString : 0); - } - void BelowMinimum(uint64_t actual, const SValue& expected, bool exclusive) { - AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, - exclusive ? &SchemaType::GetExclusiveMinimumString : 0); - } - void BelowMinimum(double actual, const SValue& expected, bool exclusive) { - AddNumberError(SchemaType::GetMinimumString(), ValueType(actual).Move(), expected, - exclusive ? &SchemaType::GetExclusiveMinimumString : 0); - } - - void TooLong(const Ch* str, SizeType length, SizeType expected) { - AddNumberError(SchemaType::GetMaxLengthString(), - ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); - } - void TooShort(const Ch* str, SizeType length, SizeType expected) { - AddNumberError(SchemaType::GetMinLengthString(), - ValueType(str, length, GetStateAllocator()).Move(), SValue(expected).Move()); - } - void DoesNotMatch(const Ch* str, SizeType length) { - currentError_.SetObject(); - currentError_.AddMember(GetActualString(), ValueType(str, length, GetStateAllocator()).Move(), GetStateAllocator()); - AddCurrentError(SchemaType::GetPatternString()); - } - - void DisallowedItem(SizeType index) { - currentError_.SetObject(); - currentError_.AddMember(GetDisallowedString(), ValueType(index).Move(), GetStateAllocator()); - AddCurrentError(SchemaType::GetAdditionalItemsString(), true); - } - void TooFewItems(SizeType actualCount, SizeType expectedCount) { - AddNumberError(SchemaType::GetMinItemsString(), - ValueType(actualCount).Move(), SValue(expectedCount).Move()); - } - void TooManyItems(SizeType actualCount, SizeType expectedCount) { - AddNumberError(SchemaType::GetMaxItemsString(), - ValueType(actualCount).Move(), SValue(expectedCount).Move()); - } - void DuplicateItems(SizeType index1, SizeType index2) { - ValueType duplicates(kArrayType); - duplicates.PushBack(index1, GetStateAllocator()); - duplicates.PushBack(index2, GetStateAllocator()); - currentError_.SetObject(); - currentError_.AddMember(GetDuplicatesString(), duplicates, GetStateAllocator()); - AddCurrentError(SchemaType::GetUniqueItemsString(), true); - } - - void TooManyProperties(SizeType actualCount, SizeType expectedCount) { - AddNumberError(SchemaType::GetMaxPropertiesString(), - ValueType(actualCount).Move(), SValue(expectedCount).Move()); - } - void TooFewProperties(SizeType actualCount, SizeType expectedCount) { - AddNumberError(SchemaType::GetMinPropertiesString(), - ValueType(actualCount).Move(), SValue(expectedCount).Move()); - } - void StartMissingProperties() { - currentError_.SetArray(); - } - void AddMissingProperty(const SValue& name) { - currentError_.PushBack(ValueType(name, GetStateAllocator()).Move(), GetStateAllocator()); - } - bool EndMissingProperties() { - if (currentError_.Empty()) - return false; - ValueType error(kObjectType); - error.AddMember(GetMissingString(), currentError_, GetStateAllocator()); - currentError_ = error; - AddCurrentError(SchemaType::GetRequiredString()); - return true; - } - void PropertyViolations(ISchemaValidator** subvalidators, SizeType count) { - for (SizeType i = 0; i < count; ++i) - MergeError(static_cast(subvalidators[i])->GetError()); - } - void DisallowedProperty(const Ch* name, SizeType length) { - currentError_.SetObject(); - currentError_.AddMember(GetDisallowedString(), ValueType(name, length, GetStateAllocator()).Move(), GetStateAllocator()); - AddCurrentError(SchemaType::GetAdditionalPropertiesString(), true); - } - - void StartDependencyErrors() { - currentError_.SetObject(); - } - void StartMissingDependentProperties() { - missingDependents_.SetArray(); - } - void AddMissingDependentProperty(const SValue& targetName) { - missingDependents_.PushBack(ValueType(targetName, GetStateAllocator()).Move(), GetStateAllocator()); - } - void EndMissingDependentProperties(const SValue& sourceName) { - if (!missingDependents_.Empty()) - currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), - missingDependents_, GetStateAllocator()); - } - void AddDependencySchemaError(const SValue& sourceName, ISchemaValidator* subvalidator) { - currentError_.AddMember(ValueType(sourceName, GetStateAllocator()).Move(), - static_cast(subvalidator)->GetError(), GetStateAllocator()); - } - bool EndDependencyErrors() { - if (currentError_.ObjectEmpty()) - return false; - ValueType error(kObjectType); - error.AddMember(GetErrorsString(), currentError_, GetStateAllocator()); - currentError_ = error; - AddCurrentError(SchemaType::GetDependenciesString()); - return true; - } - - void DisallowedValue() { - currentError_.SetObject(); - AddCurrentError(SchemaType::GetEnumString()); - } - void StartDisallowedType() { - currentError_.SetArray(); - } - void AddExpectedType(const typename SchemaType::ValueType& expectedType) { - currentError_.PushBack(ValueType(expectedType, GetStateAllocator()).Move(), GetStateAllocator()); - } - void EndDisallowedType(const typename SchemaType::ValueType& actualType) { - ValueType error(kObjectType); - error.AddMember(GetExpectedString(), currentError_, GetStateAllocator()); - error.AddMember(GetActualString(), ValueType(actualType, GetStateAllocator()).Move(), GetStateAllocator()); - currentError_ = error; - AddCurrentError(SchemaType::GetTypeString()); - } - void NotAllOf(ISchemaValidator** subvalidators, SizeType count) { - for (SizeType i = 0; i < count; ++i) { - MergeError(static_cast(subvalidators[i])->GetError()); - } - } - void NoneOf(ISchemaValidator** subvalidators, SizeType count) { - AddErrorArray(SchemaType::GetAnyOfString(), subvalidators, count); - } - void NotOneOf(ISchemaValidator** subvalidators, SizeType count) { - AddErrorArray(SchemaType::GetOneOfString(), subvalidators, count); - } - void Disallowed() { - currentError_.SetObject(); - AddCurrentError(SchemaType::GetNotString()); - } - -#define RAPIDJSON_STRING_(name, ...) \ - static const StringRefType& Get##name##String() {\ - static const Ch s[] = { __VA_ARGS__, '\0' };\ - static const StringRefType v(s, static_cast(sizeof(s) / sizeof(Ch) - 1)); \ - return v;\ - } - - RAPIDJSON_STRING_(InstanceRef, 'i', 'n', 's', 't', 'a', 'n', 'c', 'e', 'R', 'e', 'f') - RAPIDJSON_STRING_(SchemaRef, 's', 'c', 'h', 'e', 'm', 'a', 'R', 'e', 'f') - RAPIDJSON_STRING_(Expected, 'e', 'x', 'p', 'e', 'c', 't', 'e', 'd') - RAPIDJSON_STRING_(Actual, 'a', 'c', 't', 'u', 'a', 'l') - RAPIDJSON_STRING_(Disallowed, 'd', 'i', 's', 'a', 'l', 'l', 'o', 'w', 'e', 'd') - RAPIDJSON_STRING_(Missing, 'm', 'i', 's', 's', 'i', 'n', 'g') - RAPIDJSON_STRING_(Errors, 'e', 'r', 'r', 'o', 'r', 's') - RAPIDJSON_STRING_(Duplicates, 'd', 'u', 'p', 'l', 'i', 'c', 'a', 't', 'e', 's') - -#undef RAPIDJSON_STRING_ - -#if RAPIDJSON_SCHEMA_VERBOSE -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() \ -RAPIDJSON_MULTILINEMACRO_BEGIN\ - *documentStack_.template Push() = '\0';\ - documentStack_.template Pop(1);\ - internal::PrintInvalidDocument(documentStack_.template Bottom());\ -RAPIDJSON_MULTILINEMACRO_END -#else -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_() -#endif - -#define RAPIDJSON_SCHEMA_HANDLE_BEGIN_(method, arg1)\ - if (!valid_) return false; \ - if (!BeginValue() || !CurrentSchema().method arg1) {\ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_();\ - return valid_ = false;\ - } - -#define RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2)\ - for (Context* context = schemaStack_.template Bottom(); context != schemaStack_.template End(); context++) {\ - if (context->hasher)\ - static_cast(context->hasher)->method arg2;\ - if (context->validators)\ - for (SizeType i_ = 0; i_ < context->validatorCount; i_++)\ - static_cast(context->validators[i_])->method arg2;\ - if (context->patternPropertiesValidators)\ - for (SizeType i_ = 0; i_ < context->patternPropertiesValidatorCount; i_++)\ - static_cast(context->patternPropertiesValidators[i_])->method arg2;\ - } - -#define RAPIDJSON_SCHEMA_HANDLE_END_(method, arg2)\ - return valid_ = EndValue() && (!outputHandler_ || outputHandler_->method arg2) - -#define RAPIDJSON_SCHEMA_HANDLE_VALUE_(method, arg1, arg2) \ - RAPIDJSON_SCHEMA_HANDLE_BEGIN_ (method, arg1);\ - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(method, arg2);\ - RAPIDJSON_SCHEMA_HANDLE_END_ (method, arg2) - - bool Null() { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Null, (CurrentContext()), ( )); } - bool Bool(bool b) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Bool, (CurrentContext(), b), (b)); } - bool Int(int i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int, (CurrentContext(), i), (i)); } - bool Uint(unsigned u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint, (CurrentContext(), u), (u)); } - bool Int64(int64_t i) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Int64, (CurrentContext(), i), (i)); } - bool Uint64(uint64_t u) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Uint64, (CurrentContext(), u), (u)); } - bool Double(double d) { RAPIDJSON_SCHEMA_HANDLE_VALUE_(Double, (CurrentContext(), d), (d)); } - bool RawNumber(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - bool String(const Ch* str, SizeType length, bool copy) - { RAPIDJSON_SCHEMA_HANDLE_VALUE_(String, (CurrentContext(), str, length, copy), (str, length, copy)); } - - bool StartObject() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartObject, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartObject, ()); - return valid_ = !outputHandler_ || outputHandler_->StartObject(); - } - - bool Key(const Ch* str, SizeType len, bool copy) { - if (!valid_) return false; - AppendToken(str, len); - if (!CurrentSchema().Key(CurrentContext(), str, len, copy)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(Key, (str, len, copy)); - return valid_ = !outputHandler_ || outputHandler_->Key(str, len, copy); - } - - bool EndObject(SizeType memberCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndObject, (memberCount)); - if (!CurrentSchema().EndObject(CurrentContext(), memberCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndObject, (memberCount)); - } - - bool StartArray() { - RAPIDJSON_SCHEMA_HANDLE_BEGIN_(StartArray, (CurrentContext())); - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(StartArray, ()); - return valid_ = !outputHandler_ || outputHandler_->StartArray(); - } - - bool EndArray(SizeType elementCount) { - if (!valid_) return false; - RAPIDJSON_SCHEMA_HANDLE_PARALLEL_(EndArray, (elementCount)); - if (!CurrentSchema().EndArray(CurrentContext(), elementCount)) return valid_ = false; - RAPIDJSON_SCHEMA_HANDLE_END_(EndArray, (elementCount)); - } - -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_VERBOSE_ -#undef RAPIDJSON_SCHEMA_HANDLE_BEGIN_ -#undef RAPIDJSON_SCHEMA_HANDLE_PARALLEL_ -#undef RAPIDJSON_SCHEMA_HANDLE_VALUE_ - - // Implementation of ISchemaStateFactory - virtual ISchemaValidator* CreateSchemaValidator(const SchemaType& root) { - return new (GetStateAllocator().Malloc(sizeof(GenericSchemaValidator))) GenericSchemaValidator(*schemaDocument_, root, documentStack_.template Bottom(), documentStack_.GetSize(), -#if RAPIDJSON_SCHEMA_VERBOSE - depth_ + 1, -#endif - &GetStateAllocator()); - } - - virtual void DestroySchemaValidator(ISchemaValidator* validator) { - GenericSchemaValidator* v = static_cast(validator); - v->~GenericSchemaValidator(); - StateAllocator::Free(v); - } - - virtual void* CreateHasher() { - return new (GetStateAllocator().Malloc(sizeof(HasherType))) HasherType(&GetStateAllocator()); - } - - virtual uint64_t GetHashCode(void* hasher) { - return static_cast(hasher)->GetHashCode(); - } - - virtual void DestroryHasher(void* hasher) { - HasherType* h = static_cast(hasher); - h->~HasherType(); - StateAllocator::Free(h); - } - - virtual void* MallocState(size_t size) { - return GetStateAllocator().Malloc(size); - } - - virtual void FreeState(void* p) { - StateAllocator::Free(p); - } - -private: - typedef typename SchemaType::Context Context; - typedef GenericValue, StateAllocator> HashCodeArray; - typedef internal::Hasher HasherType; - - GenericSchemaValidator( - const SchemaDocumentType& schemaDocument, - const SchemaType& root, - const char* basePath, size_t basePathSize, -#if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth, -#endif - StateAllocator* allocator = 0, - size_t schemaStackCapacity = kDefaultSchemaStackCapacity, - size_t documentStackCapacity = kDefaultDocumentStackCapacity) - : - schemaDocument_(&schemaDocument), - root_(root), - stateAllocator_(allocator), - ownStateAllocator_(0), - schemaStack_(allocator, schemaStackCapacity), - documentStack_(allocator, documentStackCapacity), - outputHandler_(0), - error_(kObjectType), - currentError_(), - missingDependents_(), - valid_(true) -#if RAPIDJSON_SCHEMA_VERBOSE - , depth_(depth) -#endif - { - if (basePath && basePathSize) - memcpy(documentStack_.template Push(basePathSize), basePath, basePathSize); - } - - StateAllocator& GetStateAllocator() { - if (!stateAllocator_) - stateAllocator_ = ownStateAllocator_ = RAPIDJSON_NEW(StateAllocator)(); - return *stateAllocator_; - } - - bool BeginValue() { - if (schemaStack_.Empty()) - PushSchema(root_); - else { - if (CurrentContext().inArray) - internal::TokenHelper, Ch>::AppendIndexToken(documentStack_, CurrentContext().arrayElementIndex); - - if (!CurrentSchema().BeginValue(CurrentContext())) - return false; - - SizeType count = CurrentContext().patternPropertiesSchemaCount; - const SchemaType** sa = CurrentContext().patternPropertiesSchemas; - typename Context::PatternValidatorType patternValidatorType = CurrentContext().valuePatternValidatorType; - bool valueUniqueness = CurrentContext().valueUniqueness; - RAPIDJSON_ASSERT(CurrentContext().valueSchema); - PushSchema(*CurrentContext().valueSchema); - - if (count > 0) { - CurrentContext().objectPatternValidatorType = patternValidatorType; - ISchemaValidator**& va = CurrentContext().patternPropertiesValidators; - SizeType& validatorCount = CurrentContext().patternPropertiesValidatorCount; - va = static_cast(MallocState(sizeof(ISchemaValidator*) * count)); - for (SizeType i = 0; i < count; i++) - va[validatorCount++] = CreateSchemaValidator(*sa[i]); - } - - CurrentContext().arrayUniqueness = valueUniqueness; - } - return true; - } - - bool EndValue() { - if (!CurrentSchema().EndValue(CurrentContext())) - return false; - -#if RAPIDJSON_SCHEMA_VERBOSE - GenericStringBuffer sb; - schemaDocument_->GetPointer(&CurrentSchema()).Stringify(sb); - - *documentStack_.template Push() = '\0'; - documentStack_.template Pop(1); - internal::PrintValidatorPointers(depth_, sb.GetString(), documentStack_.template Bottom()); -#endif - - uint64_t h = CurrentContext().arrayUniqueness ? static_cast(CurrentContext().hasher)->GetHashCode() : 0; - - PopSchema(); - - if (!schemaStack_.Empty()) { - Context& context = CurrentContext(); - if (context.valueUniqueness) { - HashCodeArray* a = static_cast(context.arrayElementHashCodes); - if (!a) - CurrentContext().arrayElementHashCodes = a = new (GetStateAllocator().Malloc(sizeof(HashCodeArray))) HashCodeArray(kArrayType); - for (typename HashCodeArray::ConstValueIterator itr = a->Begin(); itr != a->End(); ++itr) - if (itr->GetUint64() == h) { - DuplicateItems(static_cast(itr - a->Begin()), a->Size()); - RAPIDJSON_INVALID_KEYWORD_RETURN(SchemaType::GetUniqueItemsString()); - } - a->PushBack(h, GetStateAllocator()); - } - } - - // Remove the last token of document pointer - while (!documentStack_.Empty() && *documentStack_.template Pop(1) != '/') - ; - - return true; - } - - void AppendToken(const Ch* str, SizeType len) { - documentStack_.template Reserve(1 + len * 2); // worst case all characters are escaped as two characters - *documentStack_.template PushUnsafe() = '/'; - for (SizeType i = 0; i < len; i++) { - if (str[i] == '~') { - *documentStack_.template PushUnsafe() = '~'; - *documentStack_.template PushUnsafe() = '0'; - } - else if (str[i] == '/') { - *documentStack_.template PushUnsafe() = '~'; - *documentStack_.template PushUnsafe() = '1'; - } - else - *documentStack_.template PushUnsafe() = str[i]; - } - } - - RAPIDJSON_FORCEINLINE void PushSchema(const SchemaType& schema) { new (schemaStack_.template Push()) Context(*this, *this, &schema); } - - RAPIDJSON_FORCEINLINE void PopSchema() { - Context* c = schemaStack_.template Pop(1); - if (HashCodeArray* a = static_cast(c->arrayElementHashCodes)) { - a->~HashCodeArray(); - StateAllocator::Free(a); - } - c->~Context(); - } - - void AddErrorLocation(ValueType& result, bool parent) { - GenericStringBuffer sb; - PointerType instancePointer = GetInvalidDocumentPointer(); - ((parent && instancePointer.GetTokenCount() > 0) - ? PointerType(instancePointer.GetTokens(), instancePointer.GetTokenCount() - 1) - : instancePointer).StringifyUriFragment(sb); - ValueType instanceRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), - GetStateAllocator()); - result.AddMember(GetInstanceRefString(), instanceRef, GetStateAllocator()); - sb.Clear(); - memcpy(sb.Push(CurrentSchema().GetURI().GetStringLength()), - CurrentSchema().GetURI().GetString(), - CurrentSchema().GetURI().GetStringLength() * sizeof(Ch)); - GetInvalidSchemaPointer().StringifyUriFragment(sb); - ValueType schemaRef(sb.GetString(), static_cast(sb.GetSize() / sizeof(Ch)), - GetStateAllocator()); - result.AddMember(GetSchemaRefString(), schemaRef, GetStateAllocator()); - } - - void AddError(ValueType& keyword, ValueType& error) { - typename ValueType::MemberIterator member = error_.FindMember(keyword); - if (member == error_.MemberEnd()) - error_.AddMember(keyword, error, GetStateAllocator()); - else { - if (member->value.IsObject()) { - ValueType errors(kArrayType); - errors.PushBack(member->value, GetStateAllocator()); - member->value = errors; - } - member->value.PushBack(error, GetStateAllocator()); - } - } - - void AddCurrentError(const typename SchemaType::ValueType& keyword, bool parent = false) { - AddErrorLocation(currentError_, parent); - AddError(ValueType(keyword, GetStateAllocator(), false).Move(), currentError_); - } - - void MergeError(ValueType& other) { - for (typename ValueType::MemberIterator it = other.MemberBegin(), end = other.MemberEnd(); it != end; ++it) { - AddError(it->name, it->value); - } - } - - void AddNumberError(const typename SchemaType::ValueType& keyword, ValueType& actual, const SValue& expected, - const typename SchemaType::ValueType& (*exclusive)() = 0) { - currentError_.SetObject(); - currentError_.AddMember(GetActualString(), actual, GetStateAllocator()); - currentError_.AddMember(GetExpectedString(), ValueType(expected, GetStateAllocator()).Move(), GetStateAllocator()); - if (exclusive) - currentError_.AddMember(ValueType(exclusive(), GetStateAllocator()).Move(), true, GetStateAllocator()); - AddCurrentError(keyword); - } - - void AddErrorArray(const typename SchemaType::ValueType& keyword, - ISchemaValidator** subvalidators, SizeType count) { - ValueType errors(kArrayType); - for (SizeType i = 0; i < count; ++i) - errors.PushBack(static_cast(subvalidators[i])->GetError(), GetStateAllocator()); - currentError_.SetObject(); - currentError_.AddMember(GetErrorsString(), errors, GetStateAllocator()); - AddCurrentError(keyword); - } - - const SchemaType& CurrentSchema() const { return *schemaStack_.template Top()->schema; } - Context& CurrentContext() { return *schemaStack_.template Top(); } - const Context& CurrentContext() const { return *schemaStack_.template Top(); } - - static const size_t kDefaultSchemaStackCapacity = 1024; - static const size_t kDefaultDocumentStackCapacity = 256; - const SchemaDocumentType* schemaDocument_; - const SchemaType& root_; - StateAllocator* stateAllocator_; - StateAllocator* ownStateAllocator_; - internal::Stack schemaStack_; //!< stack to store the current path of schema (BaseSchemaType *) - internal::Stack documentStack_; //!< stack to store the current path of validating document (Ch) - OutputHandler* outputHandler_; - ValueType error_; - ValueType currentError_; - ValueType missingDependents_; - bool valid_; -#if RAPIDJSON_SCHEMA_VERBOSE - unsigned depth_; -#endif -}; - -typedef GenericSchemaValidator SchemaValidator; - -/////////////////////////////////////////////////////////////////////////////// -// SchemaValidatingReader - -//! A helper class for parsing with validation. -/*! - This helper class is a functor, designed as a parameter of \ref GenericDocument::Populate(). - - \tparam parseFlags Combination of \ref ParseFlag. - \tparam InputStream Type of input stream, implementing Stream concept. - \tparam SourceEncoding Encoding of the input stream. - \tparam SchemaDocumentType Type of schema document. - \tparam StackAllocator Allocator type for stack. -*/ -template < - unsigned parseFlags, - typename InputStream, - typename SourceEncoding, - typename SchemaDocumentType = SchemaDocument, - typename StackAllocator = CrtAllocator> -class SchemaValidatingReader { -public: - typedef typename SchemaDocumentType::PointerType PointerType; - typedef typename InputStream::Ch Ch; - typedef GenericValue ValueType; - - //! Constructor - /*! - \param is Input stream. - \param sd Schema document. - */ - SchemaValidatingReader(InputStream& is, const SchemaDocumentType& sd) : is_(is), sd_(sd), invalidSchemaKeyword_(), error_(kObjectType), isValid_(true) {} - - template - bool operator()(Handler& handler) { - GenericReader reader; - GenericSchemaValidator validator(sd_, handler); - parseResult_ = reader.template Parse(is_, validator); - - isValid_ = validator.IsValid(); - if (isValid_) { - invalidSchemaPointer_ = PointerType(); - invalidSchemaKeyword_ = 0; - invalidDocumentPointer_ = PointerType(); - error_.SetObject(); - } - else { - invalidSchemaPointer_ = validator.GetInvalidSchemaPointer(); - invalidSchemaKeyword_ = validator.GetInvalidSchemaKeyword(); - invalidDocumentPointer_ = validator.GetInvalidDocumentPointer(); - error_.CopyFrom(validator.GetError(), allocator_); - } - - return parseResult_; - } - - const ParseResult& GetParseResult() const { return parseResult_; } - bool IsValid() const { return isValid_; } - const PointerType& GetInvalidSchemaPointer() const { return invalidSchemaPointer_; } - const Ch* GetInvalidSchemaKeyword() const { return invalidSchemaKeyword_; } - const PointerType& GetInvalidDocumentPointer() const { return invalidDocumentPointer_; } - const ValueType& GetError() const { return error_; } - -private: - InputStream& is_; - const SchemaDocumentType& sd_; - - ParseResult parseResult_; - PointerType invalidSchemaPointer_; - const Ch* invalidSchemaKeyword_; - PointerType invalidDocumentPointer_; - StackAllocator allocator_; - ValueType error_; - bool isValid_; -}; - -RAPIDJSON_NAMESPACE_END -RAPIDJSON_DIAG_POP - -#endif // RAPIDJSON_SCHEMA_H_ diff --git a/src/native/external/rapidjson/stream.h b/src/native/external/rapidjson/stream.h index 7f2643e481425..1fd70915c5472 100644 --- a/src/native/external/rapidjson/stream.h +++ b/src/native/external/rapidjson/stream.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/stringbuffer.h b/src/native/external/rapidjson/stringbuffer.h index 4e38b82c3d984..82ad3ca6bbfe8 100644 --- a/src/native/external/rapidjson/stringbuffer.h +++ b/src/native/external/rapidjson/stringbuffer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at diff --git a/src/native/external/rapidjson/writer.h b/src/native/external/rapidjson/writer.h index 6f5b6903467ad..632e02ce74a55 100644 --- a/src/native/external/rapidjson/writer.h +++ b/src/native/external/rapidjson/writer.h @@ -1,6 +1,6 @@ // Tencent is pleased to support the open source community by making RapidJSON available. // -// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. +// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. // // Licensed under the MIT License (the "License"); you may not use this file except // in compliance with the License. You may obtain a copy of the License at @@ -16,6 +16,7 @@ #define RAPIDJSON_WRITER_H_ #include "stream.h" +#include "internal/clzll.h" #include "internal/meta.h" #include "internal/stack.h" #include "internal/strfunc.h" @@ -66,6 +67,7 @@ enum WriteFlag { kWriteNoFlags = 0, //!< No flags are set. kWriteValidateEncodingFlag = 1, //!< Validate encoding of JSON strings. kWriteNanAndInfFlag = 2, //!< Allow writing of Infinity, -Infinity and NaN. + kWriteNanAndInfNullFlag = 4, //!< Allow writing of Infinity, -Infinity and NaN as null. kWriteDefaultFlags = RAPIDJSON_WRITE_DEFAULT_FLAGS //!< Default write flags. Can be customized by defining RAPIDJSON_WRITE_DEFAULT_FLAGS }; @@ -226,7 +228,7 @@ class Writer { return Key(str.data(), SizeType(str.size())); } #endif - + bool EndObject(SizeType memberCount = 0) { (void)memberCount; RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); // not inside an Object @@ -282,6 +284,8 @@ class Writer { os_->Flush(); } + static const size_t kDefaultLevelDepth = 32; + protected: //! Information for each nested level struct Level { @@ -290,8 +294,6 @@ class Writer { bool inArray; //!< true if in array, otherwise in object }; - static const size_t kDefaultLevelDepth = 32; - bool WriteNull() { PutReserve(*os_, 4); PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); return true; @@ -347,8 +349,13 @@ class Writer { bool WriteDouble(double d) { if (internal::Double(d).IsNanOrInf()) { - if (!(writeFlags & kWriteNanAndInfFlag)) + if (!(writeFlags & kWriteNanAndInfFlag) && !(writeFlags & kWriteNanAndInfNullFlag)) return false; + if (writeFlags & kWriteNanAndInfNullFlag) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); + return true; + } if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); @@ -547,6 +554,11 @@ inline bool Writer::WriteDouble(double d) { // Note: This code path can only be reached if (RAPIDJSON_WRITE_DEFAULT_FLAGS & kWriteNanAndInfFlag). if (!(kWriteDefaultFlags & kWriteNanAndInfFlag)) return false; + if (kWriteDefaultFlags & kWriteNanAndInfNullFlag) { + PutReserve(*os_, 4); + PutUnsafe(*os_, 'n'); PutUnsafe(*os_, 'u'); PutUnsafe(*os_, 'l'); PutUnsafe(*os_, 'l'); + return true; + } if (internal::Double(d).IsNan()) { PutReserve(*os_, 3); PutUnsafe(*os_, 'N'); PutUnsafe(*os_, 'a'); PutUnsafe(*os_, 'N'); @@ -668,19 +680,19 @@ inline bool Writer::ScanWriteUnescapedString(StringStream& is, siz x = vorrq_u8(x, vcltq_u8(s, s3)); x = vrev64q_u8(x); // Rev in 64 - uint64_t low = vgetq_lane_u64(reinterpret_cast(x), 0); // extract - uint64_t high = vgetq_lane_u64(reinterpret_cast(x), 1); // extract + uint64_t low = vgetq_lane_u64(vreinterpretq_u64_u8(x), 0); // extract + uint64_t high = vgetq_lane_u64(vreinterpretq_u64_u8(x), 1); // extract SizeType len = 0; bool escaped = false; if (low == 0) { if (high != 0) { - unsigned lz = (unsigned)__builtin_clzll(high); + uint32_t lz = internal::clzll(high); len = 8 + (lz >> 3); escaped = true; } } else { - unsigned lz = (unsigned)__builtin_clzll(low); + uint32_t lz = internal::clzll(low); len = lz >> 3; escaped = true; } diff --git a/src/tasks/WorkloadBuildTasks/PackageInstaller.cs b/src/tasks/WorkloadBuildTasks/PackageInstaller.cs index 0d2f5db329065..a1b8583bd6e13 100644 --- a/src/tasks/WorkloadBuildTasks/PackageInstaller.cs +++ b/src/tasks/WorkloadBuildTasks/PackageInstaller.cs @@ -54,9 +54,18 @@ private bool InstallActual(PackageReference[] references, bool stopOnMissing) Directory.CreateDirectory(projecDir); - File.WriteAllText(Path.Combine(projecDir, "Directory.Build.props"), ""); - File.WriteAllText(Path.Combine(projecDir, "Directory.Packages.props"), ""); - File.WriteAllText(Path.Combine(projecDir, "Directory.Build.targets"), ""); + File.WriteAllText(Path.Combine(projecDir, "Directory.Build.props"), """ + + + + + false + false + + + +"""); File.WriteAllText(projectPath, GenerateProject(references)); File.WriteAllText(Path.Combine(projecDir, "nuget.config"), _nugetConfigContents); diff --git a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs index 67a398d357e11..d4b81bafcd4c8 100644 --- a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs +++ b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.cs @@ -26,6 +26,9 @@ public class ErrorHandlingTests [DllImport(SwiftLib, EntryPoint = "$s18SwiftErrorHandling05getMyB7Message4from13messageLengthSPys6UInt16VGSgs0B0_p_s5Int32VztF")] public unsafe static extern void* GetErrorMessage(void* handle, out int length); + [DllImport(SwiftLib, EntryPoint = "$s18SwiftErrorHandling16freeStringBuffer6bufferySpys6UInt16VG_tF")] + public unsafe static extern void FreeErrorMessageBuffer(void* stringPtr); + [Fact] public unsafe static void TestSwiftErrorThrown() { @@ -99,7 +102,7 @@ private unsafe static string GetErrorMessageFromSwift(SwiftError error) { void* pointer = GetErrorMessage(error.Value, out int messageLength); string errorMessage = Marshal.PtrToStringUni((IntPtr)pointer, messageLength); - NativeMemory.Free((void*)pointer); + FreeErrorMessageBuffer(pointer); return errorMessage; } } diff --git a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj index 49be10b939391..89eda99352fd2 100644 --- a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj +++ b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.csproj @@ -5,8 +5,6 @@ true true - - true diff --git a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift index 20022c0dba3e2..5058014a42ce3 100644 --- a/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift +++ b/src/tests/Interop/Swift/SwiftErrorHandling/SwiftErrorHandling.swift @@ -33,3 +33,7 @@ public func getMyErrorMessage(from error: Error, messageLength: inout Int32) -> } return nil } + +public func freeStringBuffer(buffer: UnsafeMutablePointer) { + buffer.deallocate() +} diff --git a/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs b/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs index 9d7e4816522c7..0d18e7bf53514 100644 --- a/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs +++ b/src/tests/JIT/opt/Structs/MemsetMemcpyNullref.cs @@ -39,6 +39,14 @@ public static void MemsetMemcpyThrowNullRefonNull() MemoryCopyByref(ref Unsafe.NullRef(), ref valid, 0); MemoryCopyByref(ref valid, ref Unsafe.NullRef(), 0); MemoryCopyByref(ref Unsafe.NullRef(), ref Unsafe.NullRef(), 0); + + byte valid2 = 0; + MemoryInitByrefZeroLen(ref valid); + MemoryInitByrefZeroLen(ref Unsafe.NullRef()); + MemoryCopyByrefZeroLen(ref valid, ref valid2); + MemoryCopyByrefZeroLen(ref valid, ref Unsafe.NullRef()); + MemoryCopyByrefZeroLen(ref Unsafe.NullRef(), ref valid2); + MemoryCopyByrefZeroLen(ref Unsafe.NullRef(), ref Unsafe.NullRef()); } [MethodImpl(MethodImplOptions.NoInlining)] @@ -61,4 +69,12 @@ private struct HugeStruct { public fixed byte Data[20_000]; } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MemoryCopyByrefZeroLen(ref byte dst, ref byte src) => + Unsafe.CopyBlockUnaligned(ref dst, ref src, 0); + + [MethodImpl(MethodImplOptions.NoInlining)] + private static void MemoryInitByrefZeroLen(ref byte dst) => + Unsafe.InitBlockUnaligned(ref dst, 42, 0); } diff --git a/src/tests/issues.targets b/src/tests/issues.targets index 18c6df9e56ab2..2ab3b36bf3fb7 100644 --- a/src/tests/issues.targets +++ b/src/tests/issues.targets @@ -75,7 +75,10 @@ https://github.com/dotnet/runtime/issues/88586 - + + https://github.com/dotnet/runtime/issues/93631 + + https://github.com/dotnet/runtime/issues/93631 @@ -1160,16 +1163,6 @@ - - - - https://github.com/dotnet/runtime/issues/95517 - - - https://github.com/dotnet/runtime/issues/95517 - - - diff --git a/src/tests/profiler/multiple/multiple.cs b/src/tests/profiler/multiple/multiple.cs index aa0388fa0eb29..0d686ae0691a9 100644 --- a/src/tests/profiler/multiple/multiple.cs +++ b/src/tests/profiler/multiple/multiple.cs @@ -35,9 +35,9 @@ public static int RunTest(String[] args) } Console.WriteLine("Waiting for profilers to all detach"); - if (!_profilerDone.WaitOne(TimeSpan.FromMinutes(5))) + if (!_profilerDone.WaitOne(TimeSpan.FromMinutes(10))) { - Console.WriteLine("Profiler did not set the callback, test will fail."); + throw new Exception("Test timed out waiting for the profilers to set the callback, test will fail."); } return 100; diff --git a/src/tools/illink/illink.sln b/src/tools/illink/illink.sln index 87e99d208c7fb..9b8d904942baa 100644 --- a/src/tools/illink/illink.sln +++ b/src/tools/illink/illink.sln @@ -223,6 +223,7 @@ Global SolutionGuid = {E43A3901-42B0-48CA-BB36-5CD40A99A6EE} EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution + test\Trimming.Tests.Shared\Trimming.Tests.Shared.projitems*{400a1561-b6b6-482d-9e4c-3ddaede5bd07}*SharedItemsImports = 5 src\ILLink.Shared\ILLink.Shared.projitems*{dd28e2b1-057b-4b4d-a04d-b2ebd9e76e46}*SharedItemsImports = 5 src\ILLink.Shared\ILLink.Shared.projitems*{f1a44a78-34ee-408b-8285-9a26f0e7d4f2}*SharedItemsImports = 5 src\ILLink.Shared\ILLink.Shared.projitems*{ff598e93-8e9e-4091-9f50-61a7572663ae}*SharedItemsImports = 13 diff --git a/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj b/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj index 7db05c1cde9af..bc410523d5d71 100644 --- a/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj +++ b/src/tools/illink/src/ILLink.RoslynAnalyzer/ILLink.RoslynAnalyzer.csproj @@ -1,4 +1,4 @@ - + netstandard2.0 @@ -9,6 +9,12 @@ Latest $(NoWarn);CS8524 cs + + true diff --git a/src/tools/illink/src/linker/CompatibilitySuppressions.xml b/src/tools/illink/src/linker/CompatibilitySuppressions.xml index 4a0a6296c4e2f..7bf0a1e0ce697 100644 --- a/src/tools/illink/src/linker/CompatibilitySuppressions.xml +++ b/src/tools/illink/src/linker/CompatibilitySuppressions.xml @@ -253,6 +253,10 @@ CP0001 T:Mono.Linker.ILogger + + CP0001 + T:Mono.Linker.InterfaceImplementor + CP0001 T:Mono.Linker.InternalErrorException @@ -1481,10 +1485,6 @@ CP0002 M:Mono.Linker.OverrideInformation.get_IsOverrideOfInterfaceMember - - CP0002 - M:Mono.Linker.OverrideInformation.get_IsStaticInterfaceMethodPair - CP0002 M:Mono.Linker.Steps.BaseStep.get_MarkingHelpers diff --git a/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs b/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs index 0d0543dcdcf56..32fd98cbe039c 100644 --- a/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs +++ b/src/tools/illink/src/linker/Linker.Steps/MarkStep.cs @@ -701,17 +701,16 @@ void ProcessVirtualMethod (MethodDefinition method) var defaultImplementations = Annotations.GetDefaultInterfaceImplementations (method); if (defaultImplementations is not null) { foreach (var dimInfo in defaultImplementations) { - ProcessDefaultImplementation (dimInfo.ImplementingType, dimInfo.InterfaceImpl, dimInfo.DefaultInterfaceMethod); + ProcessDefaultImplementation (dimInfo); - var ov = new OverrideInformation (method, dimInfo.DefaultInterfaceMethod, Context, dimInfo.InterfaceImpl); - if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (ov, dimInfo.ImplementingType)) - MarkMethod (ov.Override, new DependencyInfo (DependencyKind.Override, ov.Base), ScopeStack.CurrentScope.Origin); + if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (dimInfo)) + MarkMethod (dimInfo.Override, new DependencyInfo (DependencyKind.Override, dimInfo.Base), ScopeStack.CurrentScope.Origin); } } var overridingMethods = Annotations.GetOverrides (method); if (overridingMethods is not null) { - foreach (var ov in overridingMethods) { - if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (ov, ov.Override.DeclaringType)) + foreach (OverrideInformation ov in overridingMethods) { + if (IsInterfaceImplementationMethodNeededByTypeDueToInterface (ov)) MarkMethod (ov.Override, new DependencyInfo (DependencyKind.Override, ov.Base), ScopeStack.CurrentScope.Origin); } } @@ -819,13 +818,14 @@ bool RequiresInterfaceRecursively (TypeDefinition typeToExamine, TypeDefinition return false; } - void ProcessDefaultImplementation (TypeDefinition typeWithDefaultImplementedInterfaceMethod, InterfaceImplementation implementation, MethodDefinition implementationMethod) + void ProcessDefaultImplementation (OverrideInformation ov) { - if ((!implementationMethod.IsStatic && !Annotations.IsInstantiated (typeWithDefaultImplementedInterfaceMethod)) - || implementationMethod.IsStatic && !Annotations.IsRelevantToVariantCasting (typeWithDefaultImplementedInterfaceMethod)) + Debug.Assert (ov.IsOverrideOfInterfaceMember); + if ((!ov.Override.IsStatic && !Annotations.IsInstantiated (ov.InterfaceImplementor.Implementor)) + || ov.Override.IsStatic && !Annotations.IsRelevantToVariantCasting (ov.InterfaceImplementor.Implementor)) return; - MarkInterfaceImplementation (implementation); + MarkInterfaceImplementation (ov.InterfaceImplementor.InterfaceImplementation); } void MarkMarshalSpec (IMarshalInfoProvider spec, in DependencyInfo reason) @@ -2549,11 +2549,11 @@ bool IsMethodNeededByTypeDueToPreservedScope (MethodDefinition method) /// /// Returns true if the override method is required due to the interface that the base method is declared on. See doc at for explanation of logic. /// - bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformation overrideInformation, TypeDefinition typeThatImplsInterface) + bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformation overrideInformation) { var @base = overrideInformation.Base; var method = overrideInformation.Override; - Debug.Assert (@base.DeclaringType.IsInterface); + Debug.Assert (overrideInformation.IsOverrideOfInterfaceMember); if (@base is null || method is null || @base.DeclaringType is null) return false; @@ -2562,7 +2562,7 @@ bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformat // If the interface implementation is not marked, do not mark the implementation method // A type that doesn't implement the interface isn't required to have methods that implement the interface. - InterfaceImplementation? iface = overrideInformation.MatchingInterfaceImplementation; + InterfaceImplementation? iface = overrideInformation.InterfaceImplementor.InterfaceImplementation; if (!((iface is not null && Annotations.IsMarked (iface)) || IsInterfaceImplementationMarkedRecursively (method.DeclaringType, @base.DeclaringType))) return false; @@ -2580,12 +2580,12 @@ bool IsInterfaceImplementationMethodNeededByTypeDueToInterface (OverrideInformat // If the method is static and the implementing type is relevant to variant casting, mark the implementation method. // A static method may only be called through a constrained call if the type is relevant to variant casting. if (@base.IsStatic) - return Annotations.IsRelevantToVariantCasting (typeThatImplsInterface) + return Annotations.IsRelevantToVariantCasting (overrideInformation.InterfaceImplementor.Implementor) || IgnoreScope (@base.DeclaringType.Scope); // If the implementing type is marked as instantiated, mark the implementation method. // If the type is not instantiated, do not mark the implementation method - return Annotations.IsInstantiated (typeThatImplsInterface); + return Annotations.IsInstantiated (overrideInformation.InterfaceImplementor.Implementor); } static bool IsSpecialSerializationConstructor (MethodDefinition method) @@ -3256,7 +3256,7 @@ protected virtual void ProcessMethod (MethodDefinition method, in DependencyInfo // Only if the interface method is referenced, then all the methods which implemented must be kept, but not the other way round. if (!markAllOverrides && Context.Resolve (@base) is MethodDefinition baseDefinition - && new OverrideInformation.OverridePair (baseDefinition, method).IsStaticInterfaceMethodPair ()) + && baseDefinition.DeclaringType.IsInterface && baseDefinition.IsStatic && method.IsStatic) continue; MarkMethod (@base, new DependencyInfo (DependencyKind.MethodImplOverride, method), ScopeStack.CurrentScope.Origin); MarkExplicitInterfaceImplementation (method, @base); diff --git a/src/tools/illink/src/linker/Linker/Annotations.cs b/src/tools/illink/src/linker/Linker/Annotations.cs index 8f7747cba3543..a7b3198265e81 100644 --- a/src/tools/illink/src/linker/Linker/Annotations.cs +++ b/src/tools/illink/src/linker/Linker/Annotations.cs @@ -462,7 +462,7 @@ public bool IsPublic (IMetadataTokenProvider provider) /// DefaultInterfaceMethod is the method that implements . /// /// The interface method to find default implementations for - public IEnumerable<(TypeDefinition ImplementingType, InterfaceImplementation InterfaceImpl, MethodDefinition DefaultInterfaceMethod)>? GetDefaultInterfaceImplementations (MethodDefinition method) + public IEnumerable? GetDefaultInterfaceImplementations (MethodDefinition method) { return TypeMapInfo.GetDefaultInterfaceImplementations (method); } diff --git a/src/tools/illink/src/linker/Linker/InterfaceImplementor.cs b/src/tools/illink/src/linker/Linker/InterfaceImplementor.cs new file mode 100644 index 0000000000000..e981ce872703f --- /dev/null +++ b/src/tools/illink/src/linker/Linker/InterfaceImplementor.cs @@ -0,0 +1,59 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using Mono.Cecil; + +namespace Mono.Linker +{ + public class InterfaceImplementor + { + /// + /// The type that implements . + /// + public TypeDefinition Implementor { get; } + /// + /// The .interfaceimpl on that points to + /// + public InterfaceImplementation InterfaceImplementation { get; } + /// + /// The type of the interface that is implemented by + /// + public TypeDefinition InterfaceType { get; } + + public InterfaceImplementor (TypeDefinition implementor, InterfaceImplementation interfaceImplementation, TypeDefinition interfaceType, IMetadataResolver resolver) + { + Implementor = implementor; + InterfaceImplementation = interfaceImplementation; + InterfaceType = interfaceType; + Debug.Assert(resolver.Resolve (interfaceImplementation.InterfaceType) == interfaceType); + } + + public static InterfaceImplementor Create(TypeDefinition implementor, TypeDefinition interfaceType, IMetadataResolver resolver) + { + foreach(InterfaceImplementation iface in implementor.Interfaces) { + if (resolver.Resolve(iface.InterfaceType) == interfaceType) { + return new InterfaceImplementor(implementor, iface, interfaceType, resolver); + } + } + + Queue ifacesToCheck = new (); + ifacesToCheck.Enqueue(implementor); + while (ifacesToCheck.Count > 0) { + var currentIface = ifacesToCheck.Dequeue (); + + foreach(InterfaceImplementation ifaceImpl in currentIface.Interfaces) { + var iface = resolver.Resolve (ifaceImpl.InterfaceType); + if (iface == interfaceType) { + return new InterfaceImplementor(implementor, ifaceImpl, interfaceType, resolver); + } + ifacesToCheck.Enqueue (iface); + } + } + throw new InvalidOperationException ($"Type '{implementor.FullName}' does not implement interface '{interfaceType.FullName}' directly or through any interfaces"); + } + } +} diff --git a/src/tools/illink/src/linker/Linker/OverrideInformation.cs b/src/tools/illink/src/linker/Linker/OverrideInformation.cs index 077353eb2ee7b..0727d5d25c19a 100644 --- a/src/tools/illink/src/linker/Linker/OverrideInformation.cs +++ b/src/tools/illink/src/linker/Linker/OverrideInformation.cs @@ -3,71 +3,39 @@ using System.Diagnostics; using Mono.Cecil; +using System.Diagnostics.CodeAnalysis; namespace Mono.Linker { [DebuggerDisplay ("{Override}")] public class OverrideInformation { - readonly ITryResolveMetadata resolver; - readonly OverridePair _pair; - private InterfaceImplementation? _matchingInterfaceImplementation; + public MethodDefinition Base { get; } - public OverrideInformation (MethodDefinition @base, MethodDefinition @override, ITryResolveMetadata resolver, InterfaceImplementation? matchingInterfaceImplementation = null) - { - _pair = new OverridePair (@base, @override); - _matchingInterfaceImplementation = matchingInterfaceImplementation; - this.resolver = resolver; - } - public readonly record struct OverridePair (MethodDefinition Base, MethodDefinition Override) - { - public bool IsStaticInterfaceMethodPair () => Base.DeclaringType.IsInterface && Base.IsStatic && Override.IsStatic; - public InterfaceImplementation? GetMatchingInterfaceImplementation (ITryResolveMetadata resolver) - { - if (!Base.DeclaringType.IsInterface) - return null; - var interfaceType = Base.DeclaringType; - foreach (var @interface in Override.DeclaringType.Interfaces) { - if (resolver.TryResolve (@interface.InterfaceType)?.Equals (interfaceType) == true) { - return @interface; - } - } - return null; - } - } + public MethodDefinition Override { get; } - public MethodDefinition Base { get => _pair.Base; } - public MethodDefinition Override { get => _pair.Override; } - public InterfaceImplementation? MatchingInterfaceImplementation { - get { - if (_matchingInterfaceImplementation is not null) - return _matchingInterfaceImplementation; - _matchingInterfaceImplementation = _pair.GetMatchingInterfaceImplementation (resolver); - return _matchingInterfaceImplementation; - } - } + internal InterfaceImplementor? InterfaceImplementor { get; } - public bool IsOverrideOfInterfaceMember { - get { - if (MatchingInterfaceImplementation != null) - return true; - - return Base.DeclaringType.IsInterface; - } + internal OverrideInformation (MethodDefinition @base, MethodDefinition @override, InterfaceImplementor? interfaceImplementor = null) + { + Base = @base; + Override = @override; + InterfaceImplementor = interfaceImplementor; + // Ensure we have an interface implementation if the base method is from an interface and the override method is on a class + Debug.Assert(@base.DeclaringType.IsInterface && interfaceImplementor != null + || !@base.DeclaringType.IsInterface && interfaceImplementor == null); + // Ensure the interfaceImplementor is for the interface we expect + Debug.Assert (@base.DeclaringType.IsInterface ? interfaceImplementor!.InterfaceType == @base.DeclaringType : true); } - public TypeDefinition? InterfaceType { - get { - if (!IsOverrideOfInterfaceMember) - return null; + public InterfaceImplementation? MatchingInterfaceImplementation + => InterfaceImplementor?.InterfaceImplementation; - if (MatchingInterfaceImplementation != null) - return resolver.TryResolve (MatchingInterfaceImplementation.InterfaceType); - - return Base.DeclaringType; - } - } + public TypeDefinition? InterfaceType + => InterfaceImplementor?.InterfaceType; - public bool IsStaticInterfaceMethodPair => _pair.IsStaticInterfaceMethodPair (); + [MemberNotNullWhen (true, nameof (InterfaceImplementor), nameof (MatchingInterfaceImplementation))] + public bool IsOverrideOfInterfaceMember + => InterfaceImplementor != null; } } diff --git a/src/tools/illink/src/linker/Linker/TypeMapInfo.cs b/src/tools/illink/src/linker/Linker/TypeMapInfo.cs index ef58e69c36fd2..bb2836a804d7b 100644 --- a/src/tools/illink/src/linker/Linker/TypeMapInfo.cs +++ b/src/tools/illink/src/linker/Linker/TypeMapInfo.cs @@ -29,9 +29,11 @@ // WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. // +using System; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Linq; using Mono.Cecil; namespace Mono.Linker @@ -43,7 +45,7 @@ public class TypeMapInfo readonly LinkContext context; protected readonly Dictionary> base_methods = new Dictionary> (); protected readonly Dictionary> override_methods = new Dictionary> (); - protected readonly Dictionary> default_interface_implementations = new Dictionary> (); + protected readonly Dictionary> default_interface_implementations = new Dictionary> (); public TypeMapInfo (LinkContext context) { @@ -92,41 +94,41 @@ public void EnsureProcessed (AssemblyDefinition assembly) /// DefaultInterfaceMethod is the method that implements . /// /// The interface method to find default implementations for - public IEnumerable<(TypeDefinition ImplementingType, InterfaceImplementation InterfaceImpl, MethodDefinition DefaultImplementationMethod)>? GetDefaultInterfaceImplementations (MethodDefinition baseMethod) + public IEnumerable? GetDefaultInterfaceImplementations (MethodDefinition baseMethod) { default_interface_implementations.TryGetValue (baseMethod, out var ret); return ret; } - public void AddBaseMethod (MethodDefinition method, MethodDefinition @base, InterfaceImplementation? matchingInterfaceImplementation) + public void AddBaseMethod (MethodDefinition method, MethodDefinition @base, InterfaceImplementor? interfaceImplementor) { if (!base_methods.TryGetValue (method, out List? methods)) { methods = new List (); base_methods[method] = methods; } - methods.Add (new OverrideInformation (@base, method, context, matchingInterfaceImplementation)); + methods.Add (new OverrideInformation (@base, method, interfaceImplementor)); } - public void AddOverride (MethodDefinition @base, MethodDefinition @override, InterfaceImplementation? matchingInterfaceImplementation = null) + public void AddOverride (MethodDefinition @base, MethodDefinition @override, InterfaceImplementor? interfaceImplementor = null) { if (!override_methods.TryGetValue (@base, out List? methods)) { methods = new List (); override_methods.Add (@base, methods); } - methods.Add (new OverrideInformation (@base, @override, context, matchingInterfaceImplementation)); + methods.Add (new OverrideInformation (@base, @override, interfaceImplementor)); } - public void AddDefaultInterfaceImplementation (MethodDefinition @base, TypeDefinition implementingType, (InterfaceImplementation, MethodDefinition) matchingInterfaceImplementation) + public void AddDefaultInterfaceImplementation (MethodDefinition @base, InterfaceImplementor interfaceImplementor, MethodDefinition defaultImplementationMethod) { Debug.Assert(@base.DeclaringType.IsInterface); if (!default_interface_implementations.TryGetValue (@base, out var implementations)) { - implementations = new List<(TypeDefinition, InterfaceImplementation, MethodDefinition)> (); + implementations = new List (); default_interface_implementations.Add (@base, implementations); } - implementations.Add ((implementingType, matchingInterfaceImplementation.Item1, matchingInterfaceImplementation.Item2)); + implementations.Add (new (@base, defaultImplementationMethod, interfaceImplementor)); } protected virtual void MapType (TypeDefinition type) @@ -168,14 +170,14 @@ void MapInterfaceMethodsInTypeHierarchy (TypeDefinition type) // Try to find an implementation with a name/sig match on the current type MethodDefinition? exactMatchOnType = TryMatchMethod (type, interfaceMethod); if (exactMatchOnType != null) { - AnnotateMethods (resolvedInterfaceMethod, exactMatchOnType); + AnnotateMethods (resolvedInterfaceMethod, exactMatchOnType, new (type, interfaceImpl.OriginalImpl, resolvedInterfaceMethod.DeclaringType, context)); continue; } // Next try to find an implementation with a name/sig match in the base hierarchy var @base = GetBaseMethodInTypeHierarchy (type, interfaceMethod); if (@base != null) { - AnnotateMethods (resolvedInterfaceMethod, @base, interfaceImpl.OriginalImpl); + AnnotateMethods (resolvedInterfaceMethod, @base, new (type, interfaceImpl.OriginalImpl, resolvedInterfaceMethod.DeclaringType, context)); continue; } } @@ -211,24 +213,29 @@ void MapVirtualMethod (MethodDefinition method) if (@base == null) return; + Debug.Assert(!@base.DeclaringType.IsInterface); + AnnotateMethods (@base, method); } void MapOverrides (MethodDefinition method) { - foreach (MethodReference override_ref in method.Overrides) { - MethodDefinition? @override = context.TryResolve (override_ref); - if (@override == null) + foreach (MethodReference baseMethodRef in method.Overrides) { + MethodDefinition? baseMethod = context.TryResolve (baseMethodRef); + if (baseMethod == null) continue; - - AnnotateMethods (@override, method); + if (baseMethod.DeclaringType.IsInterface) { + AnnotateMethods (baseMethod, method, InterfaceImplementor.Create (method.DeclaringType, baseMethod.DeclaringType, context)); + } else { + AnnotateMethods (baseMethod, method); + } } } - void AnnotateMethods (MethodDefinition @base, MethodDefinition @override, InterfaceImplementation? matchingInterfaceImplementation = null) + void AnnotateMethods (MethodDefinition @base, MethodDefinition @override, InterfaceImplementor? interfaceImplementor = null) { - AddBaseMethod (@override, @base, matchingInterfaceImplementation); - AddOverride (@base, @override, matchingInterfaceImplementation); + AddBaseMethod (@override, @base, interfaceImplementor); + AddOverride (@base, @override, interfaceImplementor); } MethodDefinition? GetBaseMethodInTypeHierarchy (MethodDefinition method) @@ -290,7 +297,7 @@ void AnnotateMethods (MethodDefinition @base, MethodDefinition @override, Interf /// /// The InterfaceImplementation on that points to the DeclaringType of . /// - void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplementsInterface, TypeDefinition typeThatMayHaveDIM, MethodDefinition interfaceMethodToBeImplemented, InterfaceImplementation implOfInterface) + void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplementsInterface, TypeDefinition typeThatMayHaveDIM, MethodDefinition interfaceMethodToBeImplemented, InterfaceImplementation originalInterfaceImpl) { // Go over all interfaces, trying to find a method that is an explicit MethodImpl of the // interface method in question. @@ -305,7 +312,7 @@ void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplement foreach (var potentialImplMethod in potentialImplInterface.Methods) { if (potentialImplMethod == interfaceMethodToBeImplemented && !potentialImplMethod.IsAbstract) { - AddDefaultInterfaceImplementation (interfaceMethodToBeImplemented, typeThatImplementsInterface, (implOfInterface, potentialImplMethod)); + AddDefaultInterfaceImplementation (interfaceMethodToBeImplemented, new (typeThatImplementsInterface, originalInterfaceImpl, interfaceMethodToBeImplemented.DeclaringType, context), potentialImplMethod); foundImpl = true; break; } @@ -314,9 +321,9 @@ void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplement continue; // This method is an override of something. Let's see if it's the method we are looking for. - foreach (var @override in potentialImplMethod.Overrides) { - if (context.TryResolve (@override) == interfaceMethodToBeImplemented) { - AddDefaultInterfaceImplementation (interfaceMethodToBeImplemented, typeThatImplementsInterface, (implOfInterface, potentialImplMethod)); + foreach (var baseMethod in potentialImplMethod.Overrides) { + if (context.TryResolve (baseMethod) == interfaceMethodToBeImplemented) { + AddDefaultInterfaceImplementation (interfaceMethodToBeImplemented, new (typeThatImplementsInterface, originalInterfaceImpl, interfaceMethodToBeImplemented.DeclaringType, context), @potentialImplMethod); foundImpl = true; break; } @@ -330,7 +337,7 @@ void FindAndAddDefaultInterfaceImplementations (TypeDefinition typeThatImplement // We haven't found a MethodImpl on the current interface, but one of the interfaces // this interface requires could still provide it. if (!foundImpl) { - FindAndAddDefaultInterfaceImplementations (typeThatImplementsInterface, potentialImplInterface, interfaceMethodToBeImplemented, implOfInterface); + FindAndAddDefaultInterfaceImplementations (typeThatImplementsInterface, potentialImplInterface, interfaceMethodToBeImplemented, originalInterfaceImpl); } } } diff --git a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs index 2e1a2bbcb3454..649b8449527f7 100644 --- a/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs +++ b/src/tools/illink/test/ILLink.RoslynAnalyzer.Tests/generated/ILLink.RoslynAnalyzer.Tests.Generator/ILLink.RoslynAnalyzer.Tests.TestCaseGenerator/Inheritance.InterfacesTests.g.cs @@ -15,6 +15,12 @@ public Task CanDisableUnusedInterfaces () return RunTest (allowMissingWarnings: true); } + [Fact] + public Task InterfaceImplementedThroughBaseInterface () + { + return RunTest (allowMissingWarnings: true); + } + [Fact] public Task InterfaceOnUninstantiatedTypeRemoved () { diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/Dependencies/InterfaceImplementedThroughBaseInterface.il b/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/Dependencies/InterfaceImplementedThroughBaseInterface.il new file mode 100644 index 0000000000000..61080f8b7d066 --- /dev/null +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/Dependencies/InterfaceImplementedThroughBaseInterface.il @@ -0,0 +1,48 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +.assembly extern mscorlib { } + +.assembly 'library' { } + +.class interface public auto ansi abstract beforefieldinit IBase +{ + // Methods + .method public hidebysig newslot abstract virtual + instance void M () cil managed + { + } // end of method IBase::M + +} // end of class IBase + +.class interface public auto ansi abstract beforefieldinit IDerived + implements IBase +{ +} // end of class IDerived + +.class public auto ansi beforefieldinit C + extends [System.Runtime]System.Object + implements IDerived +{ + // Methods + .method private final hidebysig newslot virtual + instance void IBase.M () cil managed + { + .override method instance void IBase::M() + // Method begins at RVA 0x2050 + // Code size 2 (0x2) + .maxstack 8 + + IL_0001: ret + } // end of method C::IBase.M + + .method public hidebysig specialname rtspecialname + instance void .ctor () cil managed + { + // Method begins at RVA 0x2053 + // Code size 8 (0x8) + .maxstack 8 + + IL_0007: ret + } // end of method C::.ctor +} diff --git a/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/InterfaceImplementedThroughBaseInterface.cs b/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/InterfaceImplementedThroughBaseInterface.cs new file mode 100644 index 0000000000000..e701fb9c28ba6 --- /dev/null +++ b/src/tools/illink/test/Mono.Linker.Tests.Cases/Inheritance.Interfaces/InterfaceImplementedThroughBaseInterface.cs @@ -0,0 +1,34 @@ +// Copyright (c) .NET Foundation and contributors. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Mono.Linker.Tests.Cases.Expectations.Assertions; +using Mono.Linker.Tests.Cases.Expectations.Metadata; + +namespace Mono.Linker.Tests.Cases.Inheritance.Interfaces +{ + [SetupLinkerArgument ("--skip-unresolved", "true")] + [SetupLinkerArgument ("-a", "test.exe", "library")] + [SetupLinkerArgument ("-a", "library.dll", "library")] + [TestCaseRequirements (TestRunCharacteristics.SupportsDefaultInterfaceMethods, "Requires support for default interface methods")] + [Define ("IL_ASSEMBLY_AVAILABLE")] + [SetupCompileBefore ("library.dll", new[] { "Dependencies/InterfaceImplementedThroughBaseInterface.il" })] + [SkipILVerify] + +#if IL_ASSEMBLY_AVAILABLE + [KeptMemberInAssembly ("library.dll", typeof(C), "IBase.M()")] +#endif + [KeptMember(".ctor()")] + public class InterfaceImplementedThroughBaseInterface + { + public static void Main () + { + } + } +} + +