InstrProf: Mark BiasLI as invariant. #95588
Merged
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Bias doesn't change after startup.
|
@llvm/pr-subscribers-llvm-transforms Author: NAKAMURA Takumi (chapuni) ChangesBias doesn't change after startup. The test is enhanced for optimized sequences and atomic ops. Full diff: https://github.com/llvm/llvm-project/pull/95588.diff 3 Files Affected:
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 0c79eaa812b5f..8ea76f89b7dd2 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -927,6 +927,9 @@ Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
Bias->setComdat(M.getOrInsertComdat(Bias->getName()));
}
BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
+ // Bias doesn't change after startup.
+ BiasLI->setMetadata(LLVMContext::MD_invariant_load,
+ MDNode::get(M.getContext(), std::nullopt));
}
auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
return Builder.CreateIntToPtr(Add, Addr->getType());
diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
index 44a2efd4a959a..3f0dbef387d2f 100644
--- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
@@ -1,25 +1,49 @@
; RUN: opt < %s -S -passes=instrprof | FileCheck %s
; RUN: opt < %s -S -passes=instrprof -runtime-counter-relocation | FileCheck -check-prefixes=RELOC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation | FileCheck -check-prefixes=RELOC,RELOCOPT %s
+; RUN: opt < %s -S -passes=instrprof -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC,ATOMICOPT %s
target triple = "x86_64-unknown-linux-gnu"
@__profn_foo = private constant [3 x i8] c"foo"
+@__profn_bar = private constant [3 x i8] c"bar"
; RELOC: $__llvm_profile_counter_bias = comdat any
; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat
; CHECK-LABEL: define void @foo
-; CHECK-NEXT: %pgocount = load i64, ptr @__profc_foo
-; CHECK-NEXT: %1 = add i64 %pgocount, 1
-; CHECK-NEXT: store i64 %1, ptr @__profc_foo
+; CHECK-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr @__profc_foo
+; CHECK-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
+; CHECK-NEXT: store i64 %[[PGOCOUNTINC]], ptr @__profc_foo
; RELOC-LABEL: define void @foo
-; RELOC-NEXT: %1 = load i64, ptr @__llvm_profile_counter_bias
-; RELOC-NEXT: %2 = add i64 ptrtoint (ptr @__profc_foo to i64), %1
-; RELOC-NEXT: %3 = inttoptr i64 %2 to ptr
-; RELOC-NEXT: %pgocount = load i64, ptr %3
-; RELOC-NEXT: %4 = add i64 %pgocount, 1
-; RELOC-NEXT: store i64 %4, ptr %3
+; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; RELOC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; RELOC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; RELOC-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr %[[PROFC_ADDR]]
+; RELOC-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
+; RELOC-NEXT: store i64 %[[PGOCOUNTINC]], ptr %[[PROFC_ADDR]]
+; RELOCOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; RELOCOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; RELOCOPT-NEXT: %[[PGOCOUNT1:.+]] = load i64, ptr %[[PROFC_ADDR1]]
+; RELOCOPT-NEXT: %[[PGOCOUNTINC1:.+]] = add i64 %[[PGOCOUNT1]], 1
+; RELOCOPT-NEXT: store i64 %[[PGOCOUNTINC1]], ptr %[[PROFC_ADDR1]]
+; ATOMIC-LABEL: define void @foo
+; ATOMIC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; ATOMIC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; ATOMIC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; ATOMIC-NEXT: %[[PGOCOUNTINC:.+]] = atomicrmw add ptr %[[PROFC_ADDR]], i64 1 monotonic
+; ATOMICOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; ATOMICOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; ATOMICOPT-NEXT: %[[PGOCOUNTINC1:.+]] = atomicrmw add ptr %[[PROFC_ADDR1]], i64 1 monotonic
+
+define void @bar() {
+ call void @llvm.instrprof.increment(ptr @__profn_bar, i64 0, i32 1, i32 0)
+ ret void
+}
+
define void @foo() {
call void @llvm.instrprof.increment(ptr @__profn_foo, i64 0, i32 1, i32 0)
+ call void @bar()
ret void
}
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll b/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
index c489a9f8eea84..7b1da764e776e 100644
--- a/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-unknown-linux-gnu"
define void @foo(i1 %c) {
entry:
-; CHECK: %[[BIAS:[0-9]+]] = load i64, ptr @__llvm_profile_counter_bias
+; CHECK: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias
br label %while.cond
while.cond: ; preds = %land.rhs, %while.cond.preheader
|
|
@llvm/pr-subscribers-pgo Author: NAKAMURA Takumi (chapuni) ChangesBias doesn't change after startup. The test is enhanced for optimized sequences and atomic ops. Full diff: https://github.com/llvm/llvm-project/pull/95588.diff 3 Files Affected:
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 0c79eaa812b5f..8ea76f89b7dd2 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -927,6 +927,9 @@ Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
Bias->setComdat(M.getOrInsertComdat(Bias->getName()));
}
BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
+ // Bias doesn't change after startup.
+ BiasLI->setMetadata(LLVMContext::MD_invariant_load,
+ MDNode::get(M.getContext(), std::nullopt));
}
auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
return Builder.CreateIntToPtr(Add, Addr->getType());
diff --git a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
index 44a2efd4a959a..3f0dbef387d2f 100644
--- a/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
+++ b/llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
@@ -1,25 +1,49 @@
; RUN: opt < %s -S -passes=instrprof | FileCheck %s
; RUN: opt < %s -S -passes=instrprof -runtime-counter-relocation | FileCheck -check-prefixes=RELOC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation | FileCheck -check-prefixes=RELOC,RELOCOPT %s
+; RUN: opt < %s -S -passes=instrprof -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC %s
+; RUN: opt < %s -S -passes=instrprof,inline,gvn -runtime-counter-relocation -instrprof-atomic-counter-update-all | FileCheck -check-prefixes=ATOMIC,ATOMICOPT %s
target triple = "x86_64-unknown-linux-gnu"
@__profn_foo = private constant [3 x i8] c"foo"
+@__profn_bar = private constant [3 x i8] c"bar"
; RELOC: $__llvm_profile_counter_bias = comdat any
; RELOC: @__llvm_profile_counter_bias = linkonce_odr hidden global i64 0, comdat
; CHECK-LABEL: define void @foo
-; CHECK-NEXT: %pgocount = load i64, ptr @__profc_foo
-; CHECK-NEXT: %1 = add i64 %pgocount, 1
-; CHECK-NEXT: store i64 %1, ptr @__profc_foo
+; CHECK-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr @__profc_foo
+; CHECK-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
+; CHECK-NEXT: store i64 %[[PGOCOUNTINC]], ptr @__profc_foo
; RELOC-LABEL: define void @foo
-; RELOC-NEXT: %1 = load i64, ptr @__llvm_profile_counter_bias
-; RELOC-NEXT: %2 = add i64 ptrtoint (ptr @__profc_foo to i64), %1
-; RELOC-NEXT: %3 = inttoptr i64 %2 to ptr
-; RELOC-NEXT: %pgocount = load i64, ptr %3
-; RELOC-NEXT: %4 = add i64 %pgocount, 1
-; RELOC-NEXT: store i64 %4, ptr %3
+; RELOC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; RELOC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; RELOC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; RELOC-NEXT: %[[PGOCOUNT:.+]] = load i64, ptr %[[PROFC_ADDR]]
+; RELOC-NEXT: %[[PGOCOUNTINC:.+]] = add i64 %[[PGOCOUNT]], 1
+; RELOC-NEXT: store i64 %[[PGOCOUNTINC]], ptr %[[PROFC_ADDR]]
+; RELOCOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; RELOCOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; RELOCOPT-NEXT: %[[PGOCOUNT1:.+]] = load i64, ptr %[[PROFC_ADDR1]]
+; RELOCOPT-NEXT: %[[PGOCOUNTINC1:.+]] = add i64 %[[PGOCOUNT1]], 1
+; RELOCOPT-NEXT: store i64 %[[PGOCOUNTINC1]], ptr %[[PROFC_ADDR1]]
+; ATOMIC-LABEL: define void @foo
+; ATOMIC-NEXT: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias, align 8, !invariant.load !0
+; ATOMIC-NEXT: %[[PROFC_BIAS:.+]] = add i64 ptrtoint (ptr @__profc_foo to i64), %[[BIAS]]
+; ATOMIC-NEXT: %[[PROFC_ADDR:.+]] = inttoptr i64 %[[PROFC_BIAS]] to ptr
+; ATOMIC-NEXT: %[[PGOCOUNTINC:.+]] = atomicrmw add ptr %[[PROFC_ADDR]], i64 1 monotonic
+; ATOMICOPT-NEXT: %[[PROFC_BIAS1:.+]] = add i64 ptrtoint (ptr @__profc_bar to i64), %[[BIAS]]
+; ATOMICOPT-NEXT: %[[PROFC_ADDR1:.+]] = inttoptr i64 %[[PROFC_BIAS1]] to ptr
+; ATOMICOPT-NEXT: %[[PGOCOUNTINC1:.+]] = atomicrmw add ptr %[[PROFC_ADDR1]], i64 1 monotonic
+
+define void @bar() {
+ call void @llvm.instrprof.increment(ptr @__profn_bar, i64 0, i32 1, i32 0)
+ ret void
+}
+
define void @foo() {
call void @llvm.instrprof.increment(ptr @__profn_foo, i64 0, i32 1, i32 0)
+ call void @bar()
ret void
}
diff --git a/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll b/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
index c489a9f8eea84..7b1da764e776e 100644
--- a/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
+++ b/llvm/test/Transforms/PGOProfile/counter_promo_with_bias.ll
@@ -6,7 +6,7 @@ target triple = "x86_64-unknown-linux-gnu"
define void @foo(i1 %c) {
entry:
-; CHECK: %[[BIAS:[0-9]+]] = load i64, ptr @__llvm_profile_counter_bias
+; CHECK: %[[BIAS:.+]] = load i64, ptr @__llvm_profile_counter_bias
br label %while.cond
while.cond: ; preds = %land.rhs, %while.cond.preheader
|
Conflicts: llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp llvm/test/Instrumentation/InstrProfiling/runtime-counter-relocation.ll
chapuni
commented
Jun 21, 2024
Comment on lines
930
to
931
| /// using the index represented by the a temp value into a bitmap. | ||
| void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins); |
There was a problem hiding this comment.
I think as long as you don't rely on Fn->getEntryBlock().front() it should be fine.
There was a problem hiding this comment.
I'll do in the next time. I have to reconfirm the effort of "counter promotion".
There was a problem hiding this comment.
@ellishg I meant, "I'll do in the next opportunity."
I tried sinking LoadLI from the entry to each used block. It made things worse since they weren't optimized out in larger functions.
For now, I'll leave LoadLI untouched. I have to investigate and re-evaluate register spills.
|
This eliminates 679 |
|
@ellishg Ping. Any other concerns? |
|
@ellishg Thanks! |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/190/builds/2237 Here is the relevant piece of the build log for the reference: |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/154/builds/1661 Here is the relevant piece of the build log for the reference: |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/65/builds/1669 Here is the relevant piece of the build log for the reference: |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/13/builds/922 Here is the relevant piece of the build log for the reference: |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/161/builds/735 Here is the relevant piece of the build log for the reference: |
|
LLVM Buildbot has detected a new failure on builder Full details are available at: https://lab.llvm.org/buildbot/#/builders/193/builds/1141 Here is the relevant piece of the build log for the reference: |
|
Sorry, I'll fix shortly. |
chapuni
added a commit
that referenced
this pull request
Jul 20, 2024
|
Nice load optimization!
|
yuxuanchen1997
pushed a commit
that referenced
this pull request
Jul 25, 2024
Summary: Bias doesn't change after startup. The test is enhanced for optimized sequences and atomic ops. Test Plan: Reviewers: Subscribers: Tasks: Tags: Differential Revision: https://phabricator.intern.facebook.com/D60251439
yuxuanchen1997
pushed a commit
that referenced
this pull request
Jul 25, 2024
Summary: This shall fix aarch64 builders. Test Plan: Reviewers: Subscribers: Tasks: Tags: Differential Revision: https://phabricator.intern.facebook.com/D60251402
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
5 participants
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Bias doesn't change after startup.
The test is enhanced for optimized sequences and atomic ops.