Skip to content

Commit f9c561b

Browse files
mtrofinmtrofin
authored
[profcheck] Fix encoding of 0 loopEstimatedTrip count (#174896)
We currently encode an estimated trip count of 0 as the latch having branch probabilities 0-0. That's an invalid pair of weights. The probability of a branch is computed as a fraction of its corresponding weight and the sum of the weights. In fact, `BranchProbabilityInfo::calcMetadataWeights` will convert this to a 1-1, meaning 50% - 50%, which isn't quite what we want. To indicate the loop is never taken, we just need to initialize the exit probability to non-zero (hence, 1) Related: https://reviews.llvm.org/D67905 Issue #147390
1 parent 26624d5 commit f9c561b

File tree

4 files changed

+12
-9
lines changed

4 files changed

+12
-9
lines changed

‎llvm/lib/Transforms/Utils/LoopUtils.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ using namespace llvm::PatternMatch;
5454

5555
static const char *LLVMLoopDisableNonforced = "llvm.loop.disable_nonforced";
5656
static const char *LLVMLoopDisableLICM = "llvm.licm.disable";
57+
namespace llvm {
58+
extern cl::opt<bool> ProfcheckDisableMetadataFixes;
59+
} // namespace llvm
5760

5861
bool llvm::formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI,
5962
MemorySSAUpdater *MSSAU,
@@ -967,7 +970,7 @@ bool llvm::setLoopEstimatedTripCount(
967970
return true;
968971

969972
// Calculate taken and exit weights.
970-
unsigned LatchExitWeight = 0;
973+
unsigned LatchExitWeight = ProfcheckDisableMetadataFixes ? 0 : 1;
971974
unsigned BackedgeTakenWeight = 0;
972975

973976
if (EstimatedTripCount != 0) {

‎llvm/test/Transforms/LoopVectorize/AArch64/check-prof-info.ll‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ define void @foo_i8(i64 %n) {
184184
; CHECK-V2-IC1: [[VEC_EPILOG_PH]]:
185185
; CHECK-V2-IC1: br label %[[VEC_EPILOG_VECTOR_BODY:.*]]
186186
; CHECK-V2-IC1: [[VEC_EPILOG_VECTOR_BODY]]:
187-
; CHECK-V2-IC1: br i1 [[TMP11:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
187+
; CHECK-V2-IC1: br i1 [[TMP9:%.*]], label %[[VEC_EPILOG_MIDDLE_BLOCK:.*]], label %[[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
188188
; CHECK-V2-IC1: [[VEC_EPILOG_MIDDLE_BLOCK]]:
189189
; CHECK-V2-IC1: br i1 [[CMP_N7:%.*]], label %[[FOR_COND_CLEANUP]], label %[[VEC_EPILOG_SCALAR_PH]], !prof [[PROF6]]
190190
; CHECK-V2-IC1: [[VEC_EPILOG_SCALAR_PH]]:
@@ -346,7 +346,7 @@ for.cond.cleanup: ; preds = %for.body
346346
; CHECK-V1-IC1: [[META3]] = !{!"llvm.loop.unroll.runtime.disable"}
347347
; CHECK-V1-IC1: [[META4]] = !{!"llvm.loop.estimated_trip_count", i32 128}
348348
; CHECK-V1-IC1: [[PROF5]] = !{!"branch_weights", i32 1, i32 7}
349-
; CHECK-V1-IC1: [[PROF6]] = !{!"branch_weights", i32 0, i32 0}
349+
; CHECK-V1-IC1: [[PROF6]] = !{!"branch_weights", i32 1, i32 0}
350350
; CHECK-V1-IC1: [[LOOP7]] = distinct !{[[LOOP7]], [[META3]], [[META2]], [[META8:![0-9]+]]}
351351
; CHECK-V1-IC1: [[META8]] = !{!"llvm.loop.estimated_trip_count", i32 0}
352352
; CHECK-V1-IC1: [[PROF9]] = !{!"branch_weights", i32 1, i32 31}
@@ -369,7 +369,7 @@ for.cond.cleanup: ; preds = %for.body
369369
; CHECK-V1-IC1-FORCE-EPI4: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META8:![0-9]+]], [[META3]]}
370370
; CHECK-V1-IC1-FORCE-EPI4: [[META8]] = !{!"llvm.loop.estimated_trip_count", i32 0}
371371
; CHECK-V1-IC1-FORCE-EPI4: [[PROF9]] = !{!"branch_weights", i32 1, i32 3}
372-
; CHECK-V1-IC1-FORCE-EPI4: [[PROF10]] = !{!"branch_weights", i32 0, i32 0}
372+
; CHECK-V1-IC1-FORCE-EPI4: [[PROF10]] = !{!"branch_weights", i32 1, i32 0}
373373
; CHECK-V1-IC1-FORCE-EPI4: [[LOOP11]] = distinct !{[[LOOP11]], [[META3]], [[META2]], [[META8]]}
374374
; CHECK-V1-IC1-FORCE-EPI4: [[PROF12]] = !{!"branch_weights", i32 1, i32 31}
375375
; CHECK-V1-IC1-FORCE-EPI4: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META3]], [[META14:![0-9]+]]}
@@ -388,7 +388,7 @@ for.cond.cleanup: ; preds = %for.body
388388
; CHECK-V2-IC1: [[META4]] = !{!"llvm.loop.unroll.runtime.disable"}
389389
; CHECK-V2-IC1: [[META5]] = !{!"llvm.loop.estimated_trip_count", i32 256}
390390
; CHECK-V2-IC1: [[PROF6]] = !{!"branch_weights", i32 1, i32 3}
391-
; CHECK-V2-IC1: [[PROF7]] = !{!"branch_weights", i32 0, i32 0}
391+
; CHECK-V2-IC1: [[PROF7]] = !{!"branch_weights", i32 1, i32 0}
392392
; CHECK-V2-IC1: [[LOOP8]] = distinct !{[[LOOP8]], [[META4]], [[META3]], [[META9:![0-9]+]]}
393393
; CHECK-V2-IC1: [[META9]] = !{!"llvm.loop.estimated_trip_count", i32 0}
394394
; CHECK-V2-IC1: [[PROF10]] = !{!"branch_weights", i32 1, i32 63}
@@ -412,7 +412,7 @@ for.cond.cleanup: ; preds = %for.body
412412
; CHECK-V2-IC4: [[LOOP8]] = distinct !{[[LOOP8]], [[META3]], [[META9:![0-9]+]], [[META4]]}
413413
; CHECK-V2-IC4: [[META9]] = !{!"llvm.loop.estimated_trip_count", i32 0}
414414
; CHECK-V2-IC4: [[PROF10]] = !{!"branch_weights", i32 1, i32 3}
415-
; CHECK-V2-IC4: [[PROF11]] = !{!"branch_weights", i32 0, i32 0}
415+
; CHECK-V2-IC4: [[PROF11]] = !{!"branch_weights", i32 1, i32 0}
416416
; CHECK-V2-IC4: [[LOOP12]] = distinct !{[[LOOP12]], [[META4]], [[META3]], [[META9]]}
417417
; CHECK-V2-IC4: [[LOOP13]] = distinct !{[[LOOP13]], [[META3]], [[META4]], [[META14:![0-9]+]]}
418418
; CHECK-V2-IC4: [[META14]] = !{!"llvm.loop.estimated_trip_count", i32 16}

‎llvm/test/Transforms/LoopVectorize/branch-weights.ll‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ exit:
126126
; MAINVF4IC1_EPI4: [[META7]] = !{!"llvm.loop.estimated_trip_count", i32 308}
127127
; MAINVF4IC1_EPI4: [[PROF8]] = !{!"branch_weights", i32 1, i32 3}
128128
; MAINVF4IC1_EPI4: [[PROF9]] = !{!"branch_weights", i32 4, i32 0}
129-
; MAINVF4IC1_EPI4: [[PROF10]] = !{!"branch_weights", i32 0, i32 0}
129+
; MAINVF4IC1_EPI4: [[PROF10]] = !{!"branch_weights", i32 1, i32 0}
130130
; MAINVF4IC1_EPI4: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]], [[META6]], [[META12:![0-9]+]]}
131131
; MAINVF4IC1_EPI4: [[META12]] = !{!"llvm.loop.estimated_trip_count", i32 0}
132132
; MAINVF4IC1_EPI4: [[PROF13]] = !{!"branch_weights", i32 2, i32 1}
@@ -143,7 +143,7 @@ exit:
143143
; MAINVF4IC2_EPI4: [[META7]] = !{!"llvm.loop.estimated_trip_count", i32 154}
144144
; MAINVF4IC2_EPI4: [[PROF8]] = !{!"branch_weights", i32 1, i32 7}
145145
; MAINVF4IC2_EPI4: [[PROF9]] = !{!"branch_weights", i32 4, i32 4}
146-
; MAINVF4IC2_EPI4: [[PROF10]] = !{!"branch_weights", i32 0, i32 0}
146+
; MAINVF4IC2_EPI4: [[PROF10]] = !{!"branch_weights", i32 1, i32 0}
147147
; MAINVF4IC2_EPI4: [[LOOP11]] = distinct !{[[LOOP11]], [[META5]], [[META6]], [[META12:![0-9]+]]}
148148
; MAINVF4IC2_EPI4: [[META12]] = !{!"llvm.loop.estimated_trip_count", i32 0}
149149
; MAINVF4IC2_EPI4: [[PROF13]] = !{!"branch_weights", i32 1, i32 3}

‎llvm/unittests/Transforms/Utils/LoopUtilsTest.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ TEST(LoopUtils, zeroEstimatedTripCount) {
239239
SmallVector<uint32_t, 2> Weights;
240240
EXPECT_EQ(extractBranchWeights(LatchBranch, Weights), true);
241241
EXPECT_EQ(Weights[0], 0u);
242-
EXPECT_EQ(Weights[1], 0u);
242+
EXPECT_EQ(Weights[1], 1u);
243243
EXPECT_EQ(getOptionalIntLoopAttribute(L, "foo"), Foo);
244244
EXPECT_EQ(getOptionalIntLoopAttribute(L, LLVMLoopEstimatedTripCount),
245245
0);

0 commit comments

Comments
 (0)