From 1601affe2e119bb0cb9017d86d24cbec2703db27 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Sat, 17 May 2025 13:48:41 +0100 Subject: [PATCH 1/2] [SCEVPatternMatch] Introduce m_scev_AddRec Introduce m_scev_AddRec to match affine AddRecs, a class_match for SCEVConstant, and demonstrate their utility in LSR and SCEV. While at it, rename m_Specific to m_scev_Specific for clarity. --- .../Analysis/ScalarEvolutionPatternMatch.h | 12 +- llvm/lib/Analysis/ScalarEvolution.cpp | 21 ++-- .../Transforms/Scalar/LoopStrengthReduce.cpp | 119 ++++++++---------- 3 files changed, 72 insertions(+), 80 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index 5e53a4f502153..e663cb798b685 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -61,6 +61,9 @@ template struct class_match { }; inline class_match m_SCEV() { return class_match(); } +inline class_match m_SCEVConstant() { + return class_match(); +} template struct bind_ty { Class *&VR; @@ -95,7 +98,7 @@ struct specificscev_ty { }; /// Match if we have a specific specified SCEV. -inline specificscev_ty m_Specific(const SCEV *S) { return S; } +inline specificscev_ty m_scev_Specific(const SCEV *S) { return S; } struct is_specific_cst { uint64_t CV; @@ -192,6 +195,13 @@ inline SCEVBinaryExpr_match m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1) { return m_scev_Binary(Op0, Op1); } + +// Matches affine AddRecs. +template +inline SCEVBinaryExpr_match +m_scev_AddRec(const Op0_t &Op0, const Op1_t &Op1) { + return m_scev_Binary(Op0, Op1); +} } // namespace SCEVPatternMatch } // namespace llvm diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 69714a112310e..f54c2caa664e7 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -12480,26 +12480,21 @@ static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, if (!ICmpInst::isRelational(Pred)) return false; - const SCEVAddRecExpr *LAR = dyn_cast(LHS); - if (!LAR) - return false; - const SCEVAddRecExpr *RAR = dyn_cast(RHS); - if (!RAR) + const SCEV *LStart, *RStart, *Step; + if (!match(LHS, m_scev_AddRec(m_SCEV(LStart), m_SCEV(Step))) || + !match(RHS, m_scev_AddRec(m_SCEV(RStart), m_scev_Specific(Step)))) return false; + const SCEVAddRecExpr *LAR = cast(LHS); + const SCEVAddRecExpr *RAR = cast(RHS); if (LAR->getLoop() != RAR->getLoop()) return false; - if (!LAR->isAffine() || !RAR->isAffine()) - return false; - - if (LAR->getStepRecurrence(SE) != RAR->getStepRecurrence(SE)) - return false; SCEV::NoWrapFlags NW = ICmpInst::isSigned(Pred) ? SCEV::FlagNSW : SCEV::FlagNUW; if (!LAR->getNoWrapFlags(NW) || !RAR->getNoWrapFlags(NW)) return false; - return SE.isKnownPredicate(Pred, LAR->getStart(), RAR->getStart()); + return SE.isKnownPredicate(Pred, LStart, RStart); } /// Is LHS `Pred` RHS true on the virtue of LHS or RHS being a Min or Max @@ -12716,7 +12711,7 @@ static bool isKnownPredicateExtendIdiom(CmpPredicate Pred, const SCEV *LHS, case ICmpInst::ICMP_SLE: { // If operand >=s 0 then ZExt == SExt. If operand =u 0 then ZExt == SExt. If operand using namespace llvm; +using namespace SCEVPatternMatch; #define DEBUG_TYPE "loop-reduce" @@ -556,16 +557,17 @@ static void DoInitialMatch(const SCEV *S, Loop *L, } // Look at addrec operands. - if (const SCEVAddRecExpr *AR = dyn_cast(S)) - if (!AR->getStart()->isZero() && AR->isAffine()) { - DoInitialMatch(AR->getStart(), L, Good, Bad, SE); - DoInitialMatch(SE.getAddRecExpr(SE.getConstant(AR->getType(), 0), - AR->getStepRecurrence(SE), - // FIXME: AR->getNoWrapFlags() - AR->getLoop(), SCEV::FlagAnyWrap), - L, Good, Bad, SE); - return; - } + const SCEV *Start, *Step; + if (match(S, m_scev_AddRec(m_SCEV(Start), m_SCEV(Step))) && + !Start->isZero()) { + DoInitialMatch(Start, L, Good, Bad, SE); + DoInitialMatch(SE.getAddRecExpr(SE.getConstant(S->getType(), 0), Step, + // FIXME: AR->getNoWrapFlags() + cast(S)->getLoop(), + SCEV::FlagAnyWrap), + L, Good, Bad, SE); + return; + } // Handle a multiplication by -1 (negation) if it didn't fold. if (const SCEVMulExpr *Mul = dyn_cast(S)) @@ -1436,22 +1438,16 @@ void Cost::RateRegister(const Formula &F, const SCEV *Reg, unsigned LoopCost = 1; if (TTI->isIndexedLoadLegal(TTI->MIM_PostInc, AR->getType()) || TTI->isIndexedStoreLegal(TTI->MIM_PostInc, AR->getType())) { - - // If the step size matches the base offset, we could use pre-indexed - // addressing. - if (AMK == TTI::AMK_PreIndexed && F.BaseOffset.isFixed()) { - if (auto *Step = dyn_cast(AR->getStepRecurrence(*SE))) - if (Step->getAPInt() == F.BaseOffset.getFixedValue()) - LoopCost = 0; - } else if (AMK == TTI::AMK_PostIndexed) { - const SCEV *LoopStep = AR->getStepRecurrence(*SE); - if (isa(LoopStep)) { - const SCEV *LoopStart = AR->getStart(); - if (!isa(LoopStart) && - SE->isLoopInvariant(LoopStart, L)) - LoopCost = 0; - } - } + const SCEV *Start; + const SCEVConstant *Step; + if (match(AR, m_scev_AddRec(m_SCEV(Start), m_SCEVConstant(Step)))) + // If the step size matches the base offset, we could use pre-indexed + // addressing. + if ((AMK == TTI::AMK_PreIndexed && F.BaseOffset.isFixed() && + Step->getAPInt() == F.BaseOffset.getFixedValue()) || + (AMK == TTI::AMK_PostIndexed && !isa(Start) && + SE->isLoopInvariant(Start, L))) + LoopCost = 0; } C.AddRecCost += LoopCost; @@ -2544,13 +2540,10 @@ ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { // Check the relevant induction variable for conformance to // the pattern. const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); - const SCEVAddRecExpr *AR = dyn_cast(IV); - if (!AR || !AR->isAffine() || - AR->getStart() != One || - AR->getStepRecurrence(SE) != One) + if (!match(IV, m_scev_AddRec(m_scev_SpecificInt(1), m_scev_SpecificInt(1)))) return Cond; - assert(AR->getLoop() == L && + assert(cast(IV)->getLoop() == L && "Loop condition operand is an addrec in a different loop!"); // Check the right operand of the select, and remember it, as it will @@ -3345,7 +3338,7 @@ void LSRInstance::CollectChains() { void LSRInstance::FinalizeChain(IVChain &Chain) { assert(!Chain.Incs.empty() && "empty IV chains are not allowed"); LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n"); - + for (const IVInc &Inc : Chain) { LLVM_DEBUG(dbgs() << " Inc: " << *Inc.UserInst << "\n"); auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand); @@ -3848,26 +3841,27 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); } return nullptr; - } else if (const SCEVAddRecExpr *AR = dyn_cast(S)) { + } + const SCEV *Start, *Step; + if (match(S, m_scev_AddRec(m_SCEV(Start), m_SCEV(Step)))) { // Split a non-zero base out of an addrec. - if (AR->getStart()->isZero() || !AR->isAffine()) + if (Start->isZero()) return S; - const SCEV *Remainder = CollectSubexprs(AR->getStart(), - C, Ops, L, SE, Depth+1); + const SCEV *Remainder = CollectSubexprs(Start, C, Ops, L, SE, Depth + 1); // Split the non-zero AddRec unless it is part of a nested recurrence that // does not pertain to this loop. - if (Remainder && (AR->getLoop() == L || !isa(Remainder))) { + if (Remainder && (cast(S)->getLoop() == L || + !isa(Remainder))) { Ops.push_back(C ? SE.getMulExpr(C, Remainder) : Remainder); Remainder = nullptr; } - if (Remainder != AR->getStart()) { + if (Remainder != Start) { if (!Remainder) - Remainder = SE.getConstant(AR->getType(), 0); - return SE.getAddRecExpr(Remainder, - AR->getStepRecurrence(SE), - AR->getLoop(), - //FIXME: AR->getNoWrapFlags(SCEV::FlagNW) + Remainder = SE.getConstant(S->getType(), 0); + return SE.getAddRecExpr(Remainder, Step, + cast(S)->getLoop(), + // FIXME: AR->getNoWrapFlags(SCEV::FlagNW) SCEV::FlagAnyWrap); } } else if (const SCEVMulExpr *Mul = dyn_cast(S)) { @@ -3895,17 +3889,13 @@ static bool mayUsePostIncMode(const TargetTransformInfo &TTI, if (LU.Kind != LSRUse::Address || !LU.AccessTy.getType()->isIntOrIntVectorTy()) return false; - const SCEVAddRecExpr *AR = dyn_cast(S); - if (!AR) - return false; - const SCEV *LoopStep = AR->getStepRecurrence(SE); - if (!isa(LoopStep)) + const SCEV *Start; + if (!match(S, m_scev_AddRec(m_SCEV(Start), m_SCEVConstant()))) return false; // Check if a post-indexed load/store can be used. - if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, AR->getType()) || - TTI.isIndexedStoreLegal(TTI.MIM_PostInc, AR->getType())) { - const SCEV *LoopStart = AR->getStart(); - if (!isa(LoopStart) && SE.isLoopInvariant(LoopStart, L)) + if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, S->getType()) || + TTI.isIndexedStoreLegal(TTI.MIM_PostInc, S->getType())) { + if (!isa(Start) && SE.isLoopInvariant(Start, L)) return true; } return false; @@ -4164,18 +4154,15 @@ void LSRInstance::GenerateConstantOffsetsImpl( // base pointer for each iteration of the loop, resulting in no extra add/sub // instructions for pointer updating. if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) { - if (auto *GAR = dyn_cast(G)) { - if (auto *StepRec = - dyn_cast(GAR->getStepRecurrence(SE))) { - const APInt &StepInt = StepRec->getAPInt(); - int64_t Step = StepInt.isNegative() ? - StepInt.getSExtValue() : StepInt.getZExtValue(); - - for (Immediate Offset : Worklist) { - if (Offset.isFixed()) { - Offset = Immediate::getFixed(Offset.getFixedValue() - Step); - GenerateOffset(G, Offset); - } + const APInt *StepInt; + if (match(G, m_scev_AddRec(m_SCEV(), m_scev_APInt(StepInt)))) { + int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue() + : StepInt->getZExtValue(); + + for (Immediate Offset : Worklist) { + if (Offset.isFixed()) { + Offset = Immediate::getFixed(Offset.getFixedValue() - Step); + GenerateOffset(G, Offset); } } } @@ -6647,7 +6634,7 @@ struct SCEVDbgValueBuilder { if (Op.getOp() != dwarf::DW_OP_LLVM_arg) { Op.appendToVector(DestExpr); continue; - } + } DestExpr.push_back(dwarf::DW_OP_LLVM_arg); // `DW_OP_LLVM_arg n` represents the nth LocationOp in this SCEV, From 1cdd536ce48eeac1e4f95476d3b9c7ec9a5cb159 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Sat, 17 May 2025 14:31:55 +0100 Subject: [PATCH 2/2] [SCEVPM] Rename to AffineAddRec; avoid footguns --- .../llvm/Analysis/ScalarEvolutionPatternMatch.h | 3 +-- llvm/lib/Analysis/ScalarEvolution.cpp | 4 ++-- llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp | 13 +++++++------ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index e663cb798b685..cfb1b4c6ea6b4 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -196,10 +196,9 @@ m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1) { return m_scev_Binary(Op0, Op1); } -// Matches affine AddRecs. template inline SCEVBinaryExpr_match -m_scev_AddRec(const Op0_t &Op0, const Op1_t &Op1) { +m_scev_AffineAddRec(const Op0_t &Op0, const Op1_t &Op1) { return m_scev_Binary(Op0, Op1); } } // namespace SCEVPatternMatch diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index f54c2caa664e7..342c8e39e2d3c 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -12481,8 +12481,8 @@ static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, return false; const SCEV *LStart, *RStart, *Step; - if (!match(LHS, m_scev_AddRec(m_SCEV(LStart), m_SCEV(Step))) || - !match(RHS, m_scev_AddRec(m_SCEV(RStart), m_scev_Specific(Step)))) + if (!match(LHS, m_scev_AffineAddRec(m_SCEV(LStart), m_SCEV(Step))) || + !match(RHS, m_scev_AffineAddRec(m_SCEV(RStart), m_scev_Specific(Step)))) return false; const SCEVAddRecExpr *LAR = cast(LHS); const SCEVAddRecExpr *RAR = cast(RHS); diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index ddcdce6eb3b1a..3c34bc089d900 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -558,7 +558,7 @@ static void DoInitialMatch(const SCEV *S, Loop *L, // Look at addrec operands. const SCEV *Start, *Step; - if (match(S, m_scev_AddRec(m_SCEV(Start), m_SCEV(Step))) && + if (match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step))) && !Start->isZero()) { DoInitialMatch(Start, L, Good, Bad, SE); DoInitialMatch(SE.getAddRecExpr(SE.getConstant(S->getType(), 0), Step, @@ -1440,7 +1440,7 @@ void Cost::RateRegister(const Formula &F, const SCEV *Reg, TTI->isIndexedStoreLegal(TTI->MIM_PostInc, AR->getType())) { const SCEV *Start; const SCEVConstant *Step; - if (match(AR, m_scev_AddRec(m_SCEV(Start), m_SCEVConstant(Step)))) + if (match(AR, m_scev_AffineAddRec(m_SCEV(Start), m_SCEVConstant(Step)))) // If the step size matches the base offset, we could use pre-indexed // addressing. if ((AMK == TTI::AMK_PreIndexed && F.BaseOffset.isFixed() && @@ -2540,7 +2540,8 @@ ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) { // Check the relevant induction variable for conformance to // the pattern. const SCEV *IV = SE.getSCEV(Cond->getOperand(0)); - if (!match(IV, m_scev_AddRec(m_scev_SpecificInt(1), m_scev_SpecificInt(1)))) + if (!match(IV, + m_scev_AffineAddRec(m_scev_SpecificInt(1), m_scev_SpecificInt(1)))) return Cond; assert(cast(IV)->getLoop() == L && @@ -3843,7 +3844,7 @@ static const SCEV *CollectSubexprs(const SCEV *S, const SCEVConstant *C, return nullptr; } const SCEV *Start, *Step; - if (match(S, m_scev_AddRec(m_SCEV(Start), m_SCEV(Step)))) { + if (match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEV(Step)))) { // Split a non-zero base out of an addrec. if (Start->isZero()) return S; @@ -3890,7 +3891,7 @@ static bool mayUsePostIncMode(const TargetTransformInfo &TTI, !LU.AccessTy.getType()->isIntOrIntVectorTy()) return false; const SCEV *Start; - if (!match(S, m_scev_AddRec(m_SCEV(Start), m_SCEVConstant()))) + if (!match(S, m_scev_AffineAddRec(m_SCEV(Start), m_SCEVConstant()))) return false; // Check if a post-indexed load/store can be used. if (TTI.isIndexedLoadLegal(TTI.MIM_PostInc, S->getType()) || @@ -4155,7 +4156,7 @@ void LSRInstance::GenerateConstantOffsetsImpl( // instructions for pointer updating. if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) { const APInt *StepInt; - if (match(G, m_scev_AddRec(m_SCEV(), m_scev_APInt(StepInt)))) { + if (match(G, m_scev_AffineAddRec(m_SCEV(), m_scev_APInt(StepInt)))) { int64_t Step = StepInt->isNegative() ? StepInt->getSExtValue() : StepInt->getZExtValue();