|
1 | 1 | #include "LLVMExtra.h" |
2 | 2 |
|
3 | 3 | #include <llvm/ADT/APInt.h> |
| 4 | +#include <llvm/ADT/SmallPtrSet.h> |
4 | 5 | #include <llvm/Analysis/AssumptionCache.h> |
5 | 6 | #include <llvm/Analysis/ConstantFolding.h> |
6 | 7 | #include <llvm/Analysis/DomTreeUpdater.h> |
| 8 | +#include <llvm/Analysis/InlineCost.h> |
| 9 | +#include <llvm/Analysis/InstructionSimplify.h> |
7 | 10 | #include <llvm/Analysis/LazyValueInfo.h> |
8 | 11 | #include <llvm/Analysis/LoopInfo.h> |
9 | 12 | #include <llvm/Analysis/ScalarEvolution.h> |
|
13 | 16 | #include <llvm/Analysis/SimplifyQuery.h> |
14 | 17 | #endif |
15 | 18 | #include <llvm/IR/Attributes.h> |
| 19 | +#include <llvm/IR/CFG.h> |
16 | 20 | #include <llvm/IR/ConstantRange.h> |
17 | 21 | #include <llvm/IR/Dominators.h> |
18 | 22 | #include <llvm/IR/IntrinsicInst.h> |
|
24 | 28 | #include <llvm/IR/Operator.h> |
25 | 29 | #include <llvm/Support/KnownBits.h> |
26 | 30 | #include <llvm/Transforms/Utils/BasicBlockUtils.h> |
| 31 | +#include <llvm/Transforms/Utils/Cloning.h> |
27 | 32 | #include <llvm/Transforms/Utils/Local.h> |
28 | 33 | #include <llvm/Transforms/Utils/PromoteMemToReg.h> |
29 | 34 |
|
@@ -807,22 +812,79 @@ void LLVMExtraDominatorTreeRecalculate(LLVMDominatorTreeRef Tree, LLVMValueRef F |
807 | 812 | unwrap(Tree)->recalculate(*unwrap<Function>(F)); |
808 | 813 | } |
809 | 814 |
|
| 815 | +// the `!prof` node for the given branch weights, or NULL without weights |
| 816 | +static MDNode *createBranchWeights(LLVMContext &C, const unsigned *Weights, |
| 817 | + unsigned NumWeights) { |
| 818 | + if (NumWeights == 0) |
| 819 | + return nullptr; |
| 820 | + SmallVector<uint32_t, 2> W(Weights, Weights + NumWeights); |
| 821 | + return MDBuilder(C).createBranchWeights(W); |
| 822 | +} |
| 823 | + |
| 824 | +// whether a dominator tree is one of the function containing an instruction. the root of a |
| 825 | +// tree is the entry block of its function (compared without dereferencing it, as the tree |
| 826 | +// may be stale), and LLVM asserts that blocks are of that function |
| 827 | +static bool isTreeOf(LLVMDominatorTreeRef Tree, Instruction *I) { |
| 828 | + return unwrap(Tree)->getRoot() == &I->getFunction()->getEntryBlock(); |
| 829 | +} |
| 830 | + |
810 | 831 | LLVMValueRef LLVMExtraSplitBlockAndInsertIfThen(LLVMValueRef Cond, LLVMValueRef SplitBefore, |
811 | 832 | LLVMBool Unreachable, |
| 833 | + const unsigned *BranchWeights, |
| 834 | + unsigned NumBranchWeights, |
812 | 835 | LLVMDominatorTreeRef Tree) { |
813 | 836 | Instruction *I = unwrap<Instruction>(SplitBefore); |
814 | 837 | std::optional<DomTreeUpdater> DTU; |
815 | 838 | if (Tree) { |
816 | | - // the root of a tree is the entry block of its function (compared without dereferencing |
817 | | - // it, as the tree may be stale), and LLVM asserts that blocks are of that function |
818 | | - if (unwrap(Tree)->getRoot() != &I->getFunction()->getEntryBlock()) |
| 839 | + if (!isTreeOf(Tree, I)) |
819 | 840 | return nullptr; |
820 | 841 | DTU.emplace(*unwrap(Tree), DomTreeUpdater::UpdateStrategy::Eager); |
821 | 842 | } |
| 843 | + MDNode *Weights = createBranchWeights(I->getContext(), BranchWeights, NumBranchWeights); |
822 | 844 | // the overload taking an instruction and a DomTreeUpdater exists in every version |
823 | 845 | DomTreeUpdater *Updater = DTU ? &*DTU : nullptr; |
824 | | - return wrap(SplitBlockAndInsertIfThen(unwrap(Cond), I, Unreachable, |
825 | | - /*BranchWeights=*/nullptr, Updater)); |
| 846 | + return wrap(SplitBlockAndInsertIfThen(unwrap(Cond), I, Unreachable, Weights, Updater)); |
| 847 | +} |
| 848 | + |
| 849 | +LLVMBool LLVMExtraSplitBlockAndInsertIfThenElse(LLVMValueRef Cond, LLVMValueRef SplitBefore, |
| 850 | + const unsigned *BranchWeights, |
| 851 | + unsigned NumBranchWeights, |
| 852 | + LLVMDominatorTreeRef Tree, |
| 853 | + LLVMValueRef *ThenTerm, |
| 854 | + LLVMValueRef *ElseTerm) { |
| 855 | + Instruction *I = unwrap<Instruction>(SplitBefore); |
| 856 | + if (Tree && !isTreeOf(Tree, I)) |
| 857 | + return false; |
| 858 | + MDNode *Weights = createBranchWeights(I->getContext(), BranchWeights, NumBranchWeights); |
| 859 | + Instruction *Then, *Else; |
| 860 | +#if LLVM_VERSION_MAJOR >= 16 |
| 861 | + std::optional<DomTreeUpdater> DTU; |
| 862 | + if (Tree) |
| 863 | + DTU.emplace(*unwrap(Tree), DomTreeUpdater::UpdateStrategy::Eager); |
| 864 | + DomTreeUpdater *Updater = DTU ? &*DTU : nullptr; |
| 865 | + SplitBlockAndInsertIfThenElse(unwrap(Cond), I, &Then, &Else, Weights, Updater); |
| 866 | +#else |
| 867 | + // LLVM 15 can't update a dominator tree here, so apply the updates that LLVM 16 does |
| 868 | + BasicBlock *Head = I->getParent(); |
| 869 | + SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors(succ_begin(Head), succ_end(Head)); |
| 870 | + SplitBlockAndInsertIfThenElse(unwrap(Cond), I, &Then, &Else, Weights); |
| 871 | + if (Tree) { |
| 872 | + BasicBlock *Tail = I->getParent(); |
| 873 | + SmallVector<DominatorTree::UpdateType, 8> Updates; |
| 874 | + for (BasicBlock *Succ : successors(Head)) { |
| 875 | + Updates.push_back({DominatorTree::Insert, Head, Succ}); |
| 876 | + Updates.push_back({DominatorTree::Insert, Succ, Tail}); |
| 877 | + } |
| 878 | + for (BasicBlock *Succ : UniqueOrigSuccessors) |
| 879 | + Updates.push_back({DominatorTree::Insert, Tail, Succ}); |
| 880 | + for (BasicBlock *Succ : UniqueOrigSuccessors) |
| 881 | + Updates.push_back({DominatorTree::Delete, Head, Succ}); |
| 882 | + DomTreeUpdater(*unwrap(Tree), DomTreeUpdater::UpdateStrategy::Eager).applyUpdates(Updates); |
| 883 | + } |
| 884 | +#endif |
| 885 | + *ThenTerm = wrap(Then); |
| 886 | + *ElseTerm = wrap(Else); |
| 887 | + return true; |
826 | 888 | } |
827 | 889 |
|
828 | 890 |
|
@@ -886,3 +948,61 @@ LLVMBool LLVMExtraConstantFoldCastOperand(LLVMOpcode Opcode, LLVMValueRef C, |
886 | 948 | *Result = wrap(ConstantFoldCastOperand(Op, Const, Ty, *unwrap(TD))); |
887 | 949 | return true; |
888 | 950 | } |
| 951 | + |
| 952 | +LLVMValueRef LLVMExtraSimplifyInstruction(LLVMValueRef Inst) { |
| 953 | + Instruction *I = unwrap<Instruction>(Inst); |
| 954 | + return wrap(simplifyInstruction(I, SimplifyQuery(I->getModule()->getDataLayout(), I))); |
| 955 | +} |
| 956 | + |
| 957 | + |
| 958 | +// Control flow |
| 959 | + |
| 960 | +LLVMBasicBlockRef LLVMExtraSplitBasicBlock(LLVMBasicBlockRef BB, LLVMValueRef Before, |
| 961 | + LLVMBool Head, const char *Name) { |
| 962 | + BasicBlock::iterator It = unwrap<Instruction>(Before)->getIterator(); |
| 963 | +#if LLVM_VERSION_MAJOR >= 19 |
| 964 | + It.setHeadBit(Head); |
| 965 | +#endif |
| 966 | + return wrap(unwrap(BB)->splitBasicBlock(It, Name)); |
| 967 | +} |
| 968 | + |
| 969 | + |
| 970 | +// Inlining |
| 971 | + |
| 972 | +LLVMBool LLVMExtraInlineFunction(LLVMValueRef Call, LLVMBool MergeAttributes, |
| 973 | + LLVMBool InsertLifetime, char **FailureReason) { |
| 974 | + CallBase *CB = unwrap<CallBase>(Call); |
| 975 | + Function *Caller = CB->getCaller(); |
| 976 | + InlineFunctionInfo IFI; |
| 977 | +#if LLVM_VERSION_MAJOR < 22 |
| 978 | + // before checking the personality functions, InlineFunction gives a caller without a GC |
| 979 | + // strategy the one of the callee, which it doesn't undo when it then fails. it changes |
| 980 | + // nothing else before failing (it only sets the personality function of a caller that |
| 981 | + // has none, after which it can't fail anymore), so undo that to leave the IR unchanged. |
| 982 | + bool CallerHadGC = Caller->hasGC(); |
| 983 | +#endif |
| 984 | +#if LLVM_VERSION_MAJOR >= 16 |
| 985 | + InlineResult Result = |
| 986 | + InlineFunction(*CB, IFI, MergeAttributes, /*CalleeAAR=*/nullptr, InsertLifetime); |
| 987 | +#else |
| 988 | + // InlineFunction doesn't merge the attributes yet, which its callers did instead |
| 989 | + Function *Callee = CB->getCalledFunction(); |
| 990 | + InlineResult Result = InlineFunction(*CB, IFI, /*CalleeAAR=*/nullptr, InsertLifetime); |
| 991 | + if (Result.isSuccess() && MergeAttributes) |
| 992 | + AttributeFuncs::mergeAttributesForInlining(*Caller, *Callee); |
| 993 | +#endif |
| 994 | +#if LLVM_VERSION_MAJOR < 22 |
| 995 | + if (!Result.isSuccess() && !CallerHadGC) |
| 996 | + Caller->clearGC(); |
| 997 | +#endif |
| 998 | + if (FailureReason) |
| 999 | + *FailureReason = Result.isSuccess() ? nullptr : strdup(Result.getFailureReason()); |
| 1000 | + return Result.isSuccess(); |
| 1001 | +} |
| 1002 | + |
| 1003 | +LLVMBool LLVMExtraIsInlineViable(LLVMValueRef Fn, char **FailureReason) { |
| 1004 | + InlineResult Result = isInlineViable(*unwrap<Function>(Fn)); |
| 1005 | + if (FailureReason) |
| 1006 | + *FailureReason = Result.isSuccess() ? nullptr : strdup(Result.getFailureReason()); |
| 1007 | + return Result.isSuccess(); |
| 1008 | +} |
0 commit comments