#include "polly/Support/SCEVAffinator.h"
#include "polly/Options.h"
#include "polly/ScopInfo.h"
#include "polly/Support/GICHelper.h"
#include "polly/Support/SCEVValidator.h"
#include "llvm/IR/DataLayout.h"
#include "isl/aff.h"
#include "isl/local_space.h"
#include "isl/set.h"
#include "isl/val.h"
using namespace llvm;
using namespace polly;
static cl::opt<bool> IgnoreIntegerWrapping(
"polly-ignore-integer-wrapping",
cl::desc("Do not build run-time checks to proof absence of integer "
"wrapping"),
cl::Hidden, cl::cat(PollyCategory));
static int const MaxDisjunctionsInPwAff = 100;
static unsigned const MaxSmallBitWidth = 7;
static isl_stat addNumBasicSets(__isl_take isl_set *Domain,
__isl_take isl_aff *Aff, void *User) {
auto *NumBasicSets = static_cast<unsigned *>(User);
*NumBasicSets += isl_set_n_basic_set(Domain);
isl_set_free(Domain);
isl_aff_free(Aff);
return isl_stat_ok;
}
static bool isTooComplex(PWACtx PWAC) {
unsigned NumBasicSets = 0;
isl_pw_aff_foreach_piece(PWAC.first.get(), addNumBasicSets, &NumBasicSets);
if (NumBasicSets <= MaxDisjunctionsInPwAff)
return false;
return true;
}
static SCEV::NoWrapFlags getNoWrapFlags(const SCEV *Expr) {
if (auto *NAry = dyn_cast<SCEVNAryExpr>(Expr))
return NAry->getNoWrapFlags();
return SCEV::NoWrapMask;
}
static PWACtx combine(PWACtx PWAC0, PWACtx PWAC1,
__isl_give isl_pw_aff *(Fn)(__isl_take isl_pw_aff *,
__isl_take isl_pw_aff *)) {
PWAC0.first = isl::manage(Fn(PWAC0.first.release(), PWAC1.first.release()));
PWAC0.second = PWAC0.second.unite(PWAC1.second);
return PWAC0;
}
static __isl_give isl_pw_aff *getWidthExpValOnDomain(unsigned Width,
__isl_take isl_set *Dom) {
auto *Ctx = isl_set_get_ctx(Dom);
auto *WidthVal = isl_val_int_from_ui(Ctx, Width);
auto *ExpVal = isl_val_2exp(WidthVal);
return isl_pw_aff_val_on_domain(Dom, ExpVal);
}
SCEVAffinator::SCEVAffinator(Scop *S, LoopInfo &LI)
: S(S), Ctx(S->getIslCtx().get()), SE(*S->getSE()), LI(LI),
TD(S->getFunction().getDataLayout()) {}
Loop *SCEVAffinator::getScope() { return BB ? LI.getLoopFor(BB) : nullptr; }
void SCEVAffinator::interpretAsUnsigned(PWACtx &PWAC, unsigned Width) {
auto *NonNegDom = isl_pw_aff_nonneg_set(PWAC.first.copy());
auto *NonNegPWA =
isl_pw_aff_intersect_domain(PWAC.first.copy(), isl_set_copy(NonNegDom));
auto *ExpPWA = getWidthExpValOnDomain(Width, isl_set_complement(NonNegDom));
PWAC.first = isl::manage(isl_pw_aff_union_add(
NonNegPWA, isl_pw_aff_add(PWAC.first.release(), ExpPWA)));
}
void SCEVAffinator::takeNonNegativeAssumption(
PWACtx &PWAC, RecordedAssumptionsTy *RecordedAssumptions) {
this->RecordedAssumptions = RecordedAssumptions;
auto *NegPWA = isl_pw_aff_neg(PWAC.first.copy());
auto *NegDom = isl_pw_aff_pos_set(NegPWA);
PWAC.second =
isl::manage(isl_set_union(PWAC.second.release(), isl_set_copy(NegDom)));
auto *Restriction = BB ? NegDom : isl_set_params(NegDom);
auto DL = BB ? BB->getTerminator()->getDebugLoc() : DebugLoc();
recordAssumption(RecordedAssumptions, UNSIGNED, isl::manage(Restriction), DL,
AS_RESTRICTION, BB);
}
PWACtx SCEVAffinator::getPWACtxFromPWA(isl::pw_aff PWA) {
return std::make_pair(PWA, isl::set::empty(isl::space(Ctx, 0, NumIterators)));
}
PWACtx SCEVAffinator::getPwAff(const SCEV *Expr, BasicBlock *BB,
RecordedAssumptionsTy *RecordedAssumptions) {
this->BB = BB;
this->RecordedAssumptions = RecordedAssumptions;
if (BB) {
auto *DC = S->getDomainConditions(BB).release();
NumIterators = isl_set_n_dim(DC);
isl_set_free(DC);
} else
NumIterators = 0;
return visit(Expr);
}
PWACtx SCEVAffinator::checkForWrapping(const SCEV *Expr, PWACtx PWAC) const {
if (IgnoreIntegerWrapping || (getNoWrapFlags(Expr) & SCEV::FlagNSW))
return PWAC;
isl::pw_aff PWAMod = addModuloSemantic(PWAC.first, Expr->getType());
isl::set NotEqualSet = PWAC.first.ne_set(PWAMod);
PWAC.second = PWAC.second.unite(NotEqualSet).coalesce();
const DebugLoc &Loc = BB ? BB->getTerminator()->getDebugLoc() : DebugLoc();
if (!BB)
NotEqualSet = NotEqualSet.params();
NotEqualSet = NotEqualSet.coalesce();
if (!NotEqualSet.is_empty())
recordAssumption(RecordedAssumptions, WRAPPING, NotEqualSet, Loc,
AS_RESTRICTION, BB);
return PWAC;
}
isl::pw_aff SCEVAffinator::addModuloSemantic(isl::pw_aff PWA,
Type *ExprType) const {
unsigned Width = TD.getTypeSizeInBits(ExprType);
auto ModVal = isl::val::int_from_ui(Ctx, Width);
ModVal = ModVal.pow2();
isl::set Domain = PWA.domain();
isl::pw_aff AddPW =
isl::manage(getWidthExpValOnDomain(Width - 1, Domain.release()));
return PWA.add(AddPW).mod(ModVal).sub(AddPW);
}
bool SCEVAffinator::hasNSWAddRecForLoop(Loop *L) const {
for (const auto &CachedPair : CachedExpressions) {
auto *AddRec = dyn_cast<SCEVAddRecExpr>(CachedPair.first.first);
if (!AddRec)
continue;
if (AddRec->getLoop() != L)
continue;
if (AddRec->getNoWrapFlags() & SCEV::FlagNSW)
return true;
}
return false;
}
bool SCEVAffinator::computeModuloForExpr(const SCEV *Expr) {
unsigned Width = TD.getTypeSizeInBits(Expr->getType());
if (auto *NAry = dyn_cast<SCEVNAryExpr>(Expr))
if (NAry->getNoWrapFlags() & SCEV::FlagNSW)
return false;
return Width <= MaxSmallBitWidth;
}
PWACtx SCEVAffinator::visit(const SCEV *Expr) {
auto Key = std::make_pair(Expr, BB);
PWACtx PWAC = CachedExpressions[Key];
if (!PWAC.first.is_null())
return PWAC;
auto ConstantAndLeftOverPair = extractConstantFactor(Expr, SE);
auto *Factor = ConstantAndLeftOverPair.first;
Expr = ConstantAndLeftOverPair.second;
auto *Scope = getScope();
S->addParams(getParamsInAffineExpr(&S->getRegion(), Scope, Expr, SE));
if (isl_id *Id = S->getIdForParam(Expr).release()) {
isl_space *Space = isl_space_set_alloc(Ctx.get(), 1, NumIterators);
Space = isl_space_set_dim_id(Space, isl_dim_param, 0, Id);
isl_set *Domain = isl_set_universe(isl_space_copy(Space));
isl_aff *Affine = isl_aff_zero_on_domain(isl_local_space_from_space(Space));
Affine = isl_aff_add_coefficient_si(Affine, isl_dim_param, 0, 1);
PWAC = getPWACtxFromPWA(isl::manage(isl_pw_aff_alloc(Domain, Affine)));
} else {
PWAC = SCEVVisitor<SCEVAffinator, PWACtx>::visit(Expr);
if (computeModuloForExpr(Expr))
PWAC.first = addModuloSemantic(PWAC.first, Expr->getType());
else
PWAC = checkForWrapping(Expr, PWAC);
}
if (!Factor->getType()->isIntegerTy(1)) {
PWAC = combine(PWAC, visitConstant(Factor), isl_pw_aff_mul);
if (computeModuloForExpr(Key.first))
PWAC.first = addModuloSemantic(PWAC.first, Expr->getType());
}
PWAC.first = PWAC.first.coalesce();
if (!computeModuloForExpr(Key.first))
PWAC = checkForWrapping(Key.first, PWAC);
CachedExpressions[Key] = PWAC;
return PWAC;
}
PWACtx SCEVAffinator::visitConstant(const SCEVConstant *Expr) {
ConstantInt *Value = Expr->getValue();
isl_val *v;
v = isl_valFromAPInt(Ctx.get(), Value->getValue(), true);
isl_space *Space = isl_space_set_alloc(Ctx.get(), 0, NumIterators);
isl_local_space *ls = isl_local_space_from_space(Space);
return getPWACtxFromPWA(
isl::manage(isl_pw_aff_from_aff(isl_aff_val_on_domain(ls, v))));
}
PWACtx SCEVAffinator::visitVScale(const SCEVVScale *VScale) {
llvm_unreachable("SCEVVScale not yet supported");
}
PWACtx SCEVAffinator::visitPtrToIntExpr(const SCEVPtrToIntExpr *Expr) {
return visit(Expr->getOperand(0));
}
PWACtx SCEVAffinator::visitTruncateExpr(const SCEVTruncateExpr *Expr) {
auto *Op = Expr->getOperand();
auto OpPWAC = visit(Op);
unsigned Width = TD.getTypeSizeInBits(Expr->getType());
if (computeModuloForExpr(Expr))
return OpPWAC;
auto *Dom = OpPWAC.first.domain().release();
auto *ExpPWA = getWidthExpValOnDomain(Width - 1, Dom);
auto *GreaterDom =
isl_pw_aff_ge_set(OpPWAC.first.copy(), isl_pw_aff_copy(ExpPWA));
auto *SmallerDom =
isl_pw_aff_lt_set(OpPWAC.first.copy(), isl_pw_aff_neg(ExpPWA));
auto *OutOfBoundsDom = isl_set_union(SmallerDom, GreaterDom);
OpPWAC.second = OpPWAC.second.unite(isl::manage_copy(OutOfBoundsDom));
if (!BB) {
assert(isl_set_dim(OutOfBoundsDom, isl_dim_set) == 0 &&
"Expected a zero dimensional set for non-basic-block domains");
OutOfBoundsDom = isl_set_params(OutOfBoundsDom);
}
recordAssumption(RecordedAssumptions, UNSIGNED, isl::manage(OutOfBoundsDom),
DebugLoc(), AS_RESTRICTION, BB);
return OpPWAC;
}
PWACtx SCEVAffinator::visitZeroExtendExpr(const SCEVZeroExtendExpr *Expr) {
auto *Op = Expr->getOperand();
auto OpPWAC = visit(Op);
if (!computeModuloForExpr(Op)) {
takeNonNegativeAssumption(OpPWAC, RecordedAssumptions);
return OpPWAC;
}
unsigned Width = TD.getTypeSizeInBits(Op->getType());
interpretAsUnsigned(OpPWAC, Width);
return OpPWAC;
}
PWACtx SCEVAffinator::visitSignExtendExpr(const SCEVSignExtendExpr *Expr) {
return visit(Expr->getOperand());
}
PWACtx SCEVAffinator::visitAddExpr(const SCEVAddExpr *Expr) {
PWACtx Sum = visit(Expr->getOperand(0));
for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
Sum = combine(Sum, visit(Expr->getOperand(i)), isl_pw_aff_add);
if (isTooComplex(Sum))
return complexityBailout();
}
return Sum;
}
PWACtx SCEVAffinator::visitMulExpr(const SCEVMulExpr *Expr) {
PWACtx Prod = visit(Expr->getOperand(0));
for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
Prod = combine(Prod, visit(Expr->getOperand(i)), isl_pw_aff_mul);
if (isTooComplex(Prod))
return complexityBailout();
}
return Prod;
}
PWACtx SCEVAffinator::visitAddRecExpr(const SCEVAddRecExpr *Expr) {
assert(Expr->isAffine() && "Only affine AddRecurrences allowed");
auto Flags = Expr->getNoWrapFlags();
if (Expr->getStart()->isZero()) {
assert(S->contains(Expr->getLoop()) &&
"Scop does not contain the loop referenced in this AddRec");
PWACtx Step = visit(Expr->getOperand(1));
isl_space *Space = isl_space_set_alloc(Ctx.get(), 0, NumIterators);
isl_local_space *LocalSpace = isl_local_space_from_space(Space);
unsigned loopDimension = S->getRelativeLoopDepth(Expr->getLoop());
isl_aff *LAff = isl_aff_set_coefficient_si(
isl_aff_zero_on_domain(LocalSpace), isl_dim_in, loopDimension, 1);
isl_pw_aff *LPwAff = isl_pw_aff_from_aff(LAff);
Step.first = Step.first.mul(isl::manage(LPwAff));
return Step;
}
const SCEV *ZeroStartExpr =
SE.getAddRecExpr(SE.getConstant(Expr->getStart()->getType(), 0),
Expr->getStepRecurrence(SE), Expr->getLoop(), Flags);
PWACtx Result = visit(ZeroStartExpr);
PWACtx Start = visit(Expr->getStart());
Result = combine(Result, Start, isl_pw_aff_add);
return Result;
}
PWACtx SCEVAffinator::visitSMaxExpr(const SCEVSMaxExpr *Expr) {
PWACtx Max = visit(Expr->getOperand(0));
for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
Max = combine(Max, visit(Expr->getOperand(i)), isl_pw_aff_max);
if (isTooComplex(Max))
return complexityBailout();
}
return Max;
}
PWACtx SCEVAffinator::visitSMinExpr(const SCEVSMinExpr *Expr) {
PWACtx Min = visit(Expr->getOperand(0));
for (int i = 1, e = Expr->getNumOperands(); i < e; ++i) {
Min = combine(Min, visit(Expr->getOperand(i)), isl_pw_aff_min);
if (isTooComplex(Min))
return complexityBailout();
}
return Min;
}
PWACtx SCEVAffinator::visitUMaxExpr(const SCEVUMaxExpr *Expr) {
llvm_unreachable("SCEVUMaxExpr not yet supported");
}
PWACtx SCEVAffinator::visitUMinExpr(const SCEVUMinExpr *Expr) {
llvm_unreachable("SCEVUMinExpr not yet supported");
}
PWACtx
SCEVAffinator::visitSequentialUMinExpr(const SCEVSequentialUMinExpr *Expr) {
llvm_unreachable("SCEVSequentialUMinExpr not yet supported");
}
PWACtx SCEVAffinator::visitUDivExpr(const SCEVUDivExpr *Expr) {
auto *Dividend = Expr->getLHS();
auto *Divisor = Expr->getRHS();
assert(isa<SCEVConstant>(Divisor) &&
"UDiv is no parameter but has a non-constant RHS.");
auto DividendPWAC = visit(Dividend);
auto DivisorPWAC = visit(Divisor);
if (SE.isKnownNegative(Divisor)) {
unsigned Width = TD.getTypeSizeInBits(Expr->getType());
auto *DivisorDom = DivisorPWAC.first.domain().release();
auto *WidthExpPWA = getWidthExpValOnDomain(Width, DivisorDom);
DivisorPWAC.first = DivisorPWAC.first.add(isl::manage(WidthExpPWA));
}
takeNonNegativeAssumption(DividendPWAC, RecordedAssumptions);
DividendPWAC = combine(DividendPWAC, DivisorPWAC, isl_pw_aff_div);
DividendPWAC.first = DividendPWAC.first.floor();
return DividendPWAC;
}
PWACtx SCEVAffinator::visitSDivInstruction(Instruction *SDiv) {
assert(SDiv->getOpcode() == Instruction::SDiv && "Assumed SDiv instruction!");
auto *Scope = getScope();
auto *Divisor = SDiv->getOperand(1);
auto *DivisorSCEV = SE.getSCEVAtScope(Divisor, Scope);
auto DivisorPWAC = visit(DivisorSCEV);
assert(isa<SCEVConstant>(DivisorSCEV) &&
"SDiv is no parameter but has a non-constant RHS.");
auto *Dividend = SDiv->getOperand(0);
auto *DividendSCEV = SE.getSCEVAtScope(Dividend, Scope);
auto DividendPWAC = visit(DividendSCEV);
DividendPWAC = combine(DividendPWAC, DivisorPWAC, isl_pw_aff_tdiv_q);
return DividendPWAC;
}
PWACtx SCEVAffinator::visitSRemInstruction(Instruction *SRem) {
assert(SRem->getOpcode() == Instruction::SRem && "Assumed SRem instruction!");
auto *Scope = getScope();
auto *Divisor = SRem->getOperand(1);
auto *DivisorSCEV = SE.getSCEVAtScope(Divisor, Scope);
auto DivisorPWAC = visit(DivisorSCEV);
assert(isa<ConstantInt>(Divisor) &&
"SRem is no parameter but has a non-constant RHS.");
auto *Dividend = SRem->getOperand(0);
auto *DividendSCEV = SE.getSCEVAtScope(Dividend, Scope);
auto DividendPWAC = visit(DividendSCEV);
DividendPWAC = combine(DividendPWAC, DivisorPWAC, isl_pw_aff_tdiv_r);
return DividendPWAC;
}
PWACtx SCEVAffinator::visitUnknown(const SCEVUnknown *Expr) {
if (Instruction *I = dyn_cast<Instruction>(Expr->getValue())) {
switch (I->getOpcode()) {
case Instruction::IntToPtr:
return visit(SE.getSCEVAtScope(I->getOperand(0), getScope()));
case Instruction::SDiv:
return visitSDivInstruction(I);
case Instruction::SRem:
return visitSRemInstruction(I);
default:
break;
}
}
if (isa<ConstantPointerNull>(Expr->getValue())) {
isl::val v{Ctx, 0};
isl::space Space{Ctx, 0, NumIterators};
isl::local_space ls{Space};
return getPWACtxFromPWA(isl::aff(ls, v));
}
llvm_unreachable("Unknowns SCEV was neither a parameter, a constant nor a "
"valid instruction.");
}
PWACtx SCEVAffinator::complexityBailout() {
const DebugLoc &Loc = BB ? BB->getTerminator()->getDebugLoc() : DebugLoc();
S->invalidate(COMPLEXITY, Loc);
return visit(SE.getZero(Type::getInt32Ty(S->getFunction().getContext())));
}