#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "llvm/Support/MathExtras.h"
namespace mlir {
namespace linalg {
namespace {
constexpr float G_2x2_3x3[] = {
-1, 0, 0,
1./2, -1./2, 1./2,
1./2, 1./2, 1./2,
0, 0, 1
};
constexpr float GT_2x2_3x3[] = {
-1, 1./2, 1./2, 0,
0, -1./2, 1./2, 0,
0, 1./2, 1./2, 1
};
constexpr float BT_2x2_3x3[] = {
-1, 0, 1, 0,
0, -1, 1, 0,
0, 1, 1, 0,
0, -1, 0, 1
};
constexpr float B_2x2_3x3[] = {
-1, 0, 0, 0,
0, -1, 1, -1,
1, 1, 1, 0,
0, 0, 0, 1
};
constexpr float AT_2x2_3x3[] = {
1, 1, 1, 0,
0, -1, 1, 1
};
constexpr float A_2x2_3x3[] = {
1, 0,
1, -1,
1, 1,
0, 1
};
constexpr float G_4x4_3x3[] = {
1, 0, 0,
-1./3, 1./3, -1./3,
-1./3, -1./3, -1./3,
1./12, -1./6, 1./3,
1./12, 1./6, 1./3,
0, 0, 1
};
constexpr float GT_4x4_3x3[] = {
1, -1./3, -1./3, 1./12, 1./12, 0,
0, 1./3, -1./3, -1./6, 1./6, 0,
0, -1./3, -1./3, 1./3, 1./3, 1
};
constexpr float BT_4x4_3x3[] = {
1./4, 0, -5./16, 0, 1./16, 0,
0, 1./4, -1./4, -1./16, 1./16, 0,
0, -1./4, -1./4, 1./16, 1./16, 0,
0, 1./4, -1./8, -1./4, 1./8, 0,
0, -1./4, -1./8, 1./4, 1./8, 0,
0, 1./4, 0, -5./16, 0, 1./16
};
constexpr float B_4x4_3x3[] = {
1./4, 0, 0, 0, 0, 0,
0, 1./4, -1./4, 1./4, -1./4, 1./4,
-5./16, -1./4, -1./4, -1./8, -1./8, 0,
0, -1./16, 1./16, -1./4, 1./4, -5./16,
1./16, 1./16, 1./16, 1./8, 1./8, 0,
0, 0, 0, 0, 0, 1./16
};
constexpr float AT_4x4_3x3[] = {
1./8, 1./4, 1./4, 1./8, 1./8, 0,
0, -1./4, 1./4, -1./4, 1./4, 0,
0, 1./4, 1./4, 1./2, 1./2, 0,
0, -1./4, 1./4, -1, 1, 1./2
};
constexpr float A_4x4_3x3[] = {
1./8, 0, 0, 0,
1./4, -1./4, 1./4, -1./4,
1./4, 1./4, 1./4, 1./4,
1./8, -1./4, 1./2, -1,
1./8, 1./4, 1./2, 1,
0, 0, 0, 1./2
};
constexpr float G_2x2_5x5[] = {
1, 0, 0, 0, 0,
1./6, -1./6, 1./6, -1./6, 1./6,
-1./6, -1./6, -1./6, -1./6, -1./6,
-4./15, 2./15, -1./15, 1./30, -1./60,
1./60, 1./30, 1./15, 2./15, 4./15,
0, 0, 0, 0, 1
};
constexpr float GT_2x2_5x5[] = {
1, 1./6, -1./6, -4./15, 1./60, 0,
0, -1./6, -1./6, 2./15, 1./30, 0,
0, 1./6, -1./6, -1./15, 1./15, 0,
0, -1./6, -1./6, 1./30, 2./15, 0,
0, 1./6, -1./6, -1./60, 4./15, 1
};
constexpr float BT_2x2_5x5[] = {
1./8, 3./16, -1./4, -3./16, 1./8, 0,
0, 1./8, 1./16, -5./16, 1./8, 0,
0, -1./8, -5./16, -1./16, 1./8, 0,
0, 1./4, -1./8, -1./4, 1./8, 0,
0, -1./8, -1./4, 1./8, 1./4, 0,
0, 1./8, 3./16, -1./4, -3./16, 1./8
};
constexpr float B_2x2_5x5[] = {
1./8, 0, 0, 0, 0, 0,
3./16, 1./8, -1./8, 1./4, -1./8, 1./8,
-1./4, 1./16, -5./16, -1./8, -1./4, 3./16,
-3./16, -5./16, -1./16, -1./4, 1./8, -1./4,
1./8, 1./8, 1./8, 1./8, 1./4, -3./16,
0, 0, 0, 0, 0, 1./8
};
constexpr float AT_2x2_5x5[] = {
1./2, 1, 1, 2, 1, 0,
0, -1, 1, -1, 2, 1./2
};
constexpr float A_2x2_5x5[] = {
1./2, 0,
1, -1,
1, 1,
2, -1,
1, 2,
0, 1./2
};
struct TransformMatrix {
TransformMatrix(ArrayRef<float> table, int64_t rows, int64_t cols,
int64_t scalarFactor = 1)
: table(table), rows(rows), cols(cols), scalarFactor(scalarFactor) {}
ArrayRef<float> table;
int64_t rows;
int64_t cols;
int64_t scalarFactor;
};
Value create2DTransformMatrix(OpBuilder &builder, Location loc,
TransformMatrix transform, Type type) {
assert(transform.table.size() ==
static_cast<size_t>(transform.rows * transform.cols));
assert(type.isFloat() && "Only floats are supported by Winograd");
ArrayRef<float> constVec(transform.table.data(),
transform.rows * transform.cols);
auto constAttrVec =
llvm::map_to_vector<>(constVec, [&](const float v) -> Attribute {
return builder.getFloatAttr(type, v);
});
SmallVector<int64_t, 2> shape{transform.rows, transform.cols};
return arith::ConstantOp::create(
builder, loc,
DenseFPElementsAttr::get(RankedTensorType::get(shape, type),
constAttrVec));
}
Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
Value loopNorFIndex, Value loopCorFIndex,
Value heightOffset, Value widthOffset,
int64_t extractHeight, int64_t extractWidth,
int64_t loopNorFIdx, int64_t loopCorFIdx,
int64_t heightIdx, int64_t widthIdx) {
auto sourceType = cast<ShapedType>(source.getType());
Type elementType = sourceType.getElementType();
int64_t srcSize = sourceType.getRank();
auto oneIndex = builder.getIndexAttr(1);
SmallVector<OpFoldResult> offsets;
offsets.resize(srcSize);
offsets[loopNorFIdx] = loopNorFIndex;
offsets[loopCorFIdx] = loopCorFIndex;
offsets[heightIdx] = heightOffset;
offsets[widthIdx] = widthOffset;
SmallVector<OpFoldResult> sizes(srcSize, oneIndex);
sizes[heightIdx] = builder.getIndexAttr(extractHeight);
sizes[widthIdx] = builder.getIndexAttr(extractWidth);
SmallVector<OpFoldResult> strides(srcSize, oneIndex);
auto extractFilterType =
RankedTensorType::get({extractHeight, extractWidth}, elementType);
auto extractFilterOp = tensor::ExtractSliceOp::create(
builder, loc, extractFilterType, source, offsets, sizes, strides);
return extractFilterOp;
}
Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
Value tileHIndex, Value tileWIndex,
Value loopNorFIndex, Value loopCorFIndex,
int64_t tileHIdx, int64_t tileWIdx,
int64_t loopNorFIdx, int64_t loopCorFIdx,
int64_t heightIdx, int64_t widthIdx) {
auto sourceType = cast<ShapedType>(source.getType());
Type elementType = sourceType.getElementType();
auto sourceShape = sourceType.getShape();
int64_t srcSize = sourceType.getRank();
int64_t height = sourceShape[heightIdx];
int64_t width = sourceShape[widthIdx];
auto zeroIndex = builder.getIndexAttr(0);
auto oneIndex = builder.getIndexAttr(1);
SmallVector<OpFoldResult> offsets(srcSize, zeroIndex);
offsets.resize(srcSize);
offsets[tileHIdx] = tileHIndex;
offsets[tileWIdx] = tileWIndex;
offsets[loopNorFIdx] = loopNorFIndex;
offsets[loopCorFIdx] = loopCorFIndex;
SmallVector<OpFoldResult> sizes(srcSize, oneIndex);
sizes[heightIdx] = builder.getIndexAttr(height);
sizes[widthIdx] = builder.getIndexAttr(width);
SmallVector<OpFoldResult> strides(srcSize, oneIndex);
auto extractFilterType = RankedTensorType::get({height, width}, elementType);
auto extractFilterOp = tensor::ExtractSliceOp::create(
builder, loc, extractFilterType, source, offsets, sizes, strides);
return extractFilterOp;
}
Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
Value dest, Value loopNorFIndex, Value loopCorFIndex,
Value heightOffset, Value widthOffset, int64_t height,
int64_t width, int64_t loopNorFIdx, int64_t loopCorFIdx,
int64_t heightIdx, int64_t widthIdx) {
int64_t destSize = cast<ShapedType>(dest.getType()).getRank();
auto oneIndex = builder.getIndexAttr(1);
SmallVector<OpFoldResult> retOffsets;
retOffsets.resize(destSize);
retOffsets[loopNorFIdx] = loopNorFIndex;
retOffsets[loopCorFIdx] = loopCorFIndex;
retOffsets[heightIdx] = heightOffset;
retOffsets[widthIdx] = widthOffset;
SmallVector<OpFoldResult> retSizes(destSize, oneIndex);
retSizes[heightIdx] = builder.getIndexAttr(height);
retSizes[widthIdx] = builder.getIndexAttr(width);
SmallVector<OpFoldResult> strides(destSize, oneIndex);
auto insertSliceOp = tensor::InsertSliceOp::create(
builder, loc, source, dest, retOffsets, retSizes, strides);
return insertSliceOp;
}
Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
Value dest, Value tileHIndex, Value tileWIndex,
Value loopNorFIndex, Value loopCorFIndex, int64_t height,
int64_t width, int64_t tileHIdx, int64_t tileWIdx,
int64_t loopNorFIdx, int64_t loopCorFIdx,
int64_t heightIdx, int64_t widthIdx) {
int64_t destSize = cast<ShapedType>(dest.getType()).getRank();
auto zeroIndex = builder.getIndexAttr(0);
auto oneIndex = builder.getIndexAttr(1);
SmallVector<OpFoldResult> retOffsets(destSize, zeroIndex);
retOffsets.resize(destSize);
retOffsets[tileHIdx] = tileHIndex;
retOffsets[tileWIdx] = tileWIndex;
retOffsets[loopNorFIdx] = loopNorFIndex;
retOffsets[loopCorFIdx] = loopCorFIndex;
SmallVector<OpFoldResult> retSizes(destSize, oneIndex);
retSizes[heightIdx] = builder.getIndexAttr(height);
retSizes[widthIdx] = builder.getIndexAttr(width);
SmallVector<OpFoldResult> strides(destSize, oneIndex);
auto insertSliceOp = tensor::InsertSliceOp::create(
builder, loc, source, dest, retOffsets, retSizes, strides);
return insertSliceOp;
}
Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
Value retValue, WinogradConv2DFmr fmr,
bool leftTransform = true, bool rightTransform = true) {
static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
GMatrices = {
{WinogradConv2DFmr::F_2_3, TransformMatrix(G_2x2_3x3, 4, 3)},
{WinogradConv2DFmr::F_4_3, TransformMatrix(G_4x4_3x3, 6, 3)},
{WinogradConv2DFmr::F_2_5, TransformMatrix(G_2x2_5x5, 6, 5)},
};
static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
GTMatrices = {
{WinogradConv2DFmr::F_2_3, TransformMatrix(GT_2x2_3x3, 3, 4)},
{WinogradConv2DFmr::F_4_3, TransformMatrix(GT_4x4_3x3, 3, 6)},
{WinogradConv2DFmr::F_2_5, TransformMatrix(GT_2x2_5x5, 5, 6)},
};
auto filterType = cast<ShapedType>(filter.getType());
Type elementType = filterType.getElementType();
auto filterShape = filterType.getShape();
int64_t filterF = filterShape[0];
int64_t filterH = filterShape[1];
int64_t filterW = filterShape[2];
int64_t filterC = filterShape[3];
int64_t m, r;
std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
if (filterH != r && filterH != 1)
return Value();
if (filterW != r && filterW != 1)
return Value();
Value zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
ValueRange args) -> scf::ValueVector {
Value FIter = ivs[0];
Value CIter = ivs[1];
auto extractFilter =
extract2DDataFrom4D(builder, loc, filter, FIter, CIter, zeroIdx,
zeroIdx, filterH, filterW, 0,
3, 1, 2);
int64_t retRows = 1;
Value matmulRetValue = extractFilter;
Value zero = arith::ConstantOp::create(builder, loc,
rewriter.getZeroAttr(elementType));
if (leftTransform) {
auto it = GMatrices.find(fmr);
if (it == GMatrices.end())
return {};
const TransformMatrix &GMatrix = it->second;
retRows = GMatrix.rows;
auto matmulType = RankedTensorType::get({retRows, filterW}, elementType);
auto empty = tensor::EmptyOp::create(builder, loc, matmulType.getShape(),
elementType)
.getResult();
auto init =
linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
ValueRange{G, extractFilter},
ValueRange{init});
matmulRetValue = matmulOp.getResult(0);
}
if (rightTransform) {
auto it = GTMatrices.find(fmr);
if (it == GTMatrices.end())
return {};
const TransformMatrix >Matrix = it->second;
auto matmulType =
RankedTensorType::get({retRows, GTMatrix.cols}, elementType);
auto empty = tensor::EmptyOp::create(builder, loc, matmulType.getShape(),
elementType)
.getResult();
auto init =
linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
ValueRange{matmulRetValue, GT},
ValueRange{init});
matmulRetValue = matmulOp.getResult(0);
}
int64_t retHeight = leftTransform ? m + r - 1 : 1;
int64_t retWidth = rightTransform ? m + r - 1 : 1;
auto insertSliceOp =
insert2DDataTo4D(builder, loc, matmulRetValue, args[0], FIter, CIter,
zeroIdx, zeroIdx, retHeight, retWidth,
3, 2,
0, 1);
return {insertSliceOp};
};
auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterF);
auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, filterC);
auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
scf::LoopNest loops = scf::buildLoopNest(
rewriter, loc, {zeroIdx, zeroIdx}, {fUpperBound, cUpperBound},
{oneStep, oneStep}, {retValue}, buildBody);
return loops.results[0];
}
Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
Value retValue, WinogradConv2DFmr fmr,
bool leftTransform = true, bool rightTransform = true) {
static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
BTMatrices = {
{WinogradConv2DFmr::F_2_3, TransformMatrix(BT_2x2_3x3, 4, 4)},
{WinogradConv2DFmr::F_4_3, TransformMatrix(BT_4x4_3x3, 6, 6)},
{WinogradConv2DFmr::F_2_5, TransformMatrix(BT_2x2_5x5, 6, 6)},
};
static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
BMatrices = {
{WinogradConv2DFmr::F_2_3, TransformMatrix(B_2x2_3x3, 4, 4)},
{WinogradConv2DFmr::F_4_3, TransformMatrix(B_4x4_3x3, 6, 6)},
{WinogradConv2DFmr::F_2_5, TransformMatrix(B_2x2_5x5, 6, 6)},
};
int64_t m, r;
std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
auto inputType = cast<ShapedType>(input.getType());
Type elementType = inputType.getElementType();
auto inputShape = inputType.getShape();
int64_t inputN = inputShape[0];
int64_t inputC = inputShape[3];
auto valueType = cast<ShapedType>(retValue.getType());
auto valueShape = valueType.getShape();
int64_t tileH = valueShape[2];
int64_t tileW = valueShape[3];
int64_t alphaH = leftTransform ? m + r - 1 : 1;
int64_t alphaW = rightTransform ? m + r - 1 : 1;
auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
ValueRange args) -> scf::ValueVector {
Value tileHIter = ivs[0];
Value tileWIter = ivs[1];
Value NIter = ivs[2];
Value CIter = ivs[3];
auto *context = builder.getContext();
auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
auto affineMap =
AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
Value heightOffset = affine::AffineApplyOp::create(
builder, loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
Value widthOffset = affine::AffineApplyOp::create(
builder, loc, rightTransform ? affineMap : identityAffineMap,
tileWIter);
auto extractInput =
extract2DDataFrom4D(builder, loc, input, NIter, CIter, heightOffset,
widthOffset, alphaH, alphaW, 0,
3, 1, 2);
int64_t retRows = 1;
int64_t retCols = 1;
Value matmulRetValue = extractInput;
Value zero = arith::ConstantOp::create(builder, loc,
rewriter.getZeroAttr(elementType));
if (leftTransform) {
auto it = BTMatrices.find(fmr);
if (it == BTMatrices.end())
return {};
const TransformMatrix &BTMatrix = it->second;
retRows = BTMatrix.rows;
auto matmulType = RankedTensorType::get({retRows, alphaW}, elementType);
auto empty = tensor::EmptyOp::create(builder, loc, matmulType.getShape(),
elementType)
.getResult();
auto init =
linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
Value BT = create2DTransformMatrix(builder, loc, BTMatrix, elementType);
auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
ValueRange{BT, matmulRetValue},
ValueRange{init});
matmulRetValue = matmulOp.getResult(0);
}
if (rightTransform) {
auto it = BMatrices.find(fmr);
if (it == BMatrices.end())
return {};
const TransformMatrix &BMatrix = it->second;
retCols = BMatrix.cols;
auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
auto empty = tensor::EmptyOp::create(builder, loc, matmulType.getShape(),
elementType)
.getResult();
auto init =
linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
Value B = create2DTransformMatrix(builder, loc, BMatrix, elementType);
auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
ValueRange{matmulRetValue, B},
ValueRange{init});
matmulRetValue = matmulOp.getResult(0);
}
auto combinedVal = insert2DDataTo6D(
builder, loc, matmulRetValue, args[0], tileHIter, tileWIter, NIter,
CIter, retRows, retCols, 2, 3, 4, 5,
0, 1);
return {combinedVal};
};
auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tileH);
auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputN);
auto cUpperBound = arith::ConstantIndexOp::create(rewriter, loc, inputC);
auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
scf::LoopNest loops = scf::buildLoopNest(
rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
{tileHBound, tileWBound, nUpperBound, cUpperBound},
{oneStep, oneStep, oneStep, oneStep}, {retValue}, buildBody);
return loops.results[0];
}
static Value matrixMultiply(RewriterBase &rewriter, Location loc,
Value transformedFilter, Value transformedInput,
Type outputElementType) {
auto filterType = cast<ShapedType>(transformedFilter.getType());
assert(filterType.hasStaticShape() && "only support static shapes.");
ArrayRef<int64_t> filterShape = filterType.getShape();
Type filterElementType = filterType.getElementType();
auto filterReassocType = RankedTensorType::get(
{filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
filterElementType);
SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
Value collapseFilter = tensor::CollapseShapeOp::create(
rewriter, loc, filterReassocType, transformedFilter, filterReassoc);
auto inputType = cast<ShapedType>(transformedInput.getType());
assert(inputType.hasStaticShape() && "only support static shapes.");
ArrayRef<int64_t> inputShape = inputType.getShape();
Type inputElementType = inputType.getElementType();
auto inputReassocType = RankedTensorType::get(
{inputShape[0] * inputShape[1],
inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
inputElementType);
SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
Value collapseInput = tensor::CollapseShapeOp::create(
rewriter, loc, inputReassocType, transformedInput, inputReassoc);
auto matmulType = RankedTensorType::get(
{inputShape[0] * inputShape[1],
inputShape[2] * inputShape[3] * inputShape[4], filterShape[3]},
outputElementType);
Value empty = tensor::EmptyOp::create(rewriter, loc, matmulType.getShape(),
outputElementType)
.getResult();
Value zero = arith::ConstantOp::create(
rewriter, loc, rewriter.getZeroAttr(outputElementType));
Value init = linalg::FillOp::create(rewriter, loc, zero, empty).getResult(0);
auto matmulOp = linalg::BatchMatmulOp::create(
rewriter, loc, matmulType, ValueRange({collapseInput, collapseFilter}),
ValueRange{init});
SmallVector<ReassociationIndices> outputReassoc = {{0, 1}, {2, 3, 4}, {5}};
auto outputReassocType =
RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
inputShape[3], inputShape[4], filterShape[3]},
outputElementType);
auto expandOutput = tensor::ExpandShapeOp::create(
rewriter, loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
return expandOutput;
}
Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
Value output, WinogradConv2DFmr fmr,
bool leftTransform = true, bool rightTransform = true) {
static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
ATMatrices = {
{WinogradConv2DFmr::F_2_3, TransformMatrix(AT_2x2_3x3, 2, 4)},
{WinogradConv2DFmr::F_4_3, TransformMatrix(AT_4x4_3x3, 4, 6, 32)},
{WinogradConv2DFmr::F_2_5, TransformMatrix(AT_2x2_5x5, 2, 6, 16)},
};
static const llvm::SmallDenseMap<WinogradConv2DFmr, TransformMatrix>
AMatrices = {
{WinogradConv2DFmr::F_2_3, TransformMatrix(A_2x2_3x3, 4, 2)},
{WinogradConv2DFmr::F_4_3, TransformMatrix(A_4x4_3x3, 6, 4, 32)},
{WinogradConv2DFmr::F_2_5, TransformMatrix(A_2x2_5x5, 6, 2, 16)},
};
int64_t m, r;
std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
auto valueType = cast<ShapedType>(value.getType());
Type elementType = valueType.getElementType();
auto valueShape = valueType.getShape();
int64_t valueH = valueShape[0];
int64_t valueW = valueShape[1];
int64_t valueN = valueShape[4];
int64_t valueF = valueShape[5];
int64_t alphaH = leftTransform ? m + r - 1 : 1;
int64_t alphaW = rightTransform ? m + r - 1 : 1;
if (valueH != alphaH && valueH != 1)
return Value();
if (valueW != alphaW && valueW != 1)
return Value();
auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
ValueRange args) -> scf::ValueVector {
auto *context = builder.getContext();
Value tileHIter = ivs[0];
Value tileWIter = ivs[1];
Value NIter = ivs[2];
Value FIter = ivs[3];
auto extractValue =
extract2DDataFrom6D(builder, loc, value, tileHIter, tileWIter, NIter,
FIter, 2, 3, 4,
5, 0, 1);
const TransformMatrix &AMatrix = AMatrices.at(fmr);
const TransformMatrix &ATMatrix = ATMatrices.at(fmr);
int64_t scalarFactor = (rightTransform ? AMatrix.scalarFactor : 1) *
(leftTransform ? ATMatrix.scalarFactor : 1);
int64_t retCols = rightTransform ? AMatrix.cols : 1;
int64_t retRows = leftTransform ? ATMatrix.rows : 1;
Value matmulRetValue = extractValue;
Value zero = arith::ConstantOp::create(builder, loc,
rewriter.getZeroAttr(elementType));
auto identityAffineMap = rewriter.getMultiDimIdentityMap(1);
auto affineMap =
AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
Value heightOffset = affine::AffineApplyOp::create(
builder, loc, leftTransform ? affineMap : identityAffineMap, tileHIter);
Value widthOffset = affine::AffineApplyOp::create(
builder, loc, rightTransform ? affineMap : identityAffineMap,
tileWIter);
Value outInitVal =
extract2DDataFrom4D(builder, loc, args[0], NIter, FIter, heightOffset,
widthOffset, retRows, retCols,
0,
3, 1,
2);
if (leftTransform) {
auto matmulType = RankedTensorType::get({retRows, valueW}, elementType);
Value init = outInitVal;
if (rightTransform || scalarFactor != 1) {
auto empty = tensor::EmptyOp::create(builder, loc,
matmulType.getShape(), elementType)
.getResult();
init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
}
Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
ValueRange{AT, matmulRetValue},
ValueRange{init});
matmulRetValue = matmulOp.getResult(0);
}
if (rightTransform) {
auto matmulType =
RankedTensorType::get({retRows, AMatrix.cols}, elementType);
Value init = outInitVal;
if (scalarFactor != 1) {
auto empty = tensor::EmptyOp::create(builder, loc,
matmulType.getShape(), elementType)
.getResult();
init = linalg::FillOp::create(builder, loc, zero, empty).getResult(0);
}
Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
auto matmulOp = linalg::MatmulOp::create(builder, loc, matmulType,
ValueRange{matmulRetValue, A},
ValueRange{init});
matmulRetValue = matmulOp.getResult(0);
}
if (scalarFactor != 1) {
Value scalarFactorValue = arith::ConstantOp::create(
builder, loc, FloatAttr::get(elementType, scalarFactor));
auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
SmallVector<AffineMap> affineMaps = {
AffineMap::get(2, 0, context), identityAffineMap, identityAffineMap};
matmulRetValue =
linalg::GenericOp::create(
rewriter, loc, matmulType,
ValueRange{scalarFactorValue, matmulRetValue},
ValueRange{outInitVal}, affineMaps,
llvm::ArrayRef<utils::IteratorType>{
utils::IteratorType::parallel, utils::IteratorType::parallel},
[&](OpBuilder &nestedBuilder, Location nestedLoc,
ValueRange args) {
auto mulf = arith::MulFOp::create(nestedBuilder, nestedLoc,
args[0], args[1]);
auto addf = arith::AddFOp::create(nestedBuilder, nestedLoc,
mulf.getResult(), args[2]);
linalg::YieldOp::create(nestedBuilder, nestedLoc,
addf.getResult());
})
.getResult(0);
}
Value combinedVal =
insert2DDataTo4D(builder, loc, matmulRetValue, args[0], NIter, FIter,
heightOffset, widthOffset, retRows, retCols,
0,
3, 1,
2);
return {combinedVal};
};
int64_t tilwH = valueShape[2];
int64_t tileW = valueShape[3];
auto zeroIdx = arith::ConstantIndexOp::create(rewriter, loc, 0);
auto tileHBound = arith::ConstantIndexOp::create(rewriter, loc, tilwH);
auto tileWBound = arith::ConstantIndexOp::create(rewriter, loc, tileW);
auto nUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueN);
auto fUpperBound = arith::ConstantIndexOp::create(rewriter, loc, valueF);
auto oneStep = arith::ConstantIndexOp::create(rewriter, loc, 1);
scf::LoopNest loops = scf::buildLoopNest(
rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
{tileHBound, tileWBound, nUpperBound, fUpperBound},
{oneStep, oneStep, oneStep, oneStep}, {output}, buildBody);
return loops.results[0];
}
static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
Value value, ArrayRef<int64_t> alignedShape) {
auto valueType = cast<ShapedType>(value.getType());
Type elementType = valueType.getElementType();
auto alignedType = RankedTensorType::get(alignedShape, elementType);
Value padValue = arith::ConstantOp::create(rewriter, loc, elementType,
rewriter.getZeroAttr(elementType));
return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
padValue, false);
}
static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
Value value,
RankedTensorType extractedType) {
OpFoldResult zeroIndex = rewriter.getIndexAttr(0);
OpFoldResult oneIndex = rewriter.getIndexAttr(1);
SmallVector<OpFoldResult, 4> offsets(4, zeroIndex);
SmallVector<OpFoldResult, 4> strides(4, oneIndex);
ArrayRef<int64_t> extractedShape = extractedType.getShape();
SmallVector<OpFoldResult> sizes =
getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
return tensor::ExtractSliceOp::create(rewriter, loc, extractedType, value,
offsets, sizes, strides);
}
static bool hasAllOneValues(DenseIntElementsAttr attr) {
return llvm::all_of(
attr, [](const APInt &element) { return element.getSExtValue() == 1; });
}
static FailureOr<Operation *>
winogradConv2DHelper(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp convOp,
WinogradConv2DFmr fmr) {
if (!convOp.hasPureTensorSemantics())
return rewriter.notifyMatchFailure(
convOp, "expected pure tensor semantics for linalg.conv_2d_nhwc_fhwc");
Value input = convOp.getInputs()[0];
Value filter = convOp.getInputs()[1];
Value output = convOp.getOutputs()[0];
auto inputType = cast<ShapedType>(input.getType());
auto filterType = cast<ShapedType>(filter.getType());
auto outputType = cast<ShapedType>(output.getType());
if (!inputType.hasStaticShape())
return rewriter.notifyMatchFailure(convOp,
"expected a static shape for the input");
if (!filterType.hasStaticShape())
return rewriter.notifyMatchFailure(
convOp, "expected a static shape for the filter");
if (!hasAllOneValues(convOp.getDilations()))
return rewriter.notifyMatchFailure(convOp,
"expected all ones for dilations");
if (!hasAllOneValues(convOp.getStrides()))
return rewriter.notifyMatchFailure(convOp, "expected all ones for strides");
ArrayRef<int64_t> filterShape = filterType.getShape();
int64_t filterF = filterShape[0];
int64_t filterH = filterShape[1];
int64_t filterW = filterShape[2];
int64_t filterC = filterShape[3];
ArrayRef<int64_t> inputShape = inputType.getShape();
int64_t inputN = inputShape[0];
int64_t inputH = inputShape[1];
int64_t inputW = inputShape[2];
int64_t inputC = inputShape[3];
ArrayRef<int64_t> outputShape = outputType.getShape();
int64_t outputN = outputShape[0];
int64_t outputH = outputShape[1];
int64_t outputW = outputShape[2];
int64_t outputF = outputShape[3];
int64_t m, r;
std::tie(m, r) = getFmrFromWinogradConv2DFmr(fmr);
bool isSupportedFilter = false;
if (filterH == filterW && filterH == r)
isSupportedFilter = true;
if (filterH == r && filterW == 1)
isSupportedFilter = true;
if (filterH == 1 && filterW == r)
isSupportedFilter = true;
if (!isSupportedFilter)
return rewriter.notifyMatchFailure(
convOp, "only support filter (r x r), (r x 1) or (1 x r)");
Location loc = convOp.getLoc();
bool leftTransform = filterH != 1;
bool rightTransform = filterW != 1;
int64_t heightM = leftTransform ? m : 1;
int64_t widthM = rightTransform ? m : 1;
int64_t heightR = leftTransform ? r : 1;
int64_t widthR = rightTransform ? r : 1;
Type filterElementType = filterType.getElementType();
int64_t alphaH = heightM + heightR - 1;
int64_t alphaW = widthM + widthR - 1;
int64_t tileH = llvm::divideCeilSigned(outputH, heightM);
int64_t tileW = llvm::divideCeilSigned(outputW, widthM);
auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
filterElementType);
Value retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
filterElementType);
auto transformedFilter = linalg::WinogradFilterTransformOp::create(
rewriter, loc, retType, filter, retValue, fmr);
Type inputElementType = inputType.getElementType();
int64_t alignedInputH = tileH * heightM + (heightR - 1);
int64_t alignedInputW = tileW * widthM + (widthR - 1);
if (alignedInputH != inputH || alignedInputW != inputW) {
input = padToAlignedTensor(rewriter, loc, input,
{inputN, alignedInputH, alignedInputW, inputC});
}
retType = RankedTensorType::get(
{alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
retValue = tensor::EmptyOp::create(rewriter, loc, retType.getShape(),
inputElementType);
auto transformedInput = linalg::WinogradInputTransformOp::create(
rewriter, loc, retType, input, retValue, fmr);
Type outputElementType = outputType.getElementType();
Value matmulRet = matrixMultiply(rewriter, loc, transformedFilter,
transformedInput, outputElementType);
int64_t alignedOutputH = tileH * heightM;
int64_t alignedOutputW = tileW * widthM;
bool isOutputUnaligned =
((alignedOutputH != outputH) || (alignedOutputW != outputW));
if (isOutputUnaligned) {
auto alignedOutputType = RankedTensorType::get(
{outputN, alignedOutputH, alignedOutputW, outputF}, outputElementType);
output =
padToAlignedTensor(rewriter, loc, output, alignedOutputType.getShape());
outputType = alignedOutputType;
}
Value transformedOutput = linalg::WinogradOutputTransformOp::create(
rewriter, loc, outputType, matmulRet, output, fmr);
if (isOutputUnaligned) {
transformedOutput = extractFromAlignedTensor(
rewriter, loc, transformedOutput,
RankedTensorType::get({outputN, outputH, outputW, outputF},
outputElementType));
}
rewriter.replaceOp(convOp, transformedOutput);
return transformedOutput.getDefiningOp();
}
FailureOr<Operation *>
decomposeWinogradFilterTransformHelper(RewriterBase &rewriter,
linalg::WinogradFilterTransformOp op) {
Location loc = op.getLoc();
Value filter = op.getFilter();
auto filterType = cast<ShapedType>(filter.getType());
auto filterShape = filterType.getShape();
int64_t filterH = filterShape[1];
int64_t filterW = filterShape[2];
bool leftTransform = filterH != 1;
bool rightTransform = filterW != 1;
Value transformedFilter =
filterTransform(rewriter, loc, filter, op.getOutput(), op.getFmr(),
leftTransform, rightTransform);
if (!transformedFilter)
return failure();
rewriter.replaceOp(op, transformedFilter);
return transformedFilter.getDefiningOp();
}
FailureOr<Operation *>
decomposeWinogradInputTransformHelper(RewriterBase &rewriter,
linalg::WinogradInputTransformOp op) {
Location loc = op.getLoc();
Value output = op.getOutput();
auto outputType = cast<ShapedType>(output.getType());
auto outputShape = outputType.getShape();
int64_t outputH = outputShape[0];
int64_t outputW = outputShape[1];
bool leftTransform = outputH != 1;
bool rightTransform = outputW != 1;
Value transformedInput =
inputTransform(rewriter, loc, op.getInput(), op.getOutput(), op.getFmr(),
leftTransform, rightTransform);
if (!transformedInput)
return failure();
rewriter.replaceOp(op, transformedInput);
return transformedInput.getDefiningOp();
}
FailureOr<Operation *>
decomposeWinogradOutputTransformHelper(RewriterBase &rewriter,
linalg::WinogradOutputTransformOp op) {
Location loc = op.getLoc();
Value value = op.getValue();
auto valueType = cast<ShapedType>(value.getType());
auto valueShape = valueType.getShape();
int64_t valueH = valueShape[0];
int64_t valueW = valueShape[1];
bool leftTransform = valueH != 1;
bool rightTransform = valueW != 1;
Value transformedOutput =
outputTransform(rewriter, loc, value, op.getOutput(), op.getFmr(),
leftTransform, rightTransform);
if (!transformedOutput)
return failure();
rewriter.replaceOp(op, transformedOutput);
return transformedOutput.getDefiningOp();
}
class DecomposeWinogradFilterTransform final
: public OpRewritePattern<linalg::WinogradFilterTransformOp> {
public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(linalg::WinogradFilterTransformOp op,
PatternRewriter &rewriter) const override {
return decomposeWinogradFilterTransformHelper(rewriter, op);
}
};
class DecomposeWinogradInputTransform final
: public OpRewritePattern<linalg::WinogradInputTransformOp> {
public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(linalg::WinogradInputTransformOp op,
PatternRewriter &rewriter) const override {
return decomposeWinogradInputTransformHelper(rewriter, op);
}
};
class DecomposeWinogradOutputTransform final
: public OpRewritePattern<linalg::WinogradOutputTransformOp> {
public:
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(linalg::WinogradOutputTransformOp op,
PatternRewriter &rewriter) const override {
return decomposeWinogradOutputTransformHelper(rewriter, op);
}
};
class WinogradConv2DNhwcFhwc final
: public OpRewritePattern<linalg::Conv2DNhwcFhwcOp> {
public:
using OpRewritePattern::OpRewritePattern;
WinogradConv2DNhwcFhwc(mlir::MLIRContext *context, WinogradConv2DFmr fmr)
: OpRewritePattern(context), fmr(fmr) {}
LogicalResult matchAndRewrite(linalg::Conv2DNhwcFhwcOp convOp,
PatternRewriter &rewriter) const override {
if (failed(winogradConv2DHelper(rewriter, convOp, fmr)))
return failure();
return success();
}
private:
WinogradConv2DFmr fmr;
};
}
FailureOr<Operation *> winogradConv2D(RewriterBase &rewriter,
linalg::Conv2DNhwcFhwcOp op,
linalg::WinogradConv2DFmr fmr) {
return winogradConv2DHelper(rewriter, op, fmr);
}
FailureOr<Operation *>
decomposeWinogradFilterTransformOp(RewriterBase &rewriter,
linalg::WinogradFilterTransformOp op) {
return decomposeWinogradFilterTransformHelper(rewriter, op);
}
FailureOr<Operation *>
decomposeWinogradInputTransformOp(RewriterBase &rewriter,
linalg::WinogradInputTransformOp op) {
return decomposeWinogradInputTransformHelper(rewriter, op);
}
FailureOr<Operation *>
decomposeWinogradOutputTransformOp(RewriterBase &rewriter,
linalg::WinogradOutputTransformOp op) {
return decomposeWinogradOutputTransformHelper(rewriter, op);
}
void populateWinogradConv2DPatterns(RewritePatternSet &patterns,
WinogradConv2DFmr fmr) {
MLIRContext *context = patterns.getContext();
patterns.insert<WinogradConv2DNhwcFhwc>(context, fmr);
}
void populateDecomposeWinogradOpsPatterns(RewritePatternSet &patterns) {
MLIRContext *context = patterns.getContext();
patterns
.insert<DecomposeWinogradFilterTransform, DecomposeWinogradInputTransform,
DecomposeWinogradOutputTransform>(context);
}
}
}