from functools import reduce
import sympy as sympy
from torch._inductor.codegen.simd import (EnableReduction, DisableReduction)
from torch._inductor.codegen.triton import TritonKernel
from torch._inductor.loop_body import MemoryUsageType
from torch._inductor.runtime.runtime_utils import next_power_of_2
from torch._inductor.utils import ModularIndexing, sympy_subs
from torch._inductor.virtualized import V
from torch_npu._compat.inductor import get_sizevars_backed_var_to_val
from .kernel_analysis import IndexAnalysis
from .triton_utils import get_aligned_numel
from ..config import num_vector_core, log
class SplitTiling:
def __init__(self, kernel: TritonKernel):
self.kernel = kernel
self.indexing = []
kernel.sorted_axis = [x for x in kernel.range_tree_nodes.values()]
kernel.sorted_axis.sort(reverse=True, key=self.key)
for i, dim in enumerate(kernel.sorted_axis):
dim.sorted_order = i
self.find_lowest_dimension()
self.should_outer_reduce = False
self.possible_need_permute = self.find_possible_permutes()
def find_possible_permutes(self):
if len(self.kernel.low_dims) <= 1:
return False
var_lists = []
low_dims = [self.kernel.sorted_axis[x].symbol() for x in self.kernel.low_dims]
for index in self.indexing:
var_stride = [
(key, coeff)
for key, coeff in index.as_coefficients_dict().items()
if not isinstance(key, sympy.Integer)
]
var_stride.sort(key=lambda x: x[1])
var_list = tuple([x[0] for x in var_stride if x[0] in low_dims])
var_lists.append(var_list)
for i, var_list in enumerate(var_lists):
if len(var_list) < len(low_dims):
continue
for j, other in enumerate(var_lists):
if i == j or len(other) < len(low_dims):
continue
if var_list != other:
return True
return False
@classmethod
def key(cls, x):
if x.name[0] == 'w' or x.name[0] == 'v' or x.name[0] == 't':
return "zz" + x.name
elif isinstance(x.expr, ModularIndexing):
return x.name[0] + "0" + x.name[1:]
else:
return x.name
@classmethod
def total_split_numels(cls, axis_list):
numels = [x.length for x in axis_list]
return reduce(lambda x, y: x * y, numels) if numels else 1
def select_split_axis(self):
self.kernel.split_axis.clear()
def meet_stop_condition():
if self.total_split_numels(self.kernel.split_axis) >= num_vector_core:
return True
if len(self.kernel.split_axis) == 3:
return True
return False
def select_one_split_axis(not_reduction=True, not_low_dims=True):
for axis in self.kernel.sorted_axis:
if not_reduction and axis.prefix == "r":
continue
if not_low_dims and axis.sorted_order in self.kernel.low_dims:
continue
if axis in self.kernel.split_axis:
continue
axis.is_split_axis = True
return axis
return None
count = 0
while not meet_stop_condition():
count += 1
axis = select_one_split_axis(not_reduction=True, not_low_dims=True)
if axis is not None:
self.kernel.split_axis.append(axis)
continue
axis = select_one_split_axis(not_reduction=True, not_low_dims=False)
if axis is not None:
self.kernel.split_axis.append(axis)
continue
if count > 10:
break
if not self.kernel.split_axis and self.kernel.sorted_axis:
self.kernel.split_axis.append(self.kernel.sorted_axis[0])
self.kernel.split_axis.sort(reverse=True, key=self.key)
for i, x in enumerate(self.kernel.split_axis):
x.split_order = i
def select_tiling_axis(self):
self.kernel.tiling_axis.clear()
def meet_stop_condition():
total_numel = reduce(lambda x, y: x + y,
map(lambda x: x.length, self.kernel.sorted_axis)) if self.kernel.sorted_axis else 1
tiling_numel = reduce(lambda x, y: x + y,
map(lambda x: x.length, self.kernel.tiling_axis)) if self.kernel.tiling_axis else 1
if self.kernel.numof_reduction_axis() > 1 and all(
self.kernel.range_tree_nodes[var].is_tiling_axis for var in self.kernel.reduction_axis_list()):
return True
max_transpose_dims = 2
if (self.possible_need_permute or tiling_numel / total_numel >= 0.8) and \
len(self.kernel.tiling_axis) >= min(max_transpose_dims, len(self.kernel.sorted_axis)):
return True
return False
def select_tiling(low_dim=True, reduction=True):
for axis in reversed(self.kernel.sorted_axis):
if low_dim and axis.sorted_order in self.kernel.low_dims and axis not in self.kernel.tiling_axis:
axis.is_tiling_axis = True
self.kernel.tiling_axis.append(axis)
if reduction and axis.prefix == 'r' and axis not in self.kernel.tiling_axis:
axis.is_tiling_axis = True
self.kernel.tiling_axis.append(axis)
if low_dim or reduction:
continue
longest = axis
if longest and longest not in self.kernel.tiling_axis:
self.kernel.tiling_axis.append(longest)
longest.is_tiling_axis = True
if meet_stop_condition():
break
select_tiling(low_dim=True, reduction=True)
count = 0
while not meet_stop_condition():
select_tiling(low_dim=False, reduction=False)
count += 1
if count > 10:
break
self.kernel.tiling_axis.sort(reverse=True, key=self.key)
for i, x in enumerate(self.kernel.tiling_axis):
x.tiling_order = i
def select_split_tiling_axis(self):
self.select_split_axis()
self.select_tiling_axis()
def should_outer_reduce_me(self, x):
should_outer = self.kernel.is_higher_order_reduction(True) and SplitTiling.great_than(x.length,
32768) and x.is_loop
if should_outer:
self.should_outer_reduce = True
self.kernel.split_axis = x
self.kernel.split_axis.is_split_axis = True
return should_outer
def find_longest_dimension(self, check_in_tiling=False):
longest = None
for axis in self.kernel.sorted_axis:
if (longest is None or axis.length > longest.length) and \
(not check_in_tiling or axis not in self.kernel.tiling_axis):
longest = axis
return longest
def is_lowest_dimension(self, x):
return x.sorted_order in self.kernel.low_dims
def find_lowest_dimension(self):
def construct_low_dim():
for index in self.indexing:
coefficients_dict = index.as_coefficients_dict()
for key, value in coefficients_dict.items():
if not key.free_symbols:
continue
key = list(key.free_symbols)[0]
if key not in self.kernel.range_tree_nodes:
continue
if value == sympy.Integer(1):
axis = self.kernel.range_tree_nodes[key]
self.kernel.low_dims.add(axis.sorted_order)
buf_names = [
node.node.name
for node in self.kernel.node_schedule
if node not in (EnableReduction, DisableReduction)
]
for node in self.kernel.node_schedule:
if node in (EnableReduction, DisableReduction):
continue
names = []
for read in node._body.memory_usage[MemoryUsageType.LOAD]:
name = read.index_name
arg = read.buffer_name
read_is_inptr = False if arg[:3] != 'arg' and arg in buf_names else True
if read_is_inptr:
names.append(name)
for key, index in node._body.indexing.items():
if key in names and index not in self.indexing:
self.indexing.append(index)
if self.kernel.inside_reduction:
construct_low_dim()
return
for node in self.kernel.node_schedule:
if node in (EnableReduction, DisableReduction):
continue
names = []
for write in node._body.memory_usage[MemoryUsageType.STORE]:
names.append(write.index_name)
for write in node._body.memory_usage[MemoryUsageType.STORE_REDUCTION]:
names.append(write.index_name)
for key, index in node._body.indexing.items():
if key in names and index not in self.indexing:
self.indexing.append(index)
construct_low_dim()
@staticmethod
def convert(x, y):
xnumel = x
ynumel = y
if isinstance(xnumel, (sympy.Symbol, sympy.Expr)) and not isinstance(xnumel, sympy.Integer):
xnumel = xnumel.subs(get_sizevars_backed_var_to_val(V.graph.sizevars))
if isinstance(ynumel, (sympy.Symbol, sympy.Expr)) and not isinstance(ynumel, sympy.Integer):
ynumel = ynumel.subs(get_sizevars_backed_var_to_val(V.graph.sizevars))
if isinstance(xnumel, sympy.Integer) and isinstance(ynumel, int):
ynumel = sympy.Integer(ynumel)
if isinstance(ynumel, sympy.Integer) and isinstance(xnumel, int):
xnumel = sympy.Integer(xnumel)
return (xnumel, ynumel)
@staticmethod
def less_than(x, y):
xnumel, ynumel = SplitTiling.convert(x, y)
return xnumel < ynumel
@staticmethod
def great_than(x, y):
xnumel, ynumel = SplitTiling.convert(x, y)
return xnumel > ynumel
@staticmethod
def ge_than(x, y):
xnumel, ynumel = SplitTiling.convert(x, y)
return xnumel >= ynumel