diff --git a/.github/workflows/pytorchsim_test.yml b/.github/workflows/pytorchsim_test.yml index 7cc76a5b..318c7945 100644 --- a/.github/workflows/pytorchsim_test.yml +++ b/.github/workflows/pytorchsim_test.yml @@ -57,6 +57,24 @@ jobs: -e TOGSIM_CONFIG="${{ inputs.togsim_config }}" \ ${{ inputs.image_name }} python3 PyTorchSim/tests/ops/elementwise/test_transcendental.py + test_pointwise: + name: Run test_pointwise.py + runs-on: ubuntu-latest + steps: + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Run test_pointwise.py + run: | + echo "Running test_pointwise.py" + docker run --rm \ + -e TOGSIM_CONFIG="${{ inputs.togsim_config }}" \ + ${{ inputs.image_name }} python3 PyTorchSim/tests/ops/elementwise/test_pointwise.py + test_activation: name: Run test_activation.py runs-on: ubuntu-latest diff --git a/PyTorchSimFrontend/mlir/mlir_cat_template.py b/PyTorchSimFrontend/mlir/mlir_cat_template.py index 7abdfee6..b922e51b 100644 --- a/PyTorchSimFrontend/mlir/mlir_cat_template.py +++ b/PyTorchSimFrontend/mlir/mlir_cat_template.py @@ -209,6 +209,22 @@ def _compute_excluded_dims(self, tile_sizes: list) -> list: tile_sizes[idx] = 1 return excluded + @staticmethod + def _largest_divisor_leq(extent, cap): + """Largest divisor of ``extent`` that does not exceed ``cap`` (>= 1). + + The concat-dim copy loop steps by the per-input tile over ``[0, extent)``. + If the tile does not divide ``extent`` the final iteration is ragged and, + because the DMA carries no remainder mask, over-reads the input and + over-writes the output. Snapping the tile down to a divisor keeps every + iteration full-width and in-bounds. + """ + cap = min(cap, extent) + for d in range(cap, 0, -1): + if extent % d == 0: + return d + return 1 + def _calculate_input_tile_sizes(self, kernel, input_sizes, tile_sizes, num_inputs, rank, precision_bytes): """Calculate tile sizes along the concat dimension for each input.""" non_dim_tile_elements = math.prod(tile_sizes) if tile_sizes else 1 @@ -218,7 +234,9 @@ def _calculate_input_tile_sizes(self, kernel, input_sizes, tile_sizes, num_input input_tile_sizes_dim = [] for i in range(num_inputs): if extra_concat > 0 and non_dim_tile_elements > 0: - tile_dim = min(input_sizes[i][self.dim], extra_concat) + # Snap to a divisor of the input's concat extent so the copy loop + # never emits a ragged (unmasked) tail tile. + tile_dim = self._largest_divisor_leq(input_sizes[i][self.dim], extra_concat) extra_concat -= tile_dim else: tile_dim = 1 diff --git a/PyTorchSimFrontend/mlir/mlir_codegen_backend.py b/PyTorchSimFrontend/mlir/mlir_codegen_backend.py index 1a6422a2..bd04a72d 100644 --- a/PyTorchSimFrontend/mlir/mlir_codegen_backend.py +++ b/PyTorchSimFrontend/mlir/mlir_codegen_backend.py @@ -848,7 +848,9 @@ def _index_expr(self, tile_desc, renamed_expression, index, base_vector_index): if idx == tile_desc.vmap.vlane_split_axis: # Need to add vector lane offset stride_dim = ops.remainder(dim, vlane_stride_vec) outer_dim = ops.remainder(ops.truncdiv(dim, vlane_stride_vec), vlane_outer_vec) - dim = ops.add(stride_dim, ops.mul(outer_dim, nr_vector_lane_vec)) + # Next sublane-row stride is vector_lane*vlane_stride, not vector_lane alone. + row_stride_vec = ops.mul(nr_vector_lane_vec, vlane_stride_vec) + dim = ops.add(stride_dim, ops.mul(outer_dim, row_stride_vec)) with self.override_buffer_cse(buffer=self.const_buffer, cse=self.const_cse): vlane_offset = ops.vlane_offset(vlane_vec, vlane_vec, attributes={"vlane_offset": offset}, comment="vlane offset") @@ -1399,38 +1401,26 @@ def _masked_fill_bits(self, dtype, index): return bits & ((1 << (t.element_size() * 8)) - 1) def _masked_bounds(self, name, index, dram_stride, local_tile_desc, is_load, buffer, local_dims): - """Per tile-axis [low, high) clamp for a masked DMA. Per axis the valid GLOBAL range - is [glo, ghi) (store: [0, output_extent); padded load: [pad, pad + input_extent)), and - _emit_clamp turns it into the tile-local low/high SSA vars. See - docs/design/masked-dma-descriptor.md. Returns [(tile_axis, low_var, high_var), ...]. + """Per tile-axis [low, high) clamp for a masked DMA -- ONLY the trailing tail of a + non-dividing loop extent (valid GLOBAL range per axis is [0, loop_extent)). _emit_clamp + turns it into the tile-local low/high SSA vars. Returns [(tile_axis, low_var, high_var)]. + + A padded load reads out-of-bounds positions, but we do NOT clamp those here: the + consumer already yields the correct value at pad positions (a compute-side arith.select + for a padded gather, or the pad op's own fill). Reverse-engineering per-dim padding from + the single flat index offset is ill-posed -- the offset mixes the tap shift with the + padding, and under channels_last the stride-1 (channel) axis absorbs the offset + remainder, yielding an impossible clamp (e.g. [4, 8) on a size-2 channel tile) that + zeroed the whole load and made channels_last depthwise conv 99.9% wrong. See + docs/design/masked-dma-descriptor.md. """ tile_size = local_tile_desc.get_tile_size() - const = int(index.as_coeff_Add()[0]) if not index.is_number else int(index) - # index const = -sum(pad_d * dram_stride[d]); recover per-dim pad greedily (desc stride). - pad = [0] * len(tile_size) - if is_load and const < 0: - rem = -const - for d in sorted(range(len(dram_stride)), key=lambda x: -dram_stride[x]): - s = dram_stride[d] - if s > 0 and d < len(pad): - pad[d] = rem // s - rem -= pad[d] * s - in_shape, in_stride = self.buffer_types[name][2], self.buffer_types[name][3] axes = [] for d, k in enumerate(local_dims): if d >= len(tile_size) or k >= len(self.ranges): continue iv = str(self.itervars[k]) - # A padded load (const < 0, i.e. index shifted by -pad) reads a smaller input: - # clamp per-dim to [pad_d, pad_d + input_extent_d). Every other DMA (stores and - # non-padded/contiguous loads, including collapsed tensors) is valid over the - # whole loop extent -> only the trailing tail needs clamping. - if is_load and const < 0: - ext = next((int(in_shape[j]) for j, st in enumerate(in_stride) - if int(st) == int(dram_stride[d])), None) - glo, ghi = (pad[d], pad[d] + ext) if ext is not None else (0, int(self.ranges[k])) - else: - glo, ghi = 0, int(self.ranges[k]) + glo, ghi = 0, int(self.ranges[k]) axes.append((d, iv, glo, ghi, int(tile_size[d]))) return self._emit_clamp(axes, buffer) @@ -1646,6 +1636,12 @@ def convert_indirect_indexing(self, index :sympy.Expr): if arg.is_Mul and arg.args[0].is_number: offset_stride = int(arg.args[0]) index = index.replace(arg, 0) + # A bare indirect index (e.g. x[idx]: index IS the symbol, so index.args is empty) + # is not caught by the loop above. Zero any remaining indirect symbol so the dram + # base offset does not reference the loop-local index value (the per-position gather + # is carried by the offset spad); otherwise the DMA offset affine.apply uses %sym + # before it is defined. + index = index.subs({s: 0 for s in index.free_symbols if str(s) in self.indirect_symbols}) sram_var, _, _, _, tile_shape, _ = self.spad_buffer_dict[first_dim] return index, (sram_var, tile_shape, offset_stride) diff --git a/PyTorchSimFrontend/mlir/mlir_common.py b/PyTorchSimFrontend/mlir/mlir_common.py index 99f22762..9d610bdc 100644 --- a/PyTorchSimFrontend/mlir/mlir_common.py +++ b/PyTorchSimFrontend/mlir/mlir_common.py @@ -411,7 +411,8 @@ def init_tile_size(ranges, vlane_stride, vector_lane): return [1] tile_size = [1] * nr_dim if nr_dim == 1: - tile_size[0] = 1 if ranges[0] == 1 else 2 * vlane_stride * vector_lane + # Cap to extent so the tile never over-reads the DRAM buffer (extent 128 vs tile 512 -> 384 garbage folded into the reduction). No-op when extent >= tile. + tile_size[0] = 1 if ranges[0] == 1 else min(2 * vlane_stride * vector_lane, ranges[0]) elif nr_dim == 2: tile_size[-1] = vlane_stride * vector_lane tile_size[-2] = 2 * vector_lane diff --git a/PyTorchSimFrontend/mlir/mlir_lowering.py b/PyTorchSimFrontend/mlir/mlir_lowering.py index 7f33d956..34f40f68 100644 --- a/PyTorchSimFrontend/mlir/mlir_lowering.py +++ b/PyTorchSimFrontend/mlir/mlir_lowering.py @@ -324,6 +324,19 @@ def _mlir_custom_sort_default( stable=stable_required, ) sorted_values = mlir_template.generate(template_buffer_node=value).output_node() + + def _unwrap(t): + t = t.data if isinstance(t, ir.TensorBox) else t + t = t.data if isinstance(t, ir.StorageBox) else t + return t + # The sort kernel writes `indices` in place (mlir_sort_template make_inplace), but only + # `sorted_values` is a scheduler-visible output. Advertise the indices write as a + # MutationOutput OWNED by the sort op (see MLIRTemplateBuffer) so the scheduler keeps the + # kernel alive when only indices are consumed (argsort: the values output is dead and + # would otherwise DCE the whole sort). + sort_op = _unwrap(sorted_values) + idx_buf = _unwrap(indices) + sort_op.add_mutation_output(ir.MutationOutput(idx_buf.get_layout(), idx_buf, sort_op)) return sorted_values, indices diff --git a/PyTorchSimFrontend/mlir/mlir_ops.py b/PyTorchSimFrontend/mlir/mlir_ops.py index f1fb4186..d21cc1d0 100644 --- a/PyTorchSimFrontend/mlir/mlir_ops.py +++ b/PyTorchSimFrontend/mlir/mlir_ops.py @@ -145,7 +145,7 @@ def where(condition, operand1, operand2, *args, **kwargs): operand2 = ops.broadcast(operand2, cond_type[0]) tile_size, ret_type = V.kernel.var_info[operand1] shape = f"vector<{tile_size}x{ret_type}>" if tile_size > 1 else ret_type - cond_shape = f"vector<{tile_size}xi1>" if tile_size > 1 else "" + cond_shape = f"vector<{tile_size}xi1>" if tile_size > 1 else "i1" op_str = f"arith.select %{condition}, %{operand1}, %{operand2}" shape = f"{cond_shape}, {shape}" @@ -309,7 +309,10 @@ def binary_elementwise_common(operand1, operand2): @staticmethod def abs(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + shape = f"vector<{tile_size}x{dtype}>" if tile_size > 1 else dtype + opcode = "math.absf" if dtype.startswith("f") else "math.absi" + return format_mlir_op(f'{opcode} %{operand}', shape, **kwargs), [tile_size, dtype] @staticmethod def exp(operand, *args, **kwargs): @@ -463,68 +466,263 @@ def erf(operand, *args, **kwargs): @staticmethod def cosh(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.cosh(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + exp_pos = ops.exp(operand) + exp_neg = ops.exp(ops.neg(operand)) + + sum_exp = ops.add(exp_pos, exp_neg) + half_const = ops.constant(0.5, dtype) + + res = ops.mul(sum_exp, half_const) + return res, V.kernel.var_info[res] @staticmethod def sinh(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.sinh(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + exp_pos = ops.exp(operand) + exp_neg = ops.exp(ops.neg(operand)) + + sub_exp = ops.sub(exp_pos, exp_neg) + half_const = ops.constant(0.5, dtype) + + res = ops.mul(sub_exp, half_const) + return res, V.kernel.var_info[res] @staticmethod def tanh(operand, *args, **kwargs): op_type = V.kernel.var_info[operand] + tile_size = op_type[0] + dtype = op_type[1] # Check scalar - op_type = V.kernel.var_info[operand] if op_type[0] == 1: operand = ops.broadcast(operand, 4) val = ops.tanh(operand) result = ops.extractelement(val, 0) return result, V.kernel.var_info[result] - op_type = V.kernel.var_info[operand] - tile_size = op_type[0] - dtype = op_type[1] - # Type check & auto cast - if dtype.startswith("f"): + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): operand = ops.to_dtype(operand, "f32") + dtype = "f32" shape = f"vector<{tile_size}x{dtype}>" if tile_size > 1 else dtype return format_mlir_op(f'math.tanh %{operand}', shape, **kwargs), [tile_size, dtype] @staticmethod def acos(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.acos(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + asin_val = ops.asin(operand) + res = ops.sub(ops.constant(math.pi / 2, dtype), asin_val) + return res, V.kernel.var_info[res] @staticmethod def acosh(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.acosh(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + x2 = ops.square(operand) + val = ops.sub(x2, ops.constant(1.0, dtype)) + sqrt_val = ops.sqrt(val) + sum_val = ops.add(operand, sqrt_val) + + res = ops.log(sum_val) + return res, V.kernel.var_info[res] @staticmethod def asin(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.asin(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + x2 = ops.square(operand) + denom = ops.sqrt(ops.sub(ops.constant(1.0, dtype), x2)) + res = ops.atan(ops.truediv(operand, denom)) + return res, V.kernel.var_info[res] @staticmethod def asinh(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.asinh(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + x2 = ops.square(operand) + val = ops.add(x2, ops.constant(1.0, dtype)) + sqrt_val = ops.sqrt(val) + sum_val = ops.add(operand, sqrt_val) + + res = ops.log(sum_val) + return res, V.kernel.var_info[res] @staticmethod def atan2(operand1, operand2, *args, **kwargs): - raise NotImplementedError + tile_size, ret_type, y, x = ExtensionOverrides.binary_elementwise_common(operand1, operand2) + if not ret_type.startswith("f"): + y = ops.to_dtype(y, "f32") + x = ops.to_dtype(x, "f32") + ret_type = "f32" + + pi = ops.constant(math.pi, ret_type) + zero = ops.constant(0.0, ret_type) + + base = ops.atan(ops.truediv(y, x)) + corrected = ops.add(base, ops.copysign(pi, y)) + res = ops.where(ops.lt(x, zero), corrected, base) + return res, [tile_size, ret_type] @staticmethod def atan(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.atan(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + shape = f"vector<{tile_size}x{dtype}>" if tile_size > 1 else dtype + return format_mlir_op(f'math.atan %{operand}', shape, **kwargs), [tile_size, dtype] @staticmethod def atanh(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + # Check scalar + if tile_size == 1: + operand = ops.broadcast(operand, 4) + val = ops.atanh(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): + operand = ops.to_dtype(operand, "f32") + dtype = "f32" + + one_const = ops.constant(1.0, dtype) + val1 = ops.add(one_const, operand) + val2 = ops.sub(one_const, operand) + div_val = ops.truediv(val1, val2) + + half_const = ops.constant(0.5, dtype) + res = ops.mul(ops.log(div_val), half_const) + return res, V.kernel.var_info[res] @staticmethod def copysign(operand1, operand2, *args, **kwargs): - raise NotImplementedError + tile_size, ret_type, operand1, operand2 = ExtensionOverrides.binary_elementwise_common(operand1, operand2) + if not ret_type.startswith("f"): + raise ValueError("copysign is only supported for floats") + shape = f"vector<{tile_size}x{ret_type}>" if tile_size > 1 else ret_type + op_str = f'math.copysign %{operand1}, %{operand2}' + return format_mlir_op(op_str, shape, **kwargs), [tile_size, ret_type] @staticmethod def erfc(operand, *args, **kwargs): - raise NotImplementedError + """ + There is no direct MLIR operation for erfc, so we can implement it using the relationship: + erfc(x) = 1 - erf(x) + """ + op_type = V.kernel.var_info[operand] + + # Check scalar + if op_type[0] == 1: + operand = ops.broadcast(operand, 4) + val = ops.erfc(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] + + tile_size = op_type[0] + dtype = op_type[1] + erf_val = ops.erf(operand) + one_const = ops.constant(1.0, dtype) + res = ops.sub(one_const, erf_val) + + return res, V.kernel.var_info[res] @staticmethod def erfinv(operand, *args, **kwargs): @@ -536,7 +734,15 @@ def frexp(operand, *args, **kwargs): @staticmethod def hypot(operand1, operand2, *args, **kwargs): - raise NotImplementedError + tile_size, ret_type, operand1, operand2 = ExtensionOverrides.binary_elementwise_common(operand1, operand2) + if not ret_type.startswith("f"): + raise ValueError("hypot is only supported for floats") + + x_sq = ops.square(operand1) + y_sq = ops.square(operand2) + sum_sq = ops.add(x_sq, y_sq) + res = ops.sqrt(sum_sq) + return res, V.kernel.var_info[res] @staticmethod def log10(operand, *args, **kwargs): @@ -564,12 +770,20 @@ def log2(operand, *args, **kwargs): @staticmethod def log(operand, *args, **kwargs): op_type = V.kernel.var_info[operand] + if op_type[0] == 1: + operand = ops.broadcast(operand, 4) + val = ops.log(operand) + result = ops.extractelement(val, 0) + return result, V.kernel.var_info[result] tile_size = op_type[0] dtype = op_type[1] # Type check & auto cast - if dtype.startswith("f"): + # Float-only instruction: promote non-float inputs (e.g. integers) to f32 + # to run it. Native float widths (f16/f64) are left untouched. + if not dtype.startswith("f"): operand = ops.to_dtype(operand, "f32") + dtype = "f32" shape = f"vector<{tile_size}x{dtype}>" if tile_size > 1 else dtype return format_mlir_op(f'math.log %{operand}', shape, **kwargs), [tile_size, dtype] @@ -660,11 +874,21 @@ def bitwise_xor(operand1, operand2, *args, **kwargs): @staticmethod def bitwise_left_shift(operand1, operand2, *args, **kwargs): - raise NotImplementedError + tile_size, ret_type, operand1, operand2 = ExtensionOverrides.binary_elementwise_common(operand1, operand2) + if ret_type.startswith("f"): + raise ValueError("Bitwise left shift not supported for floats") + shape = f"vector<{tile_size}x{ret_type}>" if tile_size > 1 else ret_type + op_str = f'arith.shli %{operand1}, %{operand2}' + return format_mlir_op(op_str, shape, **kwargs), [tile_size, ret_type] @staticmethod def bitwise_right_shift(operand1, operand2, *args, **kwargs): - raise NotImplementedError + tile_size, ret_type, operand1, operand2 = ExtensionOverrides.binary_elementwise_common(operand1, operand2) + if ret_type.startswith("f"): + raise ValueError("Bitwise right shift not supported for floats") + shape = f"vector<{tile_size}x{ret_type}>" if tile_size > 1 else ret_type + op_str = f'arith.shrsi %{operand1}, %{operand2}' + return format_mlir_op(op_str, shape, **kwargs), [tile_size, ret_type] @staticmethod def rsqrt(operand, *args, **kwargs): @@ -689,15 +913,51 @@ def sigmoid(operand, *args, **kwargs): @staticmethod def fmod(operand1, operand2, *args, **kwargs): - raise NotImplementedError + tile_size, ret_type, operand1, operand2 = ExtensionOverrides.binary_elementwise_common(operand1, operand2) + + if ret_type.startswith("f"): + div_val = ops.truediv(operand1, operand2) + trunc_val = ops.trunc(div_val) + mul_val = ops.mul(trunc_val, operand2) + res = ops.sub(operand1, mul_val) + return res, V.kernel.var_info[res] + else: + shape = f"vector<{tile_size}x{ret_type}>" if tile_size > 1 else ret_type + op_str = f'arith.remsi %{operand1}, %{operand2}' + return format_mlir_op(op_str, shape, **kwargs), [tile_size, ret_type] @staticmethod def isinf(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + if dtype.startswith("f"): + abs_val = ops.abs(operand) + inf_val = ops.constant("inf", dtype) + res = ops.eq(abs_val, inf_val) + return res, V.kernel.var_info[res] + else: + const_false = ops.constant(False, "i1") + if tile_size > 1: + const_false = ops.broadcast(const_false, tile_size) + return const_false, V.kernel.var_info[const_false] @staticmethod def isnan(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + if dtype.startswith("f"): + # Unordered comparison (uno) to detect NaN (uno returns true if either operand is NaN) + operand_shape = f"vector<{tile_size}x{dtype}>" if tile_size > 1 else dtype + op_str = f"arith.cmpf uno, %{operand}, %{operand}" + res = format_mlir_op(op_str, operand_shape, **kwargs) + + V.kernel.var_info[res] = [tile_size, "i1"] + return res, [tile_size, "i1"] + else: + # Integers cannot be NaN + const_false = ops.constant(False, "i1") + if tile_size > 1: + const_false = ops.broadcast(const_false, tile_size) + return const_false, V.kernel.var_info[const_false] @staticmethod def round(operand, *args, **kwargs): @@ -723,7 +983,30 @@ def floor(operand, *args, **kwargs): @staticmethod def sign(operand, *args, **kwargs): - raise NotImplementedError + tile_size, dtype = V.kernel.var_info[operand] + + if dtype.startswith("f"): + v_zero, v_one, v_neg_one = 0.0, 1.0, -1.0 + else: + v_zero, v_one, v_neg_one = 0, 1, -1 + + const_zero = ops.constant(v_zero, dtype) + const_one = ops.constant(v_one, dtype) + const_neg_one = ops.constant(v_neg_one, dtype) + + if tile_size > 1: + const_zero = ops.broadcast(const_zero, tile_size) + const_one = ops.broadcast(const_one, tile_size) + const_neg_one = ops.broadcast(const_neg_one, tile_size) + + is_pos = ops.gt(operand, const_zero) + is_neg = ops.lt(operand, const_zero) + + V.kernel.var_info[is_pos] = [tile_size, "i1"] + V.kernel.var_info[is_neg] = [tile_size, "i1"] + + res = ops.where(is_pos, const_one, ops.where(is_neg, const_neg_one, const_zero)) + return res, V.kernel.var_info[res] @staticmethod def trunc(operand, *args, **kwargs): @@ -933,11 +1216,11 @@ def xor(operand1, operand2, *args, **kwargs): @staticmethod def lshift(operand1, operand2, *args, **kwargs): - raise NotImplementedError + return ops.bitwise_left_shift(operand1, operand2) @staticmethod def rshift(operand1, operand2, *args, **kwargs): - raise NotImplementedError + return ops.bitwise_right_shift(operand1, operand2) @staticmethod def truncdiv(operand1, operand2, *args, **kwargs): diff --git a/PyTorchSimFrontend/mlir/mlir_sort_template.py b/PyTorchSimFrontend/mlir/mlir_sort_template.py index 24b3a460..338f9636 100644 --- a/PyTorchSimFrontend/mlir/mlir_sort_template.py +++ b/PyTorchSimFrontend/mlir/mlir_sort_template.py @@ -3,6 +3,7 @@ from torch._inductor.ir import Buffer, IRNode from torch._inductor.virtualized import _ops as ops +from torch._inductor.virtualized import V from torch._inductor.codegen import common from PyTorchSimFrontend.mlir import mlir_common @@ -38,7 +39,9 @@ {{ BITONIC_BODY }} {{ kernel.def_dma_op("MVOUT", "XI", [], XI_TILE_DESC, indent_size=INDENT_SIZE, dram_stride=XI_DRAM_STRIDE, dram_offset="xi_dram_offset") }} + {%- if YV_LIVE %} {{ kernel.def_dma_op("MVOUT", "YV", [], YV_TILE_DESC, indent_size=INDENT_SIZE, dram_stride=YV_DRAM_STRIDE, dram_offset="yv_dram_offset") }} + {%- endif %} {%- for d in range(RANK-1) %} } { outer_loop=true } {%- endfor %} @@ -433,6 +436,9 @@ def render( X=x, XI=xi, YV=yv, + # In argsort the sorted-values output is dead and dropped from the kernel args; + # skip its MVOUT so the body does not reference the pruned %YV (undeclared SSA). + YV_LIVE=yv.get_name() not in V.graph.removed_buffers, X_TILE_DESC=x_tile_desc, XI_TILE_DESC=xi_tile_desc, YV_TILE_DESC=yv_tile_desc, diff --git a/PyTorchSimFrontend/mlir/mlir_template.py b/PyTorchSimFrontend/mlir/mlir_template.py index e6a19f2d..1f73ad21 100644 --- a/PyTorchSimFrontend/mlir/mlir_template.py +++ b/PyTorchSimFrontend/mlir/mlir_template.py @@ -15,9 +15,9 @@ from PyTorchSimFrontend import extension_config from torch._inductor.codegen.common import KernelTemplate, CSE, DeferredLine -from torch._inductor.ir import Buffer, IRNode, TemplateBuffer, ChoiceCaller, ir_node_to_tensor +from torch._inductor.ir import Buffer, IRNode, TemplateBuffer, ChoiceCaller, ir_node_to_tensor, TensorBox from torch._inductor.select_algorithm import PartialRender -from torch._inductor.codegen.cuda.cuda_kernel import CUDATemplateCaller +from torch._inductor.codegen.cuda.cuda_kernel import CUDATemplateCaller, CUDATemplateBuffer from torch._inductor.autotune_process import TensorMeta from torch._inductor.virtualized import V, NullHandler, _ops as ops from torch._inductor.utils import IndentedBuffer @@ -1311,7 +1311,41 @@ def set_tile_size(self, template_fusion_info, prologue=False): self.compute_body_loop.step = tile_desc.get_compute_vec_size() return tile_desc +class MLIRTemplateBuffer(CUDATemplateBuffer): + """Template output buffer that can own extra MutationOutputs beyond its primary output. + + The sort kernel's primary scheduler-visible output is the sorted values, but it also + writes the indices buffer in place (mlir_sort_template make_inplace). Registering that + write as a MutationOutput owned here -- so get_outputs() advertises it -- lets the + scheduler keep the kernel alive when only the indices are consumed (argsort), instead of + DCE-ing the whole sort. OperationBuffer.get_outputs() otherwise hardcodes [self]. + """ + + def add_mutation_output(self, mutation: IRNode) -> None: + if not hasattr(self, "_extra_mutation_outputs"): + self._extra_mutation_outputs = [] + self._extra_mutation_outputs.append(mutation) + + def get_outputs(self) -> List[Buffer]: + return [self, *getattr(self, "_extra_mutation_outputs", [])] + + class MLIRTemplateCaller(CUDATemplateCaller): + def output_node(self) -> TensorBox: + # Same as CUDATemplateCaller.output_node but builds a mutation-aware buffer so + # callers (e.g. the sort lowering) can attach in-place-write MutationOutputs. + self.bmreq.update_workspace_size() + return TensorBox.create( + MLIRTemplateBuffer( + layout=self.layout, + inputs=self.input_nodes, + make_kernel_render=self.make_kernel_render, + workspace_size=self.bmreq.workspace_size, + supports_epilogue_fusion=self.supports_epilogue_fusion, + template=self.template, + ) + ) + def __init__(self, name, category, input_nodes, layout, make_kernel_render, supports_epilogue_fusion, template, info_kwargs, description): bmreq = MLIRBenchmarkRequest( kernel_name=name, diff --git a/PyTorchSimFrontend/mlir/passes/lower_to_vcix.py b/PyTorchSimFrontend/mlir/passes/lower_to_vcix.py index d32ada38..ccaca976 100644 --- a/PyTorchSimFrontend/mlir/passes/lower_to_vcix.py +++ b/PyTorchSimFrontend/mlir/passes/lower_to_vcix.py @@ -1,7 +1,7 @@ """Python port of the C++ `-test-pytorchsim-to-vcix` conversion pass (TestPyTorchSimToVCIXConversion.cpp). -Lowers `linalg.matmul` and the transcendental math ops (exp/erf/tanh/sin/cos) to +Lowers `linalg.matmul` and the transcendental math ops (exp/erf/tanh/sin/cos/log/atan) to VCIX dialect ops (RISC-V vector custom instructions). The C++ pass is a dialect-conversion (`applyPartialConversion`); the MLIR Python bindings expose no conversion framework, so each matchAndRewrite is reimplemented as imperative IR @@ -14,7 +14,7 @@ `allow_unregistered_dialects` -- so emitting generic vcix ops here is consistent with the current pipeline. -Covers all 6 C++ patterns: linalg.matmul (gemm + conv2d) and exp/erf/tanh/sin/cos. +Covers all 8 C++ patterns: linalg.matmul (gemm + conv2d) and exp/erf/tanh/sin/cos/log/atan. Wired into extension_codecache (run_to_vcix) after fine-grained, before the standard lowering; mlir-opt then runs only -test-loop-padding. Validated structurally against `mlir-opt -test-pytorchsim-to-vcix` (non-constant ops byte-identical incl. dma_wait tag @@ -31,15 +31,21 @@ from ._mlir_util import walk_ops, i32, i64, attr_bool -MARKERS = ("linalg.matmul", "math.exp", "math.erf", "math.tanh", "math.sin", "math.cos") +MARKERS = ("linalg.matmul", "math.exp", "math.erf", "math.tanh", "math.sin", + "math.cos", "math.log", "math.atan") # math op name -> (opcode, imm) for the vcix.v.iv lowering (mirror Math*ToVCIX). +# The sf.vc.v.iv opcode field is only 2 bits (uimm2, 0-3): erf(0)/tanh(1)/ +# sin,cos,log,atan(2)/exp(3). Opcode 2 is shared and disambiguated by imm +# (sin=0, cos=1, log=2, atan=3), matching the spike rs1 / gem5 VS1 sub-opcodes. _MATH_VIV = { "math.exp": (0b000011, 0), "math.erf": (0b000000, 0), "math.tanh": (0b000001, 0), "math.sin": (0b000010, 0), "math.cos": (0b000010, 1), + "math.log": (0b000010, 2), + "math.atan": (0b000010, 3), } diff --git a/tests/ops/elementwise/test_pointwise.py b/tests/ops/elementwise/test_pointwise.py new file mode 100644 index 00000000..74e6656d --- /dev/null +++ b/tests/ops/elementwise/test_pointwise.py @@ -0,0 +1,214 @@ +import torch +import os + +def clear_caches(): + from torch._functorch._aot_autograd.autograd_cache import AOTAutogradCache + from torch._inductor.codecache import FxGraphCache + AOTAutogradCache.clear() + torch._dynamo.reset() + os.environ["TORCHINDUCTOR_CACHE"] = "0" + FxGraphCache.clear() + +def test_result(name, out, cpu_out, rtol=1e-4, atol=1e-4): + if torch.allclose(out.cpu(), cpu_out, rtol=rtol, atol=atol): + message = f"|{name} Test Passed|" + print("-" * len(message)) + print(message) + print("-" * len(message)) + else: + message = f"|{name} Test Failed|" + print("-" * len(message)) + print(message) + print("-" * len(message)) + print("custom out: ", out.cpu()) + print("cpu out: ", cpu_out) + exit(1) + +def run_test(name, device, fn, inputs, size_desc, rtol=1e-4, atol=1e-4): + """ + Harness function to compile, execute on NPU, compare with CPU, and print details. + inputs: single tensor or tuple/list of tensors (on CPU) + """ + if not isinstance(inputs, (tuple, list)): + inputs = [inputs] + + npu_inputs = [x.to(device=device) for x in inputs] + cpu_inputs = [x.clone() for x in inputs] + + clear_caches() + + opt_fn = torch.compile(dynamic=False)(fn) + res = opt_fn(*npu_inputs) + out = fn(*cpu_inputs) + + # Print input / output slices (up to 10 elements) + for idx, x in enumerate(inputs): + label = f"X" if len(inputs) == 1 else f"X{idx+1}" + val = x.flatten()[:10].tolist() if x.numel() > 1 else x.item() + print(f"[{size_desc}] {name} Input {label}: {val}") + + out_val = res.flatten()[:10].tolist() if res.numel() > 1 else res.item() + print(f"[{size_desc}] {name} Output: {out_val}") + + test_result(f"{name}_{size_desc}", res, out, rtol=rtol, atol=atol) + + +# aligned, scalar, small tail, large tail +STD_SIZES = [(128, 128), (1, 1), (15, 15), (129, 129)] + + +def run_op(name, device, fn, make_input, cases=None, sizes=STD_SIZES, rtol=1e-4, atol=1e-4): + torch.manual_seed(0) + for rows, cols in sizes: + run_test(name, device, fn, make_input(rows, cols), f"{rows}x{cols}", rtol=rtol, atol=atol) + for desc, inputs in (cases or []): + run_test(name, device, fn, inputs, desc, rtol=rtol, atol=atol) + + +def test_abs(device): + run_op("Abs", device, torch.abs, lambda r, c: torch.randn(r, c) * 10, + cases=[ + ("int", torch.randint(-100, 100, (128, 128), dtype=torch.int32)), + ("strided_transposed", (torch.randn(128, 128) * 10).t()), + ("strided_sliced", (torch.randn(128, 256) * 10)[:, ::2]), + ]) + +def test_sign(device): + def make(r, c): + x = torch.randn(r, c) + x[x.abs() < 0.3] = 0.0 + return x + run_op("Sign", device, torch.sign, make, + cases=[ + ("zero", torch.tensor([[0.0]])), + ("nonzero", torch.tensor([[-4.5]])), + ("int", torch.randint(-5, 5, (128, 128), dtype=torch.int32)), + ]) + +def test_isnan(device): + def make(r, c): + x = torch.randn(r, c) + x[x.abs() < 0.3] = float('nan') + return x + run_op("IsNaN", device, torch.isnan, make, + cases=[("scalar_nan", torch.tensor([[float('nan')]]))]) + +def test_isinf(device): + def make(r, c): + x = torch.randn(r, c) + x[x > 1.0] = float('inf') + x[x < -1.0] = float('-inf') + return x + run_op("IsInf", device, torch.isinf, make, + cases=[("scalar_inf", torch.tensor([[float('-inf')]]))]) + +def test_fmod(device): + run_op("Fmod", device, torch.fmod, + lambda r, c: (torch.randn(r, c) * 10, torch.randn(r, c) * 3 + 1), + cases=[("broadcast", (torch.randn(128, 1) * 10, torch.randn(1, 128) * 3 + 1))]) + +def test_lshift(device): + run_op("LShift", device, torch.bitwise_left_shift, + lambda r, c: (torch.randint(1, 100, (r, c), dtype=torch.int32), + torch.randint(1, 5, (r, c), dtype=torch.int32)), + cases=[("broadcast", (torch.randint(1, 100, (128, 1), dtype=torch.int32), + torch.randint(1, 5, (1, 128), dtype=torch.int32)))]) + +def test_rshift(device): + run_op("RShift", device, torch.bitwise_right_shift, + lambda r, c: (torch.randint(10, 1000, (r, c), dtype=torch.int32), + torch.randint(1, 5, (r, c), dtype=torch.int32)), + cases=[("broadcast", (torch.randint(10, 1000, (128, 1), dtype=torch.int32), + torch.randint(1, 5, (1, 128), dtype=torch.int32)))]) + +def test_copysign(device): + run_op("Copysign", device, torch.copysign, + lambda r, c: (torch.randn(r, c) * 10, torch.randn(r, c)), + cases=[ + ("negzero", (torch.tensor([[3.0]]), torch.tensor([[-0.0]]))), + ("broadcast", (torch.randn(128, 1) * 10, torch.randn(1, 128))), + ]) + +def test_erfc(device): + run_op("Erfc", device, torch.erfc, lambda r, c: torch.randn(r, c), + cases=[("large", torch.tensor([[4.5]]))]) + +def test_hypot(device): + run_op("Hypot", device, torch.hypot, + lambda r, c: (torch.randn(r, c), torch.randn(r, c)), + cases=[ + ("3-4-5", (torch.tensor([[3.0]]), torch.tensor([[4.0]]))), + ("broadcast", (torch.randn(128, 1), torch.randn(1, 128))), + ]) + +def test_cosh(device): + run_op("Cosh", device, torch.cosh, lambda r, c: torch.randn(r, c), + cases=[("large", torch.tensor([[4.5]]))]) + +def test_sinh(device): + run_op("Sinh", device, torch.sinh, lambda r, c: torch.randn(r, c), + cases=[("large", torch.tensor([[4.5]]))]) + +def test_log(device): + # domain: x > 0 + run_op("Log", device, torch.log, lambda r, c: torch.rand(r, c) + 1e-5, + cases=[("e", torch.tensor([[2.71828]]))]) + +def test_acosh(device): + # domain: x >= 1 + run_op("Acosh", device, torch.acosh, lambda r, c: torch.rand(r, c) + 1, + cases=[("one", torch.tensor([[1.0]]))]) + +def test_asinh(device): + run_op("Asinh", device, torch.asinh, lambda r, c: torch.randn(r, c)) + +def test_atanh(device): + # domain: (-1, 1) + run_op("Atanh", device, torch.atanh, lambda r, c: torch.rand(r, c) * 2 - 1) + +def test_atan(device): + # domain: all reals; boundary: zero, unit, large |x| (range-reduction path) + run_op("Atan", device, torch.atan, lambda r, c: torch.randn(r, c) * 5, + cases=[("boundary", torch.tensor([[0.0, 1.0, -1.0, 1e3, -1e3, 1e-4]]))]) + +def test_asin(device): + # domain: [-1, 1]; boundary includes the +/-1 endpoints + run_op("Asin", device, torch.asin, lambda r, c: torch.rand(r, c) * 2 - 1, + cases=[("boundary", torch.tensor([[0.0, 1.0, -1.0, 0.5, -0.5]]))]) + +def test_acos(device): + # domain: [-1, 1]; boundary includes the +/-1 endpoints + run_op("Acos", device, torch.acos, lambda r, c: torch.rand(r, c) * 2 - 1, + cases=[("boundary", torch.tensor([[0.0, 1.0, -1.0, 0.5, -0.5]]))]) + +def test_atan2(device): + # atan2(y, x); boundary: axes and diagonals across all four quadrants + # (0, 0) excluded: composition yields atan(0/0)=nan while torch returns 0 + y = torch.tensor([[1.0, 0.0, -1.0, 0.0, 1.0, -1.0, 1.0, -1.0]]) + x = torch.tensor([[0.0, 1.0, 0.0, -1.0, 1.0, -1.0, -1.0, 1.0]]) + run_op("Atan2", device, torch.atan2, lambda r, c: (torch.randn(r, c), torch.randn(r, c)), + cases=[("boundary", (y, x))]) + +if __name__ == "__main__": + device = torch.device("npu:0") + + test_abs(device) + test_sign(device) + test_isnan(device) + test_isinf(device) + test_fmod(device) + test_lshift(device) + test_rshift(device) + test_copysign(device) + test_erfc(device) + test_hypot(device) + test_cosh(device) + test_sinh(device) + test_log(device) + test_acosh(device) + test_asinh(device) + test_atanh(device) + test_atan(device) + test_asin(device) + test_acos(device) + test_atan2(device) \ No newline at end of file diff --git a/thirdparty/github-releases.json b/thirdparty/github-releases.json index 2618c236..1265d380 100644 --- a/thirdparty/github-releases.json +++ b/thirdparty/github-releases.json @@ -3,7 +3,7 @@ "pytorch_image": "ubuntu:22.04", "gem5": { "repository": "PSAL-POSTECH/gem5", - "release_tag": "v1.0.1", + "release_tag": "v1.0.2", "asset_name": "gem5-release.tar.gz" }, "llvm_project": { @@ -13,7 +13,7 @@ }, "spike": { "repository": "PSAL-POSTECH/riscv-isa-sim", - "release_tag": "v1.0.5", + "release_tag": "v1.0.6", "asset_name": "spike-release.tar.gz" } }