Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

JIT: Use faster mod for uint16 values #111535

Closed
wants to merge 11 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions src/coreclr/jit/assertionprop.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,16 @@ bool IntegralRange::Contains(int64_t value) const
break;
}

case GT_STORE_LCL_VAR:
{
if (node->gtGetOp1()->OperIs(GT_CAST))
{
return ForCastOutput(node->gtGetOp1()->AsCast(), compiler);
}

break;
}

case GT_CNS_INT:
if (node->IsIntegralConst(0) || node->IsIntegralConst(1))
{
Expand All @@ -227,6 +237,16 @@ bool IntegralRange::Contains(int64_t value) const
case GT_CAST:
return ForCastOutput(node->AsCast(), compiler);

case GT_COMMA:
{
if (varTypeIsIntegral(node->gtGetOp1()))
{
return ForNode(node->gtGetOp1(), compiler);
}

break;
}

#if defined(FEATURE_HW_INTRINSICS)
case GT_HWINTRINSIC:
switch (node->AsHWIntrinsic()->GetHWIntrinsicId())
Expand Down Expand Up @@ -4116,6 +4136,7 @@ void Compiler::optAssertionProp_RangeProperties(ASSERT_VALARG_TP assertions,
// 1) Convert DIV/MOD to UDIV/UMOD if both operands are proven to be never negative
// 2) Marks DIV/UDIV/MOD/UMOD with GTF_DIV_MOD_NO_BY_ZERO if divisor is proven to be never zero
// 3) Marks DIV/UDIV/MOD/UMOD with GTF_DIV_MOD_NO_OVERFLOW if both operands are proven to be never negative
// 4) Marks UMOD with GTF_UMOD_UINT16_OPERANDS if both operands are proven to be in uint16 range
//
// Arguments:
// assertions - set of live assertions
Expand Down Expand Up @@ -4159,6 +4180,17 @@ GenTree* Compiler::optAssertionProp_ModDiv(ASSERT_VALARG_TP assertions, GenTreeO
changed = true;
}

#ifdef TARGET_AMD64
if (((tree->gtFlags & GTF_UMOD_UINT16_OPERANDS) == 0) && tree->OperIs(GT_UMOD) && op2->IsCnsIntOrI() &&
FitsIn<uint16_t>(op2->AsIntCon()->IconValue()) && op2->AsIntCon()->IconValue() > 0 &&
IntegralRange::ForType(TYP_USHORT).Contains(IntegralRange::ForNode(op1, this)))
{
JITDUMP("Both operands for UMOD are in uint16 range...\n")
tree->gtFlags |= GTF_UMOD_UINT16_OPERANDS;
changed = true;
}
#endif

return changed ? optAssertionProp_Update(tree, tree, stmt) : nullptr;
}

Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/jit/gentree.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2678,8 +2678,8 @@ bool GenTree::Compare(GenTree* op1, GenTree* op2, bool swapOK)
}
if (op1->OperIs(GT_MOD, GT_UMOD, GT_DIV, GT_UDIV))
{
if ((op1->gtFlags & (GTF_DIV_MOD_NO_BY_ZERO | GTF_DIV_MOD_NO_OVERFLOW)) !=
(op2->gtFlags & (GTF_DIV_MOD_NO_BY_ZERO | GTF_DIV_MOD_NO_OVERFLOW)))
if ((op1->gtFlags & (GTF_DIV_MOD_NO_BY_ZERO | GTF_DIV_MOD_NO_OVERFLOW | GTF_UMOD_UINT16_OPERANDS)) !=
(op2->gtFlags & (GTF_DIV_MOD_NO_BY_ZERO | GTF_DIV_MOD_NO_OVERFLOW | GTF_UMOD_UINT16_OPERANDS)))
{
return false;
}
Expand Down
2 changes: 2 additions & 0 deletions src/coreclr/jit/gentree.h
Original file line number Diff line number Diff line change
Expand Up @@ -540,6 +540,8 @@ enum GenTreeFlags : unsigned int

GTF_DIV_MOD_NO_OVERFLOW = 0x40000000, // GT_DIV, GT_MOD -- Div or mod definitely does not overflow.

GTF_UMOD_UINT16_OPERANDS = 0x80000000, // UMOD -- Both operands to a mod are in uint16 range. The divisor is non-zero constant.

GTF_CHK_INDEX_INBND = 0x80000000, // GT_BOUNDS_CHECK -- have proven this check is always in-bounds

GTF_ARRLEN_NONFAULTING = 0x20000000, // GT_ARR_LENGTH -- An array length operation that cannot fault. Same as GT_IND_NONFAULTING.
Expand Down
46 changes: 46 additions & 0 deletions src/coreclr/jit/lower.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7182,6 +7182,52 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod)
}
}

assert(divisorValue >= 3);

#ifdef TARGET_AMD64
// Replace (uint16 % uint16) with a cheaper variant of FastMod, specialized for 16-bit operands.
if ((divMod->gtFlags & GTF_UMOD_UINT16_OPERANDS) != 0)
{
assert(!isDiv);
assert(divisorValue > 0 && divisorValue <= UINT16_MAX);
assert(!comp->opts.MinOpts());

// uint multiplier = uint.MaxValue / divisor + 1;
// ulong result = ((ulong)(dividend * multiplier) * divisor) >> 32;
// return (int)result;

// multiplier = uint.MaxValue / divisor + 1
GenTree* multiplier = comp->gtNewIconNode((UINT32_MAX / divisorValue) + 1, TYP_INT);

// (dividend * multiplier)
GenTree* mul1 = comp->gtNewOperNode(GT_MUL, TYP_INT, dividend, multiplier);
mul1->SetUnsigned();

// (ulong)(dividend * multiplier)
GenTree* cast = comp->gtNewCastNode(TYP_LONG, mul1, true, TYP_LONG);

// ((ulong)(dividend * multiplier) * divisor)
GenTree* mul2 = comp->gtNewOperNode(GT_MUL, TYP_LONG, cast, divisor);
mul2->SetUnsigned();

// ((ulong)(dividend * multiplier) * divisor) >> 32
GenTree* shiftAmount = comp->gtNewIconNode(32, TYP_INT);
GenTree* shift = comp->gtNewOperNode(GT_RSZ, TYP_LONG, mul2, shiftAmount);

BlockRange().InsertBefore(divMod, multiplier, mul1, cast, shiftAmount);
BlockRange().InsertBefore(divMod, mul2, shift);

// (int)result or (long)result
divMod->ChangeOper(GT_CAST);
divMod->AsCast()->gtCastType = type;
divMod->gtOp1 = shift;
divMod->gtOp2 = nullptr;
divMod->SetUnsigned();
ContainCheckRange(multiplier, divMod);
return true;
}
#endif // TARGET_AMD64

// TODO-ARM-CQ: Currently there's no GT_MULHI for ARM32
#if defined(TARGET_XARCH) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
if (!comp->opts.MinOpts() && (divisorValue >= 3))
Expand Down
169 changes: 169 additions & 0 deletions src/tests/JIT/opt/Divide/Regressions/Regression4_Divide.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.Intrinsics.X86;
using Xunit;

public class Program
{
private static ushort s_field1;
private static ulong s_field2;

[MethodImpl(MethodImplOptions.NoInlining)]
public static uint Umod_U4_CharByZero(char c)
{
return (uint)c % 0;
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static ushort Umod_U2_CharByConst(char c)
{
return (ushort)(c % 42);
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static int Umod_I4_CharByConst(char c)
{
return c % 42;
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static uint Umod_U4_CharByConst(char c)
{
return (uint)c % 42;
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static long Umod_I8_CharByConst(char c)
{
return (long)c % 42;
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static ulong Umod_U8_CharByConst(char c)
{
return (ulong)c % 42;
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static bool TestIsWhiteSpace(char c)
{
ReadOnlySpan<char> HashEntries = [' ', ' ', '\u00A0', ' ', ' ', ' ', ' ', ' ', ' ', '\t', '\n', '\v', '\f', '\r', ' ', ' ', '\u2028', '\u2029', ' ', ' ', ' ', ' ', ' ', '\u202F', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '\u3000', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '\u0085', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', ' ', ' ', ' ', ' ', ' ', '\u205F', '\u1680', ' ', ' ', ' ', ' ', ' ', ' '];
return HashEntries[c % HashEntries.Length] == c;
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static ushort Umod_TZC(ulong value)
{
return (ushort)(BitOperations.TrailingZeroCount(value) % 42);
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static ushort Umod_TZC_Intrinsic(ulong value)
{
return (ushort)(Bmi1.X64.TrailingZeroCount(value) % 42);
}

[MethodImpl(MethodImplOptions.NoInlining)]
public static int Umod_UInt16Range_ByConst(int value)
{
if (value is > 0 and < 1234)
{
return value % 123;
}

return -1;
}

[MethodImpl(MethodImplOptions.AggressiveOptimization)]
private static void Test1()
{
for (int i = 0; i < 2; i++)
{
Core();
}

[MethodImpl(MethodImplOptions.AggressiveInlining)]
static void Core()
{
s_field1 = (ushort)(Bmi1.X64.TrailingZeroCount(s_field2) % 42);
}
}

[Fact]
public static int TestEntryPoint()
{
try
{
Umod_U4_CharByZero('a');
return 0;
}
catch (DivideByZeroException) { }

if (Umod_U2_CharByConst('a') != 13)
return 0;

if (Umod_I4_CharByConst('a') != 13)
return 0;

if (Umod_U4_CharByConst('a') != 13)
return 0;

if (Umod_I8_CharByConst('a') != 13)
return 0;

if (Umod_U8_CharByConst('a') != 13)
return 0;

if (!TestIsWhiteSpace(' '))
return 0;

if (!TestIsWhiteSpace('\u2029'))
return 0;

if (TestIsWhiteSpace('\0'))
return 0;

if (TestIsWhiteSpace('a'))
return 0;

if (Umod_TZC(1L << 40) != 40)
return 0;

if (Umod_TZC(1L << 50) != 8)
return 0;

if (Bmi1.X64.IsSupported)
{
if (Umod_TZC_Intrinsic(1L << 40) != 40)
return 0;

if (Umod_TZC_Intrinsic(1L << 50) != 8)
return 0;
}

if (Umod_UInt16Range_ByConst(0) != -1)
return 0;

if (Umod_UInt16Range_ByConst(42) != 42)
return 0;

if (Umod_UInt16Range_ByConst(123) != 0)
return 0;

if (Bmi1.X64.IsSupported)
{
s_field1 = 1;
s_field2 = 1L << 50;
Test1();

if (s_field1 != 8)
return 0;
}

return 100;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<DebugType>None</DebugType>
<Optimize>True</Optimize>
</PropertyGroup>
<ItemGroup>
<Compile Include="$(MSBuildProjectName).cs" />
</ItemGroup>
</Project>
Loading