Skip to content

Commit

Permalink
Fixed invalid alignment of vector loads in PTX backend.
Browse files Browse the repository at this point in the history
  • Loading branch information
m4rs-mt committed Jun 21, 2020
1 parent a7305a0 commit 829ccce
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 37 deletions.
2 changes: 1 addition & 1 deletion Src/ILGPU/Backends/PTX/PTXBackend.cs
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ protected override StringBuilder CreateKernelBuilder(
? PointerAlignments.Create(
backendContext.KernelMethod,
DefaultGlobalMemoryAlignment)
: PointerAlignments.Empty;
: null;

data = new PTXCodeGenerator.GeneratorArgs(
this,
Expand Down
14 changes: 10 additions & 4 deletions Src/ILGPU/Backends/PTX/PTXCodeGenerator.Emitter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -659,21 +659,27 @@ public void EmitComplexCommandWithOffsets<TEmitter>(
/// to resolve the correct offset in bytes within a structure.
/// </summary>
/// <typeparam name="TEmitter">The emitter type.</typeparam>
/// <param name="pointerValue">The pointer to get the alignment for.</param>
/// <param name="safeAlignment">The safe minimum alignment in bytes.</param>
/// <param name="command">The generic command to emit.</param>
/// <param name="emitter">The current emitter.</param>
/// <param name="register">The involved register.</param>
/// <param name="alignment">The base alignment in bytes.</param>
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public void EmitVectorizedCommand<TEmitter>(
Value pointerValue,
int safeAlignment,
string command,
in TEmitter emitter,
Register register,
int alignment)
Register register)
where TEmitter : IVectorizedCommandEmitter
{
if (register is CompoundRegister compoundRegister)
if (PointerAlignments != null &&
register is CompoundRegister compoundRegister)
{
// Check the provided alignment value to create vectorized instructions
int alignment = PointerAlignments.GetAlignment(
pointerValue,
safeAlignment);
var ranges = compoundRegister.Type.VectorizableFields;
for (int i = 0, e = ranges.Count; i < e; ++i)
{
Expand Down
18 changes: 6 additions & 12 deletions Src/ILGPU/Backends/PTX/PTXCodeGenerator.Values.cs
Original file line number Diff line number Diff line change
Expand Up @@ -439,15 +439,12 @@ public void GenerateCode(Load load)
var sourceType = load.Source.Type as PointerType;
var targetRegister = Allocate(load);

// Query alignment information to emit vectorized instructions
int alignment = PointerAlignments.GetAlignment(
load.Source,
sourceType.ElementType.Alignment);
EmitVectorizedCommand(
load.Source,
sourceType.ElementType.Alignment,
PTXInstructions.LoadOperation,
new LoadEmitter(sourceType, address),
targetRegister,
alignment);
targetRegister);
}

/// <summary>
Expand Down Expand Up @@ -540,15 +537,12 @@ public void GenerateCode(Store store)
var targetType = store.Target.Type as PointerType;
var value = Load(store.Value);

// Query alignment information to emit vectorized instructions
int baseAlignment = PointerAlignments.GetAlignment(
store.Target,
targetType.ElementType.Alignment);
EmitVectorizedCommand(
store.Target,
targetType.ElementType.Alignment,
PTXInstructions.StoreOperation,
new StoreEmitter(targetType, address),
value,
baseAlignment);
value);
}

/// <summary cref="IBackendCodeGenerator.GenerateCode(LoadFieldAddress)"/>
Expand Down
11 changes: 6 additions & 5 deletions Src/ILGPU/IR/Analyses/PointerAlignments.cs
Original file line number Diff line number Diff line change
Expand Up @@ -84,14 +84,15 @@ public readonly int Merge(int first, int second) =>
{
LoadFieldAddress lfa =>
AnalysisValue.Create(
context[lfa.Source].Data +
lfa.StructureType.GetOffset(lfa.FieldSpan.Access),
lfa.Type),
Math.Min(
context[lfa.Source].Data,
lfa.StructureType[lfa.FieldSpan.Access].Alignment),
lfa.Type),
LoadElementAddress lea =>
AnalysisValue.Create(
Math.Min(
Math.Max(
context[lea.Source].Data,
(lea.Type as IAddressSpaceType).ElementType.Size),
(lea.Type as IAddressSpaceType).ElementType.Alignment),
lea.Type),
_ => null,
};
Expand Down
83 changes: 68 additions & 15 deletions Src/ILGPU/IR/Types/StructureType.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
using System.Diagnostics.CodeAnalysis;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Security.Permissions;
using System.Text;

namespace ILGPU.IR.Types
Expand Down Expand Up @@ -410,19 +409,23 @@ public readonly struct VectorizableFieldCollection
/// </summary>
public struct Entry
{
#region Instance

/// <summary>
/// Constructs a new entry.
/// </summary>
internal Entry(TypeNode type, int index, int offset, int count = 1)
internal Entry(
TypeNode type,
int index,
int offset,
int count = 1)
{
Type = type;
Index = index;
Count = count;
Offset = offset;
}

#region Instance

#endregion

#region Properties
Expand All @@ -440,17 +443,39 @@ internal Entry(TypeNode type, int index, int offset, int count = 1)
/// <summary>
/// Returns the number of fields.
/// </summary>
public int Count { get; private set; }
public int Count { readonly get; private set; }

/// <summary>
/// Returns the base offset in bytes from the beginning of the field.
/// </summary>
public int Offset { get; }

/// <summary>
/// Returns the required alignment in bytes.
/// </summary>
public readonly int RequiredAlignment => Count * Type.Size;

#endregion

#region Methods

/// <summary>
/// Splits the current entry into two parts.
/// </summary>
/// <param name="first">The first part.</param>
/// <param name="second">The second part.</param>
internal void Split(out Entry first, out Entry second)
{
Type.Assert(Count > 1);
int firstCount = Count >> 1 + Count % 2;
first = new Entry(Type, Index, Offset, firstCount);
second = new Entry(
Type,
Index + firstCount,
Offset + Type.Size * firstCount,
Count - firstCount);
}

/// <summary>
/// Adds a field to this entry.
/// </summary>
Expand All @@ -463,7 +488,22 @@ internal Entry(TypeNode type, int index, int offset, int count = 1)
/// <param name="alignment">The underlying alignment in bytes.</param>
/// <returns>True, if the range is properly aligned.</returns>
public readonly bool IsAligned(int alignment) =>
(alignment + Offset) % (Type.Size * Count) == 0;
// Check for a proper alignment of the base address
alignment % RequiredAlignment == 0;

/// <summary>
/// Returns true if this entry can be properly aligned.
/// </summary>
/// <param name="parentType">The parent structure type.</param>
internal readonly bool CanBeAligned(StructureType parentType)
{
int requiredAlignment = RequiredAlignment;
return
// Check for a relative alignment inside the structure
Offset % requiredAlignment == 0 &&
// Check for a relative alignment of odd structure accesses
(Offset + parentType.Size) % requiredAlignment == 0;
}

#endregion
}
Expand Down Expand Up @@ -499,7 +539,7 @@ internal VectorizableFieldCollection(StructureType structureType)
currentOffset + nextType.Size != nextOffset)
{
// Register the current vectorizable entry
RegisterRange(current);
RegisterRange(structureType, current);
current = new Entry(
nextType,
i,
Expand All @@ -514,14 +554,17 @@ internal VectorizableFieldCollection(StructureType structureType)
}

// Add the last entry
RegisterRange(current);
RegisterRange(structureType, current);
}

/// <summary>
/// Registers the given range entry.
/// </summary>
/// <param name="structureType">The parent structure type.</param>
/// <param name="entry">The entry to register.</param>
private void RegisterRange(in Entry entry)
private void RegisterRange(
StructureType structureType,
in Entry entry)
{
int offset = entry.Offset;
for (
Expand All @@ -531,12 +574,22 @@ private void RegisterRange(in Entry entry)
{
for (; index + stepSize <= entry.Count; index += stepSize)
{
ranges.Add(
new Entry(
entry.Type,
index + entry.Index,
offset,
stepSize));
var newEntry = new Entry(
entry.Type,
index + entry.Index,
offset,
stepSize);
if (newEntry.Count > 1 && !newEntry.CanBeAligned(structureType))
{
newEntry.Split(out var first, out var second);
RegisterRange(structureType, first);
RegisterRange(structureType, second);
}
else
{
// The entry is properly aligned
ranges.Add(newEntry);
}
offset += entry.Type.Size * stepSize;
}
}
Expand Down

0 comments on commit 829ccce

Please sign in to comment.