diff --git a/src/Usmap.NET.Tests/TestFiles/br2.usmap b/src/Usmap.NET.Tests/TestFiles/br2.usmap new file mode 100644 index 0000000..5eac27a Binary files /dev/null and b/src/Usmap.NET.Tests/TestFiles/br2.usmap differ diff --git a/src/Usmap.NET.Tests/TestFiles/oo2.usmap b/src/Usmap.NET.Tests/TestFiles/oo2.usmap new file mode 100644 index 0000000..e9c2d73 Binary files /dev/null and b/src/Usmap.NET.Tests/TestFiles/oo2.usmap differ diff --git a/src/Usmap.NET.Tests/TestFiles/xx2.usmap b/src/Usmap.NET.Tests/TestFiles/xx2.usmap new file mode 100644 index 0000000..e0e3fe0 Binary files /dev/null and b/src/Usmap.NET.Tests/TestFiles/xx2.usmap differ diff --git a/src/Usmap.NET.Tests/Tests.cs b/src/Usmap.NET.Tests/Tests.cs index 98720b6..b68c473 100644 --- a/src/Usmap.NET.Tests/Tests.cs +++ b/src/Usmap.NET.Tests/Tests.cs @@ -16,14 +16,23 @@ public class Tests /*private const int ExpectedSchemas = 13890; private const int ExpectedEnums = 2367; private const int ExpectedNames = 74908;*/ + private const int ExpectedSchemas = 28697; private const int ExpectedEnums = 4391; private const int ExpectedNames = 141495; + private const int ExpectedSchemasV3 = 29520; + private const int ExpectedEnumsV3 = 4484; + private const int ExpectedNamesV3 = 144915; + private readonly string _uncompressedUsmapPath; private readonly string _brotliCompressedUsmapPath; private readonly string _oodleCompressedUsmapPath; + private readonly string _uncompressedUsmapV3Path; + private readonly string _brotliCompressedUsmapV3Path; + private readonly string _oodleCompressedUsmapV3Path; + public Tests() { var currentDir = Directory.GetCurrentDirectory(); @@ -32,10 +41,14 @@ public Tests() _uncompressedUsmapPath = Path.Combine(testFilesDir, "xx1.usmap"); _brotliCompressedUsmapPath = Path.Combine(testFilesDir, "br1.usmap"); _oodleCompressedUsmapPath = Path.Combine(testFilesDir, "oo1.usmap"); + + _uncompressedUsmapV3Path = Path.Combine(testFilesDir, "xx2.usmap"); + _brotliCompressedUsmapV3Path = Path.Combine(testFilesDir, "br2.usmap"); + _oodleCompressedUsmapV3Path = Path.Combine(testFilesDir, "oo2.usmap"); } [Fact] - public void ParseUncompressedUsmapFromFile() + public void ParseUncompressedFromFile() { var usmap = new Usmap(_uncompressedUsmapPath); Assert.Equal(ExpectedSchemas, usmap.Schemas.Length); @@ -45,7 +58,7 @@ public void ParseUncompressedUsmapFromFile() } [Fact] - public void ParseUncompressedUsmapFromStream() + public void ParseUncompressedFromStream() { var usmap = new Usmap(File.OpenRead(_uncompressedUsmapPath)); Assert.Equal(ExpectedSchemas, usmap.Schemas.Length); @@ -55,7 +68,7 @@ public void ParseUncompressedUsmapFromStream() } [Fact] - public void ParseUncompressedUsmapFromBuffer() + public void ParseUncompressedFromBuffer() { var buffer = File.ReadAllBytes(_uncompressedUsmapPath); var usmap = new Usmap(buffer); @@ -138,4 +151,111 @@ public void ParseOodleCompressedFromBuffer() Assert.Equal(ExpectedNames, usmap.Names.Length); Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); } + + // v3 + + [Fact] + public void ParseUncompressedV3FromFile() + { + var usmap = new Usmap(_uncompressedUsmapV3Path); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseUncompressedV3FromStream() + { + var usmap = new Usmap(File.OpenRead(_uncompressedUsmapV3Path)); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseUncompressedV3FromBuffer() + { + var buffer = File.ReadAllBytes(_uncompressedUsmapV3Path); + var usmap = new Usmap(buffer); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseBrotliCompressedV3FromFile() + { + var usmap = new Usmap(_brotliCompressedUsmapV3Path); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseBrotliCompressedV3FromStream() + { + var usmap = new Usmap(File.OpenRead(_brotliCompressedUsmapV3Path)); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseBrotliCompressedV3FromBuffer() + { + var buffer = File.ReadAllBytes(_brotliCompressedUsmapV3Path); + var usmap = new Usmap(buffer); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseOodleCompressedV3FromFile() + { + var options = new UsmapOptions + { + Oodle = OodleInstance + }; + var usmap = new Usmap(_oodleCompressedUsmapV3Path, options); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseOodleCompressedV3FromStream() + { + var options = new UsmapOptions + { + Oodle = OodleInstance + }; + var usmap = new Usmap(File.OpenRead(_oodleCompressedUsmapV3Path), options); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } + + [Fact] + public void ParseOodleCompressedV3FromBuffer() + { + var buffer = File.ReadAllBytes(_oodleCompressedUsmapV3Path); + var options = new UsmapOptions + { + Oodle = OodleInstance + }; + var usmap = new Usmap(buffer, options); + Assert.Equal(ExpectedSchemasV3, usmap.Schemas.Length); + Assert.Equal(ExpectedEnumsV3, usmap.Enums.Length); + Assert.Equal(ExpectedNamesV3, usmap.Names.Length); + Assert.All(usmap.Names, x => Assert.False(string.IsNullOrEmpty(x))); + } } diff --git a/src/Usmap.NET.Tests/Usmap.NET.Tests.csproj b/src/Usmap.NET.Tests/Usmap.NET.Tests.csproj index a42efa3..ad93df6 100644 --- a/src/Usmap.NET.Tests/Usmap.NET.Tests.csproj +++ b/src/Usmap.NET.Tests/Usmap.NET.Tests.csproj @@ -8,9 +8,9 @@ - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all @@ -31,18 +31,27 @@ PreserveNewest + + PreserveNewest + PreserveNewest PreserveNewest + + PreserveNewest + PreserveNewest PreserveNewest + + PreserveNewest + diff --git a/src/Usmap.NET/EUsmapCompressionMethod.cs b/src/Usmap.NET/EUsmapCompressionMethod.cs index 0a695c9..50e0cf1 100644 --- a/src/Usmap.NET/EUsmapCompressionMethod.cs +++ b/src/Usmap.NET/EUsmapCompressionMethod.cs @@ -9,11 +9,13 @@ public enum EUsmapCompressionMethod : byte Oodle, /// Brotli, + /// + ZStandard, /// MaxPlusOne, /// Max = MaxPlusOne - 1, /// - Unknown = byte.MaxValue + Unknown = 0xFF } diff --git a/src/Usmap.NET/EUsmapVersion.cs b/src/Usmap.NET/EUsmapVersion.cs index 3e05eec..1f11a66 100644 --- a/src/Usmap.NET/EUsmapVersion.cs +++ b/src/Usmap.NET/EUsmapVersion.cs @@ -5,7 +5,19 @@ public enum EUsmapVersion : byte { /// Initial, - + /// + /// Adds package versioning to aid with compatibility + /// + PackageVersioning, + /// + /// Adds support for 16-bit wide name-lengths (ushort/uint16) + /// + LongFName, + /// + /// Adds support for enums with more than 255 values + /// + LargeEnums, + /// LatestPlusOne, /// diff --git a/src/Usmap.NET/Usmap.NET.csproj b/src/Usmap.NET/Usmap.NET.csproj index 2657397..f64bf03 100644 --- a/src/Usmap.NET/Usmap.NET.csproj +++ b/src/Usmap.NET/Usmap.NET.csproj @@ -18,9 +18,9 @@ true true - 2.0.2.0 - 2.0.2.0 - 2.0.2 + 2.1.0.0 + 2.1.0.0 + 2.1.0 @@ -32,7 +32,7 @@ - + diff --git a/src/Usmap.NET/Usmap.cs b/src/Usmap.NET/Usmap.cs index 0debf15..a6e0892 100644 --- a/src/Usmap.NET/Usmap.cs +++ b/src/Usmap.NET/Usmap.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Diagnostics; using System.IO.Compression; using System.Text; @@ -10,6 +9,8 @@ namespace UsmapDotNet; /// public class Usmap { + private const ushort MagicValue = 0x30C4; + /// public string[] Names { get; } /// @@ -52,35 +53,51 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = try { - var header = usmapReader.Read(); - if (header.Magic != UsmapHeader.MagicValue) - throw new FileLoadException("Invalid .usmap magic constant"); - if (header.Version > EUsmapVersion.Initial) - throw new FileLoadException($"Invalid or unsupported .usmap version: {(int)header.Version}"); - if (header.CompressionMethod > EUsmapCompressionMethod.Max) - throw new FileLoadException( - $"Invalid or unsupported .usmap compression: {(int)header.CompressionMethod}"); - if (usmapReader.Length - usmapReader.Position < header.CompressedSize) + var magic = usmapReader.Read(); + if (magic != MagicValue) + throw new FileLoadException($"Invalid .usmap magic constant: 0x{magic:X4}, expected: 0x{MagicValue:X4}"); + var version = usmapReader.Read(); + if (version > EUsmapVersion.Latest) + throw new FileLoadException($"Invalid or unsupported .usmap version: {(int)version}"); + + var bHasVersioning = version >= EUsmapVersion.PackageVersioning && usmapReader.ReadBoolean(); + if (bHasVersioning) + { + //usmapReader.Read(); + usmapReader.Position += sizeof(int) * 2; + + //usmapReader.Read(); + var versionsLength = usmapReader.Read(); + usmapReader.Position += versionsLength * (16 /* FGuid */ + sizeof(int)); + } + + var compressionMethod = usmapReader.Read(); + var compressedSize = usmapReader.Read(); + var uncompressedSize = usmapReader.Read(); + + if (compressionMethod > EUsmapCompressionMethod.Max) + throw new FileLoadException($"Invalid or unsupported .usmap compression: {(int)compressionMethod}"); + if (usmapReader.Length - usmapReader.Position < compressedSize) throw new FileLoadException("There is not enough data in the .usmap file"); options ??= new UsmapOptions(); IGenericReader reader; - if (header.CompressionMethod == EUsmapCompressionMethod.None) + if (compressionMethod == EUsmapCompressionMethod.None) { - if (header.CompressedSize != header.UncompressedSize) + if (compressedSize != uncompressedSize) throw new FileLoadException("No .usmap compression: Compression size must be equal to decompression size"); reader = usmapReader; } else { - compressionBuffer = ArrayPool.Shared.Rent((int)(header.CompressedSize + header.UncompressedSize)); - var compressedSpan = new Span(compressionBuffer, 0, (int)header.CompressedSize); + compressionBuffer = ArrayPool.Shared.Rent((int)(compressedSize + uncompressedSize)); + var compressedSpan = new Span(compressionBuffer, 0, (int)compressedSize); usmapReader.Read(compressedSpan); - var uncompressedMemory = new Memory(compressionBuffer, (int)header.CompressedSize, (int)header.UncompressedSize); + var uncompressedMemory = new Memory(compressionBuffer, (int)compressedSize, (int)uncompressedSize); - switch (header.CompressionMethod) + switch (compressionMethod) { case EUsmapCompressionMethod.Oodle: { @@ -88,8 +105,8 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = throw new InvalidOperationException("Data is compressed and oodle instance was null"); var result = (uint)options.Oodle.Decompress(compressedSpan, uncompressedMemory.Span); - if (result != header.UncompressedSize) - throw new FileLoadException($"Invalid oodle .usmap decompress result: {result} / {header.UncompressedSize}"); + if (result != uncompressedSize) + throw new FileLoadException($"Invalid oodle .usmap decompress result: {result} / {uncompressedSize}"); break; } case EUsmapCompressionMethod.Brotli: @@ -97,11 +114,15 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = using var decoder = new BrotliDecoder(); var result = decoder.Decompress(compressedSpan, uncompressedMemory.Span, out var bytesConsumed, out var bytesWritten); if (result != OperationStatus.Done) - throw new FileLoadException($"Invalid brotli .usmap decompress result: {result} | {bytesWritten} / {header.UncompressedSize} | {bytesConsumed} / {header.CompressedSize}"); + throw new FileLoadException($"Invalid brotli .usmap decompress result: {result} | {bytesWritten} / {uncompressedSize} | {bytesConsumed} / {compressedSize}"); break; } + case EUsmapCompressionMethod.ZStandard: + { + throw new FileLoadException($"Unsupported .usmap compression: {(int)EUsmapCompressionMethod.ZStandard} (Zstandard)"); + } default: - throw new UnreachableException(); + throw new FileLoadException($"Invalid or unsupported .usmap compression: {(int)compressionMethod}"); } reader = new GenericBufferReader(uncompressedMemory); @@ -115,8 +136,10 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = for (var i = 0; i < size; ++i) { - var nameSize = reader.Read(); - var name = reader.ReadString(nameSize, Encoding.UTF8); + var nameLength = (int)(version >= EUsmapVersion.LongFName + ? reader.Read() + : reader.Read()); + var name = reader.ReadString(nameLength, Encoding.UTF8); names[i] = name; } } @@ -128,7 +151,10 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = for (var i = 0; i < size; ++i) { var idx = reader.Read(); - var enumNamesSize = reader.Read(); + var enumName = names[idx]; + var enumNamesSize = (int)(version >= EUsmapVersion.LargeEnums + ? reader.Read() + : reader.Read()); var enumNames = new string[enumNamesSize]; for (var j = 0; j < enumNamesSize; ++j) @@ -137,7 +163,7 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = enumNames[j] = names[nameIdx]; } - Enums[i] = new UsmapEnum(names[idx], enumNames); + Enums[i] = new UsmapEnum(enumName, enumNames); } } @@ -148,7 +174,11 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = for (var i = 0; i < size; ++i) { var idx = reader.Read(); + var schemaName = names[idx]; var superIdx = reader.Read(); + var schemaSuperType = superIdx == uint.MaxValue + ? null + : names[superIdx]; var propCount = reader.Read(); var serializablePropCount = reader.Read(); @@ -172,7 +202,7 @@ public Usmap(IGenericReader usmapReader, UsmapOptions? options, bool leaveOpen = } } - Schemas[i] = new UsmapSchema(names[idx], superIdx == uint.MaxValue ? null : names[superIdx], propCount, props); + Schemas[i] = new UsmapSchema(schemaName, schemaSuperType, propCount, props); } } diff --git a/src/Usmap.NET/UsmapHeader.cs b/src/Usmap.NET/UsmapHeader.cs deleted file mode 100644 index 84a733c..0000000 --- a/src/Usmap.NET/UsmapHeader.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System.Runtime.InteropServices; - -namespace UsmapDotNet; - -[StructLayout(LayoutKind.Sequential, Pack = 1)] -internal readonly struct UsmapHeader -{ - public const ushort MagicValue = 0x30C4; - - public readonly ushort Magic; - public readonly EUsmapVersion Version; - public readonly EUsmapCompressionMethod CompressionMethod; - public readonly uint CompressedSize; - public readonly uint UncompressedSize; -}