tiny amounts of cleanup (lots)

include/tulip/platform/MacosIntelConvention.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "../Platform.hpp"
+
+#if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+
+#include "../CallingConvention.hpp"
+
+#include <memory>
+#include <string>
+
+namespace tulip::hook {
+	class AbstractFunction;
+
+	class TULIP_HOOK_DLL SystemVConvention : public CallingConvention {
+	public:
+		~SystemVConvention() override;
+
+		void generateDefaultCleanup(BaseAssembler& a, AbstractFunction const& function) override;
+		void generateIntoDefault(BaseAssembler& a, AbstractFunction const& function) override;
+		void generateIntoOriginal(BaseAssembler& a, AbstractFunction const& function) override;
+		void generateOriginalCleanup(BaseAssembler& a, AbstractFunction const& function) override;
+		bool needsWrapper(AbstractFunction const& function) const override;
+
+		static std::shared_ptr<SystemVConvention> create();
+	};
+}
+
+#endif

include/tulip/platform/PlatformConvention.hpp

@@ -2,10 +2,13 @@
 #include "DefaultConvention.hpp"
 #include "Windows32Convention.hpp"
 #include "Windows64Convention.hpp"
+#include "MacosIntelConvention.hpp"
 
 namespace tulip::hook {
 #if defined(TULIP_HOOK_WINDOWS) && defined(TULIP_HOOK_X86)
 	using PlatformConvention = CdeclConvention;
+#elif defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+	using PlatformConvention = SystemVConvention;
 #else
 	using PlatformConvention = DefaultConvention;
 #endif

src/CMakeLists.txt

@@ -31,6 +31,7 @@ elseif(APPLE)
 		assembler/X86Assembler.cpp
 		assembler/X64Assembler.cpp
 		assembler/ArmV8Assembler.cpp
+		convention/MacosIntelConvention.cpp
 		generator/X86Generator.cpp
 		generator/X64Generator.cpp
 		generator/ArmV8Generator.cpp

src/Handler.cpp

@@ -29,12 +29,12 @@ Result<std::unique_ptr<Handler>> Handler::create(void* address, HandlerMetadata
 
 	TULIP_HOOK_UNWRAP_INTO(ret->m_trampoline, Target::get().allocateArea(0x100));
 
-	auto wrapperMetadata = WrapperMetadata{
-		.m_convention = metadata.m_convention,
-		.m_abstract = metadata.m_abstract,
-	};
-	TULIP_HOOK_UNWRAP_INTO(auto trampolineWrap, Wrapper::get().createWrapper(ret->m_trampoline, wrapperMetadata));
-	ret->m_wrapped = trampolineWrap;
+	// auto wrapperMetadata = WrapperMetadata{
+	// 	.m_convention = metadata.m_convention,
+	// 	.m_abstract = metadata.m_abstract,
+	// };
+	// TULIP_HOOK_UNWRAP_INTO(auto trampolineWrap, Wrapper::get().createWrapper(ret->m_trampoline, wrapperMetadata));
+	// ret->m_wrapped = trampolineWrap;
 	// std::cout << std::hex << "m_trampoline: " << (void*)ret->m_trampoline << std::endl;
 
 	return Ok(std::move(ret));
@@ -57,8 +57,7 @@ Result<> Handler::init() {
 	auto address = reinterpret_cast<uint8_t*>(Target::get().getRealPtr(m_address));
 	m_originalBytes.insert(m_originalBytes.begin(), address, address + target);
 
-	TULIP_HOOK_UNWRAP_INTO(auto trampolineOffset, generator->relocateOriginal(target));
-	TULIP_HOOK_UNWRAP(generator->generateTrampoline(trampolineOffset));
+	TULIP_HOOK_UNWRAP(generator->generateTrampoline(target));
 
 	this->addOriginal();
 
@@ -69,7 +68,7 @@ void Handler::addOriginal() {
 	auto metadata = HookMetadata{
 		.m_priority = INT32_MAX,
 	};
-	static_cast<void>(this->createHook(Target::get().getRealPtrAs(m_wrapped, m_address), metadata));
+	static_cast<void>(this->createHook(Target::get().getRealPtrAs(m_trampoline, m_address), metadata));
 }
 
 HookHandle Handler::createHook(void* address, HookMetadata m_metadata) {

src/Main.cpp

@@ -61,6 +61,10 @@ std::shared_ptr<CallingConvention> tulip::hook::createConvention(TulipConvention
 #endif
 		case TulipConvention::Default:
 		default:
+#if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+			return SystemVConvention::create();
+#else
 			return DefaultConvention::create();
+#endif
 	}
 }

src/assembler/X64Assembler.cpp

@@ -110,6 +110,14 @@ void X64Assembler::call(X64Register reg) {
 	X86Assembler::call(x86reg(reg));
 }
 
+void X64Assembler::call(int64_t address) {
+	X86Assembler::call(address);
+}
+
+void X64Assembler::call(std::string const& label) {
+	X86Assembler::call(label);
+}
+
 void X64Assembler::callip(std::string const& label) {
 	this->write8(0xff);
 	this->write8(0x15);

src/assembler/X64Assembler.hpp

@@ -43,6 +43,9 @@ namespace tulip::hook {
 	inline X64Pointer operator+(X64Register reg, int32_t offset) {
 		return X64Pointer(reg, offset);
 	}
+	inline X64Pointer operator-(X64Register reg, int32_t offset) {
+		return X64Pointer(reg, -offset);
+	}
 
 	// Use this to easily express a X64Pointer, like so:
 	// RegMem64 m;
@@ -77,6 +80,8 @@ namespace tulip::hook {
 		void jmpip(std::string const& label);
 
 		void call(X64Register reg);
+		void call(int64_t address);
+		void call(std::string const& label);
 		void callip(std::string const& label);
 
 		void movsd(X64Register reg, X64Pointer ptr);

src/assembler/X86Assembler.cpp

@@ -128,6 +128,11 @@ void X86Assembler::call(X86Register reg) {
 	this->encodeModRM(reg, 2);
 }
 
+void X86Assembler::call(std::string const& label) {
+	this->write8(0xE8);
+	this->label32(label);
+}
+
 void X86Assembler::movsd(X86Register reg, X86Pointer ptr) {
 	this->write8(0xF2);
 	this->write8(0x0F);

src/assembler/X86Assembler.hpp

@@ -35,6 +35,9 @@ namespace tulip::hook {
 	inline X86Pointer operator+(X86Register reg, int32_t offset) {
 		return X86Pointer(reg, offset);
 	}
+	inline X86Pointer operator-(X86Register reg, int32_t offset) {
+		return X86Pointer(reg, -offset);
+	}
 
 	// Use this to easily express a X86Pointer, like so:
 	// RegMem32 m;
@@ -95,6 +98,7 @@ namespace tulip::hook {
 
 		void call(X86Register reg);
 		void call(int64_t value);
+		void call(std::string const& label);
 
 		void movsd(X86Register reg, X86Pointer ptr);
 		void movsd(X86Pointer ptr, X86Register reg);

src/convention/MacosIntelConvention.cpp

@@ -0,0 +1,83 @@
+#include <Platform.hpp>
+
+#if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+
+#include <AbstractFunction.hpp>
+#include <platform/MacosIntelConvention.hpp>
+#include "../assembler/X64Assembler.hpp"
+
+using namespace tulip::hook;
+
+namespace {
+    size_t getStackParamSize(AbstractFunction const& function) {
+        size_t stackParamSize = 0;
+        int xmmCount = 0;
+        int gprCount = 0;
+        if (function.m_return.m_kind == AbstractTypeKind::Other && function.m_return.m_size > 16) {
+            gprCount += 1;
+        }
+        for (auto& param : function.m_parameters) {
+            if (param.m_kind == AbstractTypeKind::FloatingPoint) {
+                if (xmmCount < 8) {
+                    xmmCount++;
+                } else {
+                    stackParamSize += 8;
+                }
+            } else {
+                if (gprCount < 6) {
+                    gprCount++;
+                } else {
+                    stackParamSize += 8;
+                }
+            }
+        }
+        return stackParamSize;
+    }
+}
+
+SystemVConvention::~SystemVConvention() {}
+
+void SystemVConvention::generateDefaultCleanup(BaseAssembler& a_, AbstractFunction const& function) {
+    size_t stackParamSize = getStackParamSize(function);
+    if (stackParamSize > 0) {
+        auto& a = static_cast<X64Assembler&>(a_);
+        using enum X64Register;
+        auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize;
+        a.add(RSP, paddedSize);
+    }
+}
+
+// used to move the stack values to the correct places
+
+void SystemVConvention::generateIntoDefault(BaseAssembler& a_, AbstractFunction const& function) {
+    size_t stackParamSize = getStackParamSize(function);
+    if (stackParamSize > 0) {
+        auto& a = static_cast<X64Assembler&>(a_);
+        using enum X64Register;
+        RegMem64 m;
+        auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize;
+        a.sub(RSP, paddedSize);
+        int stackOffset = 0;
+
+        for (auto i = 0; i < stackParamSize; i += 8) {
+            a.mov(RAX, m[RBP + (16 + i)]);
+            a.mov(m[RSP + i], RAX);
+        }
+    }
+}
+
+void SystemVConvention::generateIntoOriginal(BaseAssembler& a_, AbstractFunction const& function) {
+}
+
+void SystemVConvention::generateOriginalCleanup(BaseAssembler& a_, AbstractFunction const& function) {
+}
+
+bool SystemVConvention::needsWrapper(AbstractFunction const& function) const {
+	return false;
+}
+
+std::shared_ptr<SystemVConvention> SystemVConvention::create() {
+	return std::make_shared<SystemVConvention>();
+}
+
+#endif

src/convention/Windows64Convention.cpp

@@ -8,10 +8,43 @@
 
 using namespace tulip::hook;
 
+namespace {
+    size_t getStackParamSize(AbstractFunction const& function) {
+        size_t stackParamSize = 0;
+        int xmmCount = 0;
+        int gprCount = 0;
+        if (function.m_return.m_kind == AbstractTypeKind::Other) {
+            gprCount += 1;
+        }
+        for (auto& param : function.m_parameters) {
+            if (param.m_kind == AbstractTypeKind::FloatingPoint) {
+                if (xmmCount < 4) {
+                    xmmCount++;
+                } else {
+                    stackParamSize += 8;
+                }
+            } else {
+                if (gprCount < 4) {
+                    gprCount++;
+                } else {
+                    stackParamSize += 8;
+                }
+            }
+        }
+        return stackParamSize;
+    }
+}
+
 ThiscallConvention::~ThiscallConvention() {}
 
 void ThiscallConvention::generateDefaultCleanup(BaseAssembler& a, AbstractFunction const& function) {
-    static_cast<X64Assembler&>(a).ret();
+    size_t stackParamSize = getStackParamSize(function);
+    if (stackParamSize > 0) {
+        auto& a = static_cast<X64Assembler&>(a_);
+        using enum X64Register;
+        auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize;
+        a.add(RSP, paddedSize);
+    }
 }
 
 // Member functions deal with struct return differently, since in the windows x64 convention
@@ -25,23 +58,38 @@ void ThiscallConvention::generateDefaultCleanup(BaseAssembler& a, AbstractFuncti
 // so to undo this we just swap the first two parameters (RCX and RDX).
 
 void ThiscallConvention::generateIntoDefault(BaseAssembler& a_, AbstractFunction const& function) {
+    auto& a = static_cast<X64Assembler&>(a_);
+    using enum X64Register;
+    RegMem64 m;
+
     if (function.m_return.m_kind == AbstractTypeKind::Other) {
-        auto& a = static_cast<X64Assembler&>(a_);
-        using enum X64Register;
         a.xchg(RCX, RDX);
     }
+
+    size_t stackParamSize = getStackParamSize(function);
+    if (stackParamSize > 0) {
+        auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize;
+        a.sub(RSP, paddedSize);
+        int stackOffset = 0;
+
+        for (auto i = 0; i < stackParamSize; i += 8) {
+            a.mov(RAX, m[RBP + (16 + i)]);
+            a.mov(m[RSP + i], RAX);
+        }
+    }
 }
 
 void ThiscallConvention::generateIntoOriginal(BaseAssembler& a_, AbstractFunction const& function) {
+    auto& a = static_cast<X64Assembler&>(a_);
+    using enum X64Register;
+    RegMem64 m;
+
     if (function.m_return.m_kind == AbstractTypeKind::Other) {
-        auto& a = static_cast<X64Assembler&>(a_);
-        using enum X64Register;
         a.xchg(RCX, RDX);
     }
 }
 
 void ThiscallConvention::generateOriginalCleanup(BaseAssembler& a, AbstractFunction const& function) {
-    static_cast<X64Assembler&>(a).ret();
 }
 
 bool ThiscallConvention::needsWrapper(AbstractFunction const& function) const {