Cleanup (#18)

* tiny amounts of cleanup (lots)

.github/workflows/build.yml

@@ -31,7 +31,7 @@ jobs:
 
         - name: 'macOS'
           id: mac
-          os: macos-latest
+          os: macos-12
           build_tests: true
           extra_flags: "-DCMAKE_BUILD_TYPE=Debug"
           out_paths: './build/src/libTulipHook.a'

include/tulip/platform/MacosIntelConvention.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "../Platform.hpp"
+
+#if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+
+#include "../CallingConvention.hpp"
+
+#include <memory>
+#include <string>
+
+namespace tulip::hook {
+	class AbstractFunction;
+
+	class TULIP_HOOK_DLL SystemVConvention : public CallingConvention {
+	public:
+		~SystemVConvention() override;
+
+		void generateDefaultCleanup(BaseAssembler& a, AbstractFunction const& function) override;
+		void generateIntoDefault(BaseAssembler& a, AbstractFunction const& function) override;
+		void generateIntoOriginal(BaseAssembler& a, AbstractFunction const& function) override;
+		void generateOriginalCleanup(BaseAssembler& a, AbstractFunction const& function) override;
+		bool needsWrapper(AbstractFunction const& function) const override;
+
+		static std::shared_ptr<SystemVConvention> create();
+	};
+}
+
+#endif

include/tulip/platform/PlatformConvention.hpp

@@ -2,10 +2,13 @@
 #include "DefaultConvention.hpp"
 #include "Windows32Convention.hpp"
 #include "Windows64Convention.hpp"
+#include "MacosIntelConvention.hpp"
 
 namespace tulip::hook {
 #if defined(TULIP_HOOK_WINDOWS) && defined(TULIP_HOOK_X86)
 	using PlatformConvention = CdeclConvention;
+#elif defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+	using PlatformConvention = SystemVConvention;
 #else
 	using PlatformConvention = DefaultConvention;
 #endif

src/CMakeLists.txt

@@ -31,6 +31,7 @@ elseif(APPLE)
 		assembler/X86Assembler.cpp
 		assembler/X64Assembler.cpp
 		assembler/ArmV8Assembler.cpp
+		convention/MacosIntelConvention.cpp
 		generator/X86Generator.cpp
 		generator/X64Generator.cpp
 		generator/ArmV8Generator.cpp

src/Handler.cpp

@@ -6,7 +6,6 @@
 #include "target/PlatformTarget.hpp"
 
 #include <algorithm>
-#include <sstream>
 #include <stack>
 
 using namespace tulip::hook;
@@ -22,21 +21,10 @@ Result<std::unique_ptr<Handler>> Handler::create(void* address, HandlerMetadata
 	if (!ret->m_content) {
 		return Err("Failed to allocate HandlerContent");
 	}
-	// std::cout << std::hex << "m_content: " << (void*)ret->m_content << std::endl;
 
 	TULIP_HOOK_UNWRAP_INTO(ret->m_handler, Target::get().allocateArea(0x300));
-	// std::cout << std::hex << "m_handler: " << (void*)ret->m_handler << std::endl;
-
 	TULIP_HOOK_UNWRAP_INTO(ret->m_trampoline, Target::get().allocateArea(0x100));
 
-	auto wrapperMetadata = WrapperMetadata{
-		.m_convention = metadata.m_convention,
-		.m_abstract = metadata.m_abstract,
-	};
-	TULIP_HOOK_UNWRAP_INTO(auto trampolineWrap, Wrapper::get().createWrapper(ret->m_trampoline, wrapperMetadata));
-	ret->m_wrapped = trampolineWrap;
-	// std::cout << std::hex << "m_trampoline: " << (void*)ret->m_trampoline << std::endl;
-
 	return Ok(std::move(ret));
 }
 
@@ -46,7 +34,7 @@ Result<> Handler::init() {
 	// printf("func addr: 0x%" PRIx64 "\n", (uint64_t)m_address);
 
 	auto generator =
-		Target::get().getHandlerGenerator(m_address, m_trampoline, m_handler, m_content, m_wrapped, m_metadata);
+		Target::get().getHandlerGenerator(m_address, m_trampoline, m_handler, m_content, m_metadata);
 
 	TULIP_HOOK_UNWRAP(generator->generateHandler());
 
@@ -57,8 +45,7 @@ Result<> Handler::init() {
 	auto address = reinterpret_cast<uint8_t*>(Target::get().getRealPtr(m_address));
 	m_originalBytes.insert(m_originalBytes.begin(), address, address + target);
 
-	TULIP_HOOK_UNWRAP_INTO(auto trampolineOffset, generator->relocateOriginal(target));
-	TULIP_HOOK_UNWRAP(generator->generateTrampoline(trampolineOffset));
+	TULIP_HOOK_UNWRAP(generator->generateTrampoline(target));
 
 	this->addOriginal();
 
@@ -69,7 +56,7 @@ void Handler::addOriginal() {
 	auto metadata = HookMetadata{
 		.m_priority = INT32_MAX,
 	};
-	static_cast<void>(this->createHook(Target::get().getRealPtrAs(m_wrapped, m_address), metadata));
+	static_cast<void>(this->createHook(Target::get().getRealPtrAs(m_trampoline, m_address), metadata));
 }
 
 HookHandle Handler::createHook(void* address, HookMetadata m_metadata) {

src/Handler.hpp

@@ -31,8 +31,6 @@ namespace tulip::hook {
 		void* m_trampoline = nullptr;
 		size_t m_trampolineSize = 0;
 
-		void* m_wrapped = nullptr;
-
 		void* m_handler = nullptr;
 		size_t m_handlerSize = 0;
 

src/Main.cpp

@@ -61,6 +61,10 @@ std::shared_ptr<CallingConvention> tulip::hook::createConvention(TulipConvention
 #endif
 		case TulipConvention::Default:
 		default:
+#if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+			return SystemVConvention::create();
+#else
 			return DefaultConvention::create();
+#endif
 	}
 }

src/assembler/ArmV7Assembler.cpp

@@ -1,7 +1,5 @@
 #include "ArmV7Assembler.hpp"
 
-#include <iostream>
-
 using namespace tulip::hook;
 
 ArmV7Assembler::ArmV7Assembler(int64_t baseAddress) :

src/assembler/BaseAssembler.hpp

@@ -19,6 +19,7 @@ namespace tulip::hook {
 		std::vector<uint8_t> m_buffer;
 		std::unordered_map<std::string, int64_t> m_labels;
 		std::vector<AssemblerLabelUpdates> m_labelUpdates;
+		std::vector<AssemblerLabelUpdates> m_absoluteLabelUpdates;
 
 		BaseAssembler(int64_t baseAddress);
 		BaseAssembler(BaseAssembler const&) = delete;

src/assembler/X64Assembler.cpp

@@ -53,6 +53,10 @@ void X64Assembler::updateLabels() {
 	for (auto const& update : m_labelUpdates) {
 		this->rewrite32(update.m_address, m_labels[update.m_name] - update.m_address - 4);
 	}
+	// absolute is not absolute in 64 bit
+	for (auto const& update : m_absoluteLabelUpdates) {
+		this->rewrite32(update.m_address, m_labels[update.m_name] - update.m_address - 4);
+	}
 }
 
 using enum X64Register;
@@ -110,6 +114,14 @@ void X64Assembler::call(X64Register reg) {
 	X86Assembler::call(x86reg(reg));
 }
 
+void X64Assembler::call(int64_t address) {
+	X86Assembler::call(address);
+}
+
+void X64Assembler::call(std::string const& label) {
+	X86Assembler::call(label);
+}
+
 void X64Assembler::callip(std::string const& label) {
 	this->write8(0xff);
 	this->write8(0x15);

src/assembler/X64Assembler.hpp

@@ -43,6 +43,9 @@ namespace tulip::hook {
 	inline X64Pointer operator+(X64Register reg, int32_t offset) {
 		return X64Pointer(reg, offset);
 	}
+	inline X64Pointer operator-(X64Register reg, int32_t offset) {
+		return X64Pointer(reg, -offset);
+	}
 
 	// Use this to easily express a X64Pointer, like so:
 	// RegMem64 m;
@@ -77,6 +80,8 @@ namespace tulip::hook {
 		void jmpip(std::string const& label);
 
 		void call(X64Register reg);
+		void call(int64_t address);
+		void call(std::string const& label);
 		void callip(std::string const& label);
 
 		void movsd(X64Register reg, X64Pointer ptr);

src/assembler/X86Assembler.cpp

@@ -20,8 +20,16 @@ void X86Assembler::label32(std::string const& name) {
 	this->write32(0);
 }
 
+void X86Assembler::abslabel32(std::string const& name) {
+	m_absoluteLabelUpdates.push_back({this->currentAddress(), name, 4});
+	this->write32(0);
+}
+
 void X86Assembler::updateLabels() {
 	for (auto const& update : m_labelUpdates) {
+		this->rewrite32(update.m_address, m_labels[update.m_name] - update.m_address - 4);
+	}
+	for (auto const& update : m_absoluteLabelUpdates) {
 		this->rewrite32(update.m_address, m_labels[update.m_name]);
 	}
 }
@@ -128,6 +136,11 @@ void X86Assembler::call(X86Register reg) {
 	this->encodeModRM(reg, 2);
 }
 
+void X86Assembler::call(std::string const& label) {
+	this->write8(0xE8);
+	this->label32(label);
+}
+
 void X86Assembler::movsd(X86Register reg, X86Pointer ptr) {
 	this->write8(0xF2);
 	this->write8(0x0F);
@@ -171,7 +184,7 @@ void X86Assembler::movaps(X86Pointer ptr, X86Register reg) {
 void X86Assembler::lea(X86Register reg, std::string const& label) {
 	this->write8(0x8D);
 	this->write8(0x05 | regIdx(reg) << 3);
-	this->label32(label);
+	this->abslabel32(label);
 }
 
 void X86Assembler::mov(X86Register reg, int32_t value) {
@@ -197,7 +210,7 @@ void X86Assembler::mov(X86Register dst, X86Register src) {
 void X86Assembler::mov(X86Register reg, std::string const& label) {
 	this->write8(0x8B);
 	this->write8(0x05 | regIdx(reg) << 3);
-	this->label32(label);
+	this->abslabel32(label);
 }
 
 void X86Assembler::fstps(X86Pointer ptr) {

src/assembler/X86Assembler.hpp

@@ -35,6 +35,9 @@ namespace tulip::hook {
 	inline X86Pointer operator+(X86Register reg, int32_t offset) {
 		return X86Pointer(reg, offset);
 	}
+	inline X86Pointer operator-(X86Register reg, int32_t offset) {
+		return X86Pointer(reg, -offset);
+	}
 
 	// Use this to easily express a X86Pointer, like so:
 	// RegMem32 m;
@@ -74,6 +77,7 @@ namespace tulip::hook {
 		~X86Assembler();
 
 		void label32(std::string const& name);
+		void abslabel32(std::string const& name);
 		void updateLabels() override;
 
 		void nop();
@@ -95,6 +99,7 @@ namespace tulip::hook {
 
 		void call(X86Register reg);
 		void call(int64_t value);
+		void call(std::string const& label);
 
 		void movsd(X86Register reg, X86Pointer ptr);
 		void movsd(X86Pointer ptr, X86Register reg);

src/convention/MacosIntelConvention.cpp

@@ -0,0 +1,83 @@
+#include <Platform.hpp>
+
+#if defined(TULIP_HOOK_MACOS) && defined(TULIP_HOOK_X64)
+
+#include <AbstractFunction.hpp>
+#include <platform/MacosIntelConvention.hpp>
+#include "../assembler/X64Assembler.hpp"
+
+using namespace tulip::hook;
+
+namespace {
+    size_t getStackParamSize(AbstractFunction const& function) {
+        size_t stackParamSize = 0;
+        int xmmCount = 0;
+        int gprCount = 0;
+        if (function.m_return.m_kind == AbstractTypeKind::Other && function.m_return.m_size > 16) {
+            gprCount += 1;
+        }
+        for (auto& param : function.m_parameters) {
+            if (param.m_kind == AbstractTypeKind::FloatingPoint) {
+                if (xmmCount < 8) {
+                    xmmCount++;
+                } else {
+                    stackParamSize += 8;
+                }
+            } else {
+                if (gprCount < 6) {
+                    gprCount++;
+                } else {
+                    stackParamSize += 8;
+                }
+            }
+        }
+        return stackParamSize;
+    }
+}
+
+SystemVConvention::~SystemVConvention() {}
+
+void SystemVConvention::generateDefaultCleanup(BaseAssembler& a_, AbstractFunction const& function) {
+    size_t stackParamSize = getStackParamSize(function);
+    if (stackParamSize > 0) {
+        auto& a = static_cast<X64Assembler&>(a_);
+        using enum X64Register;
+        auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize;
+        a.add(RSP, paddedSize);
+    }
+}
+
+// used to move the stack values to the correct places
+
+void SystemVConvention::generateIntoDefault(BaseAssembler& a_, AbstractFunction const& function) {
+    size_t stackParamSize = getStackParamSize(function);
+    if (stackParamSize > 0) {
+        auto& a = static_cast<X64Assembler&>(a_);
+        using enum X64Register;
+        RegMem64 m;
+        auto const paddedSize = (stackParamSize % 16) ? stackParamSize + 8 : stackParamSize;
+        a.sub(RSP, paddedSize);
+        int stackOffset = 0;
+
+        for (auto i = 0; i < stackParamSize; i += 8) {
+            a.mov(RAX, m[RBP + (16 + i)]);
+            a.mov(m[RSP + i], RAX);
+        }
+    }
+}
+
+void SystemVConvention::generateIntoOriginal(BaseAssembler& a_, AbstractFunction const& function) {
+}
+
+void SystemVConvention::generateOriginalCleanup(BaseAssembler& a_, AbstractFunction const& function) {
+}
+
+bool SystemVConvention::needsWrapper(AbstractFunction const& function) const {
+	return false;
+}
+
+std::shared_ptr<SystemVConvention> SystemVConvention::create() {
+	return std::make_shared<SystemVConvention>();
+}
+
+#endif

src/convention/Windows32Convention.cpp

@@ -3,7 +3,6 @@
 #include <AbstractFunction.hpp>
 #include <Platform.hpp>
 #include <algorithm>
-#include <iostream>
 #include <optional>
 #include <platform/Windows32Convention.hpp>
 #include <variant>