Loading assets from disk

Cargo.toml

@@ -4,6 +4,7 @@ members = [
     "crates/launcher-attach-protocol", 
     "crates/mod-host",
     "crates/mod-protocol",
+    "crates/mod-host-assets",
 ]
 resolver = "2"
 
@@ -22,6 +23,7 @@ expect-test = "1.5.0"
 me3-launcher-attach-protocol = { path = "crates/launcher-attach-protocol" }
 me3-mod-host = { path = "crates/mod-host" }
 me3-mod-protocol = { path = "crates/mod-protocol" }
+me3-mod-host-assets = { path = "crates/mod-host-assets" }
 schemars = "0.8"
 serde = "1"
 serde_derive = "1"

crates/launcher/src/game.rs

@@ -52,7 +52,7 @@ impl Game {
         let process = OwnedProcess::from_pid(pid)?;
 
         // TODO: no hardcoded timeout.
-        let _ = process.wait_for_module_by_name("kernel32", Duration::from_secs(5));
+        let _ = process.wait_for_module_by_name("kernel32", Duration::from_secs(300));
         let injector = Syringe::for_process(process);
         let module = injector.inject(dll_path)?;
         let payload: RemotePayloadProcedure<AttachFunction> = unsafe {

crates/launcher/src/main.rs

@@ -54,14 +54,16 @@ fn run() -> LauncherResult<()> {
 
     // TODO: merge
     if let Some(mut profile) = profiles.into_iter().next() {
-        let ordered_natives = sort_dependencies(profile.natives())
-            .ok_or_eyre("failed to create dependency graph for natives")?;
-
-        let ordered_packages = sort_dependencies(profile.packages())
-            .ok_or_eyre("failed to create dependency graph for packages")?;
-
-        natives.extend(ordered_natives);
-        packages.extend(ordered_packages);
+        // let ordered_natives = sort_dependencies(profile.natives())
+        //     .ok_or_eyre("failed to create dependency graph for natives")?;
+        //
+        // let ordered_packages = sort_dependencies(profile.packages())
+        //     .ok_or_eyre("failed to create dependency graph for packages")?;
+        //
+        // natives.extend(ordered_natives);
+        // packages.extend(ordered_packages);
+        natives.extend(profile.natives());
+        packages.extend(profile.packages());
     }
 
     let request = AttachRequest { natives, packages };

crates/mod-host-assets/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "me3-mod-host-assets"
+version.workspace = true
+edition.workspace = true
+repository.workspace = true
+license.workspace = true
+
+[dependencies]
+cxx = { version = "1.0", features = ["c++17"] }
+thiserror.workspace = true
+me3-mod-protocol.workspace = true
+
+[build-dependencies]
+cxx-build = "1.0"
+
+[lints]
+workspace = true

crates/mod-host-assets/build.rs

@@ -0,0 +1,9 @@
+fn main() {
+    cxx_build::bridge("src/lib.rs")
+        .include("include")
+        .compile("test");
+
+    println!("cargo:rerun-if-changed=src/lib.rs");
+    println!("cargo:rerun-if-changed=include/dl_allocator.hpp");
+    println!("cargo:rerun-if-changed=include/dl_string_bridge.hpp");
+}

crates/mod-host-assets/include/dl_allocator.hpp

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <cstdint>
+#include <type_traits>
+
+namespace DLKR {
+class DLAllocationInterface {
+public:
+    virtual ~DLAllocationInterface() = default;
+    virtual uint32_t GetAllocatorId() = 0;
+    virtual int32_t _unk0x10() = 0;
+    virtual uint32_t& GetHeapFlags(uint32_t& out) = 0;
+    virtual uint64_t GetHeapCapacity() = 0;
+    virtual uint64_t GetHeapSize() = 0;
+    virtual uint64_t GetBackingHeapCapacity() = 0;
+    virtual uint64_t GetAllocationCount() = 0;
+    virtual uint64_t GetSizeOfAllocation(void* pAllocation) = 0;
+    virtual void* AllocateMemory(uint64_t sizeBytes) = 0;
+    virtual void* AllocateAlignedMemory(uint64_t sizeBytes, uint64_t alignment) = 0;
+    virtual void* ReallocateMemory(void* pAllocation, uint64_t sizeBytes) = 0;
+    virtual void* ReallocateAlignedMemory(void* pAllocation, uint64_t sizeBytes, uint64_t alignment) = 0;
+    virtual void FreeMemory(void* pAllocation) = 0;
+};
+
+template <typename T>
+class DLAllocatorAdapter {
+public:
+    using value_type = T;
+    using size_type = uint64_t;
+    using difference_type = int64_t;
+
+    using propagate_on_container_move_assignment = std::true_type;
+    using is_always_equal = std::false_type;
+
+    template <typename U>
+    DLAllocatorAdapter(const DLAllocatorAdapter<U>& other) noexcept : allocator(other.allocator) {}
+
+    T* allocate(size_type count) {
+        return reinterpret_cast<T*>(allocator.AllocateAlignedMemory(count * sizeof(T), alignof(T)));
+    }
+
+    void deallocate(T* pAllocation, size_type count = 0) {
+        allocator.FreeMemory(reinterpret_cast<void*>(pAllocation));
+    }
+
+    template <typename T1, typename T2>
+    friend bool operator==(const DLAllocatorAdapter<T1>& lhs, const DLAllocatorAdapter<T2>& rhs) noexcept {
+        return &lhs.allocator == &rhs.allocator;
+    }
+
+private:
+    DLAllocationInterface& allocator;
+};
+
+}

crates/mod-host-assets/include/dl_string_bridge.hpp

@@ -0,0 +1,27 @@
+#pragma once
+
+#include <string>
+#include <rust/cxx.h>
+#include "dl_allocator.hpp"
+
+#if defined(_ITERATOR_DEBUG_LEVEL) && _ITERATOR_DEBUG_LEVEL > 0
+#error "_ITERATOR_DEBUG_LEVEL" must be defined as "0" for STL containers to be compatible with the ELDEN RING ABI.
+#endif
+
+template <typename CharT>
+struct DLBasicString {
+    mutable std::basic_string<CharT, std::char_traits<CharT>, DLKR::DLAllocatorAdapter<CharT>> inner;
+    bool _unk0x28;
+};
+
+using DLWString = DLBasicString<char16_t>;
+
+rust::string get_dlwstring_contents(const DLWString& str) noexcept {
+    return rust::string(str.inner.data(), str.inner.length());
+}
+
+void set_dlwstring_contents(const DLWString& str, rust::slice<const uint16_t> contents) noexcept {
+    const char16_t* first = reinterpret_cast<const char16_t*>(contents.data());
+    size_t length = contents.size();
+    str.inner.assign(first, length);
+}

crates/mod-host-assets/src/lib.rs

@@ -0,0 +1,17 @@
+use std::{error::Error, fs::File, sync::{OnceLock, RwLock}};
+
+use ffi::DLWString;
+
+pub mod mapping;
+
+#[cxx::bridge]
+pub mod ffi {
+    unsafe extern "C++" {
+        include!("dl_string_bridge.hpp");
+
+        pub type DLWString;
+
+        pub fn get_dlwstring_contents(string: &DLWString) -> String;
+        pub fn set_dlwstring_contents(string: &DLWString, contents: &[u16]);
+    }
+}

crates/mod-host-assets/src/mapping.rs

@@ -0,0 +1,177 @@
+use std::{
+    collections::{HashMap, VecDeque},
+    fs,
+    os::windows::ffi::OsStrExt,
+    path::{Path, PathBuf, StripPrefixError},
+};
+
+use me3_mod_protocol::package::AssetOverrideSource;
+use thiserror::Error;
+
+#[derive(Debug, Default)]
+pub struct ArchiveOverrideMapping {
+    map: HashMap<String, Vec<u16>>,
+}
+
+#[derive(Debug, Error)]
+pub enum ArchiveOverrideMappingError {
+    #[error("Package source specified is not a directory {0}.")]
+    InvalidDirectory(PathBuf),
+
+    #[error("Could not read directory while discovering override assets {0}")]
+    ReadDir(std::io::Error),
+
+    #[error("Could not acquire directory entry")]
+    DirEntryAcquire(std::io::Error),
+
+    #[error("Could not acquire directory entry")]
+    StripPrefix(#[from] StripPrefixError),
+}
+
+impl ArchiveOverrideMapping {
+    pub fn scan_directories<I, S>(&mut self, sources: I) -> Result<(), ArchiveOverrideMappingError>
+    where
+        I: Iterator<Item = S>,
+        S: AssetOverrideSource,
+    {
+        sources
+            .map(|p| self.scan_directory(p.asset_path()))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        Ok(())
+    }
+
+    ///  Traverses a folder structure, mapping discovered assets into itself.
+    fn scan_directory<P: AsRef<Path>>(
+        &mut self,
+        base_directory: P,
+    ) -> Result<(), ArchiveOverrideMappingError> {
+        let base_directory = normalize_path(base_directory.as_ref());
+        if (!base_directory.is_dir()) {
+            return Err(ArchiveOverrideMappingError::InvalidDirectory(
+                base_directory.clone(),
+            ));
+        }
+
+        // Keep a dequeue to push yet-to-be-scanned paths so we can avoid
+        // recursion.
+        let mut paths_to_scan = VecDeque::from(vec![base_directory.clone()]);
+        while let Some(current_path) = paths_to_scan.pop_front() {
+            for dir_entry in
+                fs::read_dir(current_path).map_err(ArchiveOverrideMappingError::ReadDir)?
+            {
+                let dir_entry = dir_entry
+                    .map_err(ArchiveOverrideMappingError::DirEntryAcquire)?
+                    .path();
+
+                // Push to the path deque for later scanning, if it's a folder
+                // add it to the mapping otherwise.
+                if dir_entry.is_dir() {
+                    paths_to_scan.push_back(dir_entry);
+                } else {
+                    let override_path = normalize_path(dir_entry);
+                    let vfs_path = path_to_asset_lookup_key(&base_directory, &override_path)?;
+
+                    self.map.insert(
+                        vfs_path,
+                        override_path.into_os_string().encode_wide().collect(),
+                    );
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    pub fn get_override(&self, path: &str) -> Option<&[u16]> {
+        let key = path.split_once(":/").map(|r| r.1).unwrap_or(path);
+
+        self.map.get(key).map(|v| &v[..])
+    }
+}
+
+/// Normalizes paths to use / as a path seperator.
+fn normalize_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    PathBuf::from(path.as_ref().to_string_lossy().replace('\\', "/"))
+}
+
+/// Turns an asset path into an asset lookup key using the mods base path.
+fn path_to_asset_lookup_key<P: AsRef<Path>>(base: P, path: P) -> Result<String, StripPrefixError> {
+    path.as_ref()
+        .strip_prefix(base)
+        .map(|p| p.to_string_lossy().to_lowercase())
+}
+
+#[cfg(test)]
+mod test {
+    use std::{
+        collections::HashMap,
+        path::{Path, PathBuf},
+    };
+
+    use crate::mapping::{path_to_asset_lookup_key, ArchiveOverrideMapping};
+
+    #[test]
+    fn asset_path_lookup_keys() {
+        const FAKE_MOD_BASE: &str = "D:/ModBase/";
+        let base_path = PathBuf::from(FAKE_MOD_BASE);
+
+        assert_eq!(
+            path_to_asset_lookup_key(
+                &base_path,
+                &PathBuf::from(format!(
+                    "{FAKE_MOD_BASE}/parts/aet/aet007/aet007_071.tpf.dcx"
+                )),
+            )
+            .unwrap(),
+            "parts/aet/aet007/aet007_071.tpf.dcx",
+        );
+
+        assert_eq!(
+            path_to_asset_lookup_key(
+                &base_path,
+                &PathBuf::from(format!(
+                    "{FAKE_MOD_BASE}/hkxbnd/m60_42_36_00/h60_42_36_00_423601.hkx.dcx"
+                )),
+            )
+            .unwrap(),
+            "hkxbnd/m60_42_36_00/h60_42_36_00_423601.hkx.dcx",
+        );
+
+        assert_eq!(
+            path_to_asset_lookup_key(
+                &base_path,
+                &PathBuf::from(format!("{FAKE_MOD_BASE}/regulation.bin")),
+            )
+            .unwrap(),
+            "regulation.bin",
+        );
+    }
+
+    #[test]
+    fn scan_directory_and_overrides() {
+        let mut asset_mapping = ArchiveOverrideMapping::default();
+
+        let test_mod_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("test-data/test-mod");
+        asset_mapping.scan_directory(test_mod_dir).unwrap();
+
+        assert!(
+            asset_mapping
+                .get_override("data0:/regulation.bin")
+                .is_some(),
+            "override for regulation.bin was not found"
+        );
+        assert!(
+            asset_mapping
+                .get_override("data0:/event/common.emevd.dcx")
+                .is_some(),
+            "override for event/common.emevd.dcx not found"
+        );
+        assert!(
+            asset_mapping
+                .get_override("data0:/common.emevd.dcx")
+                .is_none(),
+            "event/common.emevd.dcx was found incorrectly under the regulation root"
+        );
+    }
+}

crates/mod-host-assets/test-data/test-mod/event/common.emevd.dcx

@@ -0,0 +1 @@
+Empty file for testing override discovery.

crates/mod-host-assets/test-data/test-mod/regulation.bin

@@ -0,0 +1 @@
+Empty file for testing override discovery.

crates/mod-host/Cargo.toml

@@ -15,14 +15,17 @@ eyre = "0.6"
 iced-x86 = "1.21.0"
 me3-mod-protocol.workspace = true
 me3-launcher-attach-protocol.workspace = true
+me3-mod-host-assets.workspace = true
 retour = { git = "https://github.com/Hpmason/retour-rs", features = ["static-detour"] }
 thiserror.workspace = true
 windows = { version = "0.54", features = [
     "Win32_System_Memory",
     "Win32_System_Diagnostics_Debug",
     "Win32_System_Threading",
     "Win32_System_SystemInformation",
+    "Win32_System_Kernel",
+    "Win32_Foundation",
 ] }
 
 [lints]
-workspace = true
+workspace = true