diff --git a/docs/pages/todo/brainstorm_now.md b/docs/pages/todo/brainstorm_now.md index 84f5ae751..45c9ca5d9 100644 --- a/docs/pages/todo/brainstorm_now.md +++ b/docs/pages/todo/brainstorm_now.md @@ -10,7 +10,6 @@ summary: That which we are brainstorming at the moment. # BEFORE STEAM - fix small capsule -- Demos should be lz4 compressed and keep max 100 mb worth maybe # REST diff --git a/docs/pages/todo/todo_done.md b/docs/pages/todo/todo_done.md index 53efbb43e..87a88df6b 100644 --- a/docs/pages/todo/todo_done.md +++ b/docs/pages/todo/todo_done.md @@ -6582,3 +6582,7 @@ This will discard your redo history." - i'd unassign them - internal addresses are fucked up again, no port shown and doesnt detect it +- Demos should be lz4 compressed and keep max 100 mb worth maybe + - This can be done asynchronously in main menu, exactly like with neon map regeneration + - It's good because it won't delay getting back to the main menu when client is exited, plus it will work even if the client crashes and doesn't get the chance to compress + diff --git a/src/application/gui/demo_chooser.h b/src/application/gui/demo_chooser.h index b6b7c7c13..88cc12414 100644 --- a/src/application/gui/demo_chooser.h +++ b/src/application/gui/demo_chooser.h @@ -52,7 +52,7 @@ class demo_chooser : keyboard_acquiring_popup { all_paths.clear(); auto path_adder = [this](const auto& full_path) { - if (full_path.extension() != ".dem") { + if (full_path.extension() != ".demc") { return callback_result::CONTINUE; } @@ -63,11 +63,11 @@ class demo_chooser : keyboard_acquiring_popup { try { auto t = augs::open_binary_input_stream(full_path); - decltype(demo_file_meta::server_name) name; - augs::read_bytes(t, name); + demo_file_meta file_meta; + augs::read_bytes(t, file_meta); - meta.server_name = name; - meta.write_time = augs::date_time(augs::last_write_time(full_path)).how_long_ago(); + meta.server_name = file_meta.server_name; + meta.write_time = augs::date_time::from_utc_timestamp(file_meta.when_recorded).how_long_ago(); } catch (const std::ifstream::failure&) { diff --git a/src/application/setups/client/client_setup.cpp b/src/application/setups/client/client_setup.cpp index e2bb248e7..9c4541626 100644 --- a/src/application/setups/client/client_setup.cpp +++ b/src/application/setups/client/client_setup.cpp @@ -48,13 +48,41 @@ #include "augs/readwrite/json_readwrite_errors.h" #include "application/setups/server/file_chunk_packet.h" #include "application/setups/client/direct_file_download.hpp" +#include "augs/misc/compress.h" void client_demo_player::play_demo_from(const augs::path_type& p) { source_path = p; - auto source = augs::open_binary_input_stream(source_path); + auto source_bytes = augs::file_to_bytes(source_path); + auto source = augs::make_ptr_read_stream(source_bytes); augs::read_bytes(source, meta); - augs::read_vector_until_eof(source, demo_steps); + + const auto pos = source.get_read_pos(); + + if (pos < source_bytes.size()) { + std::vector decompressed; + decompressed.resize(meta.uncompressed_size); + + augs::decompress( + source_bytes.data() + pos, + source_bytes.size() - pos, + decompressed + ); + + auto s = augs::make_ptr_read_stream(decompressed); + + try { + while (s.get_read_pos() <= decompressed.size()) { + demo_step step; + augs::read_bytes(s, step); + demo_steps.emplace_back(std::move(step)); + } + } + catch (...) { + + } + } + gui.open(); } @@ -163,6 +191,7 @@ void client_setup::flush_demo_steps() { meta.server_name = displayed_connecting_server_name; meta.server_address = last_addr.address; meta.version = hypersomnia_version(); + meta.when_recorded = augs::date_time().get_utc_timestamp(); augs::write_bytes(out, meta); const auto version_info_path = augs::path_type(recorded_demo_path).replace_extension(".version.txt"); diff --git a/src/application/setups/client/demo_file_meta.h b/src/application/setups/client/demo_file_meta.h index 9934bfc0c..c10505b95 100644 --- a/src/application/setups/client/demo_file_meta.h +++ b/src/application/setups/client/demo_file_meta.h @@ -8,5 +8,7 @@ struct demo_file_meta { server_name_type server_name; address_string_type server_address; hypersomnia_version version; + version_timestamp_string when_recorded; + std::size_t uncompressed_size = 0; // END GEN INTROSPECTOR }; diff --git a/src/augs/misc/compress.cpp b/src/augs/misc/compress.cpp index a69e7eafd..316978155 100644 --- a/src/augs/misc/compress.cpp +++ b/src/augs/misc/compress.cpp @@ -31,19 +31,28 @@ namespace augs { const std::vector& input, std::vector& output ) { + compress(state, input.data(), input.size(), output); + } + + void compress( + std::vector& state, + const std::byte* input, + const std::size_t input_size, + std::vector& output + ) { #if DISABLE_COMPRESSION (void)state; - concatenate(output, input); + output.insert(output.end(), input, input + input_size); #else - const auto size_bound = LZ4_compressBound(input.size()); + const auto size_bound = LZ4_compressBound(input_size); const auto prev_size = output.size(); output.resize(prev_size + size_bound); const auto bytes_written = LZ4_compress_fast_extState( reinterpret_cast(state.data()), - reinterpret_cast(input.data()), + reinterpret_cast(input), reinterpret_cast(output.data() + prev_size), - input.size(), + input_size, size_bound, 1 ); @@ -89,12 +98,12 @@ namespace augs { if (bytes_read < 0) { output.clear(); - throw decompression_error("Decompression failure. Failed to read any bytes."); + throw decompression_error("CHECK IF YOU PASSED CORRECT uncompressed_size! Decompression failure. Failed to read any bytes."); } if (uncompressed_size != static_cast(bytes_read)) { output.clear(); - throw decompression_error("Decompression failure. Read %x bytes, but expected %x.", bytes_read, uncompressed_size); + throw decompression_error("CHECK IF YOU PASSED CORRECT uncompressed_size! Decompression failure. Read %x bytes, but expected %x.", bytes_read, uncompressed_size); } #endif } diff --git a/src/augs/misc/compress.h b/src/augs/misc/compress.h index a597cb16c..1f3c6e66c 100644 --- a/src/augs/misc/compress.h +++ b/src/augs/misc/compress.h @@ -20,6 +20,13 @@ namespace augs { std::vector& output ); + void compress( + std::vector& state, + const std::byte* input, + const std::size_t input_size, + std::vector& output + ); + std::vector decompress( const std::vector& input, std::size_t uncompressed_size diff --git a/src/view/viewables/streaming/viewables_streaming.cpp b/src/view/viewables/streaming/viewables_streaming.cpp index a3c141bdf..43581d326 100644 --- a/src/view/viewables/streaming/viewables_streaming.cpp +++ b/src/view/viewables/streaming/viewables_streaming.cpp @@ -12,6 +12,11 @@ #include "augs/misc/imgui/imgui_control_wrappers.h" #include "augs/misc/imgui/imgui_scope_wrappers.h" #include "augs/filesystem/file.h" +#include "augs/filesystem/directory.h" +#include "augs/misc/compress.h" +#include "augs/readwrite/byte_file.h" +#include "augs/readwrite/to_bytes.h" +#include "application/setups/client/demo_file_meta.h" void viewables_streaming::request_rescan() { if (!general_atlas.empty()) { @@ -338,6 +343,10 @@ void texture_in_progress::finalize_load(augs::renderer& renderer) { } void viewables_streaming::finalize_load(viewables_finalize_input in) { + if (valid_and_is_ready(future_compressed_demos)) { + future_compressed_demos.get(); + } + const auto current_frame = in.current_frame; auto& now_all_defs = now_loaded_viewables_defs; @@ -428,6 +437,82 @@ bool viewables_streaming::general_atlas_in_progress() const { return general_atlas_progress.has_value(); } +void viewables_streaming::wait_demos_compressed() { + if (future_compressed_demos.valid()) { + future_compressed_demos.get(); + } +} + +void viewables_streaming::recompress_demos() { + future_compressed_demos = launch_async([]() { + std::vector file_list; + + augs::for_each_in_directory( + augs::path_type(DEMOS_DIR), + [](auto&&...) { return callback_result::CONTINUE; }, + [&file_list](const augs::path_type& demo) { + if (demo.extension() == ".dem") { + file_list.push_back(demo); + } + + return callback_result::CONTINUE; + } + ); + + if (file_list.size() > 0) { + LOG("%x demos to be compressed.", file_list.size()); + } + else { + LOG("All demos are already compressed."); + } + + auto state = augs::make_compression_state(); + + for (const auto& demo_path : file_list) { + LOG("Compressing: %x", demo_path); + auto contents = augs::file_to_bytes(demo_path); + + demo_file_meta meta; + + auto s = augs::make_ptr_read_stream(contents); + try { + augs::read_bytes(s, meta); + } + catch (...) { + /* Broken/deprecated demo. Just delete it and move on. */ + + augs::remove_file(demo_path); + continue; + } + + const auto pos = s.get_read_pos(); + + if (pos == contents.size()) { + LOG("Demo was empty."); + continue; + } + + meta.uncompressed_size = contents.size() - pos; + auto compressed = augs::to_bytes(meta); + + augs::compress( + state, + contents.data() + pos, + contents.size() - pos, + compressed + ); + + auto new_path = demo_path; + new_path.replace_extension(".demc"); + + augs::bytes_to_file(compressed, new_path); + augs::remove_file(demo_path); + } + + return true; + }); +} + void viewables_streaming::display_loading_progress() const { using namespace augs::imgui; @@ -470,4 +555,12 @@ void viewables_streaming::display_loading_progress() const { ImGui::ProgressBar(progress_percent, ImVec2(-1.0f,0.0f)); } } + else { + if (future_compressed_demos.valid()) { + center_next_window(ImGuiCond_Always); + auto loading_window = scoped_window("Demo file compression in progress", nullptr, ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoFocusOnAppearing); + + text_color("The game is compressing demo files. Please be patient.\n", yellow); + } + } } diff --git a/src/view/viewables/streaming/viewables_streaming.h b/src/view/viewables/streaming/viewables_streaming.h index f97668d59..3180102fb 100644 --- a/src/view/viewables/streaming/viewables_streaming.h +++ b/src/view/viewables/streaming/viewables_streaming.h @@ -78,6 +78,8 @@ struct texture_in_progress { }; class viewables_streaming { + std::future future_compressed_demos; + std::vector general_atlas_pbo_fallback; all_loaded_gui_fonts loaded_gui_fonts; @@ -141,6 +143,8 @@ class viewables_streaming { void display_loading_progress() const; void request_rescan(); + void recompress_demos(); + void wait_demos_compressed(); auto& get_general_atlas() { return general_atlas; diff --git a/src/work.cpp b/src/work.cpp index 5cc940173..51ec65615 100644 --- a/src/work.cpp +++ b/src/work.cpp @@ -1260,12 +1260,15 @@ work_result work(const int argc, const char* const * const argv) try { }); map_catalogue_gui.request_refresh(); + streaming.recompress_demos(); } }; auto launch_client = [&](const bool ignore_nat_check) { bool public_internet = false; + streaming.wait_demos_compressed(); + if (ignore_nat_check) { LOG("Finished NAT traversal. Connecting immediately."); }