From 76232985006be6e61f719da061cfd089a706d1fa Mon Sep 17 00:00:00 2001 From: geneotech Date: Sun, 22 Oct 2023 21:49:17 +0200 Subject: [PATCH] Compress demos to nearly 10% of original. --- docs/pages/todo/brainstorm_now.md | 1 - docs/pages/todo/todo_done.md | 4 + src/application/gui/demo_chooser.h | 10 +- .../setups/client/client_setup.cpp | 33 ++++++- .../setups/client/demo_file_meta.h | 2 + src/augs/misc/compress.cpp | 21 +++-- src/augs/misc/compress.h | 7 ++ .../streaming/viewables_streaming.cpp | 93 +++++++++++++++++++ .../viewables/streaming/viewables_streaming.h | 4 + src/work.cpp | 3 + 10 files changed, 164 insertions(+), 14 deletions(-) diff --git a/docs/pages/todo/brainstorm_now.md b/docs/pages/todo/brainstorm_now.md index 84f5ae751c..45c9ca5d99 100644 --- a/docs/pages/todo/brainstorm_now.md +++ b/docs/pages/todo/brainstorm_now.md @@ -10,7 +10,6 @@ summary: That which we are brainstorming at the moment. # BEFORE STEAM - fix small capsule -- Demos should be lz4 compressed and keep max 100 mb worth maybe # REST diff --git a/docs/pages/todo/todo_done.md b/docs/pages/todo/todo_done.md index 53efbb43eb..87a88df6b2 100644 --- a/docs/pages/todo/todo_done.md +++ b/docs/pages/todo/todo_done.md @@ -6582,3 +6582,7 @@ This will discard your redo history." - i'd unassign them - internal addresses are fucked up again, no port shown and doesnt detect it +- Demos should be lz4 compressed and keep max 100 mb worth maybe + - This can be done asynchronously in main menu, exactly like with neon map regeneration + - It's good because it won't delay getting back to the main menu when client is exited, plus it will work even if the client crashes and doesn't get the chance to compress + diff --git a/src/application/gui/demo_chooser.h b/src/application/gui/demo_chooser.h index b6b7c7c136..88cc12414f 100644 --- a/src/application/gui/demo_chooser.h +++ b/src/application/gui/demo_chooser.h @@ -52,7 +52,7 @@ class demo_chooser : keyboard_acquiring_popup { all_paths.clear(); auto path_adder = [this](const auto& full_path) { - if (full_path.extension() != ".dem") { + if (full_path.extension() != ".demc") { return callback_result::CONTINUE; } @@ -63,11 +63,11 @@ class demo_chooser : keyboard_acquiring_popup { try { auto t = augs::open_binary_input_stream(full_path); - decltype(demo_file_meta::server_name) name; - augs::read_bytes(t, name); + demo_file_meta file_meta; + augs::read_bytes(t, file_meta); - meta.server_name = name; - meta.write_time = augs::date_time(augs::last_write_time(full_path)).how_long_ago(); + meta.server_name = file_meta.server_name; + meta.write_time = augs::date_time::from_utc_timestamp(file_meta.when_recorded).how_long_ago(); } catch (const std::ifstream::failure&) { diff --git a/src/application/setups/client/client_setup.cpp b/src/application/setups/client/client_setup.cpp index e2bb248e7b..9c45416265 100644 --- a/src/application/setups/client/client_setup.cpp +++ b/src/application/setups/client/client_setup.cpp @@ -48,13 +48,41 @@ #include "augs/readwrite/json_readwrite_errors.h" #include "application/setups/server/file_chunk_packet.h" #include "application/setups/client/direct_file_download.hpp" +#include "augs/misc/compress.h" void client_demo_player::play_demo_from(const augs::path_type& p) { source_path = p; - auto source = augs::open_binary_input_stream(source_path); + auto source_bytes = augs::file_to_bytes(source_path); + auto source = augs::make_ptr_read_stream(source_bytes); augs::read_bytes(source, meta); - augs::read_vector_until_eof(source, demo_steps); + + const auto pos = source.get_read_pos(); + + if (pos < source_bytes.size()) { + std::vector decompressed; + decompressed.resize(meta.uncompressed_size); + + augs::decompress( + source_bytes.data() + pos, + source_bytes.size() - pos, + decompressed + ); + + auto s = augs::make_ptr_read_stream(decompressed); + + try { + while (s.get_read_pos() <= decompressed.size()) { + demo_step step; + augs::read_bytes(s, step); + demo_steps.emplace_back(std::move(step)); + } + } + catch (...) { + + } + } + gui.open(); } @@ -163,6 +191,7 @@ void client_setup::flush_demo_steps() { meta.server_name = displayed_connecting_server_name; meta.server_address = last_addr.address; meta.version = hypersomnia_version(); + meta.when_recorded = augs::date_time().get_utc_timestamp(); augs::write_bytes(out, meta); const auto version_info_path = augs::path_type(recorded_demo_path).replace_extension(".version.txt"); diff --git a/src/application/setups/client/demo_file_meta.h b/src/application/setups/client/demo_file_meta.h index 9934bfc0c1..c10505b95f 100644 --- a/src/application/setups/client/demo_file_meta.h +++ b/src/application/setups/client/demo_file_meta.h @@ -8,5 +8,7 @@ struct demo_file_meta { server_name_type server_name; address_string_type server_address; hypersomnia_version version; + version_timestamp_string when_recorded; + std::size_t uncompressed_size = 0; // END GEN INTROSPECTOR }; diff --git a/src/augs/misc/compress.cpp b/src/augs/misc/compress.cpp index a69e7eafdb..3169781555 100644 --- a/src/augs/misc/compress.cpp +++ b/src/augs/misc/compress.cpp @@ -31,19 +31,28 @@ namespace augs { const std::vector& input, std::vector& output ) { + compress(state, input.data(), input.size(), output); + } + + void compress( + std::vector& state, + const std::byte* input, + const std::size_t input_size, + std::vector& output + ) { #if DISABLE_COMPRESSION (void)state; - concatenate(output, input); + output.insert(output.end(), input, input + input_size); #else - const auto size_bound = LZ4_compressBound(input.size()); + const auto size_bound = LZ4_compressBound(input_size); const auto prev_size = output.size(); output.resize(prev_size + size_bound); const auto bytes_written = LZ4_compress_fast_extState( reinterpret_cast(state.data()), - reinterpret_cast(input.data()), + reinterpret_cast(input), reinterpret_cast(output.data() + prev_size), - input.size(), + input_size, size_bound, 1 ); @@ -89,12 +98,12 @@ namespace augs { if (bytes_read < 0) { output.clear(); - throw decompression_error("Decompression failure. Failed to read any bytes."); + throw decompression_error("CHECK IF YOU PASSED CORRECT uncompressed_size! Decompression failure. Failed to read any bytes."); } if (uncompressed_size != static_cast(bytes_read)) { output.clear(); - throw decompression_error("Decompression failure. Read %x bytes, but expected %x.", bytes_read, uncompressed_size); + throw decompression_error("CHECK IF YOU PASSED CORRECT uncompressed_size! Decompression failure. Read %x bytes, but expected %x.", bytes_read, uncompressed_size); } #endif } diff --git a/src/augs/misc/compress.h b/src/augs/misc/compress.h index a597cb16c5..1f3c6e66c2 100644 --- a/src/augs/misc/compress.h +++ b/src/augs/misc/compress.h @@ -20,6 +20,13 @@ namespace augs { std::vector& output ); + void compress( + std::vector& state, + const std::byte* input, + const std::size_t input_size, + std::vector& output + ); + std::vector decompress( const std::vector& input, std::size_t uncompressed_size diff --git a/src/view/viewables/streaming/viewables_streaming.cpp b/src/view/viewables/streaming/viewables_streaming.cpp index a3c141bdf0..43581d3268 100644 --- a/src/view/viewables/streaming/viewables_streaming.cpp +++ b/src/view/viewables/streaming/viewables_streaming.cpp @@ -12,6 +12,11 @@ #include "augs/misc/imgui/imgui_control_wrappers.h" #include "augs/misc/imgui/imgui_scope_wrappers.h" #include "augs/filesystem/file.h" +#include "augs/filesystem/directory.h" +#include "augs/misc/compress.h" +#include "augs/readwrite/byte_file.h" +#include "augs/readwrite/to_bytes.h" +#include "application/setups/client/demo_file_meta.h" void viewables_streaming::request_rescan() { if (!general_atlas.empty()) { @@ -338,6 +343,10 @@ void texture_in_progress::finalize_load(augs::renderer& renderer) { } void viewables_streaming::finalize_load(viewables_finalize_input in) { + if (valid_and_is_ready(future_compressed_demos)) { + future_compressed_demos.get(); + } + const auto current_frame = in.current_frame; auto& now_all_defs = now_loaded_viewables_defs; @@ -428,6 +437,82 @@ bool viewables_streaming::general_atlas_in_progress() const { return general_atlas_progress.has_value(); } +void viewables_streaming::wait_demos_compressed() { + if (future_compressed_demos.valid()) { + future_compressed_demos.get(); + } +} + +void viewables_streaming::recompress_demos() { + future_compressed_demos = launch_async([]() { + std::vector file_list; + + augs::for_each_in_directory( + augs::path_type(DEMOS_DIR), + [](auto&&...) { return callback_result::CONTINUE; }, + [&file_list](const augs::path_type& demo) { + if (demo.extension() == ".dem") { + file_list.push_back(demo); + } + + return callback_result::CONTINUE; + } + ); + + if (file_list.size() > 0) { + LOG("%x demos to be compressed.", file_list.size()); + } + else { + LOG("All demos are already compressed."); + } + + auto state = augs::make_compression_state(); + + for (const auto& demo_path : file_list) { + LOG("Compressing: %x", demo_path); + auto contents = augs::file_to_bytes(demo_path); + + demo_file_meta meta; + + auto s = augs::make_ptr_read_stream(contents); + try { + augs::read_bytes(s, meta); + } + catch (...) { + /* Broken/deprecated demo. Just delete it and move on. */ + + augs::remove_file(demo_path); + continue; + } + + const auto pos = s.get_read_pos(); + + if (pos == contents.size()) { + LOG("Demo was empty."); + continue; + } + + meta.uncompressed_size = contents.size() - pos; + auto compressed = augs::to_bytes(meta); + + augs::compress( + state, + contents.data() + pos, + contents.size() - pos, + compressed + ); + + auto new_path = demo_path; + new_path.replace_extension(".demc"); + + augs::bytes_to_file(compressed, new_path); + augs::remove_file(demo_path); + } + + return true; + }); +} + void viewables_streaming::display_loading_progress() const { using namespace augs::imgui; @@ -470,4 +555,12 @@ void viewables_streaming::display_loading_progress() const { ImGui::ProgressBar(progress_percent, ImVec2(-1.0f,0.0f)); } } + else { + if (future_compressed_demos.valid()) { + center_next_window(ImGuiCond_Always); + auto loading_window = scoped_window("Demo file compression in progress", nullptr, ImGuiWindowFlags_NoTitleBar | ImGuiWindowFlags_AlwaysAutoResize | ImGuiWindowFlags_NoFocusOnAppearing); + + text_color("The game is compressing demo files. Please be patient.\n", yellow); + } + } } diff --git a/src/view/viewables/streaming/viewables_streaming.h b/src/view/viewables/streaming/viewables_streaming.h index f97668d593..3180102fb3 100644 --- a/src/view/viewables/streaming/viewables_streaming.h +++ b/src/view/viewables/streaming/viewables_streaming.h @@ -78,6 +78,8 @@ struct texture_in_progress { }; class viewables_streaming { + std::future future_compressed_demos; + std::vector general_atlas_pbo_fallback; all_loaded_gui_fonts loaded_gui_fonts; @@ -141,6 +143,8 @@ class viewables_streaming { void display_loading_progress() const; void request_rescan(); + void recompress_demos(); + void wait_demos_compressed(); auto& get_general_atlas() { return general_atlas; diff --git a/src/work.cpp b/src/work.cpp index 5cc9401738..51ec656157 100644 --- a/src/work.cpp +++ b/src/work.cpp @@ -1260,12 +1260,15 @@ work_result work(const int argc, const char* const * const argv) try { }); map_catalogue_gui.request_refresh(); + streaming.recompress_demos(); } }; auto launch_client = [&](const bool ignore_nat_check) { bool public_internet = false; + streaming.wait_demos_compressed(); + if (ignore_nat_check) { LOG("Finished NAT traversal. Connecting immediately."); }