[WIP] Enable compression for world maps data, with a World menu optio…

…ns to toggle it on/off.

src/main_menu.cpp

@@ -641,6 +641,7 @@ void main_menu::init_strings()
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Sh<o|O>w World Mods" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Copy World Sett<i|I>ngs" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Character to Tem<p|P>late" ) );
+    vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "Toggle World <C|c>ompression" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "<D|d>elete World" ) );
     vWorldSubItems.emplace_back( pgettext( "Main Menu|World", "<R|r>eset World" ) );
 
@@ -1297,7 +1298,7 @@ void main_menu::world_tab( const std::string &worldname )
     uilist mmenu( string_format( _( "Manage world \"%s\"" ), worldname ), {} );
     mmenu.border_color = c_white;
     int opt_val = 0;
-    std::array<char, 5> hotkeys = { 'm', 's', 't', 'd', 'r' };
+    std::array<char, 6> hotkeys = { 'm', 's', 't', 'c', 'd', 'r' };
     for( const std::string &it : vWorldSubItems ) {
         mmenu.entries.emplace_back( opt_val, true, hotkeys[opt_val],
                                     remove_color_tags( shortcut_text( c_white, it ) ) );
@@ -1344,12 +1345,23 @@ void main_menu::world_tab( const std::string &worldname )
                 load_char_templates();
             }
             break;
-        case 3: // Delete World
+        case 3: // Toggle save compression
+            if( world_generator->get_world( worldname )->has_compression_enabled() ) {
+                if( query_yn( _( "Disable save compression?" ) ) ) {
+                    world_generator->get_world( worldname )->set_compression_enabled( false );
+                }
+            } else {
+                if( query_yn( _( "Enable save compression?" ) ) ) {
+                    world_generator->get_world( worldname )->set_compression_enabled( true );
+                }
+            }
+            break;
+        case 4: // Delete World
             if( query_yn( _( "Delete the world and all saves within?" ) ) ) {
                 clear_world( true );
             }
             break;
-        case 4: // Reset World
+        case 5: // Reset World
             if( query_yn( _( "Remove all saves and regenerate world?" ) ) ) {
                 clear_world( false );
             }

src/mapbuffer.cpp

@@ -9,20 +9,28 @@
 #include <utility>
 #include <vector>
 
+#include <zstd/zstd.h>
+#include <zstd/zdict.h>
+
 #include "cata_utility.h"
 #include "debug.h"
 #include "filesystem.h"
 #include "input.h"
 #include "json.h"
+#include "json_loader.h"
 #include "map.h"
+#include "mmap_file.h"
 #include "output.h"
 #include "overmapbuffer.h"
 #include "path_info.h"
+#include "perf.h"
 #include "popup.h"
 #include "string_formatter.h"
 #include "submap.h"
 #include "translations.h"
 #include "ui_manager.h"
+#include "worldfactory.h"
+#include "zzip.h"
 
 #define dbg(x) DebugLog((x),D_MAP) << __FILE__ << ":" << __LINE__ << ": "
 
@@ -32,9 +40,9 @@ extern std::unique_ptr<game> g;
 // NOLINTNEXTLINE(cata-static-declarations)
 extern const int savegame_version;
 
-static cata_path find_quad_path( const cata_path &dirname, const tripoint_abs_omt &om_addr )
+static std::string quad_file_name( const tripoint_abs_omt &om_addr )
 {
-    return dirname / string_format( "%d.%d.%d.map", om_addr.x(), om_addr.y(), om_addr.z() );
+    return string_format( "%d.%d.%d.map", om_addr.x(), om_addr.y(), om_addr.z() );
 }
 
 static cata_path find_dirname( const tripoint_abs_omt &om_addr )
@@ -142,8 +150,20 @@ bool mapbuffer::submap_exists_approx( const tripoint_abs_sm &p )
         try {
             const tripoint_abs_omt om_addr = project_to<coords::omt>( p );
             const cata_path dirname = find_dirname( om_addr );
-            cata_path quad_path = find_quad_path( dirname, om_addr );
-            return file_exist( quad_path );
+            std::string file_name = quad_file_name( om_addr );
+
+            if( world_generator->active_world->has_compression_enabled() ) {
+                cata_path zzip_name = dirname;
+                zzip_name += ".zzip";
+                if( !file_exist( zzip_name ) ) {
+                    return false;
+                }
+                std::shared_ptr<zzip> z = zzip::load( zzip_name.get_unrelative_path(),
+                                                      ( PATH_INFO::world_base_save_path() / "maps.dict" ).get_unrelative_path() );
+                return z->has_file( file_name );
+            } else {
+                return file_exist( dirname / file_name );
+            }
         } catch( const std::exception &err ) {
             debugmsg( "Failed to load submap %s: %s", p.to_string(), err.what() );
         }
@@ -195,7 +215,7 @@ void mapbuffer::save( bool delete_after_save )
         // We're breaking them into subdirectories so there aren't too many files per directory.
         // Might want to make a set for this one too so it's only checked once per save().
         const cata_path dirname = find_dirname( om_addr );
-        const cata_path quad_path = find_quad_path( dirname, om_addr );
+        const cata_path quad_path = dirname / quad_file_name( om_addr );
 
         bool inside_reality_bubble = here.inbounds( om_addr );
         // delete_on_save deletes everything, otherwise delete submaps
@@ -256,44 +276,58 @@ void mapbuffer::save_quad(
         }
     }
 
-    // Don't create the directory if it would be empty
-    assure_dir_exist( dirname );
-    write_to_file( filename, [&]( std::ostream & fout ) {
-        JsonOut jsout( fout );
-        jsout.start_array();
-        for( auto &submap_addr : submap_addrs ) {
-            if( submaps.count( submap_addr ) == 0 ) {
-                continue;
-            }
+    std::stringstream stringout;
+    JsonOut jsout( stringout );
+    jsout.start_array();
+    for( auto &submap_addr : submap_addrs ) {
+        if( submaps.count( submap_addr ) == 0 ) {
+            continue;
+        }
 
-            submap *sm = submaps[submap_addr].get();
+        submap *sm = submaps[submap_addr].get();
 
-            if( sm == nullptr ) {
-                continue;
-            }
+        if( sm == nullptr ) {
+            continue;
+        }
 
-            jsout.start_object();
+        jsout.start_object();
 
-            jsout.member( "version", savegame_version );
-            jsout.member( "coordinates" );
+        jsout.member( "version", savegame_version );
+        jsout.member( "coordinates" );
 
-            jsout.start_array();
-            jsout.write( submap_addr.x() );
-            jsout.write( submap_addr.y() );
-            jsout.write( submap_addr.z() );
-            jsout.end_array();
+        jsout.start_array();
+        jsout.write( submap_addr.x() );
+        jsout.write( submap_addr.y() );
+        jsout.write( submap_addr.z() );
+        jsout.end_array();
 
-            sm->store( jsout );
+        sm->store( jsout );
 
-            jsout.end_object();
+        jsout.end_object();
 
-            if( delete_after_save ) {
-                submaps_to_delete.push_back( submap_addr );
-            }
+        if( delete_after_save ) {
+            submaps_to_delete.push_back( submap_addr );
         }
+    }
 
-        jsout.end_array();
-    } );
+    jsout.end_array();
+
+    std::string s = std::move( stringout ).str();
+
+    if( world_generator->active_world->has_compression_enabled() ) {
+        cata_path zzip_name = dirname;
+        zzip_name += ".zzip";
+        std::shared_ptr<zzip> z = zzip::load( zzip_name.get_unrelative_path(),
+                                              ( PATH_INFO::world_base_save_path() / "maps.dict" ).get_unrelative_path() );
+        z->add_file( filename.get_relative_path().filename(), s );
+        z->compact( 1.5 );
+    } else {
+        // Don't create the directory if it would be empty
+        assure_dir_exist( dirname );
+        write_to_file( filename, [&]( std::ostream & fout ) {
+            fout << s;
+        } );
+    }
 
     if( all_uniform && reverted_to_uniform ) {
         fs::remove( filename.get_unrelative_path() );
@@ -307,28 +341,40 @@ submap *mapbuffer::unserialize_submaps( const tripoint_abs_sm &p )
     // Map the tripoint to the submap quad that stores it.
     const tripoint_abs_omt om_addr = project_to<coords::omt>( p );
     const cata_path dirname = find_dirname( om_addr );
-    cata_path quad_path = find_quad_path( dirname, om_addr );
-
-    if( !file_exist( quad_path ) ) {
-        // Fix for old saves where the path was generated using std::stringstream, which
-        // did format the number using the current locale. That formatting may insert
-        // thousands separators, so the resulting path is "map/1,234.7.8.map" instead
-        // of "map/1234.7.8.map".
-        std::ostringstream buffer;
-        buffer << om_addr.x() << "." << om_addr.y() << "." << om_addr.z()
-               << ".map";
-        cata_path legacy_quad_path = dirname / buffer.str();
-        if( file_exist( legacy_quad_path ) ) {
-            quad_path = std::move( legacy_quad_path );
+    std::string file_name = quad_file_name( om_addr );
+    cata_path quad_path = dirname / file_name;
+
+    if( world_generator->active_world->has_compression_enabled() ) {
+        cata_path zzip_name = dirname;
+        zzip_name += ".zzip";
+        if( !file_exist( zzip_name ) ) {
+            return nullptr;
+        }
+        std::shared_ptr<zzip> z = zzip::load( zzip_name.get_unrelative_path(),
+                                              ( PATH_INFO::world_base_save_path() / "maps.dict" ).get_unrelative_path() );
+        // TODO check if entry exists.
+        if( !z->has_file( file_name ) ) {
+            return nullptr;
+        }
+        std::vector<std::byte> contents = z->get_file( file_name );
+        std::string_view string_contents{ reinterpret_cast<char *>( contents.data() ), contents.size() };
+        JsonValue jsin = json_loader::from_string( string_contents );
+        try {
+            deserialize( jsin );
+        } catch( std::exception &err ) {
+            debugmsg( _( "Failed to read from \"%1$s\": %2$s" ), zzip_name.generic_u8string() + ":" + file_name,
+                      err.what() );
+            return nullptr;
+        }
+    } else {
+        if( !read_from_file_optional_json( quad_path, [this]( const JsonValue & jsin ) {
+        deserialize( jsin );
+        } ) ) {
+            // If it doesn't exist, trigger generating it.
+            return nullptr;
         }
     }
 
-    if( !read_from_file_optional_json( quad_path, [this]( const JsonValue & jsin ) {
-    deserialize( jsin );
-    } ) ) {
-        // If it doesn't exist, trigger generating it.
-        return nullptr;
-    }
     // fill in uniform submaps that were not serialized
     oter_id const oid = overmap_buffer.ter( om_addr );
     generate_uniform_omt( project_to<coords::sm>( om_addr ), oid );

src/options.cpp

@@ -2726,6 +2726,13 @@ void options_manager::add_options_world_default()
 
     add_empty_line();
 
+    add( "WORLD_COMPRESSION", "world_default", to_translation( "World data compression" ),
+         to_translation( "If true, new worlds store data in a compressed format." ),
+         false
+       );
+
+    add_empty_line();
+
     add_option_group( "world_default", Group( "game_world_opts", to_translation( "Game world options" ),
                       to_translation( "Options regarding game world." ) ),
     [&]( const std::string & page_id ) {

src/path_info.cpp

@@ -339,6 +339,14 @@ cata_path PATH_INFO::jsondir()
 {
     return datadir_path_value / "core";
 }
+cata_path PATH_INFO::map_memory_compression_dictionary_path()
+{
+    return base_path_path_value / "mmr.dict";
+}
+cata_path PATH_INFO::maps_compression_dictionary_path()
+{
+    return base_path_path_value / "maps.dict";
+}
 cata_path PATH_INFO::moddir()
 {
     return datadir_path_value / "mods";

src/path_info.h

@@ -74,6 +74,8 @@ cata_path lastworld();
 cata_path legacy_fontdata();
 cata_path memorialdir_path();
 cata_path achievementdir_path();
+cata_path map_memory_compression_dictionary_path();
+cata_path maps_compression_dictionary_path();
 cata_path moddir();
 cata_path mods_dev_default();
 cata_path mods_user_default();

src/worldfactory.cpp

@@ -19,6 +19,7 @@
 #include "debug.h"
 #include "enums.h"
 #include "filesystem.h"
+#include "input.h"
 #include "input_context.h"
 #include "input_popup.h"
 #include "json.h"
@@ -27,13 +28,15 @@
 #include "output.h"
 #include "path_info.h"
 #include "point.h"
+#include "popup.h"
 #include "sounds.h"
 #include "string_formatter.h"
 #include "string_input_popup.h"
 #include "text_snippets.h"
 #include "translations.h"
 #include "ui.h"
 #include "ui_manager.h"
+#include "zzip.h"
 
 // single instance of world generator
 std::unique_ptr<worldfactory> world_generator;
@@ -2053,6 +2056,73 @@ void load_external_option( const JsonObject &jo )
     options_manager::update_options_cache();
 }
 
+bool WORLD::has_compression_enabled() const
+{
+    return fs::exists( ( folder_path() / "maps.dict" ).get_unrelative_path() ) ||
+           fs::exists( ( folder_path() / "mmr.dict" ).get_unrelative_path() );
+}
+
+bool WORLD::set_compression_enabled( bool enabled ) const
+{
+    // If enabled and has_compression_enabled are both true or both false,
+    // we just return immediately. That's computed by inverting the xor.
+    if( !( enabled ^ has_compression_enabled() ) ) {
+        return true;
+    }
+    static_popup popup;
+    if( enabled ) {
+        fs::path maps_dict = PATH_INFO::maps_compression_dictionary_path().get_unrelative_path();
+        std::vector<cata_path> maps_folders = get_directories( folder_path() / "maps" );
+        size_t done = 0;
+        for( const cata_path &map_folder : maps_folders ) {
+            popup.message( _( "Compressing maps [%d/%d]" ), done++, maps_folders.size() );
+            ui_manager::redraw();
+            refresh_display();
+            inp_mngr.pump_events();
+            if( !zzip::create_from_folder( ( map_folder + ".zzip" ).get_unrelative_path(),
+                                           map_folder.get_unrelative_path(), maps_dict ) ) {
+                return false;
+            }
+        }
+        copy_file( PATH_INFO::maps_compression_dictionary_path(), folder_path() / "maps.dict" );
+        done = 0;
+        for( const cata_path &map_folder : maps_folders ) {
+            popup.message( _( "Cleaning up [%d/%d]" ), done++, maps_folders.size() );
+            ui_manager::redraw();
+            refresh_display();
+            inp_mngr.pump_events();
+            std::error_code ec;
+            fs::remove_all( map_folder.get_unrelative_path(), ec );
+        }
+    } else {
+        fs::path maps_dict = ( folder_path() / "maps.dict" ).get_unrelative_path();
+        std::vector<cata_path> zzips = get_files_from_path( "zzip", folder_path() / "maps", false, true );
+        size_t done = 0;
+        for( const cata_path &map_zzip : zzips ) {
+            popup.message( _( "Decompressing maps [%d/%d]" ), done++, zzips.size() );
+            ui_manager::redraw();
+            refresh_display();
+            inp_mngr.pump_events();
+            fs::path zzip_path = map_zzip.get_unrelative_path();
+            fs::path dest_folder_name = zzip_path.parent_path() / zzip_path.stem();
+            if( !zzip::extract_to_folder( zzip_path, dest_folder_name, maps_dict ) ) {
+                return false;
+            }
+        }
+        remove_file( maps_dict );
+        done = 0;
+        for( const cata_path &map_zzip : zzips ) {
+            popup.message( _( "Cleaning up [%d/%d]" ), done++, zzips.size() );
+            ui_manager::redraw();
+            refresh_display();
+            inp_mngr.pump_events();
+            std::error_code ec;
+            fs::remove( map_zzip.get_unrelative_path(), ec );
+        }
+    }
+    return true;
+}
+
 mod_manager &worldfactory::get_mod_manager()
 {
     return *mman;