more string heap limits

GaijinEntertainment · Apr 11, 2024 · 3106252 · 3106252
1 parent ad06a1d
commit 3106252
Show file tree

Hide file tree

Showing 28 changed files with 152 additions and 124 deletions.
diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h
@@ -113,6 +113,7 @@ namespace das
         const AnnotationArgument * find ( const string & name, Type type ) const;
         bool getBoolOption(const string & name, bool def = false) const;
         int32_t getIntOption(const string & name, int32_t def = false) const;
+        uint64_t getUInt64Option(const string & name, uint64_t def = false) const;
         void serialize ( AstSerializer & ser );
     };
 

diff --git a/include/daScript/misc/string_writer.h b/include/daScript/misc/string_writer.h
@@ -137,7 +137,7 @@ namespace das {
     public:
         virtual void output() override;
     protected:
-        int pos = 0;
+        uint64_t pos = 0;
         static mutex pmut;
     };
 
@@ -178,7 +178,7 @@ namespace das {
     public:
         LOG ( int level = LogLevel::debug ) : logLevel(level) {}
         virtual void output() override {
-            int newPos = tellp();
+            auto newPos = tellp();
             if (newPos != pos) {
                 string st(data.data() + pos, newPos - pos);
                 logger(logLevel, useMarker ? getLogMarker(logLevel) : "", st.c_str(), /*ctx*/nullptr, /*at*/nullptr);
@@ -188,7 +188,7 @@ namespace das {
             }
         }
     protected:
-        int pos = 0;
+        uint64_t pos = 0;
         int logLevel = LogLevel::debug;
         bool useMarker = true;
     };

diff --git a/include/daScript/simulate/aot.h b/include/daScript/simulate/aot.h
@@ -1815,7 +1815,7 @@ namespace das {
 
     template <typename TT>
     __forceinline char * das_lexical_cast ( TT x, Context * __context__ ) {
-        return __context__->stringHeap->allocateString(to_string(x));
+        return __context__->stringHeap->allocateString(__context__,to_string(x));
     }
 
     __forceinline char * das_string_builder ( Context * __context__, const SimNode_AotInteropBase & node ) {
@@ -1826,7 +1826,7 @@ namespace das {
         }
         auto length = writer.tellp();
         if ( length ) {
-            return __context__->stringHeap->allocateString(writer.c_str(), length);
+            return __context__->stringHeap->allocateString(__context__,writer.c_str(),uint32_t(length));
         } else {
             return nullptr;
         }
@@ -1840,7 +1840,7 @@ namespace das {
         }
         auto length = writer.tellp();
         if ( length ) {
-            auto str = __context__->stringHeap->allocateString(writer.c_str(), length);
+            auto str = __context__->stringHeap->allocateString(__context__,writer.c_str(), uint32_t(length));
             __context__->freeTempString(str);
             return str;
         } else {

diff --git a/include/daScript/simulate/aot_builtin_string.h b/include/daScript/simulate/aot_builtin_string.h
@@ -78,7 +78,7 @@ namespace das {
     __forceinline char * format ( const char * fmt, TT value, Context * context ) {
         char buf[256];
         snprintf(buf, 256, fmt ? fmt : "", value);
-        return context->stringHeap->allocateString(buf, uint32_t(strlen(buf)));
+        return context->stringHeap->allocateString(context, buf, uint32_t(strlen(buf)));
     }
 
     template <typename TT>
@@ -95,7 +95,7 @@ namespace das {
         block(writer);
         auto length = writer.tellp();
         if ( length ) {
-            return context->stringHeap->allocateString(writer.c_str(), length);
+            return context->stringHeap->allocateString(context,writer.c_str(), length);
         } else {
             return nullptr;
         }

diff --git a/include/daScript/simulate/heap.h b/include/daScript/simulate/heap.h
@@ -182,15 +182,17 @@ namespace das {
         }
     };
 
+    class Context;
+
     typedef das_hash_set<StrHashEntry,StrHashPred,StrEqPred> das_string_set;
 
     class StringHeapAllocator : public AnyHeapAllocator {
     public:
         virtual void forEachString ( const callable<void (const char *)> & fn ) = 0;
         virtual void reset() override;
     public:
-        char * allocateString ( const char * text, uint32_t length );
-        char * allocateString ( const string & str );
+        char * allocateString ( Context * context, const char * text, uint32_t length );
+        char * allocateString ( Context * context, const string & str );
         void freeString ( char * text, uint32_t length );
         void setIntern ( bool on );
         bool isIntern() const { return needIntern; }
@@ -326,9 +328,21 @@ namespace das {
     class PersistentStringAllocator final : public StringHeapAllocator {
     public:
         PersistentStringAllocator() { model.alignMask = 3; }
-        virtual char * allocate ( uint32_t size ) override { return model.allocate(size); }
+        virtual char * allocate ( uint32_t size ) override {
+            if ( limit==0 || model.bytesAllocated()+size<=limit ) {
+                return model.allocate(size);
+            } else {
+                return nullptr;
+            }
+        }
         virtual void free ( char * ptr, uint32_t size ) override { model.free(ptr,size); }
-        virtual char * reallocate ( char * ptr, uint32_t oldSize, uint32_t newSize ) override { return model.reallocate(ptr,oldSize,newSize); }
+        virtual char * reallocate ( char * ptr, uint32_t oldSize, uint32_t newSize ) override {
+            if ( limit==0 || model.bytesAllocated()+newSize-oldSize<=limit ) {
+                return model.reallocate(ptr,oldSize,newSize);
+            } else {
+                return nullptr;
+            }
+        }
         virtual int depth() const override { return model.depth(); }
         virtual uint64_t bytesAllocated() const override { return model.bytesAllocated(); }
         virtual uint64_t totalAlignedMemoryAllocated() const override { return model.totalAlignedMemoryAllocated(); }
@@ -354,9 +368,21 @@ namespace das {
     class LinearStringAllocator final : public StringHeapAllocator {
     public:
         LinearStringAllocator() { model.alignMask = 3; }
-        virtual char * allocate ( uint32_t size ) override { return model.allocate(size); }
+        virtual char * allocate ( uint32_t size ) override {
+            if ( limit==0 || model.bytesAllocated()+size<=limit ) {
+                return model.allocate(size);
+            } else {
+                return nullptr;
+            }
+        }
         virtual void free ( char * ptr, uint32_t size ) override { model.free(ptr,size); }
-        virtual char * reallocate ( char * ptr, uint32_t oldSize, uint32_t newSize ) override { return model.reallocate(ptr,oldSize,newSize); }
+        virtual char * reallocate ( char * ptr, uint32_t oldSize, uint32_t newSize ) override {
+            if ( limit==0 || model.bytesAllocated()+newSize-oldSize<=limit ) {
+                return model.reallocate(ptr,oldSize,newSize);
+            } else {
+                return nullptr;
+            }
+        }
         virtual int depth() const override { return model.depth(); }
         virtual uint64_t bytesAllocated() const override { return model.bytesAllocated(); }
         virtual uint64_t totalAlignedMemoryAllocated() const override { return model.totalAlignedMemoryAllocated(); }

diff --git a/include/daScript/simulate/simulate.h b/include/daScript/simulate/simulate.h
@@ -274,14 +274,8 @@ namespace das
 
         void makeWorkerFor(const Context & ctx);
 
-        uint32_t getGlobalSize() const {
-            return globalsSize;
-        }
-
-        uint32_t getSharedSize() const {
-            return sharedSize;
-        }
-
+        uint64_t getGlobalSize() const { return globalsSize; }
+        uint64_t getSharedSize() const { return sharedSize; }
         uint64_t getInitSemanticHash();
 
         __forceinline void * getVariable ( int index ) const {

diff --git a/modules/dasClangBind/src/dasClangBind.main.cpp b/modules/dasClangBind/src/dasClangBind.main.cpp
@@ -12,7 +12,7 @@ namespace das {
 
 char * from_CXString_to_string ( const CXString & cxs, Context * context ) {
     auto cst = clang_getCString(cxs);
-    auto res = context->stringHeap->allocateString(cst);
+    auto res = context->stringHeap->allocateString(context,cst);
     clang_disposeString(cxs);
     return res;
 }

diff --git a/modules/dasImgui b/modules/dasImgui
diff --git a/modules/dasSFML b/modules/dasSFML
diff --git a/modules/dasStdDlg/src/dasStdDlg.cpp b/modules/dasStdDlg/src/dasStdDlg.cpp
@@ -13,15 +13,15 @@ char * GetSaveFileDlg ( const char * initialFileName , const char * initialPath,
         initialPath ? initialPath : "",
         filter ? filter : ""
     );
-    return ctx->stringHeap->allocateString(sf);
+    return ctx->stringHeap->allocateString(ctx,sf);
 }
 
 char * GetOpenFileDlg ( const char * initialPath, const char * filter, Context * ctx ) {
     auto sf = GetOpenFileFromUser(
         initialPath ? initialPath : "",
         filter ? filter : ""
     );
-    return ctx->stringHeap->allocateString(sf);
+    return ctx->stringHeap->allocateString(ctx,sf);
 }
 
 class Module_StdDlg : public Module {

diff --git a/src/ast/ast_aot_cpp.cpp b/src/ast/ast_aot_cpp.cpp
@@ -955,7 +955,7 @@ namespace das {
     public:
         TextWriter                  ss, sti, stg;
     protected:
-        int                         lastNewLine = -1;
+        uint64_t                    lastNewLine = -1ul;
         int                         tab = 0;
         int                         debugInfoGlobal = 0;
         AotDebugInfoHelper          helper;
@@ -1081,7 +1081,7 @@ namespace das {
             return Visitor::visit(that);
         }
     // program body
-        virtual void preVisitProgramBody ( Program * prog, Module * that ) override {
+        virtual void preVisitProgramBody ( Program * prog, Module * ) override {
             // functions
             ss << "\n";
             prog->thisModule->functions.foreach([&](auto fn){

diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp
@@ -60,6 +60,11 @@ namespace das {
         return arg ? arg->iValue : def;
     }
 
+    uint64_t AnnotationArgumentList::getUInt64Option(const string & name, uint64_t def) const {
+        auto arg = find(name, Type::tInt);
+        return arg ? uint64_t(arg->iValue) : def;
+    }
+
     // MODULE
 
     void Module::addDependency ( Module * mod, bool pub ) {

diff --git a/src/ast/ast_print.cpp b/src/ast/ast_print.cpp
@@ -1219,7 +1219,7 @@ namespace das {
         }
     protected:
         TextWriter          ss;
-        int                 lastNewLine = -1;
+        uint64_t            lastNewLine = -1ul;
         int                 tab = 0;
     };
 

diff --git a/src/ast/ast_simulate.cpp b/src/ast/ast_simulate.cpp
@@ -3017,9 +3017,9 @@ namespace das
             context.stringHeap = make_smart<LinearStringAllocator>();
         }
         context.heap->setInitialSize ( options.getIntOption("heap_size_hint", policies.heap_size_hint) );
-        context.heap->setLimit ( options.getIntOption("heap_size_limit", policies.max_heap_allocated) );
+        context.heap->setLimit ( options.getUInt64Option("heap_size_limit", policies.max_heap_allocated) );
         context.stringHeap->setInitialSize ( options.getIntOption("string_heap_size_hint", policies.string_heap_size_hint) );
-        context.stringHeap->setLimit ( options.getIntOption("string_heap_size_limit", policies.max_string_heap_allocated) );
+        context.stringHeap->setLimit ( options.getUInt64Option("string_heap_size_limit", policies.max_string_heap_allocated) );
         context.constStringHeap = make_shared<ConstStringAllocator>();
         if ( globalStringHeapSize ) {
             context.constStringHeap->setInitialSize(globalStringHeapSize);
@@ -3046,11 +3046,11 @@ namespace das
                     gvar.debugInfo = helper.makeVariableDebugInfo(*pvar);
                     gvar.flags = 0;
                     if ( pvar->global_shared ) {
-                        gvar.offset = pvar->stackTop = context.sharedSize;
+                        gvar.offset = pvar->stackTop = (uint32_t) context.sharedSize;
                         gvar.shared = true;
                         context.sharedSize = (context.sharedSize + uint64_t(gvar.size) + 0xful) & ~0xfull;
                     } else {
-                        gvar.offset = pvar->stackTop = context.globalsSize;
+                        gvar.offset = pvar->stackTop = (uint32_t) context.globalsSize;
                         context.globalsSize = (context.globalsSize + uint64_t(gvar.size) + 0xful) & ~0xfull;
                     }
                     gvar.mangledNameHash = pvar->getMangledNameHash();
@@ -3059,11 +3059,11 @@ namespace das
             }
         }
         bool canAllocateVariables = true;
-        if ( context.globalsSize >= policies.max_static_variables_size ) {
+        if ( context.globalsSize >= policies.max_static_variables_size || context.globalsSize >= 0x100000000ul ) {
             error("Global variables size exceeds " + to_string(policies.max_static_variables_size), "Global variables size is " + to_string(context.globalsSize) + " bytes", "", LineInfo());
             canAllocateVariables = false;
         }
-        if ( context.sharedSize >= 0x100000000ul ) {
+        if ( context.sharedSize >= policies.max_static_variables_size || context.sharedSize >= 0x100000000ul ) {
             error("Shared variables size exceeds " + to_string(policies.max_static_variables_size), "Shared variables size is " + to_string(context.sharedSize) + " bytes", "", LineInfo());
             canAllocateVariables = false;
         }

diff --git a/src/builtin/module_builtin_ast.cpp b/src/builtin/module_builtin_ast.cpp
@@ -230,15 +230,15 @@ namespace das {
 
     char * ast_describe_typedecl ( smart_ptr_raw<TypeDecl> t, bool d_extra, bool d_contracts, bool d_module, Context * context, LineInfoArg * at ) {
         if ( !t ) context->throw_error_at(at, "expecting type, not null");
-        return context->stringHeap->allocateString(t->describe(
+        return context->stringHeap->allocateString(context,t->describe(
             d_extra ? TypeDecl::DescribeExtra::yes : TypeDecl::DescribeExtra::no,
             d_contracts ? TypeDecl::DescribeContracts::yes : TypeDecl::DescribeContracts::no,
             d_module ? TypeDecl::DescribeModule::yes : TypeDecl::DescribeModule::no));
     }
 
     char * ast_describe_typedecl_cpp ( smart_ptr_raw<TypeDecl> t, bool d_substitureRef, bool d_skipRef, bool d_skipConst, bool d_redundantConst, Context * context, LineInfoArg * at ) {
         if ( !t ) context->throw_error_at(at, "expecting type, not null");
-        return context->stringHeap->allocateString(describeCppType(t,
+        return context->stringHeap->allocateString(context,describeCppType(t,
             d_substitureRef ? CpptSubstitureRef::yes : CpptSubstitureRef::no,
             d_skipRef ? CpptSkipRef::yes : CpptSkipRef::no,
             d_skipConst ? CpptSkipConst::yes : CpptSkipConst::no,
@@ -249,23 +249,23 @@ namespace das {
         if ( !t ) context->throw_error_at(at, "expecting expression, not null");
         TextWriter ss;
         ss << *t;
-        return context->stringHeap->allocateString(ss.str());
+        return context->stringHeap->allocateString(context,ss.str());
     }
 
     char * ast_describe_function ( smart_ptr_raw<Function> t, Context * context, LineInfoArg * at ) {
         if ( !t ) context->throw_error_at(at, "expecting function, not null");
         TextWriter ss;
         ss << *t;
-        return context->stringHeap->allocateString(ss.str());
+        return context->stringHeap->allocateString(context,ss.str());
     }
 
     char * ast_das_to_string ( Type bt, Context * context ) {
-        return context->stringHeap->allocateString(das_to_string(bt));
+        return context->stringHeap->allocateString(context,das_to_string(bt));
     }
 
     char * ast_find_bitfield_name ( smart_ptr_raw<TypeDecl> bft, Bitfield value, Context * context, LineInfoArg * at ) {
         if ( !bft ) context->throw_error_at(at, "expecting bitfield type, not null");
-        return context->stringHeap->allocateString(bft->findBitfieldName(value));
+        return context->stringHeap->allocateString(context,bft->findBitfieldName(value));
     }
 
     int64_t ast_find_enum_value ( EnumerationPtr enu, const char * value ) {
@@ -408,22 +408,22 @@ namespace das {
 
     char * get_mangled_name ( smart_ptr_raw<Function> func, Context * context, LineInfoArg * at ) {
         if ( !func ) context->throw_error_at(at,"expecting function");
-        return context->stringHeap->allocateString(func->getMangledName());
+        return context->stringHeap->allocateString(context,func->getMangledName());
     }
 
     char * get_mangled_name_t ( smart_ptr_raw<TypeDecl> typ, Context * context, LineInfoArg * at ) {
         if ( !typ ) context->throw_error_at(at,"expecting function");
-        return context->stringHeap->allocateString(typ->getMangledName());
+        return context->stringHeap->allocateString(context,typ->getMangledName());
     }
 
     char * get_mangled_name_v ( smart_ptr_raw<Variable> var, Context * context, LineInfoArg * at ) {
         if ( !var ) context->throw_error_at(at,"expecting function");
-        return context->stringHeap->allocateString(var->getMangledName());
+        return context->stringHeap->allocateString(context,var->getMangledName());
     }
 
     char * get_mangled_name_b ( smart_ptr_raw<ExprBlock> expr, Context * context, LineInfoArg * at ) {
         if ( !expr ) context->throw_error_at(at,"expecting block");
-        return context->stringHeap->allocateString(expr->getMangledName());
+        return context->stringHeap->allocateString(context,expr->getMangledName());
     }
 
     void get_use_global_variables ( smart_ptr_raw<Function> func, const TBlock<void,VariablePtr> & block, Context * context, LineInfoArg * at ) {