Add progress on restart

Checkpoint generates, restart fails during unpacking of LinkMap
feldergast · Apr 18, 2024 · 74417a7 · 74417a7
1 parent d9d53a4
commit 74417a7
Show file tree

Hide file tree

Showing 22 changed files with 448 additions and 95 deletions.
diff --git a/src/sst/core/baseComponent.cc b/src/sst/core/baseComponent.cc
@@ -814,8 +814,13 @@ BaseComponent::getComponentProfileTools(const std::string& point)
 void
 BaseComponent::serialize_order(SST::Core::Serialization::serializer& ser)
 {
+    //printf("basecomponent = %p\n", my_info);
+    printf("BaseComponent->serialize_order %d\n", __LINE__);
     ser& my_info;
+    printf("BaseComponent->serialize_order %d\n", __LINE__);
     ser& isExtension;
+    printf("BaseComponent->serialize_order %d\n", __LINE__);
+
 
     if ( ser.mode() == SST::Core::Serialization::serializer::UNPACK ) { sim_ = Simulation_impl::getSimulation(); }
 }

diff --git a/src/sst/core/component.h b/src/sst/core/component.h
@@ -92,15 +92,15 @@ class Component : public BaseComponent
     */
     void primaryComponentOKToEndSim();
 
+    void serialize_order(SST::Core::Serialization::serializer& ser) override;
+    ImplementSerializable(SST::Component)    
+
 protected:
     friend class SubComponent;
 
-private:
     // For Serialization only
     Component();
 
-    ImplementSerializable(SST::Component)
-    void serialize_order(SST::Core::Serialization::serializer& ser) override;
 };
 
 } // namespace SST

diff --git a/src/sst/core/componentInfo.cc b/src/sst/core/componentInfo.cc
@@ -186,12 +186,19 @@ ComponentInfo::~ComponentInfo()
 void
 ComponentInfo::serialize_order(SST::Core::Serialization::serializer& ser)
 {
+    printf("componentInfo->serialize_order id %d\n", __LINE__);
     ser& const_cast<ComponentId_t&>(id);
+    printf("componentInfo->serialize_order parent_info %d\n", __LINE__);
     ser& parent_info;
+    printf("componentInfo->serialize_order name %d\n", __LINE__);
     ser& const_cast<std::string&>(name);
+    printf("componentInfo->serialize_type %d\n", __LINE__);
     ser& const_cast<std::string&>(type);
+    printf("componentInfo->serialize_order link_map %d\n", __LINE__);
     ser& link_map;
+    printf("componentInfo->serialize_order component %d\n", __LINE__);
     ser& component;
+    printf("componentInfo->serialize_order defaultTimeBase %d\n", __LINE__);
 
     // Not used after construction, no need to serialize
     // ser& params;
@@ -201,6 +208,7 @@ ComponentInfo::serialize_order(SST::Core::Serialization::serializer& ser)
     // ser& allStatConfig;
     // ser& statLoadLevel;
     // ser& coordinates;
+    printf("componentInfo->serialize_order subcomponent stuff %d\n", __LINE__);
     ser& subIDIndex;
     ser& const_cast<std::string&>(slot_name);
     ser& slot_num;

diff --git a/src/sst/core/componentInfo.h b/src/sst/core/componentInfo.h
@@ -308,6 +308,11 @@ class ComponentInfoMap
         }
         dataByID.clear();
     }
+
+    size_t size()
+    {
+        return dataByID.size();
+    }
 };
 
 } // namespace SST

diff --git a/src/sst/core/exit.cc b/src/sst/core/exit.cc
@@ -186,4 +186,27 @@ Exit::check()
     // }
 }
 
+void
+Exit::serialize_order(SST::Core::Serialization::serializer& ser)
+{
+    Action::serialize_order(ser);
+
+    ser& num_threads;
+
+    if (ser.mode() == SST::Core::Serialization::serializer::UNPACK) {
+        m_thread_counts = new unsigned int[num_threads];
+    }
+
+    for (int i = 0; i < num_threads; i++) {
+        ser& m_thread_counts[i];
+    }
+
+    ser& m_refCount;
+    ser& global_count;
+    ser& m_idSet;
+    ser& end_time;
+    ser& single_rank;
+
+    }
+
 } // namespace SST
diff --git a/src/sst/core/exit.h b/src/sst/core/exit.h
@@ -95,20 +95,7 @@ class Exit : public Action
      * TODO to enable different partitioning on restart, will need to associate m_thread_counts and
      * m_idSet back to components so that a new Exit event can be generated on restart
      */
-    void serialize_order(SST::Core::Serialization::serializer& ser) override
-    {
-        Action::serialize_order(ser);
-        ser& num_threads;
-        for ( unsigned int i = 0; i < m_refCount; i++ ) {
-            ser& m_thread_counts[i];
-        }
-        ser& m_refCount;
-        ser& global_count;
-        ser& m_idSet;
-        ser& end_time;
-        ser& single_rank;
-        // TBD spinlock -> can re-create
-    }
+    void serialize_order(SST::Core::Serialization::serializer& ser) override;
     ImplementSerializable(SST::Exit)
 private:
     Exit() {}                    // for serialization only

diff --git a/src/sst/core/heartbeat.cc b/src/sst/core/heartbeat.cc
@@ -116,4 +116,12 @@ SimulatorHeartbeat::execute(void)
     }
 }
 
+void SimulatorHeartbeat::serialize_order(SST::Core::Serialization::serializer& ser)
+{
+    Action::serialize_order(ser);
+    ser& rank;
+    ser& m_period;
+    ser& lastTime;
+}
+
 } // namespace SST
diff --git a/src/sst/core/heartbeat.h b/src/sst/core/heartbeat.h
@@ -38,9 +38,12 @@ class SimulatorHeartbeat : public Action
     SimulatorHeartbeat(Config* cfg, int this_rank, Simulation_impl* sim, TimeConverter* period);
     ~SimulatorHeartbeat();
 
-    NotSerializable(SST::SimulatorHeartbeat)
+    void serialize_order(SST::Core::Serialization::serializer& ser) override;
+    ImplementSerializable(SST::SimulatorHeartbeat)
 
-        private : SimulatorHeartbeat() {};
+private : 
+
+    SimulatorHeartbeat() {};
     SimulatorHeartbeat(const SimulatorHeartbeat&);
 
     void           operator=(SimulatorHeartbeat const&);

diff --git a/src/sst/core/link.cc b/src/sst/core/link.cc
@@ -41,6 +41,7 @@ SST::Core::Serialization::serialize_impl<Link*>::operator()(Link*& s, SST::Core:
     // 1 - Link
     // 2 - SelfLink
     int16_t type;
+    printf("Link->serialize_impl %d\n", __LINE__);
 
     switch ( ser.mode() ) {
     case serializer::SIZER:
@@ -144,15 +145,19 @@ SST::Core::Serialization::serialize_impl<Link*>::operator()(Link*& s, SST::Core:
         }
         break;
     case serializer::UNPACK:
+        printf("link->serialize_order UNPACK type %d\n", __LINE__);
         ser& type;
 
         // If we put in a nullptr, return a nullptr
         if ( type == 0 ) {
+            printf("link->serialize_order UNPACK type=0 %d\n", __LINE__);
             s = nullptr;
             return;
         }
 
         if ( type == 2 ) {
+            printf("link->serialize_order UNPACK type=1 %d\n", __LINE__);
+
             // Self link
             s = new SelfLink();
 
@@ -180,11 +185,14 @@ SST::Core::Serialization::serialize_impl<Link*>::operator()(Link*& s, SST::Core:
             // ser & s->profile_tools;
         }
         else {
+            printf("link->serialize_order UNPACK type=2 %d\n", __LINE__);
+
             // Regular link
 
             // Pull out the tags for the two links
             uintptr_t my_tag;
             uintptr_t pair_tag;
+            printf("link->serialize_order UNPACK tags %d\n", __LINE__);
 
             ser& my_tag;
             ser& pair_tag;
@@ -203,6 +211,8 @@ SST::Core::Serialization::serialize_impl<Link*>::operator()(Link*& s, SST::Core:
                 link_tracker[my_tag] = s;
             }
 
+            printf("link->serialize_order UNPACK %d\n", __LINE__);
+
             ser & s->type;
             ser & s->mode;
             ser & s->tag;
@@ -234,12 +244,14 @@ SST::Core::Serialization::serialize_impl<Link*>::operator()(Link*& s, SST::Core:
             // no need to serialize
 
             // Need to recreate the send_queue
-            if ( s->pair_link->type == Link::SYNC ) {
+            /*if ( s->pair_link->type == Link::SYNC ) {
                 // TODO: Need to reregister with the SyncManager
             }
-            else {
+            else {*/
+                        printf("link->serialize_order UNPACK %d\n", __LINE__);
+
                 s->send_queue = Simulation_impl::getSimulation()->getTimeVortex();
-            }
+            //}
 
             // Profile tools not yet supported
             // ser & s->profile_tools;

diff --git a/src/sst/core/linkMap.h b/src/sst/core/linkMap.h
@@ -36,8 +36,12 @@ class LinkMap
 
     void serialize_order(SST::Core::Serialization::serializer& ser)
     {
+
+        printf("LinkMap->serialize_order %d\n", __LINE__);
         ser& linkMap;
+        printf("LinkMap->serialize_order, %zu %d\n", __LINE__);
         ser& selfPorts;
+        printf("LinkMap->serialize_order, %zu %d\n", __LINE__);
     }
 
 

diff --git a/src/sst/core/main.cc b/src/sst/core/main.cc
@@ -633,6 +633,8 @@ start_simulation(uint32_t tid, SimThreadInfo_t& info, Core::ThreadSafe::Barrier&
     else {
         // Do restart stuff
         printf("Made it to restart section\n");
+        sim->restart(info.config);
+        exit(-1);
     }
     /* Run Simulation */
     sim->run();

diff --git a/src/sst/core/serialization/serializable.cc b/src/sst/core/serialization/serializable.cc
@@ -63,6 +63,7 @@ serializable_factory::add_builder(serializable_builder* builder, const char* nam
         abort();
     }
     current = builder;
+    printf("%s, %" PRIu32 "\n", name, hash);
     return hash;
 }
 

diff --git a/src/sst/core/serialization/serialize.h b/src/sst/core/serialization/serialize.h
@@ -53,6 +53,10 @@ class serialize_impl
             // If it falls through to the default, let's check to see if it's
             // a non-polymorphic class and try to call serialize_order
             if constexpr ( std::is_class_v<std::remove_pointer<T>> && !std::is_polymorphic_v<std::remove_pointer<T>> ) {
+                if (ser.mode() == serializer::UNPACK) {
+                    t = new typename std::remove_pointer<T>::type();
+                    ser.report_new_pointer(reinterpret_cast<uintptr_t>(t));
+                }
                 t->serialize_order(ser);
             }
             else {

diff --git a/src/sst/core/serialization/serialize_priority_queue.h b/src/sst/core/serialization/serialize_priority_queue.h
@@ -67,8 +67,10 @@ class serialize<std::priority_queue<T, S, C>>
         {
             size_t size;
             ser.unpack(size);
+            printf("Unpack priority queue, size = %zu\n", size);
             for ( size_t i = 0; i < size; ++i ) {
                 T    t = {};
+                printf("Attempt serialization of item %zu\n", i);
                 ser& t;
                 v.push(t);
             }

diff --git a/src/sst/core/serialization/serialize_vector.h b/src/sst/core/serialization/serialize_vector.h
@@ -43,16 +43,20 @@ class serialize<std::vector<T>>
         }
         case serializer::UNPACK:
         {
+            printf("serialize_vector UNPACK %d\n", __LINE__);
             size_t s;
             ser.unpack(s);
+            printf("serialize_vector UNPACK size =%zu, %d\n", s, __LINE__);            
             v.resize(s);
             break;
         }
         }
-
+
+        printf("serialize_vector UNPACK %d\n", __LINE__);
         for ( size_t i = 0; i < v.size(); ++i ) {
             ser& v[i];
         }
+        printf("serialize_vector UNPACK %d\n", __LINE__);
     }
 };