fix(cpp): incorrect num_tokens_predicted

mybigday · Oct 5, 2023 · 6dd9865 · 6dd9865
1 parent 68fe09c
commit 6dd9865
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/cpp/rn-llama.hpp b/cpp/rn-llama.hpp
@@ -329,9 +329,11 @@ struct llama_rn_context
             );
         }
 
+        bool tg = true;
         while (n_past < embd.size())
         {
             int n_eval = (int)embd.size() - n_past;
+            tg = n_eval == 1;
             if (n_eval > params.n_batch)
             {
                 n_eval = params.n_batch;
@@ -457,7 +459,9 @@ struct llama_rn_context
             }
             last_n_tokens.erase(last_n_tokens.begin());
             last_n_tokens.push_back(result.tok);
-            num_tokens_predicted++;
+            if (tg) {
+                num_tokens_predicted++;
+            }
         }
 
         // add it to the context