diff --git a/configs/stories/llama2/1m.json b/configs/stories/llama2/1m.json
index 52f1c893..4d3d7cbb 100644
--- a/configs/stories/llama2/1m.json
+++ b/configs/stories/llama2/1m.json
@@ -1,6 +1,6 @@
 {
   "model_config": {
-    "hidden_size": 84,
+    "hidden_size": 82,
     "intermediate_size": 256,
     "num_attention_heads": 8,
     "num_hidden_layers": 4,
diff --git a/configs/stories/llama2/2.5m.json b/configs/stories/llama2/2.5m.json
index 4d55904c..0be616ec 100644
--- a/configs/stories/llama2/2.5m.json
+++ b/configs/stories/llama2/2.5m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 168,
-    "intermediate_size": 384,
+    "hidden_size": 176,
+    "intermediate_size": 352,
     "num_attention_heads": 8,
     "num_hidden_layers": 4,
     "num_key_value_heads": 4
diff --git a/configs/stories/llama2/250k.json b/configs/stories/llama2/250k.json
index 7a4ed066..baa81f2e 100644
--- a/configs/stories/llama2/250k.json
+++ b/configs/stories/llama2/250k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 28,
-    "intermediate_size": 96,
+    "hidden_size": 30,
+    "intermediate_size": 68,
     "num_attention_heads": 4,
     "num_hidden_layers": 2,
     "num_key_value_heads": 2
diff --git a/configs/stories/llama2/500k.json b/configs/stories/llama2/500k.json
index c4e0ec8e..060062d8 100644
--- a/configs/stories/llama2/500k.json
+++ b/configs/stories/llama2/500k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 52,
-    "intermediate_size": 184,
+    "hidden_size": 54,
+    "intermediate_size": 144,
     "num_attention_heads": 4,
     "num_hidden_layers": 2,
     "num_key_value_heads": 2
diff --git a/configs/stories/llama2/50k.json b/configs/stories/llama2/50k.json
index 53afb500..73060f6f 100644
--- a/configs/stories/llama2/50k.json
+++ b/configs/stories/llama2/50k.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 6,
-    "intermediate_size": 24,
+    "hidden_size": 8,
+    "intermediate_size": 16,
     "num_attention_heads": 2,
     "num_hidden_layers": 1,
     "num_key_value_heads": 1
diff --git a/configs/stories/llama2/5m.json b/configs/stories/llama2/5m.json
index 839221f6..795cb768 100644
--- a/configs/stories/llama2/5m.json
+++ b/configs/stories/llama2/5m.json
@@ -1,7 +1,7 @@
 {
   "model_config": {
-    "hidden_size": 232,
-    "intermediate_size": 512,
+    "hidden_size": 240,
+    "intermediate_size": 480,
     "num_attention_heads": 12,
     "num_hidden_layers": 6,
     "num_key_value_heads": 6