Qwen2.5: fix block size for Coder series (#1856)

Lightning-AI · Dec 7, 2024 · 9750eb6 · 9750eb6
1 parent 7865e8a
commit 9750eb6
Showing 1 changed file with 4 additions and 4 deletions.
diff --git a/litgpt/config.py b/litgpt/config.py
@@ -1928,7 +1928,7 @@ def norm_class(self) -> Type:
     dict(
         name="Qwen2.5-Coder-1.5B{}",
         hf_config=dict(org="Qwen", name="Qwen2.5-Coder-1.5B{}"),
-        block_size=131072,
+        block_size=32768,
         vocab_size=151643,
         padded_vocab_size=151936,
         n_layer=28,
@@ -1970,7 +1970,7 @@ def norm_class(self) -> Type:
     dict(
         name="Qwen2.5-Coder-7B{}",
         hf_config=dict(org="Qwen", name="Qwen2.5-Coder-7B{}"),
-        block_size=131072,
+        block_size=32768,
         vocab_size=151643,
         padded_vocab_size=152064,
         n_layer=28,
@@ -1991,7 +1991,7 @@ def norm_class(self) -> Type:
     dict(
         name="Qwen2.5-Coder-14B{}",
         hf_config=dict(org="Qwen", name="Qwen2.5-Coder-14B{}"),
-        block_size=131072,
+        block_size=32768,
         vocab_size=151643,
         padded_vocab_size=152064,
         n_layer=48,
@@ -2012,7 +2012,7 @@ def norm_class(self) -> Type:
     dict(
         name="Qwen2.5-Coder-32B{}",
         hf_config=dict(org="Qwen", name="Qwen2.5-Coder-32B{}"),
-        block_size=131072,
+        block_size=32768,
         vocab_size=151643,
         padded_vocab_size=152064,
         n_layer=64,