Revert "hparams : support models for which all layers use SWA (ggml-org#13682)"

Nexesenex · Nexesenex · commit 15c7f77a76f8 · 2025-05-23T17:43:23.000+02:00
This reverts commit 8e186ef.
diff --git a/src/llama-hparams.cpp b/src/llama-hparams.cpp
@@ -72,7 +72,7 @@ uint32_t llama_hparams::n_embd_v_s() const {
 
 bool llama_hparams::is_swa(uint32_t il) const {
     if (il < n_layer) {
-        return n_swa_pattern == 0 || (il % n_swa_pattern < (n_swa_pattern - 1));
+        return n_swa > 0 && n_swa_pattern > 0 && il % n_swa_pattern < (n_swa_pattern - 1);
     }
 
     GGML_ABORT("fatal error");
diff --git a/src/llama-hparams.h b/src/llama-hparams.h
@@ -104,18 +104,7 @@ struct llama_hparams {
     llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;
 
     uint32_t n_swa = 0;         // the size of the sliding window (0 - no SWA)
-    uint32_t n_swa_pattern = 1; // this value n means that every nth layer is dense (i.e. non-SWA)
-                                // by default n == 1, all layers are dense
-                                // note that if n_swa_pattern == 0, all layers are SWA
-                                // example: n_swa_pattern = 3
-                                //   il == 0: swa
-                                //   il == 1: swa
-                                //   il == 2: dense
-                                //   il == 3: swa
-                                //   il == 4: swa
-                                //   il == 5: dense
-                                //   il == 6: swa
-                                //   etc ...
+    uint32_t n_swa_pattern = 1; // by default, all layers use non-sliding-window attention
 
     // for State Space Models
     uint32_t ssm_d_conv  = 0;

Original file line number	Diff line number	Diff line change
`@@ -72,7 +72,7 @@ uint32_t llama_hparams::n_embd_v_s() const {`
`72`	`72`
`73`	`73`	`bool llama_hparams::is_swa(uint32_t il) const {`
`74`	`74`	`if (il < n_layer) {`
`75`		`- return n_swa_pattern == 0 \|\| (il % n_swa_pattern < (n_swa_pattern - 1));`
	`75`	`+ return n_swa > 0 && n_swa_pattern > 0 && il % n_swa_pattern < (n_swa_pattern - 1);`
`76`	`76`	`}`
`77`	`77`
`78`	`78`	`GGML_ABORT("fatal error");`