Fix for negative temp (sample_softmax)

tk-master · tk-master · commit e2ab96a43332 · 2023-11-17T01:14:34.000+02:00
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -1111,7 +1111,8 @@ def sample(
             )
 
         if temp < 0.0:
-            id = self._ctx.sample_softmax(candidates=self._candidates)
+            self._ctx.sample_softmax(candidates=self._candidates)
+            id = self._candidates.candidates.data[0].id
         elif temp == 0.0:
             id = self._ctx.sample_token_greedy(candidates=self._candidates)
         elif mirostat_mode == 1:

Original file line number	Diff line number	Diff line change
`@@ -1111,7 +1111,8 @@ def sample(`
`1111`	`1111`	`)`
`1112`	`1112`
`1113`	`1113`	`if temp < 0.0:`
`1114`		`- id = self._ctx.sample_softmax(candidates=self._candidates)`
	`1114`	`+ self._ctx.sample_softmax(candidates=self._candidates)`
	`1115`	`+ id = self._candidates.candidates.data[0].id`
`1115`	`1116`	`elif temp == 0.0:`
`1116`	`1117`	`id = self._ctx.sample_token_greedy(candidates=self._candidates)`
`1117`	`1118`	`elif mirostat_mode == 1:`