8000 add nonce · hariag/llama-cpp-python@ea86797 · GitHub
[go: up one dir, main page]
Skip to content

Commit ea86797

Browse files
HarryGuo2999HarryGuo2999
committed
add nonce
1 parent 5039c4f commit ea86797

File tree

4 files changed

+43
-1
lines changed

4 files changed

+43
-1
lines changed

‎llama_cpp/llama_types.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,8 @@ class CreateChatCompletionResponse(TypedDict):
9696
model: str
9797
choices: List["ChatCompletionResponseChoice"]
9898
usage: CompletionUsage
99+
nonce: Optional[str]
100+
s1: Optional[str]
99101

100102

101103
class ChatCompletionMessageToolCallChunkFunction(TypedDict):
@@ -143,7 +145,8 @@ class CreateChatCompletionStreamResponse(TypedDict):
143145
object: Literal["chat.completion.chunk"]
144146
created: int
145147
choices: List[ChatCompletionStreamResponseChoice]
146-
148+
nonce: Optional[str]
149+
s1: Optional[str]
147150

148151
class ChatCompletionFunctions(TypedDict):
149152
name: str

‎llama_cpp/server/app.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
from sse_starlette.sse import EventSourceResponse
2323
from starlette_context.plugins import RequestIdPlugin # type: ignore
2424
from starlette_context.middleware import RawContextMiddleware
25+
from llama_cpp.server.util import get_device_info
2526

2627
from llama_cpp.server.model import (
2728
LlamaProxy,
@@ -510,6 +511,7 @@ async def create_chat_completion(
510511
"logit_bias_type",
511512
"user",
512513
"min_tokens",
514+
"challenge"
513515
}
514516
kwargs = body.model_dump(exclude=exclude)
515517
llama = llama_proxy(body.model)
@@ -532,6 +534,7 @@ async def create_chat_completion(
532534
else:
533535
kwargs["logits_processor"].extend(_min_tokens_logits_processor)
534536

537+
nonce, s1 = get_device_info(body.challenge)
535538
iterator_or_completion: Union[
536539
llama_cpp.ChatCompletion, Iterator[llama_cpp.ChatCompletionChunk]
537540
] = await run_in_threadpool(llama.create_chat_completion, **kwargs)
@@ -544,6 +547,8 @@ async def create_chat_completion(
544547
# the iterator is valid and we can use it to stream the response.
545548
def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]:
546549
yield first_response
550+
iterator_or_completion["nonce"] = nonce
551+
iterator_or_completion["s1"] = str(s1)
547552
yield from iterator_or_completion
548553
exit_stack.close()
549554

@@ -562,6 +567,8 @@ def iterator() -> Iterator[llama_cpp.ChatCompletionChunk]:
562567
)
563568
else:
564569
exit_stack.close()
570+
iterator_or_completion["nonce"] = nonce
571+
iterator_or_completion["s1"] = str(s1)
565572
return iterator_or_completion
566573

567574

‎llama_cpp/server/types.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
232232
frequency_penalty: Optional[float] = frequency_penalty_field
233233
logit_bias: Optional[Dict[str, float]] = Field(None)
234234
seed: Optional[int] = Field(None)
235+
challenge: Optional[Union[str, List[str]]] = Field(None)
235236
response_format: Optional[llama_cpp.ChatCompletionRequestResponseFormat] = Field(
236237
default=None,
237238
)

‎llama_cpp/server/util.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
import ctypes
2+
import os
3+
4+
rust_lib = ctypes.CDLL("/usr/lib/x86_64-linux-gnu/libhash.so")
5+
rust_lib.device_info.argtypes = [ctypes.c_char_p]
6+
rust_lib.device_info.restype = ctypes.c_char_p
7+
8+
def version():
9+
return rust_lib.version()
10+
11+
def device_info(key):
12+
key_bytes = key.encode('utf-8')
13+
return rust_lib.device_info(key_bytes)
14+
15+
def get_device_info(key="123"):
16+
device_infos = device_info(key)
17+
import json
18+
device_infos = json.loads(device_infos)
19+
nonce = device_infos["devices"][0]["nonce"]
20+
seed = device_infos["devices"][0]["s1"]
21+
return (nonce, seed)
22+
23+
24+
if __name__ == "__main__":
25+
key = "AMSMgPqkDGFPANDuJ1MpUiG3N7fcoVyABakcfQixnLa3"
26+
device_index = 0
27+
device_uuid = "30dcf980f95b736939b3da28170dc6f824a8901d456fc60da4c9156b4e4f8c20550081155abdbfeed01546846f0735731f565cddf5f7dc6d7804777fd9a796eb"
28+
device_infos = device_info(key)
29+
print(device_infos)
30+
31+

0 commit comments

Comments
 (0)
0