8000 server : (refactor) no more json in server_task input by ngxson · Pull Request #10691 · ggml-org/llama.cpp · GitHub
[go: up one dir, main page]
Skip to content

server : (refactor) no more json in server_task input #10691

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Dec 7, 2024
Merged

server : (refactor) no more json in server_task input #10691

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
db97c8b
server : (refactor) no more json in server_task input
ngxson Dec 6, 2024
9bb1ae6
add test for slots endpoint
ngxson Dec 7, 2024
6bf6e30
Merge branch 'master' into xsn/refactor_server_struct_input
ngxson Dec 7, 2024
e721f4c
add tests for /props and /slots
ngxson Dec 7, 2024
090a113
remove task inf_type
ngxson Dec 7, 2024
65d2e6d
fix CI by adding safe_json_to_str
ngxson Dec 7, 2024
1949f68
add "model_path" to /props
ngxson Dec 7, 2024
89c2af9
update readme
ngxson Dec 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add tests for /props and /slots
  • Loading branch information
@ngxson
ngxson committed Dec 7, 2024
commit e721f4c6b443395e168e05ca96b46fbd424d7635
20 changes: 19 additions & 1 deletion examples/server/tests/unit/test_basic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ def test_server_props():
res = server.make_request("GET", "/props")
assert res.status_code == 200
assert res.body["total_slots"] == server.n_slots
default_val = res.body["default_generation_settings"]
assert server.n_ctx is not None and server.n_slots is not None
assert default_val["n_ctx"] == server.n_ctx / server.n_slots
assert default_val["params"]["seed"] == server.seed


def test_server_models():
Expand All @@ -36,12 +40,26 @@ def test_server_models():

def test_server_slots():
global server

# without slots endpoint enabled, this should return error
server.server_slots = False
server.start()
res = server.make_request("GET", "/slots")
assert res.status_code == 501 # ERROR_TYPE_NOT_SUPPORTED
assert "error" in res.body
server.stop()

# with slots endpoint enabled, this should return slots info
server.server_slots = True
server.n_slots = 2
server.start()
res = server.make_request("GET", "/slots")
assert res.status_code == 200
assert len(res.body) == server.n_slots
assert res.body[0]["n_ctx"] > 0
assert server.n_ctx is not None and server.n_slots is not None
assert res.body[0]["n_ctx"] == server.n_ctx / server.n_slots
assert "params" in res.body[0]
assert res.body[0]["params"]["seed"] == server.seed


def test_load_split_model():
Expand Down
7 changes: 3 additions & 4 deletions examples/server/tests/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ def start(self, timeout_seconds: int = 10) -> None:
else:
server_path = "../../../build/bin/llama-server"
server_args = [
"--slots", # requires to get slot status via /slots endpoint
"--host",
self.server_host,
"--port",
Expand Down Expand Up @@ -184,7 +183,7 @@ def start(self, timeout_seconds: int = 10) -> None:
start_time = time.time()
while time.time() - start_time < timeout_seconds:
try:
response = self.make_request("GET", "/slots", headers={
response = self.make_request("GET", "/health", headers={
"Authorization": f"Bearer {self.api_key}" if self.api_key else None
})
if response.status_code == 200:
Expand Down Expand Up @@ -227,7 +226,7 @@ def make_request(
result.headers = dict(response.headers)
result.status_code = response.status_code
result.body = response.json() if parse_body else None
print("Response from server", result.body)
print("Response from server", json.dumps(result.body, indent=2))
return result

def make_stream_request(
Expand All @@ -248,7 +247,7 @@ def make_stream_request(
break
elif line.startswith('data: '):
data = json.loads(line[6:])
print("Partial response from server", data)
print("Partial response from server", json.dumps(data, indent=2))
yield data


Expand Down
Loading
0