limit_concurrency

RossAlRed · web-flow · commit 73ccef95a1db · 2023-11-03T12:07:25.000Z
diff --git a/llama_cpp/server/app.py b/llama_cpp/server/app.py
@@ -157,6 +157,10 @@ class Settings(BaseSettings):
     )
     # Server Params
     host: str = Field(default="localhost", description="Listen address")
+    limit_concurrency: int = Field(
+        default=2,
+        description="Number of concurrent processes + 1"
+    )
     port: int = Field(default=8000, description="Listen port")
     interrupt_requests: bool = Field(
         default=True,
