Description
so i am currently using LLamaSharp like this
public StatefulChatService ( IConfiguration configuration )
{
_configuration = configuration;
modelPath = Path.Combine(_configuration [ "ModelPath" ], _configuration [ "ModelName" ]);
systemPrompt = File.ReadAllText(_configuration [ "ChatWithBase" ]);
_context = new LLamaContext(new LLama.Common.ModelParams(modelPath)
{
ContextSize = 2048,
GpuLayerCount = 50,
MainGpu=0
});
_chatSession = new ChatSession(new InteractiveExecutor(_context));
_chatSession.AddInputTransform(new MyInputTransform1());
_chatSession.AddInputTransform(new MyInputTransform2());
_chatSession.AddInputTransform(new MyInputTransform3());
}
public void Dispose ()
{
_chatSession.SaveSession(_configuration [ "SavedSessionPath" ]);
_context?.Dispose();
}
public async Task<string> ChatAsync ( SendMessageInput input )
{
var userInput = input.Text;
if (!_continue)
{
userInput = systemPrompt + userInput;
Console.Write(systemPrompt);
_continue = true;
}
Console.ForegroundColor = ConsoleColor.Green;
Console.Write(input.Text);
Console.ForegroundColor = ConsoleColor.White;
var outputs = _chatSession.ChatAsync(userInput, new LLama.Common.InferenceParams()
{
RepeatPenalty = 1.0f,
AntiPrompts = new string [ ] { "User:" },
FrequencyPenalty = 1.0f,
Temperature=0.5f
});
var result = "";
await foreach (var output in outputs)
{
Console.Write(output);
result += output;
}
return result;
}
but the issue i am encountering is that i cant seem to let this run on my gpu it only uses my cpu and ram