8000 Adding TinyLlama for CPU inference · Embarcadero/llama-cpp-delphi@9a25fe5 · GitHub
[go: up one dir, main page]
Skip to content

Commit 9a25fe5

Browse files
lmbelolmbelo
committed
Adding TinyLlama for CPU inference
1 parent 07b1d59 commit 9a25fe5

File tree

5 files changed

+225
-33
lines changed

5 files changed

+225
-33
lines changed

‎samples/SimpleChatWithDownload/DownloadForm.fmx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ object FormDownload: TFormDownload
2222
Size.Height = 280.000000000000000000
2323
Size.PlatformDefault = False
2424
TabOrder = 0
25-
Viewport.Width = 396.000000000000000000
26-
Viewport.Height = 276.000000000000000000
25+
Viewport.Width = 400.000000000000000000
26+
Viewport.Height = 280.000000000000000000
2727
end
2828
object LlamaDownload1: TLlamaDownload
2929
OnWriteData = LlamaDownload1WriteData

‎samples/SimpleChatWithDownload/DownloadForm.pas

Lines changed: 40 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,13 @@ TFormDownload = class(TForm)
1515

1616
procedure Download(const ALlama: TLlama; const ATask: TFunc<string>);
1717
public
18+
// HF Auth
19+
procedure HFAuth(const AUserName, AToken: string);
20+
1821
procedure DownloadAndPrepareLlama2(const ALlama: TLlama);
1922
procedure DownloadAndPrepareLlama3(const ALlama: TLlama);
2023
procedure DownloadAndPrepareMistralLite(const ALlama: TLlama);
24+
procedure DownloadAndPrepareTinyLlama(const ALlama: TLlama);
2125
end;
2226

2327
var
@@ -40,22 +44,27 @@ procedure TFormDownload.Download(const ALlama: TLlama;
4044
+ sLineBreak + sLineBreak);
4145

4246
TTask.Run(procedure() begin
43-
ALlama.ModelPath := ATask;
44-
45-
TThread.Queue(nil, procedure() begin
46-
memoDownload.Lines.Add('Loading...');
47-
end);
48-
49-
ALlama.Init();
50-
51-
TThread.Queue(nil, procedure() begin
52-
memoDownload.Lines.Add(String.Empty);
53-
memoDownload.Lines.Add('All done!');
54-
end);
55-
56-
TThread.ForceQueue(nil, procedure() begin
57-
Self.Close();
58-
end, 500);
47+
try
48+
ALlama.ModelPath := ATask;
49+
50+
TThread.Queue(nil, procedure() begin
51+
memoDownload.Lines.Add('Loading...');
52+
end);
53+
54+
ALlama.Init();
55+
56+
TThread.Queue(nil, procedure() begin
57+
memoDownload.Lines.Add(String.Empty);
58+
memoDownload.Lines.Add('All done!');
59+
end);
60+
61+
TThread.ForceQueue(nil, procedure() begin
62+
Self.Close();
63+
end, 500);
64+
except
65+
on E: Exception do
66+
Application.ShowException(E);
67+
end;
5968
end);
6069

6170
Self.ShowModal();
@@ -97,4 +106,19 @@ procedure TFormDownload.LlamaDownload1WriteData(Sender: TObject;
97106
end);
98107
end;
99108

109+
procedure TFormDownload.DownloadAndPrepareTinyLlama(const ALlama: TLlama);
110+
begin
111+
ALlama.Settings.ChatFormat := 'zephyr';
112+
113+
Download(ALlama, function(): string begin
114+
Result := LlamaDownload1.DownloadTinyLlama_1_1B()[0];
115+
end);
116+
end;
117+
118+
procedure TFormDownload.HFAuth(const AUserName, AToken: string);
119+
begin
120+
LLamaDownload1.HuggingFace.UserName := AUserName;
121+
LLamaDownload1.HuggingFace.Token := AToken;
122+
end;
123+
100124
end.

‎samples/SimpleChatWithDownload/MainForm.fmx

Lines changed: 90 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ object FormMain: TFormMain
1616
OnCreate = FormCreate
1717
OnCloseQuery = FormCloseQuery
1818
OnShow = FormShow
19+
OnSaveState = FormSaveState
1920
DesignerMasterStyle = 1
2021
object StyleBook1: TStyleBook
2122
Styles = <
@@ -1030,7 +1031,7 @@ object FormMain: TFormMain
10301031
Size.Width = 250.000000000000000000
10311032
Size.Height = 40.000000000000000000
10321033
Size.PlatformDefault = False
1033-
TabOrder = 0
1034+
TabOrder = 6
10341035
object lbMenuHeader: TLabel
10351036
Align = Client
10361037
Size.Width = 250.000000000000000000
@@ -1047,7 +1048,7 @@ object FormMain: TFormMain
10471048
Size.Width = 250.000000000000000000
10481049
Size.Height = 50.000000000000000000
10491050
Size.PlatformDefault = False
1050-
TabOrder = 1
1051+
TabOrder = 0
10511052
object lbModel: TLabel
10521053
Align = Top
10531054
Margins.Left = 5.000000000000000000
@@ -1056,14 +1057,15 @@ object FormMain: TFormMain
10561057
Size.Height = 17.000000000000000000
10571058
Size.PlatformDefault = False
10581059
Text = 'Select a model:'
1059-
TabOrder = 0
1060+
TabOrder = 1
10601061
end
10611062
object cbModel: TComboBox
10621063
Align = Top
10631064
Items.Strings = (
10641065
'Llama 2 (~4 GB)'
10651066
'Llama 3 (~5 GB)'
1066-
'Mistral Lite (~7 GB)')
1067+
'Mistral Lite (~7 GB)'
1068+
'TinyLlama (~1 GB)')
10671069
Margins.Left = 5.000000000000000000
10681070
Margins.Top = 5.000000000000000000
10691071
Margins.Right = 5.000000000000000000
@@ -1072,7 +1074,7 @@ object FormMain: TFormMain
10721074
Size.Width = 240.000000000000000000
10731075
Size.Height = 22.000000000000000000
10741076
Size.PlatformDefault = False
1075-
TabOrder = 1
1077+
TabOrder = 0
10761078
end
10771079
end
10781080
object layContextLength: TLayout
@@ -1081,7 +1083,7 @@ object FormMain: TFormMain
10811083
Size.Width = 250.000000000000000000
10821084
Size.Height = 50.000000000000000000
10831085
Size.PlatformDefault = False
1084-
TabOrder = 4
1086+
TabOrder = 1
10851087
object lblContextLength: TLabel
10861088
Align = Top
10871089
Margins.Left = 5.000000000000000000
@@ -1090,12 +1092,12 @@ object FormMain: TFormMain
10901092
Size.Height = 17.000000000000000000
10911093
Size.PlatformDefault = False
10921094
Text = 'Context Length:'
1093-
TabOrder = 0
1095+
TabOrder = 1
10941096
end
10951097
object nbContextLength: TNumberBox
10961098
Touch.InteractiveGestures = [LongTap, DoubleTap]
10971099
Align = Top
1098-
TabOrder = 1
1100+
TabOrder = 0
10991101
Cursor = crIBeam
11001102
Max = 4096.000000000000000000
11011103
Value = 4096.000000000000000000
@@ -1115,7 +1117,7 @@ object FormMain: TFormMain
11151117
Size.Width = 250.000000000000000000
11161118
Size.Height = 21.000000000000000000
11171119
Size.PlatformDefault = False
1118-
TabOrder = 3
1120+
TabOrder = 2
11191121
object cbSeed: TCheckBox
11201122
Align = Top
11211123
IsChecked = True
@@ -1135,23 +1137,23 @@ object FormMain: TFormMain
11351137
Size.Width = 250.000000000000000000
11361138
Size.Height = 40.000000000000000000
11371139
Size.PlatformDefault = False
1138-
TabOrder = 2
1140+
TabOrder = 3
11391141
object switchCPUGPU: TSwitch
11401142
Align = Client
11411143
IsChecked = True
11421144
Margins.Left = 10.000000000000000000
11431145
Margins.Top = 8.000000000000000000
11441146
Margins.Right = 10.000000000000000000
11451147
Margins.Bottom = 8.000000000000000000
1146-
Size.Width = 132.000000000000000000
1148+
Size.Width = 131.333343505859400000
11471149
Size.Height = 24.000000000000000000
11481150
Size.PlatformDefault = False
11491151
TabOrder = 0
11501152
end
11511153
object lbCPU: TLabel
11521154
Align = Left
11531155
AutoSize = True
1154-
Size.Width = 49.000000000000000000
1156+
Size.Width = 49.333332061767580000
11551157
Size.Height = 40.000000000000000000
11561158
Size.PlatformDefault = False
11571159
TextSettings.HorzAlign = Center
@@ -1161,15 +1163,89 @@ object FormMain: TFormMain
11611163
object lbGPU: TLabel
11621164
Align = Right
11631165
AutoSize = True
1164-
Position.X = 201.000000000000000000
1165-
Size.Width = 49.000000000000000000
1166+
Position.X = 200.666671752929700000
1167+
Size.Width = 49.333328247070310000
11661168
Size.Height = 40.000000000000000000
11671169
Size.PlatformDefault = False
11681170
TextSettings.HorzAlign = Center
11691171
Text = 'GPU'
11701172
TabOrder = 1
11711173
end
11721174
end
1175+
object lblHuggingFaceAuth: TLabel
1176+
Align = Top
1177+
Margins.Top = 5.000000000000000000
1178+
Margins.Bottom = 5.000000000000000000
1179+
Position.Y = 206.000000000000000000
1180+
Size.Width = 250.000000000000000000
1181+
Size.Height = 17.000000000000000000
1182+
Size.PlatformDefault = False
1183+
TextSettings.HorzAlign = Center
1184+
Text = 'HuggingFace Authentication'
1185+
TabOrder = 7
1186+
end
1187+
object layHFUserName: TLayout
1188+
Align = Top
1189+
Position.Y = 228.000000000000000000
1190+
Size.Width = 250.000000000000000000
1191+
Size.Height = 50.000000000000000000
1192+
Size.PlatformDefault = False
1193+
TabOrder = 4
1194+
object lblHFUserName: TLabel
1195+
Align = Top
1196+
Margins.Left = 5.000000000000000000
1197+
Position.X = 5.000000000000000000
1198+
Size.Width = 245.000000000000000000
1199+
Size.Height = 17.000000000000000000
1200+
Size.PlatformDefault = False
1201+
Text = 'User Name:'
1202+
TabOrder = 0
1203+
end
1204+
object edtHFUserName: TEdit
1205+
Touch.InteractiveGestures = [LongTap, DoubleTap]
1206+
Align = Top
1207+
TabOrder = 1
1208+
Position.X = 5.000000000000000000
1209+
Position.Y = 22.000000000000000000
1210+
Margins.Left = 5.000000000000000000
1211+
Margins.Top = 5.000000000000000000
1212+
Margins.Right = 5.000000000000000000
1213+
Size.Width = 240.000000000000000000
1214+
Size.Height = 22.000000000000000000
1215+
Size.PlatformDefault = False
1216+
end
1217+
end
1218+
object layHFToken: TLayout
1219+
Align = Top
1220+
Position.Y = 278.000000000000000000
1221+
Size.Width = 250.000000000000000000
1222+
Size.Height = 50.000000000000000000
1223+
Size.PlatformDefault = False
1224+
TabOrder = 5
1225+
object lblHFToken: TLabel
1226+
Align = Top
1227+
Margins.Left = 5.000000000000000000
1228+
Position.X = 5.000000000000000000
1229+
Size.Width = 245.000000000000000000
1230+
Size.Height = 17.000000000000000000
1231+
Size.PlatformDefault = False
1232+
Text = 'Token:'
1233+
TabOrder = 0
1234+
end
1235+
object edtHFToken: TEdit
1236+
Touch.InteractiveGestures = [LongTap, DoubleTap]
1237+
Align = Top
1238+
TabOrder = 1
1239+
Position.X = 5.000000000000000000
1240+
Position.Y = 22.000000000000000000
1241+
Margins.Left = 5.000000000000000000
1242+
Margins.Top = 5.000000000000000000
1243+
Margins.Right = 5.000000000000000000
1244+
Size.Width = 240.000000000000000000
1245+
Size.Height = 22.000000000000000000
1246+
Size.PlatformDefault = False
1247+
end
1248+
end
11731249
end
11741250
object Llama1: TLlama
11751251
Settings.NGpuLayers = -1

‎samples/SimpleChatWithDownload/MainForm.pas

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,13 @@ TFormMain = class(TForm)
3636
switchCPUGPU: TSwitch;
3737
lbCPU: TLabel;
3838
lbGPU: TLabel;
39+
lblHuggingFaceAuth: TLabel;
40+
layHFUserName: TLayout;
41+
lblHFUserName: TLabel;
42+
layHFToken: TLayout;
43+
lblHFToken: TLabel;
44+
edtHFToken: TEdit;
45+
edtHFUserName: TEdit;
3946
procedure FormCreate(Sender: TObject);
4047
procedure Llama1ChatCompletionStreamComplete(Sender: TObject);
4148
procedure Llama1ChatCompletionStream(Sender: TObject;
@@ -47,6 +54,7 @@ TFormMain = class(TForm)
4754
procedure FormShow(Sender: TObject);
4855
procedure MultiView1StartHiding(Sender: TObject);
4956
procedure FormCloseQuery(Sender: TObject; var CanClose: Boolean);
57+
procedure FormSaveState(Sender: TObject);
5058
private
5159
FStatus: byte; // 0 - idle/cancelling 1 - generating
5260
FChatMessages: TArray<TChatCompletionRequestMessage>;
@@ -93,6 +101,29 @@ procedure TFormMain.FormCreate(Sender: TObject);
93101
'You''re a master in the Delphi programming language,')];
94102

95103
FDownloadForm := TFormDownload.Create(Self);
104+
105+
if SaveState.Stream.Size > 0 then begin
106+
var LReader := TBinaryReader.Create(SaveState.Stream);
107+
try
108+
edtHFUserName.Text := LReader.ReadString();
109+
edtHFToken.Text := LReader.ReadString();
110+
finally
111+
LReader.Free();
112+
end;
113+
end;
114+
end;
115+
116+
procedure TFormMain.FormSaveState(Sender: TObject);
117+
begin
118+
SaveState.Stream.Clear();
119+
120+
var LWriter := TBinaryWriter.Create(SaveState.Stream);
121+
try
122+
LWriter.Write(edtHFUserName.Text);
123+
LWriter.Write(edtHFToken.Text);
124+
finally
125+
LWriter.Free();
126+
end;
96127
end;
97128

98129
procedure TFormMain.FormShow(Sender: TObject);
@@ -156,6 +187,9 @@ procedure TFormMain.MultiView1StartHiding(Sender: TObject);
156187
if not cbModel.Enabled then
157188
Exit;
158189

190+
// HF Auth
191+
FDownloadForm.HFAuth(edtHFUserName.Text, edtHFToken.Text);
192+
159193
Llama1.Settings.NCtx := Trunc(nbContextLength.Value);
160194
if switchCPUGPU.IsChecked then
161195
Llama1.Settings.NGpuLayers := -1
@@ -174,6 +208,10 @@ procedure TFormMain.MultiView1StartHiding(Sender: TObject);
174208
2: begin
175209
FDownloadForm.DownloadAndPrepareMistralLite(Llama1);
176210
Caption := Caption + ' (MistralLite)';
211+
end;
212+
3: begin
213+
FDownloadForm.DownloadAndPrepareTinyLlama(Llama1);
214+
Caption := Caption + ' (TinyLlama)';
177215
end
178216
else raise Exception.Create('Select a model.');
179217
end;

0 commit comments

Comments
 (0)
0