8000 Support O3 OpenAI model mapping (#7394) · dotnet/machinelearning@99723e7 · GitHub
[go: up one dir, main page]
Skip to content

Commit 99723e7

Browse files
tarekghCopilot
tarekgh
and
Copilot
authored
Support O3 OpenAI model mapping (#7394)
* Support O3 OpenAI model mapping * Update test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent a99f8cb commit 99723e7

File tree

2 files changed

+7
-0
lines changed

2 files changed

+7
-0
lines changed

‎src/Microsoft.ML.Tokenizers/Model/TiktokenTokenizer.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1026,6 +1026,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
10261026
[
10271027
// chat
10281028
( "o1-", ModelEncoding.O200kBase ), // e.g. o1-mini
1029+
( "o3-", ModelEncoding.O200kBase ), // e.g. o3-mini
10291030
( "gpt-4o-", ModelEncoding.O200kBase), // e.g., gpt-4o-2024-05-13
10301031
( "gpt-4-", ModelEncoding.Cl100kBase), // e.g., gpt-4-0314, etc., plus gpt-4-32k
10311032
( "gpt-3.5-", ModelEncoding.Cl100kBase), // e.g, gpt-3.5-turbo-0301, -0401, etc.
@@ -1038,6 +1039,7 @@ private static readonly (string Prefix, ModelEncoding Encoding)[] _modelPrefixTo
10381039
// chat
10391040
{ "gpt-4o", ModelEncoding.O200kBase },
10401041
{ "o1", ModelEncoding.O200kBase },
1042+
{ "o3", ModelEncoding.O200kBase },
10411043
{ "gpt-4", ModelEncoding.Cl100kBase },
10421044
{ "gpt-3.5-turbo", ModelEncoding.Cl100kBase },
10431045
{ "gpt-3.5-turbo-16k", ModelEncoding.Cl100kBase },

‎test/Microsoft.ML.Tokenizers.Tests/TiktokenTests.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,10 @@ public void TestEncodeR50kBase()
394394
[Theory]
395395
[InlineData("o1")]
396396
[InlineData("o1-")]
397+
[InlineData("o1-mini")]
398+
[InlineData("o3")]
399+
[InlineData("o3-")]
400+
[InlineData("o3-mini")]
397401
[InlineData("gpt-4o")]
398402
[InlineData("gpt-4o-")]
399403
[InlineData(&qu 6A27 ot;gpt-4")]
@@ -496,6 +500,7 @@ public void TestEncodingNamesNegativeCases()
496500
[InlineData("gpt-4")]
497501
[InlineData("gpt-4o")]
498502
[InlineData("o1")]
503+
[InlineData("o3")]
499504
[InlineData("text-davinci-003")]
500505
[InlineData("text-curie-001")]
501506
[InlineData("text-davinci-edit-001")]

0 commit comments

Comments
 (0)
0