Merge pull request #10 from ando-masaki/o1-preview-and-mini-fix

Added o1-preview and o1-mini

tokenizer.go

@@ -13,6 +13,8 @@ package tokenizer
 //
 // Alternatively you can request a tokenizer using OpenAI's model name, the
 // following OpenAI models are supported:
+// - O1Preview
+// - O1Mini
 // - GPT4
 // - GPT35Turbo
 // - TextEmbeddingAda002
@@ -92,6 +94,8 @@ type Codec interface {
 type Model string
 
 const (
+	O1Preview                Model = "o1-preview"
+	O1Mini                   Model = "o1-mini"
 	GPT4o                    Model = "gpt-4o"
 	GPT4                     Model = "gpt-4"
 	GPT35Turbo               Model = "gpt-3.5-turbo"
@@ -140,6 +144,7 @@ const (
 )
 
 var modelPrefixToEncoding map[Model]Encoding = map[Model]Encoding{
+	"o1-":              O200kBase,
 	"gpt-4o-":          O200kBase,
 	"gpt-4-":           Cl100kBase,
 	"gpt-3.5-turbo-":   Cl100kBase,
@@ -176,7 +181,7 @@ func Get(encoding Encoding) (Codec, error) {
 // is returned.
 func ForModel(model Model) (Codec, error) {
 	switch model {
-	case GPT4o:
+	case O1Preview, O1Mini, GPT4o:
 		return Get(O200kBase)
 
 	case GPT4, GPT35, GPT35Turbo, TextEmbeddingAda002: