Fix per token atrributes bits (ggerganov#7749)

l3utterfly · Jun 4, 2024 · c90dbe0 · c90dbe0
1 parent b90dc56
commit c90dbe0
Showing 1 changed file with 10 additions and 10 deletions.
diff --git a/llama.h b/llama.h
@@ -109,16 +109,16 @@ extern "C" {
 
     enum llama_token_attr {
         LLAMA_TOKEN_ATTR_UNDEFINED    = 0,
-        LLAMA_TOKEN_ATTR_UNKNOWN      = 1 <<  1,
-        LLAMA_TOKEN_ATTR_UNUSED       = 1 <<  2,
-        LLAMA_TOKEN_ATTR_NORMAL       = 1 <<  3,
-        LLAMA_TOKEN_ATTR_CONTROL      = 1 <<  4,  // SPECIAL?
-        LLAMA_TOKEN_ATTR_USER_DEFINED = 1 <<  5,
-        LLAMA_TOKEN_ATTR_BYTE         = 1 <<  6,
-        LLAMA_TOKEN_ATTR_NORMALIZED   = 1 <<  7,
-        LLAMA_TOKEN_ATTR_LSTRIP       = 1 <<  8,
-        LLAMA_TOKEN_ATTR_RSTRIP       = 1 <<  9,
-        LLAMA_TOKEN_ATTR_SINGLE_WORD  = 1 << 10,
+        LLAMA_TOKEN_ATTR_UNKNOWN      = 1 << 0,
+        LLAMA_TOKEN_ATTR_UNUSED       = 1 << 1,
+        LLAMA_TOKEN_ATTR_NORMAL       = 1 << 2,
+        LLAMA_TOKEN_ATTR_CONTROL      = 1 << 3,  // SPECIAL?
+        LLAMA_TOKEN_ATTR_USER_DEFINED = 1 << 4,
+        LLAMA_TOKEN_ATTR_BYTE         = 1 << 5,
+        LLAMA_TOKEN_ATTR_NORMALIZED   = 1 << 6,
+        LLAMA_TOKEN_ATTR_LSTRIP       = 1 << 7,
+        LLAMA_TOKEN_ATTR_RSTRIP       = 1 << 8,
+        LLAMA_TOKEN_ATTR_SINGLE_WORD  = 1 << 9,
     };
 
     // model file types