fix: Fix lexer bug on file ending with a directive

lexer.c

@@ -183,17 +183,18 @@ typed_token *next_op(char **inp_ptr, int is_newline)
                 }
                 continue;
             }
-            if (*inp == '\n' || *inp == '\0')
-            {
-                line[sz] = '\0';
-                *inp_ptr = inp;
-                typed_token *dir_tkns = tokenize(line);
-                return new_tkn(TKN_DIRECTIVE, dir_tkns, directive_tkn_debug);
-            }
+
+            if (*inp == '\n')
+                break;
+
             line[sz] = *inp;
             sz++;
             inp++;
         }
+        line[sz] = '\0';
+        *inp_ptr = inp;
+        typed_token *dir_tkns = tokenize(line);
+        return new_tkn(TKN_DIRECTIVE, dir_tkns, directive_tkn_debug);
     }
     if (*inp == '?')
     {
@@ -625,4 +626,8 @@ typed_token *tokenize_file(char *path)
     return tokenize(data);
 ret:
     return NULL;
+}
+
+typed_token *eof_token() {
+    return new_simp_tkn(TKN_EOF);
 }

lexer.h

@@ -13,6 +13,7 @@ typed_token *new_tkn(int tkn_id, void *data, void (*debug)(typed_token *));
 void str_tkn_debug(typed_token *tkn);
 typed_token *tokenize_file(char *path);
 typed_token *tokenize(char *inp);
+typed_token *eof_token();
 
 #define TKN_EOF 0
 #define TKN_VOID 1

preprocess/preprocess.c

@@ -25,6 +25,10 @@ typed_token *chain_tokens(linked_list *tkns)
             curr_tkn->next = (typed_token *)curr->value;
             curr_tkn = curr_tkn->next;
         }
+        else
+        {
+            curr_tkn->next = eof_token();
+        }
     }
     return res_first;
 }