refactoring the scanner into two classes

Author: Cesar

CMakeLists.txt

@@ -21,4 +21,4 @@ include_directories(/lib/error)
 add_executable(lox_test tests/lox_main_test.cpp)
 target_link_libraries(lox_test PRIVATE Catch2::Catch2WithMain)
 
-add_executable(interpreter main.cpp lib/scanner.h lib/error/report.h)
+add_executable(interpreter main.cpp lib/parser/scanner.h lib/error/report.h lib/parser/tokens.h)

lib/scanner.h lib/parser/scanner.h

@@ -11,50 +11,8 @@
 #include <string>
 #include <vector>
 #include <sstream>
-#include "error/report.h"
-
-
-enum TokenType {
-    LEFT_PAREN, RIGHT_PAREN, LEFT_BRACE, RIGHT_BRACE,
-    COMMA, DOT, MINUS, PLUS, SEMICOLON, SLASH, STAR, BANG,
-    BANG_EQUAL, EQUAL, EQUAL_EQUAL, GREATER, GREATER_EQUAL,
-    LESS, LESS_EQUAL,
-
-    // Literals.
-    IDENTIFIER, STRING, NUMBER,
-
-    // Keywords.
-    AND, CLASS, ELSE, FALSE, FUN, FOR, IF, NIL, OR,
-    PRINT, RETURN, SUPER, THIS, TRUE, VAR, WHILE, _EOF,
-};
-
-class Object{};
-
-class Token {
-private:
-    TokenType type;
-    std::string lexeme;
-    Object literal;
-    int line;
-
-public:
-    Token(TokenType _type, std::string _lexeme, Object _literal, int _line):
-    type{_type},
-    lexeme(_lexeme),
-    literal(_literal),
-    line{_line}
-    {}
-
-    std::string toString(){
-        std::stringstream ss;
-        ss << "type" << " " << lexeme << " " << "literal";
-        return ss.str();
-    }
-
-    std::string getLexeme(){
-        return lexeme;
-    }
-};
+#include "../error/report.h"
+#include "tokens.h"
 
 class Scanner {
 private:
@@ -159,7 +117,6 @@ class Scanner {
 };
 
 
-
 std::vector<std::string> tokenizer(std::string basicString, char placeholder = ' '){
     auto size = basicString.size();
     std::vector<std::string> tokens;

lib/parser/tokens.h

@@ -0,0 +1,55 @@
+//
+// Created by Cesar Valdez on 13/02/2023.
+//
+
+#ifndef INTERPRETER_TOKENS_H
+#define INTERPRETER_TOKENS_H
+#include <iostream>
+
+enum TokenType {
+    LEFT_PAREN, RIGHT_PAREN, LEFT_BRACE, RIGHT_BRACE,
+    COMMA, DOT, MINUS, PLUS, SEMICOLON, SLASH, STAR, BANG,
+    BANG_EQUAL, EQUAL, EQUAL_EQUAL, GREATER, GREATER_EQUAL,
+    LESS, LESS_EQUAL,
+
+    // Literals.
+    IDENTIFIER, STRING, NUMBER,
+
+    // Keywords.
+    AND, CLASS, ELSE, FALSE, FUN, FOR, IF, NIL, OR,
+    PRINT, RETURN, SUPER, THIS, TRUE, VAR, WHILE, _EOF,
+};
+
+class Object{};
+
+class Token {
+private:
+    TokenType type;
+    std::string lexeme;
+    Object literal;
+    int line;
+
+public:
+    Token(TokenType _type, std::string _lexeme, Object _literal, int _line):
+            type{_type},
+            lexeme(_lexeme),
+            literal(_literal),
+            line{_line}
+    {}
+
+    std::string toString(){
+        std::stringstream ss;
+        ss << "type" << " " << lexeme << " " << "literal";
+        return ss.str();
+    }
+
+    std::string getLexeme(){
+        return lexeme;
+    }
+
+    TokenType getType(){
+        return type;
+    }
+};
+
+#endif //INTERPRETER_TOKENS_H

main.cpp

@@ -1,6 +1,6 @@
 #include <iostream>
 #include <vector>
-#include "lib/scanner.h"
+#include "lib/parser/scanner.h"
 
 using namespace std;
 

tests/lox_main_test.cpp

@@ -3,7 +3,7 @@
 //
 
 #include <catch2/catch_test_macros.hpp>
-#include "../lib/scanner.h"
+#include "../lib/parser/scanner.h"
 #include "catch2/matchers/catch_matchers_vector.hpp"
 
 unsigned int Factorial( unsigned int number ) {
@@ -62,5 +62,14 @@ TEST_CASE("Testing Scanner multiple operators !*+-/=<> with comments input"){
     auto t2 = tokens[1];
     REQUIRE( t2.getLexeme() == "*" );
 
+    for(auto tk : tokens){
+
+        if(tk.getType() == TokenType::_EOF)
+            break;
+
+        auto pos = code.find_first_of(tk.getLexeme());
+        REQUIRE( pos != std::string::npos );
+    }
+
     REQUIRE( tokens.size() == 11 );
 }