Do not consider indent/dedent as tokens in python

thu-cs-lab · Jul 11, 2024 · e660024 · e660024
1 parent 933a0ca
commit e660024
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/core/src/lang/tokenizer/python.rs b/core/src/lang/tokenizer/python.rs
@@ -34,8 +34,11 @@ pub fn tokenize_str(content: &str) -> anyhow::Result<Vec<Token>> {
             // skip newline or comments
             Comment(_) => continue,
             Newline => continue, // 6
-            Indent => 7,
-            Dedent => 8,
+            // skip indentation
+            Indent => continue,
+            Dedent => continue,
+            // Indent => 7,
+            // Dedent => 8,
             StartModule => 9,
             StartInteractive => 10,
             StartExpression => 11,