lexer debug utils

makefile

@@ -29,4 +29,7 @@ clean:
 run: $(TARGET)
 	$(TARGET)
 
-.PHONY: clean run
+test: $(TARGET)
+	$(TARGET) --file test/lexer/1.in --dump-tokens
+
+.PHONY: clean run test 

src/main.cpp

@@ -1,10 +1,63 @@
 #include "parser.hpp"
 #include "scanner.hpp"
 #include <iostream>
+#include <fstream>
 
-int main()
+// for debugging, and unit testing the lexer
+void dump_tokens(Scanner &scanner)
 {
-    Scanner scanner(std::cin);
+    yy::parser::semantic_type lval;
+
+    while (true)
+    {
+        int tok = scanner.lex(&lval);
+        if (tok == 0)
+        {
+            std::cout << "EOF\n";
+            break;
+        }
+
+        using Parser = yy::parser;
+        using kind = Parser::symbol_kind_type;
+
+        std::cout << Parser::symbol_name(static_cast<kind>(tok));
+
+        if (tok == Parser::token::TOK_WORD)
+        {
+            std::cout << " " << lval.as<std::string>();
+        }
+
+        std::cout << "\n";
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    std::istream *input_stream = &std::cin;
+    std::ifstream file_stream;
+
+    bool token_dump = false;
+
+    for (int i = 1; i < argc; ++i)
+    {
+        std::string arg = argv[i];
+        if (arg == "--file" && i + 1 < argc)
+        {
+            file_stream.open(argv[i + 1]);
+            input_stream = &file_stream;
+        }
+        else if (arg == "--dump-tokens")
+        {
+            token_dump = true;
+        }
+    }
+
+    Scanner scanner(*input_stream);
+    if (token_dump)
+    {
+        dump_tokens(scanner);
+        return 0;
+    }
     yy::parser parser(&scanner);
     return parser.parse();
 }

src/parser.y

@@ -2,9 +2,12 @@
 %require "3.0"
 %defines
 %define api.value.type variant
+%define api.token.raw
 %define api.parser.class { parser }
 %define api.namespace { yy }
 %define api.token.prefix {TOK_}
+%define parse.trace
+
 
 %code requires {
     #include <string>

test/lexer/1.in

@@ -0,0 +1,3 @@
+hello = 42
+name = "world"
+x + y - z

test/lexer/2.in

@@ -0,0 +1,5 @@
+def function_name():
+    if condition:
+        return value
+    for item in list:
+        print(item)

test/lexer/3.in

@@ -0,0 +1,3 @@
+x == y != z < > <= >=
+a + b - c * d / e % f
+( ) [ ] { } , : . ;

test/lexer/4.in

@@ -0,0 +1,3 @@
+42 3.14 0xFF 0b1010
+"hello" 'world' """multiline"""
+'test' "escaped\"quote"

test/lexer/5.in

@@ -0,0 +1,8 @@
+def calculate(a, b=10):
+    result = (a + b) * 2
+    if result > 100:
+        print("Large result:", result)
+    return result
+
+# This is a comment
+x = calculate(5, 15)

test/lexer/6.in

@@ -0,0 +1,5 @@
+_private = "valid"
+var123 = 456
+CamelCase = True
+snake_case = False
+very_long_variable_name_123 = "test"