lexer debug utils

src/main.cpp

@@ -1,10 +1,63 @@
 #include "parser.hpp"
 #include "scanner.hpp"
 #include <iostream>
+#include <fstream>
 
-int main()
+// for debugging, and unit testing the lexer
+void dump_tokens(Scanner &scanner)
 {
-    Scanner scanner(std::cin);
+    yy::parser::semantic_type lval;
+
+    while (true)
+    {
+        int tok = scanner.lex(&lval);
+        if (tok == 0)
+        {
+            std::cout << "EOF\n";
+            break;
+        }
+
+        using Parser = yy::parser;
+        using kind = Parser::symbol_kind_type;
+
+        std::cout << Parser::symbol_name(static_cast<kind>(tok));
+
+        if (tok == Parser::token::TOK_WORD)
+        {
+            std::cout << " " << lval.as<std::string>();
+        }
+
+        std::cout << "\n";
+    }
+}
+
+int main(int argc, char *argv[])
+{
+    std::istream *input_stream = &std::cin;
+    std::ifstream file_stream;
+
+    bool token_dump = false;
+
+    for (int i = 1; i < argc; ++i)
+    {
+        std::string arg = argv[i];
+        if (arg == "--file" && i + 1 < argc)
+        {
+            file_stream.open(argv[i + 1]);
+            input_stream = &file_stream;
+        }
+        else if (arg == "--dump-tokens")
+        {
+            token_dump = true;
+        }
+    }
+
+    Scanner scanner(*input_stream);
+    if (token_dump)
+    {
+        dump_tokens(scanner);
+        return 0;
+    }
     yy::parser parser(&scanner);
     return parser.parse();
 }

src/parser.y

@@ -2,9 +2,12 @@
 %require "3.0"
 %defines
 %define api.value.type variant
+%define api.token.raw
 %define api.parser.class { parser }
 %define api.namespace { yy }
 %define api.token.prefix {TOK_}
+%define parse.trace
+
 
 %code requires {
     #include <string>