proper eof handling

src/lexer.l

@@ -23,6 +23,7 @@ in              { return yy::parser::token::TOK_KW_IF;}
 \"              { yymore(); BEGIN(STRING); }
 [0-9]+          { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
 [_a-zA-Z]+      { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_IDENTIFIER; }
+<<EOF>>         { return yy::parser::token::TOK_EOF; }
 .               { /* the last rule, ignore anything else (there should be nothing) */}
 
 <STRING>\"\"    { yymore(); }

src/main.cpp

@@ -13,9 +13,9 @@ void dump_tokens(Scanner &scanner)
     while (true)
     {
         int tok = scanner.lex(&lval);
-        if (tok == Parser::token::TOK_YYEOF)
+        if (tok == Parser::token::TOK_EOF)
         {
-            std::cout << "NEWLINE\nEOF";
+            std::cout << Parser::symbol_name(static_cast<kind>(tok));
             break;
         }
 

test/lexer/1.in

@@ -1,3 +1,3 @@
 hello = 42
 name = "world"
-x + y - z
+x + y - z

test/lexer/2.in

@@ -2,4 +2,4 @@ def function_name():
     if condition:
         return value
     for item in list:
-        print(item)
+        print(item)

test/lexer/3.in

@@ -1,3 +1,3 @@
 x == y != z < > <= >=
 a + b - c * d / e % f
-( ) [ ] { } , : . ;
+( ) [ ] { } , : . ;

test/lexer/4.in

@@ -1,3 +1,3 @@
 42 3.14 0xFF 0b1010
 "hello" 'world' """multiline"""
-'test' "escaped\"quote"
+'test' "escaped\"quote"

test/lexer/5.in

@@ -5,4 +5,4 @@ def calculate(a, b=10):
     return result
 
 # This is a comment
-x = calculate(5, 15)
+x = calculate(5, 15)

test/lexer/6.in

@@ -2,4 +2,4 @@ _private = "valid"
 var123 = 456
 CamelCase = True
 snake_case = False
-very_long_variable_name_123 = "test"
+very_long_variable_name_123 = "test"