some progress towards test 2

src/lexer.l

@@ -8,15 +8,24 @@
 %}
 
 %%
-[0-9]+          { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
+def             { return yy::parser::token::TOK_KW_DEF;}
-[_a-zA-Z]+      { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_IDENTIFIER; }
+if              { return yy::parser::token::TOK_KW_IF;}
+return          { return yy::parser::token::TOK_KW_RETURN;}
+for             { return yy::parser::token::TOK_KW_FOR;}
+in              { return yy::parser::token::TOK_KW_IF;}
 \n              { return yy::parser::token::TOK_NEWLINE; }
+\(              { return yy::parser::token::TOK_L_PAREN; }
+\)              { return yy::parser::token::TOK_R_PAREN; }
+\:              { return yy::parser::token::TOK_COLON; }
 \=              { return yy::parser::token::TOK_EQUALS; }
 \+              { return yy::parser::token::TOK_PLUS; }
 \-              { return yy::parser::token::TOK_MINUS; }
 \"              { yymore(); BEGIN(STRING); }
+[0-9]+          { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
+[_a-zA-Z]+      { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_IDENTIFIER; }
+.               { /* the last rule, ignore anything else (there should be nothing) */}
+
 <STRING>\"\"    { yymore(); }
 <STRING>\"     { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
 <STRING>.       { yymore(); }
-.               { /* the last rule, ignore anything else (there should be nothing) */}
 %%

src/main.cpp

@@ -6,20 +6,19 @@
 // for debugging, and unit testing the lexer
 void dump_tokens(Scanner &scanner)
 {
+    using Parser = yy::parser;
+    using kind = Parser::symbol_kind_type;
     yy::parser::semantic_type lval;
 
     while (true)
     {
         int tok = scanner.lex(&lval);
-        if (tok == 0)
+        if (tok == Parser::token::TOK_YYEOF)
         {
-            std::cout << "EOF\n";
+            std::cout << "NEWLINE\nEOF";
             break;
         }
 
-        using Parser = yy::parser;
-        using kind = Parser::symbol_kind_type;
-
         std::cout << Parser::symbol_name(static_cast<kind>(tok));
 
         if (tok == Parser::token::TOK_IDENTIFIER || tok == Parser::token::TOK_NUMBER || tok == Parser::token::TOK_STRING)

src/parser.y

@@ -27,7 +27,9 @@
 %token <std::string> IDENTIFIER
 %token <std::string> STRING
 %token <int> NUMBER
-%token EQUALS NEWLINE PLUS MINUS EOF
+%token EQUALS PLUS MINUS L_PAREN R_PAREN COLON
+%token KW_DEF KW_IF KW_RETURN KW_FOR KW_IN
+%token EOF NEWLINE INDENT_UP INDENT_DOWN
 
 
 %%

test/lexer/1.out

@@ -11,4 +11,5 @@ PLUS
 IDENTIFIER y
 MINUS
 IDENTIFIER z
+NEWLINE
 EOF

test/lexer/2.out

@@ -0,0 +1,31 @@
+KW_DEF
+IDENTIFIER function_name
+L_PAREN
+R_PAREN
+COLON
+NEWLINE
+INDENT_UP
+KW_IF
+IDENTIFIER condition
+COLON
+NEWLINE
+INDENT_UP
+KW_RETURN
+IDENTIFIER value
+NEWLINE
+INDENT_DOWN
+KW_FOR
+IDENTIFIER item
+KW_IN
+IDENTIFIER list
+COLON
+NEWLINE
+INDENT_UP
+IDENTIFIER print
+L_PAREN
+IDENTIFIER item
+R_PAREN
+NEWLINE
+INDENT_DOWN
+INDENT_DOWN
+EOF