From 87695ebb20b6991b4b04e34ebd47a76fe0bff15f Mon Sep 17 00:00:00 2001
From: bronku <bronk.jakub@proton.me>
Date: Sat, 22 Nov 2025 00:31:00 +0100
Subject: [PATCH] some progress towards test 2

---
 src/lexer.l      | 15 ++++++++++++---
 src/main.cpp     |  9 ++++-----
 src/parser.y     |  4 +++-
 test/lexer/1.out |  3 ++-
 test/lexer/2.out | 31 +++++++++++++++++++++++++++++++
 5 files changed, 52 insertions(+), 10 deletions(-)
 create mode 100644 test/lexer/2.out

diff --git a/src/lexer.l b/src/lexer.l
index b18dcb6..b9209b6 100644
--- a/src/lexer.l
+++ b/src/lexer.l
@@ -8,15 +8,24 @@
 %}
 
 %%
-[0-9]+          { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
-[_a-zA-Z]+      { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_IDENTIFIER; }
+def             { return yy::parser::token::TOK_KW_DEF;}
+if              { return yy::parser::token::TOK_KW_IF;}
+return          { return yy::parser::token::TOK_KW_RETURN;}
+for             { return yy::parser::token::TOK_KW_FOR;}
+in              { return yy::parser::token::TOK_KW_IF;}
 \n              { return yy::parser::token::TOK_NEWLINE; }
+\(              { return yy::parser::token::TOK_L_PAREN; }
+\)              { return yy::parser::token::TOK_R_PAREN; }
+\:              { return yy::parser::token::TOK_COLON; }
 \=              { return yy::parser::token::TOK_EQUALS; }
 \+              { return yy::parser::token::TOK_PLUS; }
 \-              { return yy::parser::token::TOK_MINUS; }
 \"              { yymore(); BEGIN(STRING); }
+[0-9]+          { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_NUMBER; }
+[_a-zA-Z]+      { yylval->as<std::string>() = yytext; return yy::parser::token::TOK_IDENTIFIER; }
+.               { /* the last rule, ignore anything else (there should be nothing) */}
+
 <STRING>\"\"    { yymore(); }
 <STRING>\"     { BEGIN(INITIAL); yylval->as<std::string>() = yytext; return yy::parser::token::TOK_STRING; }
 <STRING>.       { yymore(); }
-.               { /* the last rule, ignore anything else (there should be nothing) */}
 %%
\ No newline at end of file
diff --git a/src/main.cpp b/src/main.cpp
index 67ca52f..d37685f 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -6,20 +6,19 @@
 // for debugging, and unit testing the lexer
 void dump_tokens(Scanner &scanner)
 {
+    using Parser = yy::parser;
+    using kind = Parser::symbol_kind_type;
     yy::parser::semantic_type lval;
 
     while (true)
     {
         int tok = scanner.lex(&lval);
-        if (tok == 0)
+        if (tok == Parser::token::TOK_YYEOF)
         {
-            std::cout << "EOF\n";
+            std::cout << "NEWLINE\nEOF";
             break;
         }
 
-        using Parser = yy::parser;
-        using kind = Parser::symbol_kind_type;
-
         std::cout << Parser::symbol_name(static_cast<kind>(tok));
 
         if (tok == Parser::token::TOK_IDENTIFIER || tok == Parser::token::TOK_NUMBER || tok == Parser::token::TOK_STRING)
diff --git a/src/parser.y b/src/parser.y
index f9a182c..dcd994a 100644
--- a/src/parser.y
+++ b/src/parser.y
@@ -27,7 +27,9 @@
 %token <std::string> IDENTIFIER
 %token <std::string> STRING
 %token <int> NUMBER
-%token EQUALS NEWLINE PLUS MINUS EOF
+%token EQUALS PLUS MINUS L_PAREN R_PAREN COLON
+%token KW_DEF KW_IF KW_RETURN KW_FOR KW_IN
+%token EOF NEWLINE INDENT_UP INDENT_DOWN
 
 
 %%
diff --git a/test/lexer/1.out b/test/lexer/1.out
index e9301a8..b3b5d8a 100644
--- a/test/lexer/1.out
+++ b/test/lexer/1.out
@@ -11,4 +11,5 @@ PLUS
 IDENTIFIER y
 MINUS
 IDENTIFIER z
-EOF
+NEWLINE
+EOF
\ No newline at end of file
diff --git a/test/lexer/2.out b/test/lexer/2.out
new file mode 100644
index 0000000..c58341f
--- /dev/null
+++ b/test/lexer/2.out
@@ -0,0 +1,31 @@
+KW_DEF
+IDENTIFIER function_name
+L_PAREN
+R_PAREN
+COLON
+NEWLINE
+INDENT_UP
+KW_IF
+IDENTIFIER condition
+COLON
+NEWLINE
+INDENT_UP
+KW_RETURN
+IDENTIFIER value
+NEWLINE
+INDENT_DOWN
+KW_FOR
+IDENTIFIER item
+KW_IN
+IDENTIFIER list
+COLON
+NEWLINE
+INDENT_UP
+IDENTIFIER print
+L_PAREN
+IDENTIFIER item
+R_PAREN
+NEWLINE
+INDENT_DOWN
+INDENT_DOWN
+EOF
\ No newline at end of file