diff --git a/src/lexer.l b/src/lexer.l index 0b52e2a..f67c656 100644 --- a/src/lexer.l +++ b/src/lexer.l @@ -13,7 +13,8 @@ ^[ \t]*\r?\n { /* Ignore lines containing whitespace only */ } ^[ \t]*#.*\n { /* Ignore comment-only lines */ } -^[ \t]+ { + +^[ \t]* { int col = indent_stack.calculateColumn(yytext, yyleng); auto action = indent_stack.updateIndentation(col); @@ -21,6 +22,13 @@ if (action == IndentAction::Dedent) return yy::parser::token::TOK_DEDENT; } +^. { + // Line starts with non-whitespace (column 0) + auto action = indent_stack.updateIndentation(0); + yyless(0); + if (action == IndentAction::Dedent) return yy::parser::token::TOK_DEDENT; +} + \n { return yy::parser::token::TOK_NEWLINE; } def { return yy::parser::token::TOK_KW_DEF; } diff --git a/test/lexer/06.in b/test/lexer/06.in index 3ae113c..4242a87 100644 --- a/test/lexer/06.in +++ b/test/lexer/06.in @@ -3,6 +3,6 @@ def calculate(a, b=10): if result > 100: print("Large result:", result) return result - -# This is a comment + +# this is a comment x = calculate(5, 15) diff --git a/test/lexer/06.out b/test/lexer/06.out index a482919..d62e9e9 100644 --- a/test/lexer/06.out +++ b/test/lexer/06.out @@ -39,7 +39,6 @@ KW_RETURN IDENTIFIER result NEWLINE DEDENT -NEWLINE IDENTIFIER x ASSIGN IDENTIFIER calculate @@ -49,4 +48,4 @@ COMMA NUMBER 15 R_PAREN NEWLINE -EOF +EOF \ No newline at end of file diff --git a/test/lexer/07.out b/test/lexer/07.out index bb61e99..8dfccba 100644 --- a/test/lexer/07.out +++ b/test/lexer/07.out @@ -18,4 +18,4 @@ IDENTIFIER very_long_variable_name_123 ASSIGN STRING "test" NEWLINE -EOF +EOF \ No newline at end of file diff --git a/test/lexer/08.out b/test/lexer/08.out index 098abc4..fe81bd9 100644 --- a/test/lexer/08.out +++ b/test/lexer/08.out @@ -16,4 +16,4 @@ NEWLINE KW_NOT IDENTIFIER flag NEWLINE -EOF +EOF \ No newline at end of file