From f0f71075f1a2d87f1472f637a8cf72e7f2f0f203 Mon Sep 17 00:00:00 2001 From: bronku Date: Thu, 27 Nov 2025 17:15:29 +0100 Subject: [PATCH] newline at eof fix --- src/parser.y | 2 +- src/scanner.hpp | 62 +++++++++++++++++++++++++++++-------------------- 2 files changed, 38 insertions(+), 26 deletions(-) diff --git a/src/parser.y b/src/parser.y index 6875349..31d1df9 100644 --- a/src/parser.y +++ b/src/parser.y @@ -69,7 +69,7 @@ stmt_list: ; stmt: - expr { $$ = std::make_unique(std::move($1)); } + expr NEWLINE { $$ = std::make_unique(std::move($1)); } ; expr: diff --git a/src/scanner.hpp b/src/scanner.hpp index b914320..83e612a 100644 --- a/src/scanner.hpp +++ b/src/scanner.hpp @@ -11,6 +11,7 @@ class Scanner : public yyFlexLexer { bool eof_emitted = false; + bool last_was_newline = true; public: IndentStack indent_stack; @@ -18,40 +19,51 @@ public: int lex(yy::parser::semantic_type *lval) { - if (indent_stack.popDedent()) - { - return yy::parser::token::TOK_DEDENT; - } - this->yylval = lval; - int token = yylex(); - - // std::cout << "lex: " << yy::parser::symbol_name(static_cast(token)) << '\n'; - - if (token == yy::parser::token::TOK_EOF) - { - auto result = indent_stack.closeBlock(); - - if (result == IndentAction::Dedent) - { - return yy::parser::token::TOK_DEDENT; - } - } - - if (token != yy::parser::token::TOK_EOF) - { - return token; - } + using Parser = yy::parser; + using kind = Parser::symbol_kind_type; if (eof_emitted) { + //std::cout << "lex: 0\n"; return 0; } - eof_emitted = true; + if (indent_stack.popDedent()) + { + //std::cout << "lex: dedent\n"; + return yy::parser::token::TOK_DEDENT; + } + + this->yylval = lval; + int token = yylex(); + + if (token == yy::parser::token::TOK_EOF) + { + // Emit synthetic NEWLINE if needed + if (!last_was_newline) + { + last_was_newline = true; + //std::cout << "lex: newline\n"; + return yy::parser::token::TOK_NEWLINE; + } + + auto result = indent_stack.closeBlock(); + if (result == IndentAction::Dedent) + { + //std::cout << "lex: dedent\n"; + return yy::parser::token::TOK_DEDENT; + } + + eof_emitted = true; + } + + last_was_newline = (token == yy::parser::token::TOK_NEWLINE); + + //std::cout << "lex: " << Parser::symbol_name(static_cast(token)) << '\n'; return token; } private: virtual int yylex() override; yy::parser::semantic_type *yylval; -}; \ No newline at end of file +};